diff --git a/.gitmodules b/.gitmodules index 64714b3..5625b91 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,7 @@ path = vendor/tinycortex url = https://github.com/tinyhumansai/tinycortex.git branch = main +[submodule "vendor/tinyagents"] + path = vendor/tinyagents + url = https://github.com/tinyhumansai/tinyagents + branch = main diff --git a/Cargo.lock b/Cargo.lock index ca4dcf9..ab07c58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,6 +49,71 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "axum-macros", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "sync_wrapper", + "tokio", + "tokio-tungstenite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "base64" version = "0.22.1" @@ -79,6 +144,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -98,6 +172,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -121,7 +197,7 @@ checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -187,6 +263,12 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "digest" version = "0.10.7" @@ -208,6 +290,37 @@ dependencies = [ "crypto-common 0.2.2", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "objc2", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -220,6 +333,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -232,6 +355,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.10" @@ -364,6 +493,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -373,11 +514,23 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", - "rand_core", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasm-bindgen", ] +[[package]] +name = "git2" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -392,19 +545,22 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "foldhash", -] [[package]] name = "hashlink" -version = "0.12.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.17.1", + "hashbrown 0.16.1", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "1.5.0" @@ -444,6 +600,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hybrid-array" version = "0.4.14" @@ -466,6 +628,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -589,6 +752,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.104" @@ -606,6 +779,27 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libgit2-sys" +version = "0.18.7+1.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libredox" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" +dependencies = [ + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.38.2" @@ -617,6 +811,24 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "lock_api" version = "0.4.14" @@ -638,12 +850,24 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "mio" version = "1.2.2" @@ -664,12 +888,68 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-contacts" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b034b578389f89a85c055eacc8d8b368be5f04a6c1b07f672bf3aec21d0ef621" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "parking_lot" version = "0.12.5" @@ -711,6 +991,15 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -734,7 +1023,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -749,14 +1038,14 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand", + "rand 0.10.2", "rand_pcg", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -785,12 +1074,39 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.2" @@ -799,7 +1115,45 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -814,7 +1168,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -826,6 +1180,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "ref-cast" version = "1.0.26" @@ -937,14 +1302,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ "hashbrown 0.16.1", - "thiserror", + "thiserror 2.0.20", ] [[package]] name = "rusqlite" -version = "0.40.2" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +checksum = "1b3492ea85308705c3a5cc24fb9b9cf77273d30590349070db42991202b214c4" dependencies = [ "bitflags", "fallible-iterator", @@ -961,6 +1326,19 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.43" @@ -1102,6 +1480,26 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -1123,6 +1521,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1151,6 +1560,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "slab" version = "0.4.12" @@ -1222,13 +1641,46 @@ dependencies = [ "futures-core", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1245,18 +1697,17 @@ dependencies = [ [[package]] name = "tinyagents" version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d9746cb9ff6f37646b2c91eebe3dee775a56083d21ed41446a54c930efaf056" dependencies = [ "async-trait", "bytes", "chrono", "futures", "reqwest", + "rusqlite", "serde", "serde_json", "sha2 0.11.0", - "thiserror", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -1268,20 +1719,26 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "dirs", "futures", + "git2", + "hex", "log", "parking_lot", - "rand", + "rand 0.10.2", "regex", + "reqwest", "rusqlite", "schemars", "serde", "serde_json", "sha2 0.10.9", - "thiserror", + "thiserror 2.0.20", "tinyagents", "tinycortex-api", - "toml", + "tokio", + "toml 0.8.23", + "tracing", "uuid", "walkdir", ] @@ -1296,7 +1753,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "thiserror", + "thiserror 2.0.20", "uuid", ] @@ -1320,11 +1777,52 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "log", + "schemars", "serde", "serde_json", "sha2 0.11.0", - "thiserror", + "thiserror 2.0.20", + "toml 0.9.12+spec-1.1.0", + "uuid", +] + +[[package]] +name = "tinymemory-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "block2", + "chrono", + "dirs", + "futures", + "git2", + "log", + "objc2", + "objc2-contacts", + "objc2-foundation", + "parking_lot", + "rand 0.8.7", + "regex", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.20", + "tinyagents", + "tinycortex", + "tinycortex-api", + "tinymemory", + "tinymemory-api", + "tokio", + "tracing", + "url", "uuid", + "walkdir", ] [[package]] @@ -1363,7 +1861,9 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -1390,6 +1890,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -1405,37 +1917,78 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.4+spec-1.1.0" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ "indexmap", "serde_core", - "serde_spanned", - "toml_datetime", + "serde_spanned 1.1.1", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", ] [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_parser" version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow", + "winnow 1.0.4", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.1.2+spec-1.1.0" @@ -1524,6 +2077,22 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "sha1", + "thiserror 2.0.20", +] + [[package]] name = "typenum" version = "1.20.1" @@ -1630,6 +2199,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -1795,13 +2373,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1813,34 +2400,67 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1853,36 +2473,95 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zeroize" version = "1.9.0" diff --git a/Cargo.toml b/Cargo.toml index 711a854..efd6540 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] -members = [".", "api", "adapters/tinycortex"] -default-members = [".", "api", "adapters/tinycortex"] +members = [".", "api", "core", "adapters/tinycortex"] +default-members = [".", "api", "core", "adapters/tinycortex"] # `vendor/` holds engine submodules (tinycortex, tinybus), each of which is its # own workspace with its own lockfile. Same exclusion `vendor/tinycortex` uses # for its own nested vendor directory. @@ -93,6 +93,22 @@ private_intra_doc_links = "warn" # of this workspace resolve them to the nested `vendor/` submodules. [patch.crates-io] tinycortex = { path = "vendor/tinycortex" } +tinycortex-api = { path = "vendor/tinycortex/api" } +# `tinymemory-core`'s summariser and embedder factory name tinyagents' chat and +# embedding model traits. It is not published, so a standalone build of this +# workspace has to be told where it is — hence its own submodule alongside +# tinycortex and tinybus. +# +# Deliberately NOT `vendor/tinycortex/vendor/tinyagents`: the engine pins v2.1.0 +# there, which predates `RECOMMENDED_OLLAMA_CONTEXT_TOKENS` that the embedder +# factory needs. Reaching through another submodule's pin also makes this +# crate's dependency an accident of the engine's, which is not a relationship +# worth having. +# +# A host that embeds this crate patches tinyagents itself — patch tables only +# apply from the workspace root being built — so this entry affects standalone +# builds and `cargo test` here, and nothing downstream. +tinyagents = { path = "vendor/tinyagents" } [profile.release] # Cross-crate optimization and smaller, faster binaries for release builds. diff --git a/api/Cargo.toml b/api/Cargo.toml index 7714275..dbcb753 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -22,6 +22,12 @@ description = "Stable public contracts for the TinyMemory memory system" # - `uuid` — `tool_memory::ToolMemoryRule::generate_id` (v4 bytes, nibble # encoded). Only the `v4` feature is needed here; the engine # crate additionally enables `serde`. +# - `schemars` — the `host::` config sections are still fields of the host's +# root `Config`, which derives `JsonSchema` to generate the +# settings schema the UI renders. Dropping the derive on the way +# down here would silently shrink that schema. `schemars` is pure +# Rust (serde + serde_json + dyn-clone + ref-cast) and carries +# none of the forbidden dependencies below. # # Nothing here may pull in `rusqlite`, `git2`, `reqwest`, `regex`, or an async # runtime. Guard with the FORWARD form, which is scoped to this package: @@ -36,8 +42,24 @@ description = "Stable public contracts for the TinyMemory memory system" anyhow = "1" async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } +# `log` is the zero-dependency logging facade, not an implementation. The +# `host::cloud_providers` legacy-field migration logs what it rewrote. +log = "0.4" serde = { version = "1", features = ["derive"] } serde_json = "1" +schemars = "1.2" sha2 = "0.11" thiserror = "2" uuid = { version = "1", features = ["v4"] } + +[dev-dependencies] +# The moved `host::` config sections are parsed from TOML in their own tests, +# exactly as the host parses them from `config.toml`. +toml = "0.9" + +[features] +default = [] +# `host::test_support::TestHostConfig` — a concrete, `Default`-able +# `MemoryHostConfig`. `tinymemory-core` enables this from its dev-dependencies; +# nothing enables it in a shipped build. +test-support = [] diff --git a/api/src/host/cloud_providers.rs b/api/src/host/cloud_providers.rs new file mode 100644 index 0000000..dfdd6c3 --- /dev/null +++ b/api/src/host/cloud_providers.rs @@ -0,0 +1,855 @@ +//! Cloud provider credential schema. +//! +//! Each entry in `Config::cloud_providers` represents one configured LLM +//! backend. Providers are keyed by a user-chosen `slug` (e.g. `"openai"`, +//! `"my-deepseek"`). The factory in `inference::provider::factory` +//! resolves workload-to-provider strings against this list at runtime using +//! the grammar `":"`. +//! +//! Legacy configs that use `type`/`default_model` are migrated in-memory on +//! load via `migrate_legacy_fields()`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BuiltinCloudProvider { + pub slug: &'static str, + pub label: &'static str, + pub endpoint: &'static str, + pub auth_style: AuthStyle, +} + +pub const BUILTIN_CLOUD_PROVIDERS: &[BuiltinCloudProvider] = &[ + BuiltinCloudProvider { + slug: "openhuman", + label: "OpenHuman", + endpoint: "https://api.openhuman.ai/v1", + auth_style: AuthStyle::OpenhumanJwt, + }, + BuiltinCloudProvider { + slug: "openai", + label: "OpenAI", + endpoint: "https://api.openai.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "anthropic", + label: "Anthropic", + endpoint: "https://api.anthropic.com/v1", + auth_style: AuthStyle::Anthropic, + }, + BuiltinCloudProvider { + slug: "openrouter", + label: "OpenRouter", + endpoint: "https://openrouter.ai/api/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "orcarouter", + label: "OrcaRouter", + endpoint: "https://api.orcarouter.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "gmi", + label: "GMI", + endpoint: "https://api.gmi-serving.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "fireworks", + label: "Fireworks", + endpoint: "https://api.fireworks.ai/inference/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "moonshot", + label: "Kimi (Moonshot)", + endpoint: "https://api.moonshot.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "groq", + label: "Groq", + endpoint: "https://api.groq.com/openai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "mistral", + label: "Mistral", + endpoint: "https://api.mistral.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "deepseek", + label: "DeepSeek", + endpoint: "https://api.deepseek.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "together", + label: "Together AI", + endpoint: "https://api.together.xyz/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "google", + label: "Google Gemini", + endpoint: "https://generativelanguage.googleapis.com/v1beta/openai", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "cerebras", + label: "Cerebras", + endpoint: "https://api.cerebras.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "xai", + label: "xAI", + endpoint: "https://api.x.ai/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "huggingface", + label: "Hugging Face", + endpoint: "https://router.huggingface.co/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "nvidia", + label: "NVIDIA", + endpoint: "https://integrate.api.nvidia.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "zai", + label: "Z.AI", + endpoint: "https://api.z.ai/api/paas/v4", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "minimax", + label: "MiniMax", + // MiniMax exposes a full OpenAI-compatible surface at `/v1` + // (`/v1/chat/completions`, `/v1/models`). The previous `/anthropic` + // base + Anthropic auth pointed at MiniMax's Messages-protocol API, + // which OpenHuman does not speak — it only builds OpenAI-style + // `/chat/completions` and `/models` — so both chat and model-listing + // 404'd (`/anthropic/chat/completions`, `/anthropic/models`). The + // 404 on model-listing was Sentry TAURI-RUST-8X3. Use the `/v1` + // OpenAI surface with Bearer auth so both paths resolve. + endpoint: "https://api.minimax.io/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "stepfun", + label: "StepFun", + endpoint: "https://api.stepfun.ai/step_plan/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "kilocode", + label: "Kilo Code", + endpoint: "https://api.kilo.ai/api/gateway", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "deepinfra", + label: "DeepInfra", + endpoint: "https://api.deepinfra.com/v1/openai", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "novita", + label: "Novita", + endpoint: "https://api.novita.ai/v3/openai", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "venice", + label: "Venice", + endpoint: "https://api.venice.ai/api/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "vercel-ai-gateway", + label: "Vercel AI Gateway", + endpoint: "https://ai-gateway.vercel.sh/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "sumopod", + label: "SumoPod", + endpoint: "https://ai.sumopod.com/v1", + auth_style: AuthStyle::Bearer, + }, + BuiltinCloudProvider { + slug: "modelscope", + label: "ModelScope", + endpoint: "https://api-inference.modelscope.cn/v1", + auth_style: AuthStyle::Bearer, + }, +]; + +fn builtin_cloud_provider(type_str: &str) -> Option<&'static BuiltinCloudProvider> { + BUILTIN_CLOUD_PROVIDERS + .iter() + .find(|provider| provider.slug == type_str) +} + +/// Whether `slug` matches a built-in cloud provider preset. +/// +/// The chat factory uses this to decide capability defaults (e.g. whether the +/// provider exposes the OpenAI Responses API) only for providers we ship and +/// therefore know the API surface of. Custom / user-defined slugs are treated +/// as unknown and keep the permissive defaults. +pub fn is_builtin_cloud_slug(slug: &str) -> bool { + builtin_cloud_provider(slug).is_some() +} + +/// Whether a built-in cloud provider exposes the OpenAI **Responses API** +/// (`/v1/responses`). +/// +/// Only OpenAI's first-party endpoint serves `/responses`; every other built-in +/// preset (DeepSeek, Groq, Mistral, Fireworks, …) is chat-completions-only. +/// Enabling the chat-completions-404 → `/responses` fallback for those +/// guarantees a second 404 against an endpoint that does not exist, which floods +/// Sentry with an empty-body `" Responses API error:"` event +/// (TAURI-RUST-5EN — same class as the local-provider TAURI-RUST-59Y fix). The +/// factory consults this to build chat-completions-only built-ins with +/// `new_no_responses_fallback`. +/// +/// Custom / unknown slugs are intentionally NOT covered here (see +/// [`is_builtin_cloud_slug`]): a user-defined OpenAI-compatible endpoint may be +/// a genuine OpenAI proxy that does support `/responses`, so the factory keeps +/// the fallback for those. +pub fn builtin_cloud_supports_responses_api(slug: &str) -> bool { + matches!(slug, "openai") +} + +/// Extract the lowercased authority host from an endpoint URL, dropping the +/// scheme, any userinfo, the port, and the path. Returns `None` when no host +/// can be parsed. Tolerant of a missing scheme and of IPv6 literals. +pub fn endpoint_host(endpoint: &str) -> Option { + let s = endpoint.trim(); + // Drop the scheme (`https://…`); tolerate a bare `host/path` form. + let after_scheme = s.split_once("://").map(|(_, rest)| rest).unwrap_or(s); + // The authority ends at the first path / query / fragment delimiter. + let authority = after_scheme + .split(['/', '?', '#']) + .next() + .unwrap_or(after_scheme); + // Strip any `user:pass@` userinfo prefix. + let host_port = authority + .rsplit_once('@') + .map(|(_, host)| host) + .unwrap_or(authority); + // Strip the port, handling bracketed IPv6 literals (`[::1]:8080`). + let host = if let Some(rest) = host_port.strip_prefix('[') { + rest.split_once(']').map(|(h, _)| h).unwrap_or(rest) + } else { + host_port + .rsplit_once(':') + .map(|(h, _)| h) + .unwrap_or(host_port) + }; + let host = host.trim().to_ascii_lowercase(); + (!host.is_empty()).then_some(host) +} + +/// Whether `host` is the authority host of any built-in cloud **inference** +/// provider (e.g. `openrouter.ai`, `api.openai.com`, `api.groq.com`). +/// +/// Derived entirely from [`BUILTIN_CLOUD_PROVIDERS`] so the set stays in sync +/// with the provider registry. `host` is compared case-insensitively against +/// each preset's [`endpoint_host`]. +/// +/// # Why this exists +/// +/// `config.api_url` is overloaded: it is the chat/inference endpoint, but +/// `api::config::effective_backend_api_url` also reuses it as the +/// OpenHuman **backend** base for team/billing/auth calls. A BYO user who +/// points `api_url` at a provider's canonical base (`https://openrouter.ai/api/v1`) +/// would otherwise have every backend domain call routed to the inference host +/// → 400/404 (TAURI-RUST-HW1: 4932 `GET /teams/me/usage` 400s from `openrouter.ai`). +/// The backend-URL resolver uses this to treat such hosts as non-backend and +/// fall back to the default backend chain — the cloud analogue of the local-AI +/// guard that fixed the Ollama case (OPENHUMAN-TAURI-51/-80/-7Z). +pub fn host_is_builtin_cloud_provider(host: &str) -> bool { + let host = host.trim().to_ascii_lowercase(); + if host.is_empty() { + return false; + } + BUILTIN_CLOUD_PROVIDERS + .iter() + .any(|p| endpoint_host(p.endpoint).as_deref() == Some(host.as_str())) +} + +/// Whether an endpoint **host** is a known cloud host that does NOT serve the +/// OpenAI Responses API (`/v1/responses`) — i.e. it is chat-completions-only, +/// regardless of which user slug points at it. +/// +/// Derived entirely from [`BUILTIN_CLOUD_PROVIDERS`]: a host is chat-only when +/// some built-in preset uses it AND no preset at that host advertises the +/// Responses API (only OpenAI's `api.openai.com` does). This closes the +/// custom-slug gap behind the builtin-slug gate +/// ([`builtin_cloud_supports_responses_api`]): a user slug pointed at, e.g., +/// `integrate.api.nvidia.com` must never attempt `/responses` (TAURI-RUST-5A1), +/// while a genuinely unknown proxy host keeps the permissive fallback so a real +/// OpenAI proxy still gets `/responses`. +pub fn endpoint_host_is_chat_completions_only(endpoint: &str) -> bool { + let Some(host) = endpoint_host(endpoint) else { + return false; + }; + let mut matched_chat_only = false; + for provider in BUILTIN_CLOUD_PROVIDERS { + if endpoint_host(provider.endpoint).as_deref() == Some(host.as_str()) { + if builtin_cloud_supports_responses_api(provider.slug) { + // A Responses-capable built-in lives at this host → not chat-only. + return false; + } + matched_chat_only = true; + } + } + matched_chat_only +} + +/// Authentication header style for a cloud provider. +/// +/// Wire format is lowercase (e.g. `"bearer"`). Determines which HTTP headers +/// are attached when calling the provider's API. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum AuthStyle { + /// OpenAI-compatible: `Authorization: Bearer ` + #[default] + Bearer, + /// Anthropic: `x-api-key: ` + `anthropic-version: 2023-06-01` + Anthropic, + /// OpenHuman session JWT (injected by the backend provider, not stored here). + OpenhumanJwt, + /// No auth header — e.g. local Ollama. + None, +} + +impl AuthStyle { + pub fn as_str(&self) -> &'static str { + match self { + Self::Bearer => "bearer", + Self::Anthropic => "anthropic", + Self::OpenhumanJwt => "openhuman_jwt", + Self::None => "none", + } + } +} + +/// Endpoint config for one cloud LLM provider. +/// +/// **Note on secrets**: API keys are NOT stored on this struct. They live in +/// `auth-profiles.json` via `security::credentials::AuthService`, +/// keyed by `provider:` (falling back to bare `` for legacy +/// entries). The factory looks up the token at call time via +/// `inference::provider::factory::auth_key_for_slug`. +/// +/// ## Back-compat +/// +/// Old configs may have `type` and `default_model` fields. These are +/// tolerated on read (via `legacy_type` / `default_model`) but never written. +/// Call `migrate_legacy_fields()` after deserialising. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(default)] +pub struct CloudProviderCreds { + /// Opaque stable id, e.g. `"p_openai_a8c3f"`. Never shown in the UI. + /// Generated once by [`generate_provider_id`] and never changes. + pub id: String, + /// Routing key chosen by the user or seeded from the legacy type. + /// Lower-case alphanumeric + `-`. Must be unique per config and not in the + /// reserved list (see [`is_slug_reserved`]). The factory resolves + /// `":"` strings against this field. + pub slug: String, + /// Human-readable display label, supplied by the frontend. Not used in routing. + pub label: String, + /// OpenAI-compatible base URL (`/models`, `/chat/completions` etc. are appended). + pub endpoint: String, + /// Authentication header style. + pub auth_style: AuthStyle, + + // ── Back-compat: old `type` field ─────────────────────────────────────── + /// Legacy discriminator written by older builds. Read-only; never emitted. + #[serde(rename = "type", default, skip_serializing)] + pub legacy_type: Option, + + // ── Back-compat: old `default_model` field ────────────────────────────── + /// Legacy default model written by older builds. Read-only; never emitted. + #[serde(default, skip_serializing)] + pub default_model: Option, +} + +impl Default for CloudProviderCreds { + fn default() -> Self { + Self { + id: String::new(), + slug: String::new(), + label: String::new(), + endpoint: String::new(), + auth_style: AuthStyle::Bearer, + legacy_type: None, + default_model: None, + } + } +} + +/// Reserved slugs that may not be used for user-configured providers. +/// These are sentinels in the factory's routing grammar. +/// +/// `ollama` is deliberately NOT reserved: the AI settings panel registers an +/// `ollama` `cloud_providers` entry so `list_configured_models` can resolve +/// the user's chosen base_url for the model dropdown. The factory's chat +/// routing is unaffected — the `ollama:` prefix branch in +/// `factory::create_chat_provider_from_string` fires before the +/// `:` cloud-provider lookup, so a synthetic `ollama` entry +/// never reaches `make_cloud_provider_by_slug`. When no `cloud_providers` +/// row exists (config drift, upgrade from a build that only persisted +/// `config.local_ai.base_url`, flush-vs-probe race), +/// `inference::provider::ops::list_configured_models` +/// falls back to a synthetic entry via `synthesize_local_runtime_entry` +/// (Sentry TAURI-RUST-28Z fix). The same fallback applies to `lmstudio`. +pub fn is_slug_reserved(s: &str) -> bool { + matches!(s.trim(), "" | "cloud" | "openhuman" | "pid") +} + +/// Apply legacy field migration in-place. +/// +/// Idempotent: only fills in empty fields from the legacy `type`/`default_model` +/// values. Safe to call on already-migrated entries. +pub fn migrate_legacy_fields(entry: &mut CloudProviderCreds) { + let legacy_type = entry.legacy_type.clone().unwrap_or_default(); + let lt = legacy_type.trim(); + + // Slug from legacy type when missing. + if entry.slug.is_empty() && !lt.is_empty() { + entry.slug = lt.to_string(); + log::debug!( + "[config][cloud_providers] migrated slug from legacy type='{}' id={}", + lt, + entry.id + ); + } + + // Label from static map when missing. + if entry.label.is_empty() { + entry.label = legacy_label_for(if entry.slug.is_empty() { + lt + } else { + &entry.slug + }) + .to_string(); + log::debug!( + "[config][cloud_providers] migrated label='{}' for slug='{}' id={}", + entry.label, + entry.slug, + entry.id + ); + } + + // Endpoint from legacy defaults when missing. + if entry.endpoint.is_empty() { + let ep = legacy_default_endpoint(lt); + if !ep.is_empty() { + entry.endpoint = ep.to_string(); + } + } + + // Auth style from legacy type when still at default Bearer. + if entry.auth_style == AuthStyle::Bearer { + if let Some(provider) = builtin_cloud_provider(lt) { + entry.auth_style = provider.auth_style; + } + } +} + +/// Map a legacy type string (or slug) to a human-readable label. +fn legacy_label_for(type_str: &str) -> &'static str { + builtin_cloud_provider(type_str) + .map(|provider| provider.label) + .unwrap_or("Custom") +} + +/// Map a legacy type string to its well-known default endpoint. +fn legacy_default_endpoint(type_str: &str) -> &'static str { + builtin_cloud_provider(type_str) + .map(|provider| provider.endpoint) + .unwrap_or("") +} + +/// Generate a short opaque id for a new provider entry. +/// +/// Format: `"p__<5 random alphanumerics>"`, e.g. `"p_openai_a8c3f"`. +/// The random suffix is not cryptographically strong — it only needs to be +/// unique within a single user's config file. +pub fn generate_provider_id(slug: &str) -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + // Cheap pseudo-random from timestamp nanoseconds — adequate for local + // config uniqueness without pulling in a PRNG crate. + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .subsec_nanos(); + let chars: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; + let mut suffix = String::with_capacity(5); + let mut seed = nanos as usize; + for _ in 0..5 { + suffix.push(chars[seed % chars.len()] as char); + seed = seed + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + seed = (seed >> 33) ^ seed; + } + // Sanitise slug to only alphanumeric + '-' for the id prefix. + let safe_slug: String = slug + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' { + c + } else { + '_' + } + }) + .take(20) + .collect(); + format!("p_{}_{}", safe_slug, suffix) +} + +// ── Back-compat type alias ────────────────────────────────────────────────── +// Kept so existing code that imports `CloudProviderType` compiles without +// sweeping changes. New code should use `AuthStyle` directly. + +/// Legacy discriminator enum. **Deprecated**: use `AuthStyle` on new entries. +/// Retained only to satisfy callers that still pattern-match on +/// `CloudProviderType` (e.g. the migration module). Will be removed once all +/// call sites are updated to slug-keyed lookups. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum CloudProviderType { + Openhuman, + Openai, + Anthropic, + Openrouter, + Orcarouter, + Custom, +} + +impl CloudProviderType { + /// Well-known default base URL for each provider type. + pub fn default_endpoint(&self) -> &'static str { + match self { + Self::Openhuman => "https://api.openhuman.ai/v1", + Self::Openai => "https://api.openai.com/v1", + Self::Anthropic => "https://api.anthropic.com/v1", + Self::Openrouter => "https://openrouter.ai/api/v1", + Self::Orcarouter => "https://api.orcarouter.ai/v1", + Self::Custom => "", + } + } + + /// Human-readable label used in logs and error messages. + pub fn label(&self) -> &'static str { + match self { + Self::Openhuman => "OpenHuman", + Self::Openai => "OpenAI", + Self::Anthropic => "Anthropic", + Self::Openrouter => "OpenRouter", + Self::Orcarouter => "OrcaRouter", + Self::Custom => "Custom", + } + } + + /// Lowercase wire-format string (matches JSON serialisation). + pub fn as_str(&self) -> &'static str { + match self { + Self::Openhuman => "openhuman", + Self::Openai => "openai", + Self::Anthropic => "anthropic", + Self::Openrouter => "openrouter", + Self::Orcarouter => "orcarouter", + Self::Custom => "custom", + } + } + + /// Corresponding `AuthStyle`. + pub fn auth_style(&self) -> AuthStyle { + match self { + Self::Openhuman => AuthStyle::OpenhumanJwt, + Self::Anthropic => AuthStyle::Anthropic, + _ => AuthStyle::Bearer, + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + builtin_cloud_supports_responses_api, endpoint_host, + endpoint_host_is_chat_completions_only, host_is_builtin_cloud_provider, + is_builtin_cloud_slug, is_slug_reserved, migrate_legacy_fields, AuthStyle, + CloudProviderCreds, BUILTIN_CLOUD_PROVIDERS, + }; + + #[test] + fn reserved_slugs() { + for s in ["", " ", "cloud", "openhuman", "pid"] { + assert!(is_slug_reserved(s), "{s:?} must stay reserved"); + } + } + + // Regression: `ollama` was previously reserved, which made the AI settings + // panel unable to persist an `ollama` cloud_providers entry — so the + // model-list dropdown failed with "no cloud provider with id or slug + // 'ollama' found". The factory's chat routing is unaffected by this + // change because the `ollama:` prefix branch fires before any + // cloud_providers lookup. + #[test] + fn ollama_and_lmstudio_are_not_reserved() { + assert!( + !is_slug_reserved("ollama"), + "ollama must be usable as a cloud_providers slug for the /models probe" + ); + assert!( + !is_slug_reserved("lmstudio"), + "lmstudio is a free-form OpenAI-compatible slug" + ); + } + + #[test] + fn builtin_cloud_provider_defaults_cover_phase_one_presets() { + for (slug, label, endpoint, auth_style) in [ + ( + "groq", + "Groq", + "https://api.groq.com/openai/v1", + AuthStyle::Bearer, + ), + ( + "deepseek", + "DeepSeek", + "https://api.deepseek.com/v1", + AuthStyle::Bearer, + ), + ( + "minimax", + "MiniMax", + "https://api.minimax.io/v1", + AuthStyle::Bearer, + ), + ( + "sumopod", + "SumoPod", + "https://ai.sumopod.com/v1", + AuthStyle::Bearer, + ), + ( + "modelscope", + "ModelScope", + "https://api-inference.modelscope.cn/v1", + AuthStyle::Bearer, + ), + ] { + let mut entry = CloudProviderCreds { + id: format!("p_{slug}"), + legacy_type: Some(slug.to_string()), + ..Default::default() + }; + migrate_legacy_fields(&mut entry); + + assert_eq!(entry.slug, slug); + assert_eq!(entry.label, label); + assert_eq!(entry.endpoint, endpoint); + assert_eq!(entry.auth_style, auth_style); + } + } + + #[test] + fn builtin_cloud_provider_slugs_are_unique() { + let mut slugs = std::collections::HashSet::new(); + for provider in BUILTIN_CLOUD_PROVIDERS { + assert!( + slugs.insert(provider.slug), + "duplicate built-in cloud provider slug {}", + provider.slug + ); + } + } + + #[test] + fn is_builtin_cloud_slug_matches_presets_only() { + for slug in ["openai", "deepseek", "groq", "mistral"] { + assert!(is_builtin_cloud_slug(slug), "{slug} is a built-in preset"); + } + for slug in ["my-proxy", "custom-openai", "totally-unknown", ""] { + assert!( + !is_builtin_cloud_slug(slug), + "{slug:?} is not a built-in preset" + ); + } + } + + #[test] + fn only_openai_builtin_exposes_responses_api() { + assert!(builtin_cloud_supports_responses_api("openai")); + for slug in ["deepseek", "groq", "mistral", "fireworks", "together"] { + assert!( + !builtin_cloud_supports_responses_api(slug), + "{slug} is chat-completions-only and must not advertise the Responses API" + ); + } + } + + /// Drift guard (TAURI-RUST-5EN): couple the capability helper to the + /// preset list so adding a new built-in that wrongly claims the Responses + /// API — or renaming `openai` — fails CI rather than silently re-enabling + /// the guaranteed-404 `/responses` fallback. OpenAI's first-party endpoint + /// is the only built-in that serves `/v1/responses`. + #[test] + fn responses_api_capability_is_coupled_to_the_preset_list() { + for provider in BUILTIN_CLOUD_PROVIDERS { + let expected = provider.slug == "openai"; + assert_eq!( + builtin_cloud_supports_responses_api(provider.slug), + expected, + "built-in {} Responses-API capability drifted from the openai-only invariant", + provider.slug + ); + } + } + + #[test] + fn endpoint_host_parses_scheme_path_and_port() { + assert_eq!( + endpoint_host("https://integrate.api.nvidia.com/v1").as_deref(), + Some("integrate.api.nvidia.com") + ); + // Missing scheme, mixed case, trailing path. + assert_eq!( + endpoint_host("API.OpenAI.com/v1/chat").as_deref(), + Some("api.openai.com") + ); + // Userinfo + explicit port are stripped. + assert_eq!( + endpoint_host("https://user:pass@api.groq.com:443/openai/v1").as_deref(), + Some("api.groq.com") + ); + // Bracketed IPv6 literal with port. + assert_eq!( + endpoint_host("http://[::1]:8080/v1").as_deref(), + Some("::1") + ); + assert_eq!(endpoint_host(" ").as_deref(), None); + } + + /// TAURI-RUST-HW1: the backend-URL resolver uses this to reroute backend + /// domain calls away from a BYO inference host. Every built-in provider host + /// must be recognised; OpenHuman backend hosts and unknown proxies must not. + #[test] + fn host_is_builtin_cloud_provider_recognises_inference_hosts() { + for host in [ + "openrouter.ai", + "api.openai.com", + "api.anthropic.com", + "api.groq.com", + "generativelanguage.googleapis.com", + "API.OPENAI.COM", // case-insensitive + ] { + assert!( + host_is_builtin_cloud_provider(host), + "{host} is a built-in cloud inference host" + ); + } + for host in [ + "api.tinyhumans.ai", + "staging-api.tinyhumans.ai", + "my-backend.example", + "", + ] { + assert!( + !host_is_builtin_cloud_provider(host), + "{host:?} is not a built-in cloud inference host" + ); + } + // Every registry endpoint's own host must classify as builtin. + for provider in BUILTIN_CLOUD_PROVIDERS { + let host = endpoint_host(provider.endpoint).expect("preset endpoint has a host"); + assert!( + host_is_builtin_cloud_provider(&host), + "{} ({host}) must be recognised", + provider.slug + ); + } + } + + /// TAURI-RUST-5A1: a *custom* slug pointed at a known chat-only host (NVIDIA) + /// must be classified chat-only so the factory disables the guaranteed-404 + /// `/responses` fallback — the builtin-slug gate alone misses this because + /// the slug is not builtin. + #[test] + fn nvidia_host_is_chat_completions_only_regardless_of_slug() { + assert!(endpoint_host_is_chat_completions_only( + "https://integrate.api.nvidia.com/v1" + )); + // Other chat-only built-in hosts too. + for endpoint in [ + "https://api.deepseek.com/v1", + "https://api.groq.com/openai/v1", + "https://api.mistral.ai/v1", + ] { + assert!( + endpoint_host_is_chat_completions_only(endpoint), + "{endpoint} is a chat-completions-only built-in host" + ); + } + } + + #[test] + fn openai_host_and_unknown_proxies_keep_the_responses_fallback() { + // OpenAI's first-party host serves /responses — must NOT be gated off, + // even via a custom proxy slug pointed at it. + assert!(!endpoint_host_is_chat_completions_only( + "https://api.openai.com/v1" + )); + // Genuinely unknown proxy hosts keep the permissive default (they may be + // real OpenAI proxies that implement /responses). + for endpoint in [ + "https://my-llm-proxy.internal.example/v1", + "https://litellm.mycorp.dev/v1", + "", + ] { + assert!( + !endpoint_host_is_chat_completions_only(endpoint), + "{endpoint:?} is an unknown host and must keep the fallback" + ); + } + } + + /// Drift guard: the host-based gate must agree with the slug-based + /// capability for every built-in preset's own endpoint, so adding a preset + /// can't silently desync the two gates. + #[test] + fn host_gate_agrees_with_slug_capability_for_every_builtin() { + for provider in BUILTIN_CLOUD_PROVIDERS { + // OpenhumanJwt / Anthropic presets never route through the + // OpenAI-compatible Responses fallback; the gate only matters for + // the Bearer OpenAI-compatible hosts. + if provider.auth_style != AuthStyle::Bearer { + continue; + } + let host_chat_only = endpoint_host_is_chat_completions_only(provider.endpoint); + let slug_supports = builtin_cloud_supports_responses_api(provider.slug); + assert_eq!( + host_chat_only, !slug_supports, + "host gate for built-in {} disagrees with its slug capability", + provider.slug + ); + } + } +} diff --git a/api/src/host/composio.rs b/api/src/host/composio.rs new file mode 100644 index 0000000..ddb7b70 --- /dev/null +++ b/api/src/host/composio.rs @@ -0,0 +1,878 @@ +//! Composio value types — connections, capabilities, execute responses. +//! +//! Moved here from the host's `integrations::composio::types` because the +//! extracted memory sync pipelines read these fields directly on every run, and +//! a trait accessor per field would be absurd. They are inert serde data with +//! no behaviour and no dependencies beyond `serde`, so the contract crate's +//! dependency-light guarantee is unaffected. +//! +//! The Composio *client* deliberately did not come with them — see +//! `tinymemory_core::composio_host`. Its `Direct` variant wraps a host agent +//! tool, and mode dispatch is host policy. +//! +//! Domain types for the Composio integration. +//! +//! These mirror the response envelopes emitted by the openhuman backend under +//! `/agent-integrations/composio/*`. See: +//! - `src/routes/agentIntegrations/composio.ts` +//! - `src/controllers/agentIntegrations/composio/*.ts` +//! in the backend repo for the authoritative shapes. + +use serde::{Deserialize, Deserializer, Serialize}; + +/// Accepts either a JSON string or an object whose first matching field +/// (`slug`/`id`/`name`/`key`) is a string. Lets us tolerate upstream +/// shape drift where a previously-stringy field is now nested in an +/// object — e.g. `"toolkit": {"slug": "gmail", "logo": "…"}`. +fn de_string_or_object<'de, D: Deserializer<'de>>(d: D) -> Result { + use serde::de::Error; + let v = serde_json::Value::deserialize(d)?; + match v { + serde_json::Value::String(s) => Ok(s), + serde_json::Value::Object(map) => { + for key in ["slug", "id", "name", "key"] { + if let Some(serde_json::Value::String(s)) = map.get(key) { + return Ok(s.clone()); + } + } + Err(D::Error::custom( + "expected string or object with slug/id/name/key field", + )) + } + other => Err(D::Error::custom(format!( + "expected string, got {}", + match other { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "bool", + serde_json::Value::Number(_) => "number", + serde_json::Value::Array(_) => "array", + _ => "unknown", + } + ))), + } +} + +/// Like [`de_string_or_object`] but optional and resilient: missing / +/// null / unrecognized object shapes return `None` instead of erroring. +fn de_opt_string_or_object<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let v = Option::::deserialize(d)?; + Ok(match v { + None | Some(serde_json::Value::Null) => None, + Some(serde_json::Value::String(s)) => Some(s), + Some(serde_json::Value::Object(map)) => { + let mut found = None; + for key in ["state", "value", "slug", "id", "name", "key"] { + if let Some(serde_json::Value::String(s)) = map.get(key) { + found = Some(s.clone()); + break; + } + } + found + } + _ => None, + }) +} + +// ── Toolkits ──────────────────────────────────────────────────────── + +/// One toolkit from the live Composio catalog, forwarded verbatim from the +/// backend (`GET /agent-integrations/composio/toolkits`). +/// +/// The core does not interpret these fields — it passes them straight through +/// to the desktop UI so the app no longer hardcodes toolkit display metadata +/// (see the workspace `COMPOSIO_DYNAMIC_CATALOG_PLAN.md`). Everything except +/// `slug` is best-effort; backends predating the dynamic catalog omit the +/// whole `catalog` array, in which case the UI falls back to local metadata. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioToolkitCatalogEntry { + /// Toolkit slug as Composio emits it, e.g. `"googlecalendar"`. + pub slug: String, + /// Human-readable name, e.g. `"Google Calendar"`. + #[serde(default)] + pub name: String, + /// Composio-hosted logo URL (`meta.logo`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logo: Option, + /// Short description (`meta.description`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Composio category names (`meta.categories`). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub categories: Vec, + /// Whether the user can connect/use this toolkit (passed the backend gate). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub enabled: Option, +} + +/// Response body of `GET /agent-integrations/composio/toolkits`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioToolkitsResponse { + /// Server-enforced toolkit allowlist, e.g. `["gmail", "notion"]`. + #[serde(default)] + pub toolkits: Vec, + /// Rich render model from the live Composio catalog. Optional — empty when + /// the backend predates the dynamic catalog. Forwarded as-is to the UI. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub catalog: Vec, +} + +/// One row in OpenHuman's local Composio capability matrix. +/// +/// Unlike `ComposioToolkitsResponse`, this is not tied to a signed-in +/// backend/direct Composio session. It describes what this core build knows +/// how to do for each toolkit: whether the toolkit has a native provider +/// implementation, a curated tool catalog, profile/sync hooks, and memory +/// ingestion support. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioCapability { + pub toolkit: String, + pub description: String, + pub native_provider: bool, + pub curated_tools: bool, + pub curated_tool_count: usize, + pub tool_execution: bool, + pub user_profile: bool, + pub initial_sync: bool, + pub periodic_sync: bool, + pub sync_interval_secs: Option, + pub trigger_webhooks: bool, + pub memory_ingest: bool, +} + +/// Response body of `composio.list_capabilities`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioCapabilitiesResponse { + #[serde(default)] + pub capabilities: Vec, +} + +/// Response body of `composio.list_agent_ready_toolkits`. +/// +/// Sorted slugs that have a curated agent catalog — the frontend +/// uses this to decide whether to label a connected toolkit as +/// "preview / agent integration coming soon". See #2283. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioAgentReadyToolkitsResponse { + #[serde(default)] + pub toolkits: Vec, +} + +// ── Connections ───────────────────────────────────────────────────── + +/// One connected Composio account (OAuth integration instance). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioConnection { + /// Composio connection id (what you DELETE to disconnect). + pub id: String, + /// Toolkit slug, e.g. `"gmail"`. + pub toolkit: String, + /// Connection status — `"ACTIVE"`, `"CONNECTED"`, `"PENDING"`, … + pub status: String, + /// ISO timestamp (backend passes this through from Composio). + #[serde(rename = "createdAt", default, skip_serializing_if = "Option::is_none")] + pub created_at: Option, + /// Account email — populated from the cached provider profile when + /// the toolkit reports an email address (e.g. Gmail, Google Calendar, + /// Google Sheets). Lets the UI picker show "Gmail · user@example.com" + /// instead of a generic "Account N" label. + #[serde( + rename = "accountEmail", + default, + skip_serializing_if = "Option::is_none" + )] + pub account_email: Option, + /// Workspace or team display name — populated for workspace-based + /// services (e.g. Slack: user display name / team name, Notion: workspace + /// name). Used by the picker when no email is available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, + /// Screen name or handle — populated for username-based services + /// (e.g. GitHub login, Twitter handle). Used by the picker as a + /// last-resort identity hint after email and workspace. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub username: Option, +} + +impl ComposioConnection { + /// Return the toolkit slug in the canonical form used by provider + /// lookup, prompt injection, and tool-action prefix matching. + pub fn normalized_toolkit(&self) -> String { + self.toolkit.trim().to_ascii_lowercase() + } + + /// Whether this row represents a usable connection. + /// + /// The web UI already treats status case-insensitively. Keep the + /// core-side chat/runtime filters aligned so a backend spelling such + /// as `connected` cannot display as connected in Settings while + /// disappearing from the agent's integration surface. + pub fn is_active(&self) -> bool { + let status = self.status.trim(); + status.eq_ignore_ascii_case("ACTIVE") || status.eq_ignore_ascii_case("CONNECTED") + } +} + +/// Response body of `GET /agent-integrations/composio/connections`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioConnectionsResponse { + #[serde(default)] + pub connections: Vec, +} + +/// Response body of `POST /agent-integrations/composio/authorize`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioAuthorizeResponse { + /// Composio-hosted OAuth URL the user opens in a browser. + #[serde(rename = "connectUrl")] + pub connect_url: String, + /// Composio connection id created by this authorize call. + #[serde(rename = "connectionId")] + pub connection_id: String, +} + +/// Response body of `DELETE /agent-integrations/composio/connections/:id`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioDeleteResponse { + #[serde(default)] + pub deleted: bool, + #[serde(default)] + pub memory_chunks_deleted: usize, +} + +// ── Tools ─────────────────────────────────────────────────────────── + +/// OpenAI function-calling schema returned by the backend for each tool. +/// +/// The backend wraps Composio's upstream shape; we keep the `type` + +/// `function` envelope so callers can forward directly into an LLM. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioToolSchema { + #[serde(rename = "type", default = "default_function_type")] + pub kind: String, + pub function: ComposioToolFunction, +} + +fn default_function_type() -> String { + "function".to_string() +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioToolFunction { + /// Composio action slug, e.g. `"GMAIL_SEND_EMAIL"`. + pub name: String, + /// Human-readable description shown to the model. + #[serde(default)] + pub description: Option, + /// JSON schema for the tool's INPUT parameters. + #[serde(default)] + pub parameters: Option, + /// JSON schema describing the tool's OUTPUT/return-value shape, when the + /// upstream listing publishes one. Composio's v3 `/tools` endpoint calls + /// this `output_parameters` — documented as "Schema definition of return + /// values from the tool" + /// () — + /// alongside `input_parameters`. `None` means "unknown" (not "empty"): + /// the backend-proxied `/agent-integrations/composio/tools` path is + /// opaque to this crate and may not forward it, and not every Composio + /// action publishes an output schema. + #[serde(default)] + pub output_parameters: Option, +} + +/// Response body of `GET /agent-integrations/composio/tools`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioToolsResponse { + #[serde(default)] + pub tools: Vec, +} + +// ── Execute ───────────────────────────────────────────────────────── + +/// Response body of `POST /agent-integrations/composio/execute`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioExecuteResponse { + /// Raw result from the upstream provider. + #[serde(default)] + pub data: serde_json::Value, + /// Did the provider report success? + #[serde(default)] + pub successful: bool, + /// Provider error message if any. + #[serde(default)] + pub error: Option, + /// Amount charged to the caller (base + margin) in USD. + #[serde(rename = "costUsd", default)] + pub cost_usd: f64, + /// Backend-rendered compact markdown for known tools (set by + /// backend PR tinyhumansai/backend#683). When present and non-empty + /// callers should prefer this over `data` for LLM/CLI consumption. + #[serde(rename = "markdownFormatted", default)] + pub markdown_formatted: Option, +} + +// ── GitHub repos + triggers ───────────────────────────────────────── + +/// One repository returned by `GET /agent-integrations/composio/github/repos`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioGithubRepo { + pub owner: String, + pub repo: String, + #[serde(rename = "fullName")] + pub full_name: String, + #[serde(default)] + pub private: Option, + #[serde(rename = "defaultBranch", default)] + pub default_branch: Option, + #[serde(rename = "htmlUrl", default)] + pub html_url: Option, +} + +/// Response body of `GET /agent-integrations/composio/github/repos`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioGithubReposResponse { + #[serde(rename = "connectionId")] + pub connection_id: String, + #[serde(default, rename = "repositories")] + pub repositories: Vec, +} + +/// Response body of `POST /agent-integrations/composio/triggers`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioCreateTriggerResponse { + #[serde(rename = "triggerId")] + pub trigger_id: String, + #[serde(default)] + pub status: Option, +} + +// ── Trigger management (catalog + active list + enable/disable) ───── + +/// Per-repo descriptor used by GitHub-scoped available triggers. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioAvailableTriggerRepo { + pub owner: String, + pub repo: String, +} + +/// One entry in `GET /agent-integrations/composio/triggers/available`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioAvailableTrigger { + pub slug: String, + /// `"static"` or `"github_repo"`. + pub scope: String, + #[serde( + rename = "defaultConfig", + default, + skip_serializing_if = "Option::is_none" + )] + pub default_config: Option, + #[serde( + rename = "requiredConfigKeys", + default, + skip_serializing_if = "Option::is_none" + )] + pub required_config_keys: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioAvailableTriggersResponse { + #[serde(default)] + pub triggers: Vec, +} + +/// One entry in `GET /agent-integrations/composio/triggers`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioActiveTrigger { + #[serde(deserialize_with = "de_string_or_object")] + pub id: String, + #[serde(deserialize_with = "de_string_or_object")] + pub slug: String, + #[serde(deserialize_with = "de_string_or_object")] + pub toolkit: String, + #[serde(rename = "connectionId", deserialize_with = "de_string_or_object")] + pub connection_id: String, + #[serde( + rename = "triggerConfig", + default, + skip_serializing_if = "Option::is_none" + )] + pub trigger_config: Option, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "de_opt_string_or_object" + )] + pub state: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioActiveTriggersResponse { + #[serde(default)] + pub triggers: Vec, +} + +/// Response body of `POST /agent-integrations/composio/triggers` (enable). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioEnableTriggerResponse { + #[serde(rename = "triggerId")] + pub trigger_id: String, + pub slug: String, + #[serde(rename = "connectionId")] + pub connection_id: String, +} + +/// Response body of `DELETE /agent-integrations/composio/triggers/:id`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioDisableTriggerResponse { + #[serde(default)] + pub deleted: bool, +} + +// ── Triggers ──────────────────────────────────────────────────────── + +/// Payload of the `composio:trigger` Socket.IO event emitted by the backend +/// when a Composio webhook is received, HMAC-verified, and delivered to the +/// user's active sockets. +/// +/// See `src/controllers/agentIntegrations/composio/handleWebhook.ts` in the +/// backend repo. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioTriggerEvent { + /// Toolkit slug, e.g. `"gmail"`. + #[serde(default)] + pub toolkit: String, + /// Trigger slug, e.g. `"GMAIL_NEW_GMAIL_MESSAGE"`. + #[serde(default)] + pub trigger: String, + /// Trigger-specific payload (provider-defined shape). + #[serde(default)] + pub payload: serde_json::Value, + /// Metadata the backend attaches: `{ id, uuid }`. + #[serde(default)] + pub metadata: ComposioTriggerMetadata, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioTriggerMetadata { + #[serde(default)] + pub id: String, + #[serde(default)] + pub uuid: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioTriggerHistoryEntry { + /// Unix timestamp in milliseconds when the trigger reached the core. + pub received_at_ms: u64, + /// Toolkit slug, e.g. `"gmail"`. + pub toolkit: String, + /// Trigger slug, e.g. `"GMAIL_NEW_GMAIL_MESSAGE"`. + pub trigger: String, + /// Backend metadata id for this event. + pub metadata_id: String, + /// Backend metadata UUID for this event. + pub metadata_uuid: String, + /// Raw provider payload as forwarded by the backend socket event. + pub payload: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComposioTriggerHistoryResult { + /// Directory containing daily JSONL archives. + pub archive_dir: String, + /// Today's JSONL file path. + pub current_day_file: String, + /// Recent triggers, newest first. + pub entries: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn connection_is_active_matches_ui_status_normalization() { + for status in ["ACTIVE", "CONNECTED", "active", "connected", " connected "] { + let conn = ComposioConnection { + id: "c1".into(), + toolkit: "slack".into(), + status: status.into(), + created_at: None, + account_email: None, + workspace: None, + username: None, + }; + assert!(conn.is_active(), "status {status:?} should be active"); + } + + for status in ["PENDING", "INITIATED", "FAILED", ""] { + let conn = ComposioConnection { + id: "c1".into(), + toolkit: "slack".into(), + status: status.into(), + created_at: None, + account_email: None, + workspace: None, + username: None, + }; + assert!(!conn.is_active(), "status {status:?} should not be active"); + } + } + + #[test] + fn connection_normalizes_toolkit_for_runtime_matching() { + let conn = ComposioConnection { + id: "c1".into(), + toolkit: " Slack ".into(), + status: "ACTIVE".into(), + created_at: None, + account_email: None, + workspace: None, + username: None, + }; + assert_eq!(conn.normalized_toolkit(), "slack"); + } + + #[test] + fn toolkits_response_defaults_to_empty() { + let resp: ComposioToolkitsResponse = serde_json::from_str("{}").unwrap(); + assert!(resp.toolkits.is_empty()); + } + + #[test] + fn toolkits_response_roundtrips() { + let resp = ComposioToolkitsResponse { + toolkits: vec!["gmail".into(), "notion".into()], + ..Default::default() + }; + let value = serde_json::to_value(&resp).unwrap(); + // Empty catalog is skipped on the wire — back-compat with old cores. + assert_eq!(value, json!({ "toolkits": ["gmail", "notion"] })); + let back: ComposioToolkitsResponse = serde_json::from_value(value).unwrap(); + assert_eq!(back.toolkits, vec!["gmail", "notion"]); + assert!(back.catalog.is_empty()); + } + + #[test] + fn toolkits_response_forwards_catalog() { + // A backend that sends the dynamic catalog must deserialize and + // re-serialize verbatim so the field reaches the desktop UI. + let raw = json!({ + "toolkits": ["gmail"], + "catalog": [ + { + "slug": "gmail", + "name": "Gmail", + "logo": "https://logos.composio.dev/api/gmail", + "description": "Send and read email", + "categories": ["productivity"], + "enabled": true + } + ] + }); + let resp: ComposioToolkitsResponse = serde_json::from_value(raw).unwrap(); + assert_eq!(resp.catalog.len(), 1); + let entry = &resp.catalog[0]; + assert_eq!(entry.slug, "gmail"); + assert_eq!(entry.name, "Gmail"); + assert_eq!(entry.enabled, Some(true)); + assert_eq!(entry.categories, vec!["productivity".to_string()]); + + // Round-trips back out with the catalog intact. + let value = serde_json::to_value(&resp).unwrap(); + assert_eq!(value["catalog"][0]["slug"], "gmail"); + assert_eq!(value["catalog"][0]["enabled"], true); + } + + #[test] + fn connection_parses_and_serializes_camelcase_created_at() { + let raw = json!({ + "id": "conn_1", + "toolkit": "gmail", + "status": "ACTIVE", + "createdAt": "2026-02-01T00:00:00Z" + }); + let conn: ComposioConnection = serde_json::from_value(raw.clone()).unwrap(); + assert_eq!(conn.id, "conn_1"); + assert_eq!(conn.toolkit, "gmail"); + assert_eq!(conn.status, "ACTIVE"); + assert_eq!(conn.created_at.as_deref(), Some("2026-02-01T00:00:00Z")); + + // Round-trip must use camelCase too. + let serialized = serde_json::to_value(&conn).unwrap(); + assert!(serialized.get("createdAt").is_some()); + } + + #[test] + fn connection_without_created_at_omits_field_when_serialized() { + let conn = ComposioConnection { + id: "x".into(), + toolkit: "notion".into(), + status: "PENDING".into(), + created_at: None, + account_email: None, + workspace: None, + username: None, + }; + let s = serde_json::to_value(&conn).unwrap(); + assert!( + s.get("createdAt").is_none(), + "createdAt must be skipped when None" + ); + } + + #[test] + fn authorize_response_uses_camelcase_keys() { + let raw = json!({ + "connectUrl": "https://composio.dev/oauth/abc", + "connectionId": "conn_2" + }); + let resp: ComposioAuthorizeResponse = serde_json::from_value(raw).unwrap(); + assert_eq!(resp.connect_url, "https://composio.dev/oauth/abc"); + assert_eq!(resp.connection_id, "conn_2"); + + let s = serde_json::to_value(&resp).unwrap(); + assert!(s.get("connectUrl").is_some()); + assert!(s.get("connectionId").is_some()); + } + + #[test] + fn tool_schema_defaults_type_field_to_function() { + let raw = json!({ + "function": { + "name": "GMAIL_SEND_EMAIL", + "description": "Send an email", + "parameters": { "type": "object" } + } + }); + let tool: ComposioToolSchema = serde_json::from_value(raw).unwrap(); + assert_eq!(tool.kind, "function"); + assert_eq!(tool.function.name, "GMAIL_SEND_EMAIL"); + assert_eq!(tool.function.description.as_deref(), Some("Send an email")); + assert!(tool.function.parameters.is_some()); + } + + #[test] + fn tool_function_tolerates_missing_description_and_parameters() { + let raw = json!({ "function": { "name": "SLUG_ONLY" } }); + let tool: ComposioToolSchema = serde_json::from_value(raw).unwrap(); + assert_eq!(tool.function.name, "SLUG_ONLY"); + assert!(tool.function.description.is_none()); + assert!(tool.function.parameters.is_none()); + } + + #[test] + fn execute_response_parses_cost_and_error() { + let raw = json!({ + "data": { "messageId": "m-1" }, + "successful": true, + "error": null, + "costUsd": 0.0025 + }); + let resp: ComposioExecuteResponse = serde_json::from_value(raw).unwrap(); + assert!(resp.successful); + assert!(resp.error.is_none()); + assert!((resp.cost_usd - 0.0025).abs() < f64::EPSILON); + } + + #[test] + fn execute_response_defaults_when_fields_missing() { + let resp: ComposioExecuteResponse = serde_json::from_str("{}").unwrap(); + assert!(!resp.successful); + assert!(resp.error.is_none()); + assert_eq!(resp.cost_usd, 0.0); + assert!(resp.data.is_null()); + } + + #[test] + fn available_trigger_deserializes_and_serializes_camelcase_fields() { + let raw = json!({ + "slug": "GMAIL_NEW_GMAIL_MESSAGE", + "scope": "static", + "defaultConfig": { "labelIds": ["INBOX"] }, + "requiredConfigKeys": ["labelIds"], + "repo": { "owner": "acme", "repo": "inbox" } + }); + let trigger: ComposioAvailableTrigger = serde_json::from_value(raw).unwrap(); + assert_eq!(trigger.slug, "GMAIL_NEW_GMAIL_MESSAGE"); + assert_eq!(trigger.scope, "static"); + assert_eq!( + trigger.default_config, + Some(json!({ "labelIds": ["INBOX"] })) + ); + assert_eq!( + trigger.required_config_keys, + Some(vec!["labelIds".to_string()]) + ); + let repo = trigger.repo.as_ref().expect("repo"); + assert_eq!(repo.owner, "acme"); + assert_eq!(repo.repo, "inbox"); + + let value = serde_json::to_value(&trigger).unwrap(); + assert!(value.get("defaultConfig").is_some()); + assert!(value.get("requiredConfigKeys").is_some()); + } + + #[test] + fn active_trigger_parses_connection_id_and_optional_fields() { + let raw = json!({ + "id": "ti_1", + "slug": "GMAIL_NEW_GMAIL_MESSAGE", + "toolkit": "gmail", + "connectionId": "c-1", + "triggerConfig": { "labelIds": "INBOX" }, + "state": "active" + }); + let trigger: ComposioActiveTrigger = serde_json::from_value(raw).unwrap(); + assert_eq!(trigger.id, "ti_1"); + assert_eq!(trigger.slug, "GMAIL_NEW_GMAIL_MESSAGE"); + assert_eq!(trigger.connection_id, "c-1"); + assert_eq!(trigger.trigger_config, Some(json!({"labelIds":"INBOX"}))); + assert_eq!(trigger.state.as_deref(), Some("active")); + + let value = serde_json::to_value(&trigger).unwrap(); + assert!(value.get("connectionId").is_some()); + assert!(value.get("triggerConfig").is_some()); + assert!(value.get("state").is_some()); + } + + #[test] + fn trigger_enable_response_uses_camelcase_and_optional_defaults() { + let raw = json!({ + "triggerId": "ti_9", + "slug": "GMAIL_NEW_GMAIL_MESSAGE", + "connectionId": "c-9" + }); + let resp: ComposioEnableTriggerResponse = serde_json::from_value(raw).unwrap(); + assert_eq!(resp.trigger_id, "ti_9"); + assert_eq!(resp.slug, "GMAIL_NEW_GMAIL_MESSAGE"); + assert_eq!(resp.connection_id, "c-9"); + + let serialized = serde_json::to_value(&resp).unwrap(); + assert_eq!(serialized.get("triggerId").unwrap(), "ti_9"); + assert_eq!(serialized.get("connectionId").unwrap(), "c-9"); + } + + #[test] + fn delete_trigger_response_defaults_deleted_to_false() { + let raw = json!({}); + let resp: ComposioDisableTriggerResponse = serde_json::from_value(raw).unwrap(); + assert!(!resp.deleted); + } + + #[test] + fn trigger_event_defaults_empty_fields_to_empty_strings() { + let ev: ComposioTriggerEvent = serde_json::from_str("{}").unwrap(); + assert_eq!(ev.toolkit, ""); + assert_eq!(ev.trigger, ""); + assert_eq!(ev.metadata.id, ""); + assert_eq!(ev.metadata.uuid, ""); + assert!(ev.payload.is_null()); + } + + #[test] + fn trigger_event_parses_full_payload() { + let raw = json!({ + "toolkit": "gmail", + "trigger": "GMAIL_NEW_GMAIL_MESSAGE", + "payload": { "subject": "hi" }, + "metadata": { "id": "evt-1", "uuid": "uuid-1" } + }); + let ev: ComposioTriggerEvent = serde_json::from_value(raw).unwrap(); + assert_eq!(ev.toolkit, "gmail"); + assert_eq!(ev.trigger, "GMAIL_NEW_GMAIL_MESSAGE"); + assert_eq!(ev.metadata.id, "evt-1"); + assert_eq!(ev.metadata.uuid, "uuid-1"); + assert_eq!(ev.payload["subject"], "hi"); + } + + #[test] + fn active_trigger_accepts_string_fields() { + let v = json!({ + "id": "t1", + "slug": "GMAIL_NEW_MAIL", + "toolkit": "gmail", + "connectionId": "c1", + "state": "ACTIVE", + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert_eq!(trig.id, "t1"); + assert_eq!(trig.slug, "GMAIL_NEW_MAIL"); + assert_eq!(trig.toolkit, "gmail"); + assert_eq!(trig.connection_id, "c1"); + assert_eq!(trig.state.as_deref(), Some("ACTIVE")); + } + + #[test] + fn active_trigger_accepts_object_fields() { + // Mirrors upstream API drift where these fields arrive as objects + // rather than plain strings. + let v = json!({ + "id": {"id": "t1"}, + "slug": {"slug": "GMAIL_NEW_MAIL"}, + "toolkit": {"slug": "gmail", "logo": "https://…"}, + "connectionId": {"id": "c1"}, + "state": {"state": "ACTIVE", "slug": "should-be-ignored"}, + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert_eq!(trig.id, "t1"); + assert_eq!(trig.slug, "GMAIL_NEW_MAIL"); + assert_eq!(trig.toolkit, "gmail"); + assert_eq!(trig.connection_id, "c1"); + // `state` priority must prefer the literal `state` key over metadata. + assert_eq!(trig.state.as_deref(), Some("ACTIVE")); + } + + #[test] + fn active_trigger_state_falls_back_to_value() { + let v = json!({ + "id": "t1", + "slug": "X", + "toolkit": "gmail", + "connectionId": "c1", + "state": {"value": "PENDING"}, + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert_eq!(trig.state.as_deref(), Some("PENDING")); + } + + #[test] + fn active_trigger_state_missing_or_unknown_returns_none() { + let v = json!({ + "id": "t1", + "slug": "X", + "toolkit": "gmail", + "connectionId": "c1", + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert!(trig.state.is_none()); + + let v = json!({ + "id": "t1", + "slug": "X", + "toolkit": "gmail", + "connectionId": "c1", + "state": {"unrelated": 42}, + }); + let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); + assert!(trig.state.is_none()); + } + + #[test] + fn active_trigger_required_field_rejects_unsupported_object() { + // Object without any of slug/id/name/key must fail loudly so we + // notice further upstream shape drift instead of silently dropping + // the trigger. + let v = json!({ + "id": {"unrelated": 42}, + "slug": "X", + "toolkit": "gmail", + "connectionId": "c1", + }); + let err = serde_json::from_value::(v).unwrap_err(); + assert!(err.to_string().contains("expected string or object")); + } +} diff --git a/api/src/host/config.rs b/api/src/host/config.rs new file mode 100644 index 0000000..af3d79f --- /dev/null +++ b/api/src/host/config.rs @@ -0,0 +1,256 @@ +//! [`MemoryHostConfig`] — the memory subsystem's view of the host's config. +//! +//! # Why a trait and not a struct +//! +//! The host's `Config` is one giant serde struct covering voice, channels, +//! sandboxing, inference routing, the agent harness — the lot. The memory +//! subsystem reads about two dozen of its fields. Moving the whole struct into +//! this crate would drag the host's entire configuration vocabulary into a +//! contract crate that is meant to stay dependency-light; leaving it behind and +//! passing individual values would mean rewriting every function signature in +//! the extracted code. +//! +//! A trait threads the needle. `tinymemory_core::Config` is the alias +//! `dyn MemoryHostConfig`, so a function that took `config: &Config` before the +//! extraction still takes `config: &Config` after it, and the host's concrete +//! `Config` unsize-coerces at the call site with no edit at all. Only the field +//! *accesses* inside the extracted code change, from `config.workspace_dir` to +//! `config.workspace_dir()`. +//! +//! # Accessor shapes are chosen for zero churn, not for elegance +//! +//! Several accessors return `&PathBuf` / `&Vec` where `&Path` / `&[T]` would +//! be the idiomatic choice. That is deliberate: the extracted code calls +//! `.clone()` on these values in dozens of places, and `&Path`/`&[T]` would +//! silently resolve `.clone()` to the *reference*'s `Clone` impl and fail at the +//! use site with a confusing type error. Returning the owning type keeps every +//! one of those sites compiling unchanged. +//! +//! # Mutation +//! +//! Three methods take `&mut self`. They exist because the extracted code owns +//! two write paths the host does not: the composio source-caps migration and +//! the CLI's env-override re-application. Everything else is read-only. + +use std::path::PathBuf; + +use super::cloud_providers::CloudProviderCreds; +use super::local_ai::LocalAiConfig; +use super::scheduler_gate::SchedulerGateConfig; +use super::storage_memory::{MemoryConfig, MemoryTreeConfig}; + +/// Composio routing mode: proxied through the host's cloud backend. +pub const COMPOSIO_MODE_BACKEND: &str = "backend"; +/// Composio routing mode: BYO API key, calling `backend.composio.dev` directly. +pub const COMPOSIO_MODE_DIRECT: &str = "direct"; + +/// The subset of a host's Composio configuration the memory sync pipelines read. +/// +/// Passed by value rather than by reference because the host's own +/// `ComposioConfig` carries fields (toolkit triage opt-outs, the enabled flag) +/// that have nothing to do with memory, and because borrowing it would pin the +/// host's type into this contract. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ComposioMode { + /// [`COMPOSIO_MODE_BACKEND`] or [`COMPOSIO_MODE_DIRECT`]. + pub mode: String, + /// The Composio entity the host authenticates as. + pub entity_id: String, + /// Direct-mode API key, when the user hand-wrote one into `config.toml`. + /// The keychain-backed value takes precedence and is resolved host-side. + pub api_key: Option, + /// Whether the LLM triage turn is switched off for all triggers. + pub triage_disabled: bool, +} + +impl ComposioMode { + /// True when the host routes Composio calls directly rather than through + /// its cloud backend. + #[must_use] + pub fn is_direct(&self) -> bool { + self.mode.eq_ignore_ascii_case(COMPOSIO_MODE_DIRECT) + } +} + +/// The host's configuration, as the memory subsystem sees it. +/// +/// Implemented by the embedding application for its own root config type. See +/// the module docs for why this is a trait and why the accessor return types +/// are shaped the way they are. +#[async_trait::async_trait] +pub trait MemoryHostConfig: Send + Sync + std::fmt::Debug { + // ── Paths ─────────────────────────────────────────────────────────────── + + /// Root of the host's internal per-user state. Every memory database, + /// summary-tree directory and queue file is resolved beneath this. + fn workspace_dir(&self) -> &PathBuf; + + /// Absolute path of the `config.toml` this config was loaded from. + fn config_path(&self) -> &PathBuf; + + /// Where chunk `.md` files are written. Either the explicit + /// `memory_tree.content_dir` or `/memory_tree/content`. + fn memory_tree_content_root(&self) -> PathBuf; + + // ── Memory-owned sections ─────────────────────────────────────────────── + + /// The `[memory]` block — backend selection, embedding provider/model/dims, + /// relevance floor, SQLite timeouts. + fn memory(&self) -> &MemoryConfig; + + /// The `[memory_tree]` block — summary-tree embedder, extractor and + /// summariser wiring. + fn memory_tree(&self) -> &MemoryTreeConfig; + + /// The `[scheduler_gate]` block — when background LLM-bound work may run. + fn scheduler_gate(&self) -> &SchedulerGateConfig; + + // ── Host-owned sections the memory subsystem still reads ──────────────── + // + // These are the seam's rough edge (see the module docs on `host`): they are + // read only to *construct* embedding providers, which is work that belongs + // in the host. Moving the embedding factory back out of the core would let + // all four of these accessors go away. + + /// The `[local_ai]` block — whether a local runtime is enabled and which + /// model it serves. + fn local_ai(&self) -> &LocalAiConfig; + + /// Configured cloud LLM/embedding backends, keyed by user-chosen slug. + fn cloud_providers(&self) -> &Vec; + + /// `provider:model` routing string for the embeddings workload, if pinned. + fn embeddings_provider(&self) -> Option<&str>; + + /// `provider:model` routing string for the memory workload, if pinned. + fn memory_provider(&self) -> Option<&str>; + + /// The local model id for a workload, when that workload is routed to + /// Ollama (`"ollama:"`). `None` for cloud or unset workloads. + /// + /// This is the single source of truth for "is this workload local?" — + /// callers must not consult the deprecated `local_ai.usage.*` booleans or + /// `memory_tree.llm_backend`. + fn workload_local_model(&self, workload: &str) -> Option; + + // ── Scalars ───────────────────────────────────────────────────────────── + + /// The concrete config behind this trait object, for host code that needs + /// its own type back. + /// + /// The seam deliberately hands the core a `dyn MemoryHostConfig`, and that + /// is the right shape for everything the core does. But a *host* + /// implementation of one of the behavioural seams — chat-model routing, + /// Composio mode dispatch — is handed the same trait object and has to get + /// its own `Config` back: routing reads BYOK fallbacks, per-role routes and + /// credentials, none of which are on this trait and none of which should + /// be. + /// + /// Implementations return `self`. A host downcasts and, on failure, falls + /// back to whatever it was configured with — a failure means the config is + /// somebody else's type (a test double), not that something is wrong. + fn as_any(&self) -> &dyn std::any::Any; + + /// An owned, shareable handle to this config. + /// + /// `tinymemory_core::Config` is the *unsized* `dyn MemoryHostConfig`, which + /// makes `&Config` free at every call site — the host's concrete `Config` + /// unsize-coerces with no edit. The cost is that a borrow cannot be turned + /// into an owned value: background loops that outlive their caller, structs + /// that hold a config, and `spawn_blocking` bodies all need one. + /// + /// This is that escape hatch. Implementations return + /// `Arc::new(self.clone())`; callers that only read should keep taking + /// `&Config` rather than reaching for this. + fn to_arc(&self) -> std::sync::Arc; + + /// Backend base URL, used to recognise first-party endpoints. + fn api_url(&self) -> Option<&str>; + + /// The backend API URL this host actually talks to, with the host's own + /// environment and default resolution already applied. + /// + /// Distinct from [`Self::api_url`], which is the raw configured value — + /// resolution (env override, staging/prod default, trailing-slash + /// normalisation) is host logic and must not be re-derived here. + fn effective_backend_api_url(&self) -> String; + + /// The current backend session bearer, or `None` when signed out. + /// + /// Read through the trait rather than from a config field because the host + /// keeps it in its credential store, not in `config.toml`. + /// + /// # Errors + /// + /// Returns `Err` when the credential store cannot be read — distinct from + /// `Ok(None)`, which means "read fine, not signed in". + fn session_token(&self) -> Result, String>; + + /// Default chat model id. + fn default_model(&self) -> Option<&str>; + + /// Default sampling temperature for background LLM calls. + fn default_temperature(&self) -> f64; + + /// Optional language for background LLM artifacts — tree summaries, + /// extraction reasons, learning reflections. `None` keeps the default. + fn output_language(&self) -> Option<&str>; + + /// Global memory-sync cadence in seconds. `None` means "no explicit choice" + /// and callers fall back to [`super::DEFAULT_MEMORY_SYNC_INTERVAL_SECS`]; + /// `Some(0)` means manual-only. + fn memory_sync_interval_secs(&self) -> Option; + + /// Whether the user has finished onboarding. Background ingestion holds off + /// until they have. + fn onboarding_completed(&self) -> bool; + + /// Whether at-rest secret encryption is switched on for this workspace. + fn secrets_encrypt(&self) -> bool; + + /// Composio routing mode + credentials, as the sync pipelines need them. + fn composio(&self) -> ComposioMode; + + // ── Memory sources ────────────────────────────────────────────────────── + // + // Serde-mediated on purpose. `MemorySourceEntry` is defined by the *engine* + // crate (`tinycortex`), which this contract crate must not depend on — it + // would drag SQLite in and break the dependency-light guarantee this crate + // exists to hold. JSON is the narrowest waist that keeps the type where it + // belongs. + + /// The persisted `[[memory_sources]]` registry, as JSON. + /// + /// # Errors + /// Propagates a serialization failure from the host's own entry type. + fn memory_sources_json(&self) -> anyhow::Result; + + /// Replace the persisted `[[memory_sources]]` registry. Does not save to + /// disk — call [`Self::save`] afterwards. + /// + /// # Errors + /// Returns an error when `value` does not deserialize into the host's entry + /// type, in which case the registry is left untouched. + fn set_memory_sources_json(&mut self, value: serde_json::Value) -> anyhow::Result<()>; + + // ── Migration bookkeeping ─────────────────────────────────────────────── + + /// Version of the composio source-capabilities migration already applied. + fn composio_source_caps_migration_version(&self) -> u32; + + /// Record that the composio source-capabilities migration has run. + fn set_composio_source_caps_migration_version(&mut self, version: u32); + + // ── Lifecycle ─────────────────────────────────────────────────────────── + + /// Re-apply the host's environment-variable overlay over this config. + /// Used by the CLI entry points, which build a config before the host's + /// normal load path has run. + fn apply_env_overrides(&mut self); + + /// Persist this config back to [`Self::config_path`] atomically. + /// + /// # Errors + /// Propagates the host's own write/serialize failure. + async fn save(&self) -> anyhow::Result<()>; +} diff --git a/api/src/host/embedding_host.rs b/api/src/host/embedding_host.rs new file mode 100644 index 0000000..55451a7 --- /dev/null +++ b/api/src/host/embedding_host.rs @@ -0,0 +1,101 @@ +//! [`EmbeddingHost`] — provider *construction*, which the host owns. +//! +//! [`super::EmbeddingProvider`] is the contract for a provider that already +//! exists. This trait is the other half: how one comes into being. Resolving an +//! API key from the credential store, knowing which managed cloud endpoint the +//! signed-in user is entitled to, knowing where the local Ollama server is +//! listening — all of that is host policy, and none of it belongs in a memory +//! engine. +//! +//! The core reaches this through a process-global installed at startup, for the +//! same reason [`super::MemoryEventSink`] is a global: the construction sites +//! sit deep inside retrieval and sealing call stacks that already thread a +//! config and a store handle. +//! +//! # Default is failure, not silence +//! +//! Unlike the event sink, an unwired [`EmbeddingHost`] must **not** degrade +//! quietly. A missing sink drops a notification about work that already +//! happened; a missing embedding provider means vectors would be written into +//! the wrong embedding space, or a query would silently return lexical-only +//! results. Both are data corruption with a delayed fuse, so the unwired +//! accessors return `Err`/`None` and every call site is written to propagate. + +use std::sync::Arc; + +use super::EmbeddingProvider; + +/// Builds [`EmbeddingProvider`]s on the core's behalf. +/// +/// Object-safe: the core holds one as `Arc`. +pub trait EmbeddingHost: Send + Sync + std::fmt::Debug { + /// The API key for `provider`, from the host's credential store. + /// + /// Returns `None` when the provider has no stored credential — which is not + /// an error: a local provider needs none, and an unconfigured cloud one is + /// a state the caller reports rather than a failure. + fn resolve_api_key(&self, provider: &str) -> Option; + + /// Base URL of the local Ollama server, honouring the host's env override + /// and config before falling back to the default. + fn ollama_base_url(&self) -> String; + + /// The host's default provider — the managed cloud embedder. + /// + /// Constructed lazily with respect to authentication: this may be called + /// before login completes, and the first `embed()` is what fails if the + /// user is unauthenticated. + fn default_embedding_provider(&self) -> Arc; + + /// Builds a provider from an explicit provider/model/credential triple. + /// + /// # Errors + /// + /// Returns `Err` when `provider` is not one the host knows how to build, or + /// when the supplied credentials are unusable for it. + fn create_embedding_provider_with_credentials( + &self, + provider: &str, + model: &str, + dims: usize, + api_key: &str, + custom_endpoint: Option<&str>, + ) -> Result, String>; + + /// Whether `model` accepts a caller-chosen output dimensionality. + /// + /// Asking for dimensions a model does not support is rejected by the + /// provider at request time, so the core checks first rather than writing a + /// batch that will fail halfway. + fn model_supports_dimensions(&self, model: &str) -> bool; + + /// The managed cloud embedder at an explicit model and dimensionality. + /// + /// # Errors + /// + /// Returns `Err` when the host cannot reach its managed endpoint + /// configuration. + fn cloud_embedding_provider( + &self, + model: &str, + dims: usize, + ) -> Result, String>; + + /// The default model id the managed cloud embedder uses. + fn default_cloud_embedding_model(&self) -> &str; + + /// The dimensionality [`Self::default_cloud_embedding_model`] emits. + fn default_cloud_embedding_dimensions(&self) -> usize; + + /// An Ollama-backed provider at `base_url`. + /// + /// # Errors + /// + /// Returns `Err` when the host cannot construct one for `model`. + fn ollama_embedding_provider( + &self, + base_url: &str, + model: &str, + dims: usize, + ) -> Result, String>; +} diff --git a/api/src/host/embeddings.rs b/api/src/host/embeddings.rs new file mode 100644 index 0000000..76ba792 --- /dev/null +++ b/api/src/host/embeddings.rs @@ -0,0 +1,92 @@ +//! [`EmbeddingProvider`] — text → vector, supplied by the host. +//! +//! The memory subsystem embeds chunks, summaries and queries, but it does not +//! decide *how*: which provider, which credentials, which rate limit and which +//! fallback are host policy. So the core takes an `Arc` +//! and never constructs one. +//! +//! This trait deliberately lives in the contract crate rather than in +//! `tinymemory-core`, so that a host implementing it does not have to depend on +//! the engine. It carries nothing heavier than `async-trait` and `anyhow`. + +use async_trait::async_trait; + +/// Formats the canonical embedding-space signature string. +/// +/// This is the **single source of truth** for the signature format. Both the +/// live-provider [`EmbeddingProvider::signature`] and any config-derived +/// signature must route through here, so a signature computed from +/// configuration is byte-identical to one computed from an instantiated +/// provider. Drift between the two silently splits one embedding space into +/// two, and every vector written on the wrong side of the split becomes +/// unsearchable without a re-embed. +#[must_use] +pub fn format_embedding_signature(name: &str, model_id: &str, dims: usize) -> String { + format!("provider={name};model={model_id};dims={dims}") +} + +/// Converts text into numerical vectors. +#[async_trait] +pub trait EmbeddingProvider: Send + Sync { + /// Provider name, e.g. `"ollama"`, `"openai"`. + fn name(&self) -> &str; + + /// Stable model identifier used to generate embeddings. + fn model_id(&self) -> &str; + + /// Number of dimensions in the generated embeddings. + fn dimensions(&self) -> usize; + + /// Stable signature for the embedding space. + /// + /// Changing any component means existing vectors are no longer comparable + /// with newly generated ones and must be stored and queried separately + /// until a migration re-embeds them. + fn signature(&self) -> String { + format_embedding_signature(self.name(), self.model_id(), self.dimensions()) + } + + /// Generates embeddings for a batch of strings. + /// + /// # Errors + /// Propagates transport, authentication and quota failures from the + /// underlying provider. + async fn embed(&self, texts: &[&str]) -> anyhow::Result>>; + + /// Generates an embedding for a single string. + /// + /// # Errors + /// As [`Self::embed`], plus an error when the provider returns no vector. + async fn embed_one(&self, text: &str) -> anyhow::Result> { + let mut results = self.embed(&[text]).await?; + results + .pop() + .ok_or_else(|| anyhow::anyhow!("Empty embedding result")) + } +} + +/// The inert provider bound when semantic search is switched off or no +/// embedding backend is configured. Reports zero dimensions and returns one +/// empty vector per input, so keyword-only retrieval keeps working while +/// vector rerank degrades to a no-op rather than an error. +#[derive(Debug, Clone, Copy, Default)] +pub struct NoopEmbedding; + +#[async_trait] +impl EmbeddingProvider for NoopEmbedding { + fn name(&self) -> &str { + "none" + } + + fn model_id(&self) -> &str { + "none" + } + + fn dimensions(&self) -> usize { + 0 + } + + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(vec![Vec::new(); texts.len()]) + } +} diff --git a/api/src/host/error_reporter.rs b/api/src/host/error_reporter.rs new file mode 100644 index 0000000..02c874b --- /dev/null +++ b/api/src/host/error_reporter.rs @@ -0,0 +1,43 @@ +//! [`ErrorReporter`] — the host's crash/error telemetry, as the core sees it. +//! +//! The memory subsystem reports a handful of failures that are worth a +//! developer's attention: a corrupt SQLite database, host filesystem I/O +//! errors, a sync run that failed for a non-user reason. *Where* those go — +//! Sentry, a log sink, nowhere — and which of them count as expected rather +//! than exceptional is host policy, so the core states the fact and the host +//! decides what to do with it. +//! +//! # The two methods are not interchangeable +//! +//! [`ErrorReporter::report_error`] is unconditional: the caller has already +//! decided this is a real defect. [`ErrorReporter::report_error_or_expected`] +//! asks the host to classify first, so routine user- and config-caused failures +//! (an unreachable local runtime, a revoked OAuth token) do not page anyone. +//! Collapsing them into one would either spam the error channel or hide real +//! bugs, which is why both exist. + +/// Receives error reports from the memory subsystem. +/// +/// Takes the **already-rendered** message rather than a concrete error type: +/// the trait has to be object-safe, so it cannot be generic over `E: Display` +/// the way the host's own `report_error` is. The core's free functions keep +/// that generic signature and render with `{:#}` — the alternate specifier that +/// makes `anyhow::Error` print its full context chain — before crossing. +pub trait ErrorReporter: Send + Sync + std::fmt::Debug { + /// Report `error` as a defect worth investigating. + /// + /// `domain` and `operation` are stable, low-cardinality strings used for + /// grouping (`"memory"` / `"tree_jobs_worker_corrupt"`); `tags` carries + /// additional non-sensitive key/value context. + fn report_error(&self, rendered: &str, domain: &str, operation: &str, tags: &[(&str, &str)]); + + /// Report `error`, letting the host classify it as a defect or an expected + /// user/config failure and route it accordingly. + fn report_error_or_expected( + &self, + rendered: &str, + domain: &str, + operation: &str, + tags: &[(&str, &str)], + ); +} diff --git a/api/src/host/events.rs b/api/src/host/events.rs new file mode 100644 index 0000000..30942eb --- /dev/null +++ b/api/src/host/events.rs @@ -0,0 +1,228 @@ +//! [`MemoryEventSink`] — the events the memory subsystem announces. +//! +//! # Why the host's event enum does not move +//! +//! The host's `DomainEvent` is a single flat enum covering agents, channels, +//! cron, tools, webhooks and the system domain as well as memory. It is the +//! host's own vocabulary; a *memory* crate must not own it, and importing it +//! would make every other subsystem's events a transitive dependency of memory. +//! +//! So the seam runs the other way. This module defines the ~15 memory-domain +//! events the extracted code emits, as a small enum of plain data. The host +//! implements [`MemoryEventSink`] by mapping each variant onto the matching +//! `DomainEvent` and publishing it on its own bus. The core publishes into the +//! sink and never learns that a bus exists. +//! +//! # Subscribing is not part of this seam +//! +//! Several extracted modules used to *subscribe* as well as publish +//! (`sync_events.rs`, `sync/composio/bus.rs`, `conversations/bus.rs`). Those are +//! host wiring by the repository README's split — event-bus subscribers belong +//! in the host, next to the registration site that installs them. They move back +//! rather than growing a subscribe method here. + +/// Why an embedding model was reported unhealthy, and what took over. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmbeddingHealthReason { + /// The provider that failed. + pub provider: String, + /// The model that failed. + pub model: String, + /// The provider bound in its place. + pub fallback_provider: String, + /// Operator-facing explanation. Never carries credentials. + pub message: String, +} + +/// What kicked off a sync run — a schedule, a user action, a webhook. +pub type SyncTrigger = String; + +/// A memory-domain event, as announced by `tinymemory-core`. +/// +/// Field names and types mirror the host's own event payloads exactly, so the +/// host's [`MemoryEventSink`] impl is a straight structural mapping with no +/// judgement calls in it. +/// +/// Deliberately **not** `#[non_exhaustive]`: the host's mapping impl matches +/// exhaustively on purpose, so adding a variant here is a compile error at the +/// mapping site rather than an event that silently never reaches the bus. +#[derive(Debug, Clone)] +pub enum MemoryEvent { + /// A sync run moved to a new stage. + SyncStageChanged { + /// What started this run. + trigger: SyncTrigger, + /// The stage just entered. + stage: String, + /// Provider slug, when the stage is provider-scoped. + provider: Option, + /// Connection id, when the stage is connection-scoped. + connection_id: Option, + /// Free-form operator detail. + detail: Option, + /// Memory-source id, when the stage is source-scoped. + source_id: Option, + }, + /// A document entered the ingestion pipeline. + IngestionStarted { + /// Document being ingested. + document_id: String, + /// Human-readable document title. + title: String, + /// Target namespace. + namespace: String, + /// Items still queued behind this one. + queue_depth: usize, + }, + /// A document left the ingestion pipeline. + IngestionCompleted { + /// Document that was ingested. + document_id: String, + /// Target namespace. + namespace: String, + /// Whether ingestion succeeded. + success: bool, + /// Wall-clock duration. + elapsed_ms: u64, + /// Items still queued afterwards. + queue_depth: usize, + }, + /// A source document was canonicalized into chunks. + DocumentCanonicalized { + /// Source the document came from. + source_id: String, + /// Source kind (`gmail`, `slack`, `file`, …). + source_kind: String, + /// How many chunks were written. + chunks_written: usize, + /// Ids of the written chunks. + chunk_ids: Vec, + /// Unix timestamp, seconds with fraction. + canonicalized_at: f64, + /// Truncated body preview for operator UIs. + body_preview: Option, + }, + /// An hour bucket was sealed and summarized. + TreeSummarizerHourCompleted { + /// Tree namespace. + namespace: String, + /// Node that was sealed. + node_id: String, + /// Tokens in the produced summary. + token_count: u32, + }, + /// A summary was propagated up a level. + TreeSummarizerPropagated { + /// Tree namespace. + namespace: String, + /// Node that received the propagated summary. + node_id: String, + /// Level name. + level: String, + /// Tokens in the produced summary. + token_count: u32, + }, + /// A full tree rebuild finished. + TreeSummarizerRebuildCompleted { + /// Tree namespace. + namespace: String, + /// Nodes in the rebuilt tree. + total_nodes: u64, + }, + /// Progress ticks during a tree build, for the operator UI. + TreeBuildProgress { + /// Coarse phase name. + phase: String, + /// Fine step name. + step: String, + /// Which tree, when scoped. + tree_scope: Option, + /// Tree level, when levelled. + level: Option, + /// Items processed in this step. + item_count: Option, + /// Free-form operator detail. + detail: Option, + }, + /// An embedding model failed health checks and a fallback was bound. + EmbeddingModelUnhealthy(EmbeddingHealthReason), + /// The configured memory driver could not be bound, and another was used. + DriverBindFailed { + /// Driver named in config. + configured_driver: String, + /// Driver actually bound. + bound_driver: String, + /// Why the configured driver was rejected. + reason: String, + }, + /// A diff snapshot was captured for a source. + DiffSnapshotTaken { + /// The new snapshot. + snapshot_id: String, + /// Source the snapshot covers. + source_id: String, + /// Source kind. + source_kind: String, + /// Items in the snapshot. + item_count: usize, + /// What triggered the snapshot. + trigger: String, + }, + /// Diffs were acknowledged by the user. + DiffMarkedRead { + /// Sources marked read. + source_ids: Vec, + /// Snapshots marked read. + snapshot_ids: Vec, + }, + /// The set of connected Composio toolkits changed. + ComposioIntegrationsChanged { + /// Toolkit slugs now connected. + toolkits: Vec, + }, + /// The memory subsystem is asking for a sync run. + SyncRequested { + /// Channel to report progress back on, when the request came from one. + channel_id: Option, + }, + /// The local embedding runtime is unusable and the user must act outside + /// the app (start Ollama, pull the model). + /// + /// The host surfaces this in its durable user-error centre. Carries no + /// provider text, model id or endpoint — see [`LOCAL_MODEL_UNAVAILABLE_KIND`]. + LocalModelUnavailable { + /// Short, non-sensitive tag naming which producer fired + /// (`health_gate` / `embed_classify`), so the two paths stay + /// distinguishable in the log without a correlation id. + origin: String, + }, +} + +/// Stable `error_type` token for the local-embedding-runtime user error. +/// +/// Mirrors the frontend `UserErrorKind` discriminator of the same name. It is +/// defined in the contract crate because both sides name it: the host builds +/// the wire payload from it, and the core's tests assert on it. A drift on +/// either side drops the signal silently. +pub const LOCAL_MODEL_UNAVAILABLE_KIND: &str = "local_model_unavailable"; + +/// `error_source` for the memory subsystem's user errors. Drives the panel's +/// scope grouping (`socketService` maps it to the `memory` `UserErrorScope`). +pub const MEMORY_USER_ERROR_SOURCE: &str = "memory"; + +/// Receives [`MemoryEvent`]s and does something host-shaped with them. +pub trait MemoryEventSink: Send + Sync + std::fmt::Debug { + /// Announce an event. Implementations must not block and must not fail — + /// an event bus that can reject a publish turns every emit site into an + /// error path, which is not what any of the call sites want. + fn publish(&self, event: MemoryEvent); +} + +/// The sink bound when no host has installed one — in unit tests, in the +/// standalone engine build, and before startup wiring runs. Drops everything. +#[derive(Debug, Clone, Copy, Default)] +pub struct NoopEventSink; + +impl MemoryEventSink for NoopEventSink { + fn publish(&self, _event: MemoryEvent) {} +} diff --git a/api/src/host/evidence.rs b/api/src/host/evidence.rs new file mode 100644 index 0000000..7134b6d --- /dev/null +++ b/api/src/host/evidence.rs @@ -0,0 +1,49 @@ +//! [`EvidenceRef`] — a pointer to the thing a learned fact was learned from. +//! +//! Moved here from the host's `agent::learning::candidate` because it is +//! persisted *in the memory store*: `store::namespace_store::profile` writes it +//! into profile rows, and the Composio provider-profile sync reads it back. Two +//! structurally identical enums either side of the seam would round-trip +//! through serde and silently diverge on the first added variant. +//! +//! Inert serde data; the contract crate's dependency-light guarantee is +//! unaffected. **Its serde form is persisted**, so the `#[serde(tag = "type")]` +//! representation and every variant name are a compatibility surface. + +use serde::{Deserialize, Serialize}; + +/// A typed pointer back into the memory substrate from which a candidate was +/// derived. Used for provenance tracking, citation, and the `evidence_ids` +/// column in `user_profile_facets` (Phase 3+). +/// +/// Serialised with a `"type"` discriminator in snake_case so the JSON is +/// human-readable: `{"type":"episodic","episodic_id":42}`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EvidenceRef { + /// A single row in `episodic_log`. + Episodic { episodic_id: i64 }, + /// A contiguous window of rows in `episodic_log`. + EpisodicWindow { from_id: i64, to_id: i64 }, + /// A row in the tree-source summary table. + SourceSummary { summary_id: String }, + /// A node in `tree_topic`. + TreeTopic { topic_id: String }, + /// A chunk in `vector_chunks` associated with a document source. + DocumentChunk { source_id: String, chunk_id: String }, + /// A specific message in an email source. + EmailMessage { + source_id: String, + message_id: String, + }, + /// A field value from a connected provider (Composio toolkit). + Provider { + toolkit: String, + connection_id: String, + field: String, + }, + /// A tool call record within an episodic entry. + ToolCall { tool_name: String, episodic_id: i64 }, + /// A per-window weight from `tree_source`. + TreeSourceWeight { window_label: String }, +} diff --git a/api/src/host/local_ai.rs b/api/src/host/local_ai.rs new file mode 100644 index 0000000..f5cee1e --- /dev/null +++ b/api/src/host/local_ai.rs @@ -0,0 +1,285 @@ +//! Local AI runtime configuration. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Per-feature flags controlling which subsystems route through the selected +/// local runtime. All default to `false` (use cloud instead). Guarded by +/// `LocalAiConfig::runtime_enabled` — when that is `false` every helper +/// method below returns `false` regardless of these values. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +#[derive(Default)] +pub struct LocalAiUsage { + /// When true (and `runtime_enabled`), use the local model for embedding + /// generation instead of the cloud backend. + #[serde(default)] + pub embeddings: bool, + /// When true (and `runtime_enabled`), use the local model inside the + /// heartbeat loop. + #[serde(default)] + pub heartbeat: bool, + /// When true (and `runtime_enabled`), use the local model for + /// learning/reflection passes. + #[serde(default)] + pub learning_reflection: bool, + /// When true (and `runtime_enabled`), use the local model for + /// subconscious evaluation and execution. + #[serde(default)] + pub subconscious: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct LocalAiConfig { + /// Master runtime switch. Defaults to `false` — local AI is OFF by default. + /// Note: the old on-disk field was `enabled`; that key is now unknown to + /// serde and will be silently ignored on load (intentional forced reset). + #[serde(default = "default_runtime_enabled")] + pub runtime_enabled: bool, + /// Local provider identifier. Supported values are `ollama`, `lm_studio`, + /// and `omlx`; unknown values normalize to `ollama` at runtime. + #[serde(default = "default_provider")] + pub provider: String, + /// Optional provider base URL. For LM Studio this defaults to + /// `http://localhost:1234/v1`. + #[serde(default)] + pub base_url: Option, + #[serde(default)] + pub api_key: Option, + #[serde(default = "default_model_id")] + pub model_id: String, + #[serde(default = "default_chat_model_id")] + pub chat_model_id: String, + #[serde(default = "default_vision_model_id")] + pub vision_model_id: String, + #[serde(default = "default_embedding_model_id")] + pub embedding_model_id: String, + #[serde(default = "default_stt_model_id")] + pub stt_model_id: String, + #[serde(default = "default_stt_download_url")] + pub stt_download_url: Option, + /// Legacy voice STT routing string. `"cloud"` (the default) means "use + /// `voice_server.stt_engine`"; a third-party `"[:]"` overrides + /// the engine outright. The local `"whisper"` value it once accepted is + /// dead — `config::migrations` rewrites it back to `"cloud"`. + #[serde(default = "default_stt_provider")] + pub stt_provider: String, + #[serde(default = "default_tts_voice_id")] + pub tts_voice_id: String, + /// Voice TTS provider selector. `"cloud"` (default) routes through the + /// backend ElevenLabs proxy and returns rich visemes; `"piper"` runs + /// local Piper via the `PIPER_BIN` env var. + #[serde(default = "default_tts_provider")] + pub tts_provider: String, + #[serde(default = "default_tts_download_url")] + pub tts_download_url: Option, + #[serde(default = "default_tts_config_download_url")] + pub tts_config_download_url: Option, + #[serde(default = "default_quantization")] + pub quantization: String, + #[serde(default = "default_preload_vision_model")] + pub preload_vision_model: bool, + #[serde(default = "default_preload_embedding_model")] + pub preload_embedding_model: bool, + #[serde(default = "default_preload_stt_model")] + pub preload_stt_model: bool, + #[serde(default = "default_preload_tts_voice")] + pub preload_tts_voice: bool, + #[serde(default = "default_download_url")] + pub download_url: Option, + #[serde(default = "default_autosummary_debounce_ms")] + pub autosummary_debounce_ms: u64, + #[serde(default)] + pub selected_tier: Option, + /// Explicit MVP opt-in marker. Bootstrap disables local AI unless this is + /// `true`, regardless of any prior `selected_tier` value. Existing installs + /// (upgrading from pre-MVP) default to `false` and must re-opt-in from + /// Settings. Set by `apply_preset` on any non-disabled tier. + #[serde(default)] + pub opt_in_confirmed: bool, + /// Optional path to a manually-installed Ollama binary. + #[serde(default)] + pub ollama_binary_path: Option, + /// When true and Ollama is available, pass raw transcription through a + /// local LLM to fix grammar/punctuation using conversation context. + #[serde(default = "default_voice_llm_cleanup_enabled")] + pub voice_llm_cleanup_enabled: bool, + /// Ollama `options.num_ctx` override. When set, every chat request to + /// an Ollama provider includes `"options": {"num_ctx": }` so + /// the model allocates at least this much KV-cache. Ollama defaults + /// to 2048 for many models which is too small for agentic use. + #[serde(default)] + pub num_ctx: Option, + /// Per-feature flags. Each gate is AND-ed with `runtime_enabled`. + /// All default to `false` (cloud path). + #[serde(default)] + pub usage: LocalAiUsage, +} + +fn default_runtime_enabled() -> bool { + false +} + +fn default_provider() -> String { + "ollama".to_string() +} + +fn default_model_id() -> String { + "gemma3:1b-it-qat".to_string() +} + +fn default_chat_model_id() -> String { + "gemma3:1b-it-qat".to_string() +} + +fn default_vision_model_id() -> String { + String::new() +} + +fn default_embedding_model_id() -> String { + // bge-m3 (1024 dims, 8192-token context). Required by the memory tree's + // fixed on-disk embedding format (EMBEDDING_DIM=1024) — `all-minilm` + // (384 dims) and `nomic-embed-text` (768 dims) would fail the + // post-call dim validator at `memory::tree::score::embed::mod::embed`. + "bge-m3".to_string() +} + +fn default_stt_model_id() -> String { + "ggml-base-q5_1.bin".to_string() +} + +fn default_tts_voice_id() -> String { + "en_US-lessac-medium".to_string() +} + +fn default_stt_provider() -> String { + "cloud".to_string() +} + +fn default_tts_provider() -> String { + "cloud".to_string() +} + +fn default_stt_download_url() -> Option { + Some( + "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q5_1.bin?download=true" + .to_string(), + ) +} + +fn default_tts_download_url() -> Option { + Some( + "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx?download=true" + .to_string(), + ) +} + +fn default_tts_config_download_url() -> Option { + Some( + "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json?download=true" + .to_string(), + ) +} + +fn default_quantization() -> String { + "q4".to_string() +} + +fn default_preload_vision_model() -> bool { + false +} + +fn default_preload_embedding_model() -> bool { + true +} + +fn default_preload_stt_model() -> bool { + false +} + +fn default_preload_tts_voice() -> bool { + false +} + +fn default_download_url() -> Option { + None +} + +fn default_autosummary_debounce_ms() -> u64 { + 2500 +} + +fn default_voice_llm_cleanup_enabled() -> bool { + true +} + +impl LocalAiConfig { + /// Returns `true` when the local Ollama runtime is active. + /// This is the primary gate; all per-feature helpers below AND with this. + pub fn is_active(&self) -> bool { + self.runtime_enabled + } + + /// **Deprecated** — read from `Config::workload_uses_local("embeddings")` + /// instead. This helper only consults the legacy `usage.*` booleans, which + /// are no longer the source of truth after the unified AI settings + /// migration (schema_version >= 2). + #[deprecated(note = "Use Config::workload_uses_local(\"embeddings\")")] + pub fn use_local_for_embeddings(&self) -> bool { + self.runtime_enabled && self.usage.embeddings + } + + /// **Deprecated** — read from `Config::workload_uses_local("heartbeat")`. + #[deprecated(note = "Use Config::workload_uses_local(\"heartbeat\")")] + pub fn use_local_for_heartbeat(&self) -> bool { + self.runtime_enabled && self.usage.heartbeat + } + + /// **Deprecated** — read from `Config::workload_uses_local("learning")`. + #[deprecated(note = "Use Config::workload_uses_local(\"learning\")")] + pub fn use_local_for_learning(&self) -> bool { + self.runtime_enabled && self.usage.learning_reflection + } + + /// **Deprecated** — read from `Config::workload_uses_local("subconscious")`. + #[deprecated(note = "Use Config::workload_uses_local(\"subconscious\")")] + pub fn use_local_for_subconscious(&self) -> bool { + self.runtime_enabled && self.usage.subconscious + } +} + +impl Default for LocalAiConfig { + fn default() -> Self { + Self { + runtime_enabled: default_runtime_enabled(), + provider: default_provider(), + base_url: None, + api_key: None, + model_id: default_model_id(), + chat_model_id: default_chat_model_id(), + vision_model_id: default_vision_model_id(), + embedding_model_id: default_embedding_model_id(), + stt_model_id: default_stt_model_id(), + stt_download_url: default_stt_download_url(), + stt_provider: default_stt_provider(), + tts_voice_id: default_tts_voice_id(), + tts_provider: default_tts_provider(), + tts_download_url: default_tts_download_url(), + tts_config_download_url: default_tts_config_download_url(), + quantization: default_quantization(), + preload_vision_model: default_preload_vision_model(), + preload_embedding_model: default_preload_embedding_model(), + preload_stt_model: default_preload_stt_model(), + preload_tts_voice: default_preload_tts_voice(), + download_url: default_download_url(), + autosummary_debounce_ms: default_autosummary_debounce_ms(), + selected_tier: None, + opt_in_confirmed: false, + ollama_binary_path: None, + voice_llm_cleanup_enabled: default_voice_llm_cleanup_enabled(), + num_ctx: None, + usage: LocalAiUsage::default(), + } + } +} diff --git a/api/src/host/mod.rs b/api/src/host/mod.rs new file mode 100644 index 0000000..d3b4352 --- /dev/null +++ b/api/src/host/mod.rs @@ -0,0 +1,93 @@ +//! The **host seam** — everything `tinymemory-core` needs from the application +//! that embeds it, expressed as object-safe traits plus the plain serde config +//! structs the memory subsystem owns. +//! +//! # Why this module exists +//! +//! `tinymemory-core` holds the substance of a memory subsystem: the store, the +//! summary tree, the sync pipelines, ingestion, recall. Per the repository +//! README's split, the *host* keeps the RPC surface, the agent tools, the +//! security policy, the schedulers, the event bus, and config loading. That +//! split only works if the core can name what it needs from the host without +//! naming the host itself — which is what these traits are. +//! +//! # The three seams +//! +//! - [`MemoryHostConfig`] — the host's configuration, read through accessor +//! methods rather than public fields. `tinymemory_core::Config` is the type +//! alias `dyn MemoryHostConfig`, so code moved out of the host keeps writing +//! `config: &Config` and the host's concrete `Config` unsize-coerces at every +//! call site. +//! - [`EmbeddingProvider`] — text → vector. The core never builds one; the host +//! resolves provider credentials, rate limits and routing and hands an +//! `Arc` down. +//! - [`MemoryEventSink`] — the handful of domain events the memory subsystem +//! publishes. The host implements it by publishing its own event enum onto +//! its own bus; the core never learns that enum exists. +//! +//! # Config *sections* live here, config *loading* does not +//! +//! [`MemoryConfig`], [`MemoryTreeConfig`], [`MemorySubsystemConfig`] and friends +//! moved here from the host because the core reads their fields directly and a +//! trait accessor per field would be absurd. They are inert serde/`schemars` +//! data with no behaviour, and **their serde representation is persisted in +//! users' `config.toml`** — field names, defaults, and `#[serde(...)]` +//! attributes are a compatibility surface, not an implementation detail. +//! +//! Sections that are *not* memory-owned but that the core still reads +//! ([`LocalAiConfig`], [`cloud_providers`]) are here for the same mechanical +//! reason. They are the seam's rough edge: the honest fix is to move embedding +//! *construction* back into the host, at which point the core stops reading +//! them and they can go home. + +pub mod cloud_providers; +pub mod composio; +pub mod local_ai; +pub mod scheduler_gate; +pub mod storage_memory; +pub mod subsystems; + +mod config; +mod embedding_host; +mod embeddings; +mod error_reporter; +mod events; +mod evidence; +mod nlp; +mod routes; +mod usage; + +#[cfg(feature = "test-support")] +pub mod test_support; + +pub use cloud_providers::{ + endpoint_host, generate_provider_id, is_slug_reserved, migrate_legacy_fields, AuthStyle, + CloudProviderCreds, CloudProviderType, +}; +pub use config::{ComposioMode, MemoryHostConfig, COMPOSIO_MODE_BACKEND, COMPOSIO_MODE_DIRECT}; +pub use embedding_host::EmbeddingHost; +pub use embeddings::{format_embedding_signature, EmbeddingProvider, NoopEmbedding}; +pub use error_reporter::ErrorReporter; +pub use events::{ + EmbeddingHealthReason, MemoryEvent, MemoryEventSink, NoopEventSink, SyncTrigger, + LOCAL_MODEL_UNAVAILABLE_KIND, MEMORY_USER_ERROR_SOURCE, +}; +pub use evidence::EvidenceRef; +pub use local_ai::{LocalAiConfig, LocalAiUsage}; +pub use nlp::{SpacyEntity, SpacyResponse}; +pub use routes::EmbeddingRouteConfig; +pub use scheduler_gate::{PauseReason, Policy, SchedulerGateConfig, SchedulerGateMode}; +pub use storage_memory::{ + LlmBackend, MemoryConfig, MemoryTreeConfig, StorageConfig, StorageProviderConfig, + StorageProviderSection, DEFAULT_CLOUD_LLM_MODEL, +}; +pub use subsystems::{ + MemoryDriverConfig, MemoryHooksConfig, MemorySubsystemConfig, SubsystemsConfig, +}; +pub use usage::UsageInfo; + +/// Effective default global memory-sync cadence (seconds) used when +/// [`MemoryHostConfig::memory_sync_interval_secs`] is `None` — i.e. the user has +/// not explicitly picked a schedule. 24h, matching the "Sync every 24h" preset +/// surfaced in the Memory Sources UI. +pub const DEFAULT_MEMORY_SYNC_INTERVAL_SECS: u64 = 86_400; diff --git a/api/src/host/nlp.rs b/api/src/host/nlp.rs new file mode 100644 index 0000000..46e6660 --- /dev/null +++ b/api/src/host/nlp.rs @@ -0,0 +1,30 @@ +//! spaCy extraction results — the wire shape of the host's Python NLP server. +//! +//! Moved here from the host's `runtime::python_server::spacy` because the +//! summary tree's query-entity extractor consumes them directly, canonicalising +//! each entity into the same `:` namespace the indexed chunks use. +//! Inert serde data. +//! +//! Provisioning the runtime (`ensure_spacy`, `spacy_provisioned`, the model id) +//! deliberately stayed in the host: downloading and launching a Python server +//! is not something a memory engine should do. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SpacyEntity { + pub text: String, + pub label: String, + #[serde(default)] + pub start: u32, + #[serde(default)] + pub end: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SpacyResponse { + #[serde(default)] + pub entities: Vec, + #[serde(default)] + pub nouns: Vec, +} diff --git a/api/src/host/routes.rs b/api/src/host/routes.rs new file mode 100644 index 0000000..f5204e5 --- /dev/null +++ b/api/src/host/routes.rs @@ -0,0 +1,18 @@ +//! [`EmbeddingRouteConfig`] — a per-workload embedding provider override. +//! +//! Moved here from the host's `config::schema::routes` because the memory +//! store's factory reads its fields directly when resolving which embedder +//! backs a workload. Inert serde data; **its serde form is persisted** in +//! users' `config.toml`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct EmbeddingRouteConfig { + pub hint: String, + pub provider: String, + pub model: String, + #[serde(default)] + pub dimensions: Option, +} diff --git a/api/src/host/scheduler_gate.rs b/api/src/host/scheduler_gate.rs new file mode 100644 index 0000000..9d3b4e3 --- /dev/null +++ b/api/src/host/scheduler_gate.rs @@ -0,0 +1,188 @@ +//! Scheduler-gate configuration — controls when background AI work runs. +//! +//! Consumed by `openhuman::cron::scheduler_gate`. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +#[derive(Default)] +pub enum SchedulerGateMode { + /// Decide based on power + CPU + deployment-mode signals. + #[default] + Auto, + /// Always run background AI flat-out (server / power-user setting). + AlwaysOn, + /// Never run background AI. User can still trigger work explicitly. + Off, +} + +impl SchedulerGateMode { + pub fn as_str(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::AlwaysOn => "always_on", + Self::Off => "off", + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct SchedulerGateConfig { + /// Top-level mode — `auto` (default), `always_on`, or `off`. + #[serde(default)] + pub mode: SchedulerGateMode, + + /// Battery charge floor in `auto` mode, 0.0..=1.0. Below this and not on + /// AC, the gate throttles. Default: 0.80. + #[serde(default = "default_battery_floor")] + pub battery_floor: f32, + + /// CPU busy threshold (recent global usage, 0..100). Above this, the gate + /// throttles even when plugged in. Default: 70.0 (i.e. <30% headroom). + #[serde(default = "default_cpu_busy_threshold")] + pub cpu_busy_threshold_pct: f32, + + /// In `Throttled` mode, sleep this many ms before each LLM-bound job to + /// serialise workers and let the host catch up. Default: 30_000 (30s). + #[serde(default = "default_throttled_backoff_ms")] + pub throttled_backoff_ms: u64, + + /// In `Paused` mode, re-check the policy every this many ms so workers + /// resume promptly when the user toggles the gate back on. Default: + /// 60_000 (60s). + #[serde(default = "default_paused_poll_ms")] + pub paused_poll_ms: u64, + + /// Hard CPU ceiling (recent global usage, 0..100). When the host CPU + /// climbs above this in `auto` mode, the gate flips to + /// `Paused { CpuPressure }` rather than just `Throttled` — every + /// background LLM call is held until the host calms down. Distinct + /// from `cpu_busy_threshold_pct`, which only triggers `Throttled`. + /// Default: 95.0. + #[serde(default = "default_cpu_severe_pct")] + pub cpu_severe_pct: f32, + + /// When `true`, `auto` mode only runs background LLM work while the + /// laptop is on AC power. On battery the gate flips to + /// `Paused { OnBattery }` — no background inference at all, + /// regardless of charge level. + /// + /// Default `false` to preserve the prior behavior (battery-floor + /// based throttling). Power-conscious users who never want + /// background inference on battery can flip this on. + #[serde(default)] + pub require_ac_power: bool, +} + +fn default_battery_floor() -> f32 { + 0.80 +} +fn default_cpu_busy_threshold() -> f32 { + 70.0 +} +fn default_throttled_backoff_ms() -> u64 { + 30_000 +} +fn default_paused_poll_ms() -> u64 { + 60_000 +} +fn default_cpu_severe_pct() -> f32 { + 95.0 +} + +impl Default for SchedulerGateConfig { + fn default() -> Self { + Self { + mode: SchedulerGateMode::default(), + battery_floor: default_battery_floor(), + cpu_busy_threshold_pct: default_cpu_busy_threshold(), + throttled_backoff_ms: default_throttled_backoff_ms(), + paused_poll_ms: default_paused_poll_ms(), + cpu_severe_pct: default_cpu_severe_pct(), + require_ac_power: false, + } + } +} + +// ── Gate decision vocabulary ──────────────────────────────────────────────── +// +// `Policy` and `PauseReason` moved here from the host's +// `cron::scheduler_gate::policy` because the extracted sync loops read them on +// every tick to decide whether to back off. They are inert `Copy` enums with no +// dependencies; the *decision function* that produces a `Policy` from sampled +// signals stays in the host, where the signals are. + +/// Why the gate is currently paused. Carried by [`Policy::Paused`] so +/// downstream consumers (UI, logging, observability) can surface a +/// specific user-facing reason instead of a generic "paused" label. +/// +/// New variants will land alongside #1073's full power-aware work +/// (`OnBattery`, `CpuPressure`); `UserDisabled` covers the existing +/// `SchedulerGateMode::Off` path and `Unknown` is the safe fallback for +/// callers that don't have specific context yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PauseReason { + /// User explicitly turned the gate off in config. + UserDisabled, + /// Host on battery and gate's power-aware mode kicked in (#1073). + OnBattery, + /// CPU pressure exceeded the gate threshold (#1073). + CpuPressure, + /// No active app session — background AI work is suspended until the + /// user signs in again. Trumps every other signal: while signed out + /// the host should do *no* LLM-bound work, period. Set by + /// `gate::set_signed_out(true)` from the credentials lifecycle and + /// from 401-detection sites. + SignedOut, + /// Pause reason not yet classified — placeholder while #1073 is in flight. + Unknown, +} + +impl PauseReason { + pub fn as_str(self) -> &'static str { + match self { + Self::UserDisabled => "user_disabled", + Self::OnBattery => "on_battery", + Self::CpuPressure => "cpu_pressure", + Self::SignedOut => "signed_out", + Self::Unknown => "unknown", + } + } +} + +/// Background-AI scheduling tier. See module docs in `mod.rs` for semantics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Policy { + Aggressive, + Normal, + Throttled, + /// Gate paused. The `reason` is rendered to users in the memory-sync + /// status UI (#1136) and recorded in observability. + Paused { + reason: PauseReason, + }, +} + +impl Policy { + pub fn as_str(self) -> &'static str { + match self { + Self::Aggressive => "aggressive", + Self::Normal => "normal", + Self::Throttled => "throttled", + Self::Paused { .. } => "paused", + } + } + + /// `Some(reason)` when paused, `None` otherwise. Convenience for + /// callers that only need the reason and don't want to pattern-match + /// the whole enum (UI badges, log line construction). + pub fn pause_reason(self) -> Option { + match self { + Self::Paused { reason } => Some(reason), + _ => None, + } + } +} diff --git a/api/src/host/storage_memory.rs b/api/src/host/storage_memory.rs new file mode 100644 index 0000000..928f1da --- /dev/null +++ b/api/src/host/storage_memory.rs @@ -0,0 +1,561 @@ +//! Storage provider and memory configuration. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Debug, Clone, Serialize, Deserialize, Default, JsonSchema)] +#[serde(default)] +pub struct StorageConfig { + #[serde(default)] + pub provider: StorageProviderSection, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, JsonSchema)] +#[serde(default)] +pub struct StorageProviderSection { + #[serde(default)] + pub config: StorageProviderConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +#[derive(Default)] +pub struct StorageProviderConfig { + #[serde(default)] + pub provider: String, +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[allow(clippy::struct_excessive_bools)] +#[serde(default)] +pub struct MemoryConfig { + #[serde(default = "default_memory_backend")] + pub backend: String, + #[serde(default = "default_true")] + pub auto_save: bool, + #[serde(default = "default_embedding_provider")] + pub embedding_provider: String, + #[serde(default = "default_embedding_model")] + pub embedding_model: String, + #[serde(default = "default_embedding_dims")] + pub embedding_dimensions: usize, + /// Outbound embedding-request budget for cloud providers, in requests per + /// minute. Cloud backends (OpenHuman/Voyage, OpenAI, remote `custom:` + /// endpoints) cap requests per account; the client throttles to stay under + /// that quota rather than tripping 429s. `0` disables throttling. Loopback + /// endpoints are always exempt. Env override: + /// `OPENHUMAN_MEMORY_EMBED_RATE_LIMIT`. + #[serde(default = "default_embedding_rate_limit_per_min")] + pub embedding_rate_limit_per_min: u32, + #[serde(default = "default_min_relevance_score")] + pub min_relevance_score: f64, + #[serde(default)] + pub sqlite_open_timeout_secs: Option, + + /// Base URL for the `agentmemory` REST server. Honored only when + /// `backend = "agentmemory"`. Defaults to `http://localhost:3111` + /// (the agentmemory loopback default). + #[serde(default)] + pub agentmemory_url: Option, + + /// Optional bearer token sent as `Authorization: Bearer ` + /// to the agentmemory REST server. When unset, the backend speaks + /// to a local agentmemory daemon without authentication. Setting a + /// secret + a non-loopback host enables the v0.9.12 plaintext-bearer + /// guard semantics on the client side: the backend refuses to send + /// the token over plaintext HTTP when the host is not loopback. + #[serde(default)] + pub agentmemory_secret: Option, + + /// Per-request timeout for the agentmemory REST client, in + /// milliseconds. Defaults to 5000 ms. + #[serde(default)] + pub agentmemory_timeout_ms: Option, +} + +fn default_memory_backend() -> String { + "sqlite".into() +} + +fn default_true() -> bool { + true +} + +fn default_embedding_provider() -> String { + // Default to the OpenHuman backend (Voyage-backed `embedding-v1`) so a + // fresh install works without requiring a local Ollama daemon. Users + // who want fully-local embeddings can flip this to "ollama" in + // `config.toml` or enable `local_ai.usage.embeddings = true`, which is + // wired into the memory factory via `LocalAiConfig::use_local_for_embeddings`. + "cloud".into() +} +fn default_embedding_model() -> String { + // Keep this in sync with `embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_MODEL`. + "embedding-v1".into() +} +fn default_embedding_dims() -> usize { + // Keep this in sync with `embeddings::cloud::DEFAULT_CLOUD_EMBEDDING_DIMENSIONS`. + 1024 +} +fn default_embedding_rate_limit_per_min() -> u32 { + // Cloud embedding backends cap requests at ~60/min per account. Keep in + // sync with `embeddings::rate_limit::DEFAULT_EMBEDDING_RATE_LIMIT_PER_MIN`. + 60 +} +fn default_min_relevance_score() -> f64 { + 0.4 +} + +impl Default for MemoryConfig { + fn default() -> Self { + Self { + backend: default_memory_backend(), + auto_save: default_true(), + embedding_provider: default_embedding_provider(), + embedding_model: default_embedding_model(), + embedding_dimensions: default_embedding_dims(), + embedding_rate_limit_per_min: default_embedding_rate_limit_per_min(), + min_relevance_score: default_min_relevance_score(), + sqlite_open_timeout_secs: None, + agentmemory_url: None, + agentmemory_secret: None, + agentmemory_timeout_ms: None, + } + } +} + +// Manual `Debug` implementation that redacts `agentmemory_secret`. Without +// this, any `format!("{cfg:?}")` / `tracing::debug!(?cfg, ...)` / panic +// message capturing a `MemoryConfig` would dump the bearer token in +// plaintext — directly against the repo rule "Never log secrets, raw +// JWTs, API keys, credentials, or full PII in debug logs". +impl std::fmt::Debug for MemoryConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryConfig") + .field("backend", &self.backend) + .field("auto_save", &self.auto_save) + .field("embedding_provider", &self.embedding_provider) + .field("embedding_model", &self.embedding_model) + .field("embedding_dimensions", &self.embedding_dimensions) + .field( + "embedding_rate_limit_per_min", + &self.embedding_rate_limit_per_min, + ) + .field("min_relevance_score", &self.min_relevance_score) + .field("sqlite_open_timeout_secs", &self.sqlite_open_timeout_secs) + .field("agentmemory_url", &self.agentmemory_url) + .field( + "agentmemory_secret", + &self.agentmemory_secret.as_ref().map(|_| ""), + ) + .field("agentmemory_timeout_ms", &self.agentmemory_timeout_ms) + .finish() + } +} + +/// Which inference backend the memory_tree's LLM calls (extractor + +/// summariser) should use. +/// +/// - `Cloud` (default): route through `providers::router` against the +/// OpenHuman backend with the `summarization-v1` model. No local Ollama +/// required. +/// - `Local`: keep using the legacy Ollama-direct path (the +/// `llm_extractor_endpoint` / `llm_summariser_endpoint` config). Useful +/// for offline development and CI smoke tests. +/// +/// Embedder selection is unchanged — `OllamaEmbedder` (bge-m3) stays +/// local-only and isn't governed by this enum. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +#[derive(Default)] +pub enum LlmBackend { + /// Route through the OpenHuman backend (default). + #[default] + Cloud, + /// Use the local Ollama path configured via `llm_extractor_*` / + /// `llm_summariser_*`. + Local, +} + +impl LlmBackend { + /// Stable wire string for env vars / RPCs / logs. + pub fn as_str(self) -> &'static str { + match self { + Self::Cloud => "cloud", + Self::Local => "local", + } + } + + /// Inverse of [`Self::as_str`]; case-insensitive parse. + pub fn parse(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "cloud" => Ok(Self::Cloud), + "local" => Ok(Self::Local), + other => Err(format!("unknown llm (expected cloud|local): {other}")), + } + } +} + +fn default_llm_backend() -> LlmBackend { + LlmBackend::default() +} + +/// Default model identifier to use when `llm_backend = "cloud"`. Routed +/// through the OpenHuman backend; keep in sync with the backend's +/// summariser model registry. +pub const DEFAULT_CLOUD_LLM_MODEL: &str = "summarization-v1"; + +fn default_cloud_llm_model() -> Option { + Some(DEFAULT_CLOUD_LLM_MODEL.to_string()) +} + +/// Phase 4 memory-tree configuration — embedding provider wiring for the +/// hierarchical memory (#710). +/// +/// When `embedding_endpoint` and `embedding_model` are both set, ingest +/// and bucket-seal route every new chunk/summary through the Ollama +/// embedder before writing. When unset, behaviour depends on +/// `embedding_strict`: +/// - `true` (default): ingest/seal bail with a clear config error. +/// - `false`: fall back to the inert zero-vector embedder and warn. +/// +/// Env overrides apply in `openhuman::config::schema::load`: +/// - `OPENHUMAN_MEMORY_EMBED_ENDPOINT` +/// - `OPENHUMAN_MEMORY_EMBED_MODEL` +/// - `OPENHUMAN_MEMORY_EMBED_TIMEOUT_MS` +/// - `OPENHUMAN_MEMORY_EXTRACT_ENDPOINT` +/// - `OPENHUMAN_MEMORY_EXTRACT_MODEL` +/// - `OPENHUMAN_MEMORY_EXTRACT_TIMEOUT_MS` +/// - `OPENHUMAN_MEMORY_SUMMARISE_ENDPOINT` +/// - `OPENHUMAN_MEMORY_SUMMARISE_MODEL` +/// - `OPENHUMAN_MEMORY_SUMMARISE_TIMEOUT_MS` +/// - `OPENHUMAN_MEMORY_TREE_CONTENT_DIR` (Phase MD-content) +/// - `OPENHUMAN_MEMORY_TREE_LLM_BACKEND` (cloud|local) +/// - `OPENHUMAN_MEMORY_TREE_CLOUD_LLM_MODEL` +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemoryTreeConfig { + /// Ollama endpoint for the embedder (e.g. `http://localhost:11434`). + /// `None` disables the Ollama path — see `embedding_strict` for the + /// resulting behaviour. + #[serde(default = "default_memory_tree_embedding_endpoint")] + pub embedding_endpoint: Option, + + /// Embedding model name. Must produce 768-dim vectors (see + /// `memory::tree::score::embed::EMBEDDING_DIM`). `None` disables + /// the Ollama path. + #[serde(default = "default_memory_tree_embedding_model")] + pub embedding_model: Option, + + /// Per-request timeout for the embedder, in milliseconds. + #[serde(default = "default_memory_tree_embedding_timeout_ms")] + pub embedding_timeout_ms: Option, + + /// When true, ingest/seal refuse to run with embeddings disabled. + /// When false, an inert zero-vector embedder is used and retrieval + /// rerank falls back to scope + recency ordering only. + #[serde(default = "default_memory_tree_embedding_strict")] + pub embedding_strict: bool, + + /// Ollama endpoint for the LLM entity extractor + /// (`memory::tree::score::extract::llm::LlmEntityExtractor`). + /// Defaults to `Some("http://localhost:11434")` — the standard + /// Ollama listener — see `default_memory_tree_llm_endpoint`. + /// Soft failures in the LLM path fall back to regex-only for + /// that chunk. + #[serde(default = "default_memory_tree_llm_endpoint")] + pub llm_extractor_endpoint: Option, + + /// Model name for the entity extractor. Defaults to `gemma3:4b` + /// (see `default_memory_tree_llm_model` for the rationale); + /// override to a smaller model on resource-constrained hosts. + #[serde(default = "default_memory_tree_llm_model")] + pub llm_extractor_model: Option, + + /// Per-request timeout for the LLM extractor, in milliseconds. + #[serde(default = "default_memory_tree_llm_extractor_timeout_ms")] + pub llm_extractor_timeout_ms: Option, + + /// Ollama endpoint for the summariser + /// (`memory::tree::tree_source::summariser::llm::LlmSummariser`). + /// Defaults to `Some("http://localhost:11434")` — see + /// `default_memory_tree_llm_endpoint`. Soft failures fall back + /// to `InertSummariser` per seal. + #[serde(default = "default_memory_tree_llm_endpoint")] + pub llm_summariser_endpoint: Option, + + /// Model name for the summariser. Defaults to `gemma3:4b` — + /// larger Gemma tiers (`gemma3:12b-it-qat`, `gemma3:27b`) produce + /// more coherent abstractive summaries at higher latency. See + /// `default_memory_tree_llm_model`. + #[serde(default = "default_memory_tree_llm_model")] + pub llm_summariser_model: Option, + + /// Per-request timeout for the summariser, in milliseconds. Default + /// is higher than the extractor because summarisation uses more + /// tokens and therefore takes longer to generate. + #[serde(default = "default_memory_tree_llm_summariser_timeout_ms")] + pub llm_summariser_timeout_ms: Option, + + /// Phase MD-content: root directory where chunk `.md` files are stored. + /// + /// Resolved at runtime via `MemoryHostConfig::memory_tree_content_root`: + /// - `Some(path)` → use that path verbatim. + /// - `None` → default `/memory_tree/content/`. + /// + /// Env override: `OPENHUMAN_MEMORY_TREE_CONTENT_DIR` (empty string = fall + /// back to default, consistent with other memory_tree env vars). + #[serde(default = "default_memory_tree_content_dir")] + pub content_dir: Option, + + /// Backend selector for the memory_tree's LLM calls (extractor + + /// summariser). Defaults to [`LlmBackend::Cloud`] so a fresh install + /// works without requiring a local Ollama daemon. Set to + /// [`LlmBackend::Local`] (or `OPENHUMAN_MEMORY_TREE_LLM_BACKEND=local`) to + /// keep the legacy Ollama-direct path. + /// + /// The embedder is unaffected by this setting — `OllamaEmbedder` (bge-m3) + /// stays local-only. + #[serde(default = "default_llm_backend")] + pub llm_backend: LlmBackend, + + /// **Deprecated / inert.** Formerly the model identifier for managed + /// (`llm_backend = "cloud"`) summarization. The managed summarization tier is + /// now fixed at `summarization-v1` + /// (`inference::provider::factory::summarization_tier_model`) + /// and this field is no longer consumed — the hosted backend serves exactly + /// one tier for this workload. Kept for config back-compat (existing + /// `config.toml` / `OPENHUMAN_MEMORY_TREE_CLOUD_LLM_MODEL` still parse without + /// error). To run summarization on a different model, point `memory_provider` + /// at a BYOK/local provider instead, where the model rides in the provider + /// string. + /// + /// Defaults to [`DEFAULT_CLOUD_LLM_MODEL`] (`summarization-v1`). + #[serde(default = "default_cloud_llm_model")] + pub cloud_llm_model: Option, + + /// Provider:model string for the smart_walk retrieval agent (e.g. + /// `"deepseek:deepseek-chat"`). When set, the smart walk loop uses this + /// model instead of the general memory/chat provider. Fast, cheap models + /// work best here since the walker makes many short-turn calls. + /// + /// Env override: `OPENHUMAN_MEMORY_TREE_SMART_WALK_MODEL`. + #[serde(default)] + pub smart_walk_model: Option, + + /// Explicit opt-in to cloud-based summarization when local AI is disabled. + /// + /// Default `false` — "Build Summary Trees" was local-only before #002. + /// Enabling this routes workspace memory summaries to the configured cloud + /// provider. Set `memory_tree.cloud_summarization_opt_in = true` or + /// `OPENHUMAN_MEMORY_TREE_CLOUD_SUMMARIZATION=true` to acknowledge that memory + /// content will be sent to an external service. + #[serde(default)] + pub cloud_summarization_opt_in: bool, + + /// Enable the spaCy NER sidecar used by the deterministic (E2GraphRAG) + /// retriever to extract entities from a query. When `true` (default), the + /// managed Python runtime provisions spaCy on first use and serves entity + /// extraction over stdio. When `false` — or whenever Python/spaCy is + /// unavailable — query-entity extraction falls back to the in-Rust + /// regex+LLM extractor (`score::extract`). Env override: + /// `OPENHUMAN_MEMORY_TREE_SPACY_ENABLED`. + #[serde(default = "default_memory_tree_spacy_enabled")] + pub spacy_enabled: bool, +} + +fn default_memory_tree_spacy_enabled() -> bool { + // Opt-in (#5056). Default OFF so a fresh install never provisions the spaCy + // venv + `en_core_web_sm` model on first launch, and the runtime Python + // server is not spawned on every boot when no local NLP is configured. + // Query-entity extraction degrades to the in-Rust regex+LLM extractor + // (`score::extract`); operators opt in via config or + // `OPENHUMAN_MEMORY_TREE_SPACY_ENABLED=1`. + false +} + +/// Returns `None` so that existing installs that never opted into Phase 4 +/// embeddings stay on the inert zero-vector path rather than suddenly +/// attempting to reach a local Ollama daemon they haven't configured. +/// Operators enable the Ollama path by setting either `embedding_endpoint` +/// in TOML or the `OPENHUMAN_MEMORY_EMBED_ENDPOINT` env var. +fn default_memory_tree_embedding_endpoint() -> Option { + None +} + +fn default_memory_tree_embedding_model() -> Option { + None +} + +fn default_memory_tree_embedding_timeout_ms() -> Option { + Some(10_000) +} + +/// Defaults to `false` so installs without an embedding endpoint fall back +/// to the inert zero-vector embedder (with a warn log) instead of refusing +/// to run. Set to `true` in production configs that require embeddings. +fn default_memory_tree_embedding_strict() -> bool { + false +} + +/// Shared `None` default for the LLM-path fields (extractor + summariser +/// endpoints + models). Keeping the same function for all of them makes +/// the intent explicit. +/// +/// Default points at the standard Ollama localhost listener. A user +/// who sets `llm_backend = "local"` plus a `_model` is clearly opting +/// into Ollama, and forcing them to also specify the endpoint just to +/// hit `localhost:11434` was a stealth foot-gun: the +/// `OllamaChatProvider` returned an error on an empty endpoint, which +/// the summariser silently swallowed into its `InertSummariser` +/// fallback — producing concat-and-truncate "summaries" that looked +/// correct but didn't run any LLM at all. With a default endpoint in +/// place, the only signal needed to enable a local LLM seal is a +/// non-empty `_model`. Override via TOML or +/// `OPENHUMAN_MEMORY_TREE_LLM_*_ENDPOINT` to point at a different +/// Ollama host. +fn default_memory_tree_llm_endpoint() -> Option { + Some("http://localhost:11434".to_string()) +} + +fn default_memory_tree_llm_extractor_timeout_ms() -> Option { + Some(15_000) +} + +fn default_memory_tree_llm_summariser_timeout_ms() -> Option { + // 120s — large enough for small/medium local models to finish a + // seal-budget summary on a cold-loaded weight cache. Tighter + // values cause the LlmSummariser to time out and silently fall + // back to InertSummariser (no LLM signal in the resulting node). + Some(120_000) +} + +/// Returns `None` so the default `/memory_tree/content/` path is +/// used unless explicitly overridden via TOML or env var. +fn default_memory_tree_content_dir() -> Option { + None +} + +/// Default Ollama model for the memory-tree LLMs (extractor + summariser). +/// +/// `gemma3:4b` is in the Gemma 3 family (Gemma 4 isn't released yet) +/// and sits between the 1B compact tier and the 12B/27B large tiers. +/// At ~3 GB on disk and ~8 GB RAM at inference it stays inside the +/// envelope of a typical laptop and produces coherent abstractive +/// summaries on real Gmail inboxes — smaller models (≤1.5B) regress +/// to "the email says X, the email says Y" enumeration that's barely +/// better than the InertSummariser concat fallback. +/// +/// Override via `memory_tree.llm_summariser_model` / +/// `llm_extractor_model` in TOML (or `OPENHUMAN_MEMORY_TREE_LLM_*_MODEL` +/// env vars) to scale up (`gemma3:12b-it-qat`, `llama3.1:8b`) or down +/// (`gemma3:1b-it-qat`) for the host's headroom. The frontend +/// `ModelCatalog` lists the curated picks the UI offers as +/// downloadable presets. +fn default_memory_tree_llm_model() -> Option { + Some("gemma3:4b".to_string()) +} + +impl Default for MemoryTreeConfig { + fn default() -> Self { + Self { + embedding_endpoint: default_memory_tree_embedding_endpoint(), + embedding_model: default_memory_tree_embedding_model(), + embedding_timeout_ms: default_memory_tree_embedding_timeout_ms(), + embedding_strict: default_memory_tree_embedding_strict(), + llm_extractor_endpoint: default_memory_tree_llm_endpoint(), + llm_extractor_model: default_memory_tree_llm_model(), + llm_extractor_timeout_ms: default_memory_tree_llm_extractor_timeout_ms(), + llm_summariser_endpoint: default_memory_tree_llm_endpoint(), + llm_summariser_model: default_memory_tree_llm_model(), + llm_summariser_timeout_ms: default_memory_tree_llm_summariser_timeout_ms(), + content_dir: default_memory_tree_content_dir(), + llm_backend: default_llm_backend(), + cloud_llm_model: default_cloud_llm_model(), + smart_walk_model: None, + cloud_summarization_opt_in: false, + spacy_enabled: default_memory_tree_spacy_enabled(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn llm_default_is_cloud() { + assert_eq!(LlmBackend::default(), LlmBackend::Cloud); + assert_eq!(MemoryTreeConfig::default().llm_backend, LlmBackend::Cloud); + } + + #[test] + fn llm_round_trip() { + for v in [LlmBackend::Cloud, LlmBackend::Local] { + assert_eq!(LlmBackend::parse(v.as_str()).unwrap(), v); + } + } + + #[test] + fn llm_parse_is_case_insensitive() { + assert_eq!(LlmBackend::parse("CLOUD").unwrap(), LlmBackend::Cloud); + assert_eq!(LlmBackend::parse(" Local ").unwrap(), LlmBackend::Local); + } + + #[test] + fn llm_parse_rejects_unknown() { + assert!(LlmBackend::parse("hybrid").is_err()); + assert!(LlmBackend::parse("").is_err()); + } + + #[test] + fn cloud_llm_model_default_is_summarizer_v1() { + let cfg = MemoryTreeConfig::default(); + assert_eq!( + cfg.cloud_llm_model.as_deref(), + Some(DEFAULT_CLOUD_LLM_MODEL) + ); + assert_eq!(DEFAULT_CLOUD_LLM_MODEL, "summarization-v1"); + } + + /// #5056: spaCy is opt-in — a fresh install must never provision the + /// spaCy venv / `en_core_web_sm` model, nor spawn the runtime Python + /// server, without an explicit config or env-var opt-in. + #[test] + fn spacy_enabled_defaults_to_false() { + assert!(!MemoryTreeConfig::default().spacy_enabled); + assert!(!default_memory_tree_spacy_enabled()); + } + + #[test] + fn memory_tree_config_default_content_dir_is_none() { + let cfg = MemoryTreeConfig::default(); + assert!( + cfg.content_dir.is_none(), + "default content_dir must be None so workspace default path is used" + ); + } + + /// Verify that the env-var override logic correctly maps non-empty strings + /// to `Some(PathBuf)` and empty/blank strings to `None`. We test the + /// logic inline (not via `apply_env_overrides`) to avoid mutating the + /// process environment in a way that could race with parallel tests. + #[test] + fn content_dir_env_override_logic() { + // Simulate the load.rs overlay logic. + let apply = |raw: &str| -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + None + } else { + Some(PathBuf::from(trimmed)) + } + }; + + assert_eq!(apply("/tmp/foo"), Some(PathBuf::from("/tmp/foo"))); + assert_eq!(apply(" /tmp/foo "), Some(PathBuf::from("/tmp/foo"))); + assert_eq!(apply(""), None); + assert_eq!(apply(" "), None); + } +} diff --git a/api/src/host/subsystems.rs b/api/src/host/subsystems.rs new file mode 100644 index 0000000..b765dca --- /dev/null +++ b/api/src/host/subsystems.rs @@ -0,0 +1,261 @@ +//! `[subsystems.*]` config section — the uniform cross-subsystem driver-binding +//! shape defined in `docs/specs/kernel.md` §3.6 and `docs/specs/plan-memory.md` §4.5. +//! +//! GREENFIELD / ZERO BEHAVIOUR CHANGE: nothing reads this config yet. It exists +//! so `[subsystems.memory]` can be authored today and so `inference`, +//! `channels`, `sandbox`, … can slot in later as sibling fields on +//! [`SubsystemsConfig`] without reshaping this type. +//! +//! Shape (kernel.md §3.6 / plan-memory.md §4.5): +//! +//! ```toml +//! [subsystems.memory] +//! driver = "tinycortex" +//! +//! [subsystems.memory.hooks] +//! auto_recall = true +//! auto_capture = true +//! max_context_tokens = 2000 +//! recall_max_chars = 1000 +//! capture_max_chars = 500 +//! +//! [subsystems.memory.drivers.supermemory] +//! class = "external" +//! transport = "http" +//! endpoint = "https://api.supermemory.ai" +//! credential_ref = "keychain:supermemory" +//! trust_state = "untrusted" +//! ``` + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Top-level `[subsystems]` config block. Currently carries only `memory`; +/// future subsystems (`inference`, `channels`, `sandbox`, …) are added here +/// as sibling fields — see kernel.md §3.6. +#[derive(Debug, Clone, Serialize, Deserialize, Default, JsonSchema)] +#[serde(default)] +pub struct SubsystemsConfig { + #[serde(default)] + pub memory: MemorySubsystemConfig, +} + +/// `[subsystems.memory]` — which driver is bound for the memory subsystem, +/// its hook budgets, and the per-driver option table. +/// +/// `PartialEq`/`Eq` let `CoreContext::rebind_workspace` short-circuit a +/// no-op rebind by comparing the config it was handed against the one already +/// held — equality is value comparison only, so it never prints or leaks the +/// credential fields the way `Debug` would. `Hash` lets `binding` +/// key its per-workspace cache on the whole config, so a changed driver/hooks/ +/// trust for an already-bound workspace yields a fresh binding rather than a +/// stale cache hit. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemorySubsystemConfig { + /// The bound driver id (e.g. `"tinycortex"`, `"supermemory"`, `"null"`). + /// Must match a key under `drivers` when that driver needs options. + #[serde(default = "default_memory_driver")] + pub driver: String, + + #[serde(default)] + pub hooks: MemoryHooksConfig, + + /// Per-driver option tables, keyed by driver id. The embedded default + /// (`tinycortex`) needs no entry here — its options continue to live in + /// the existing `[memory]` / `[memory_tree]` / `[[memory_sources]]` + /// blocks (plan-memory.md §4.5: "no user-visible config break"). + #[serde(default)] + pub drivers: BTreeMap, +} + +fn default_memory_driver() -> String { + "tinycortex".into() +} + +impl Default for MemorySubsystemConfig { + fn default() -> Self { + Self { + driver: default_memory_driver(), + hooks: MemoryHooksConfig::default(), + drivers: BTreeMap::new(), + } + } +} + +/// Memory-hook budgets — the auto-recall / auto-capture behavior gating +/// values. Defaults reproduce today's (pre-`[subsystems]`) behavior exactly; +/// nothing reads these yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemoryHooksConfig { + #[serde(default = "default_true")] + pub auto_recall: bool, + #[serde(default = "default_true")] + pub auto_capture: bool, + #[serde(default = "default_max_context_tokens")] + pub max_context_tokens: usize, + #[serde(default = "default_recall_max_chars")] + pub recall_max_chars: usize, + #[serde(default = "default_capture_max_chars")] + pub capture_max_chars: usize, +} + +fn default_true() -> bool { + true +} +fn default_max_context_tokens() -> usize { + 2000 +} +fn default_recall_max_chars() -> usize { + 1000 +} +fn default_capture_max_chars() -> usize { + 500 +} + +impl Default for MemoryHooksConfig { + fn default() -> Self { + Self { + auto_recall: default_true(), + auto_capture: default_true(), + max_context_tokens: default_max_context_tokens(), + recall_max_chars: default_recall_max_chars(), + capture_max_chars: default_capture_max_chars(), + } + } +} + +/// One entry under `[subsystems.memory.drivers.]`. Describes an +/// external/embedded driver binding — class, transport, endpoint, and a +/// *reference* to a credential resolved via the keychain (never an inline +/// secret; plan-memory.md §4.5, kernel.md §3.6). +/// +/// `trust_state` is fail-closed `"untrusted"` per kernel.md §3.4: an external +/// driver must have its trust explicitly raised before bind succeeds. +/// +/// MUST NOT derive `Debug` — see the manual impl below. `credential_ref` is a +/// secret handle and plan-memory.md §7 Tier-3 conformance requires "credential never +/// in `Debug`/error output", mirroring `storage_memory::MemoryConfig`'s +/// manual redacting `Debug` impl for `agentmemory_secret`. +/// +/// `PartialEq`/`Eq` are safe to derive: they compare values for equality and +/// never render them, so `credential_ref` stays out of any output. +#[derive(Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemoryDriverConfig { + /// Driver class: `"embedded"` | `"external"` | `"null"`. See kernel.md §3.1. + #[serde(default)] + pub class: Option, + + /// Wire transport for external drivers, e.g. `"http"`. See plan-memory.md §4.2. + #[serde(default)] + pub transport: Option, + + /// Base endpoint URL for external/http drivers. + #[serde(default)] + pub endpoint: Option, + + /// A *reference* to a credential (e.g. `"keychain:supermemory"`), + /// resolved kernel-side through the existing keychain — never an inline + /// secret. Redacted in `Debug`/error output; see the manual `Debug` impl. + #[serde(default)] + pub credential_ref: Option, + + /// Fail-closed trust state for this driver binding. Defaults to + /// `"untrusted"`; must be explicitly raised before an external driver's + /// bind succeeds (kernel.md §3.4). + #[serde(default = "default_trust_state")] + pub trust_state: String, +} + +fn default_trust_state() -> String { + "untrusted".into() +} + +impl Default for MemoryDriverConfig { + fn default() -> Self { + Self { + class: None, + transport: None, + endpoint: None, + credential_ref: None, + trust_state: default_trust_state(), + } + } +} + +// Manual `Debug` implementation that redacts `credential_ref`. Without this, +// any `format!("{cfg:?}")` / `tracing::debug!(?cfg, ...)` / panic message +// capturing a `MemoryDriverConfig` would dump the credential reference +// verbatim. The value itself (e.g. `"keychain:supermemory"`) is only a +// *reference*, not the secret — but plan-memory.md §7 Tier-3 conformance requires it +// never appear in Debug/error output regardless, so this mirrors +// `MemoryConfig`'s `agentmemory_secret` treatment exactly. NEVER derive +// `Debug` on this struct. +impl std::fmt::Debug for MemoryDriverConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryDriverConfig") + .field("class", &self.class) + .field("transport", &self.transport) + .field("endpoint", &self.endpoint) + .field( + "credential_ref", + &self.credential_ref.as_ref().map(|_| ""), + ) + .field("trust_state", &self.trust_state) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn subsystems_config_defaults_reproduce_today_behavior() { + let cfg = SubsystemsConfig::default(); + assert_eq!(cfg.memory.driver, "tinycortex"); + assert!(cfg.memory.hooks.auto_recall); + assert!(cfg.memory.hooks.auto_capture); + assert_eq!(cfg.memory.hooks.max_context_tokens, 2000); + assert_eq!(cfg.memory.hooks.recall_max_chars, 1000); + assert_eq!(cfg.memory.hooks.capture_max_chars, 500); + assert!(cfg.memory.drivers.is_empty()); + } + + #[test] + fn absent_subsystems_block_deserializes_to_default() { + let cfg: SubsystemsConfig = toml::from_str("").expect("empty toml parses"); + assert_eq!( + serde_json::to_value(&cfg).unwrap(), + serde_json::to_value(SubsystemsConfig::default()).unwrap() + ); + } + + #[test] + fn memory_driver_config_debug_never_leaks_credential_ref() { + let driver = MemoryDriverConfig { + class: Some("external".into()), + transport: Some("http".into()), + endpoint: Some("https://api.supermemory.ai".into()), + credential_ref: Some("keychain:supermemory-super-secret-value".into()), + trust_state: "untrusted".into(), + }; + let debug_output = format!("{driver:?}"); + assert!( + !debug_output.contains("keychain:supermemory-super-secret-value"), + "Debug output must never contain the credential_ref value: {debug_output}" + ); + assert!( + debug_output.contains(""), + "Debug output should show a redaction marker: {debug_output}" + ); + } + + #[test] + fn memory_driver_config_default_trust_state_is_untrusted() { + assert_eq!(MemoryDriverConfig::default().trust_state, "untrusted"); + } +} diff --git a/api/src/host/test_support.rs b/api/src/host/test_support.rs new file mode 100644 index 0000000..486798f --- /dev/null +++ b/api/src/host/test_support.rs @@ -0,0 +1,209 @@ +//! [`TestHostConfig`] — a concrete, `Default`-able [`MemoryHostConfig`] for +//! tests. +//! +//! `tinymemory_core::Config` is `dyn MemoryHostConfig`, which cannot be +//! `Default::default()`ed. The extracted test suites build a config, tweak two +//! or three fields, and pass `&config` into the code under test — a pattern +//! that needs a real struct. This is that struct. +//! +//! It is behind the `test-support` feature and enabled from +//! `tinymemory-core`'s dev-dependencies, so it never enters a shipped build. +//! It is deliberately *not* a mock: the fields are the real config sections +//! with their real serde defaults, so a test that asserts on default behaviour +//! is asserting on the same values production loads. + +use std::path::PathBuf; + +use super::cloud_providers::CloudProviderCreds; +use super::config::{ComposioMode, MemoryHostConfig}; +use super::local_ai::LocalAiConfig; +use super::scheduler_gate::SchedulerGateConfig; +use super::storage_memory::{MemoryConfig, MemoryTreeConfig}; + +/// A concrete host config for tests. Fields are public — mutate them directly +/// rather than reaching for a builder. +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub struct TestHostConfig { + /// See [`MemoryHostConfig::workspace_dir`]. + pub workspace_dir: PathBuf, + /// See [`MemoryHostConfig::config_path`]. + pub config_path: PathBuf, + /// See [`MemoryHostConfig::memory`]. + pub memory: MemoryConfig, + /// See [`MemoryHostConfig::session_token`]. `None` is signed-out. + pub session_token: Option, + /// See [`MemoryHostConfig::memory_tree`]. + pub memory_tree: MemoryTreeConfig, + /// See [`MemoryHostConfig::scheduler_gate`]. + pub scheduler_gate: SchedulerGateConfig, + /// See [`MemoryHostConfig::local_ai`]. + pub local_ai: LocalAiConfig, + /// See [`MemoryHostConfig::cloud_providers`]. + pub cloud_providers: Vec, + /// See [`MemoryHostConfig::embeddings_provider`]. + pub embeddings_provider: Option, + /// See [`MemoryHostConfig::memory_provider`]. + pub memory_provider: Option, + /// See [`MemoryHostConfig::api_url`]. + pub api_url: Option, + /// See [`MemoryHostConfig::default_model`]. + pub default_model: Option, + /// See [`MemoryHostConfig::default_temperature`]. + pub default_temperature: f64, + /// See [`MemoryHostConfig::output_language`]. + pub output_language: Option, + /// See [`MemoryHostConfig::memory_sync_interval_secs`]. + pub memory_sync_interval_secs: Option, + /// See [`MemoryHostConfig::onboarding_completed`]. + pub onboarding_completed: bool, + /// See [`MemoryHostConfig::secrets_encrypt`]. + pub secrets_encrypt: bool, + /// See [`MemoryHostConfig::composio`]. + pub composio: ComposioMode, + /// See [`MemoryHostConfig::memory_sources_json`]. Defaults to an empty + /// array so a test that never touches sources behaves like a fresh install. + pub memory_sources: Option, + /// See [`MemoryHostConfig::composio_source_caps_migration_version`]. + pub composio_source_caps_migration_version: u32, +} + +#[async_trait::async_trait] +impl MemoryHostConfig for TestHostConfig { + fn workspace_dir(&self) -> &PathBuf { + &self.workspace_dir + } + + fn config_path(&self) -> &PathBuf { + &self.config_path + } + + fn memory_tree_content_root(&self) -> PathBuf { + self.memory_tree + .content_dir + .clone() + .unwrap_or_else(|| self.workspace_dir.join("memory_tree").join("content")) + } + + fn memory(&self) -> &MemoryConfig { + &self.memory + } + + fn memory_tree(&self) -> &MemoryTreeConfig { + &self.memory_tree + } + + fn scheduler_gate(&self) -> &SchedulerGateConfig { + &self.scheduler_gate + } + + fn local_ai(&self) -> &LocalAiConfig { + &self.local_ai + } + + fn cloud_providers(&self) -> &Vec { + &self.cloud_providers + } + + fn embeddings_provider(&self) -> Option<&str> { + self.embeddings_provider.as_deref() + } + + fn memory_provider(&self) -> Option<&str> { + self.memory_provider.as_deref() + } + + fn workload_local_model(&self, workload: &str) -> Option { + let raw = match workload { + "memory" => self.memory_provider.as_deref(), + "embeddings" => self.embeddings_provider.as_deref(), + _ => None, + }?; + let model = raw.trim().strip_prefix("ollama:")?.trim(); + if model.is_empty() { + None + } else { + Some(model.to_string()) + } + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn to_arc(&self) -> std::sync::Arc { + std::sync::Arc::new(self.clone()) + } + + fn api_url(&self) -> Option<&str> { + self.api_url.as_deref() + } + + fn effective_backend_api_url(&self) -> String { + // No resolution to do: a test config states its backend URL outright, + // and the host's env/default ladder is not something to reimplement + // here. + self.api_url.clone().unwrap_or_default() + } + + fn session_token(&self) -> Result, String> { + // `Ok(None)` — "read fine, not signed in" — rather than `Err`, so a + // test that never sets a token exercises the signed-out path instead of + // a credential-store failure. + Ok(self.session_token.clone()) + } + + fn default_model(&self) -> Option<&str> { + self.default_model.as_deref() + } + + fn default_temperature(&self) -> f64 { + self.default_temperature + } + + fn output_language(&self) -> Option<&str> { + self.output_language.as_deref() + } + + fn memory_sync_interval_secs(&self) -> Option { + self.memory_sync_interval_secs + } + + fn onboarding_completed(&self) -> bool { + self.onboarding_completed + } + + fn secrets_encrypt(&self) -> bool { + self.secrets_encrypt + } + + fn composio(&self) -> ComposioMode { + self.composio.clone() + } + + fn memory_sources_json(&self) -> anyhow::Result { + Ok(self + .memory_sources + .clone() + .unwrap_or_else(|| serde_json::Value::Array(Vec::new()))) + } + + fn set_memory_sources_json(&mut self, value: serde_json::Value) -> anyhow::Result<()> { + self.memory_sources = Some(value); + Ok(()) + } + + fn composio_source_caps_migration_version(&self) -> u32 { + self.composio_source_caps_migration_version + } + + fn set_composio_source_caps_migration_version(&mut self, version: u32) { + self.composio_source_caps_migration_version = version; + } + + fn apply_env_overrides(&mut self) {} + + async fn save(&self) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/api/src/host/usage.rs b/api/src/host/usage.rs new file mode 100644 index 0000000..11d9c45 --- /dev/null +++ b/api/src/host/usage.rs @@ -0,0 +1,33 @@ +//! [`UsageInfo`] — token accounting returned by an inference provider. +//! +//! Lives in the contract crate because both sides name it: the host's chat +//! providers produce it, and the memory subsystem's summariser threads it back +//! out so callers can attribute cost to a summarisation run. It is inert data +//! with no dependencies, so it costs the contract crate nothing. + +/// Token usage information returned by the provider after an inference call. +#[derive(Debug, Clone, Default)] +pub struct UsageInfo { + /// Number of tokens in the input/prompt. + pub input_tokens: u64, + /// Number of tokens in the output/completion. + pub output_tokens: u64, + /// Total context window size for the model (0 if unknown). + pub context_window: u64, + /// Number of input tokens that were served from the KV cache + /// (returned by backends that support prompt caching, e.g. via + /// `openhuman.usage.cached_input_tokens` or + /// `prompt_tokens_details.cached_tokens`). + pub cached_input_tokens: u64, + /// Number of input tokens written into a provider prompt/KV cache on this + /// request (cache-creation / cache-write tokens). Distinct from + /// `cached_input_tokens` (cache reads). Zero when the provider does not + /// report a cache-write breakdown. + pub cache_creation_tokens: u64, + /// Number of reasoning/thinking output tokens when the provider exposes + /// them separately from `output_tokens`. Zero when unavailable. + pub reasoning_tokens: u64, + /// Amount billed for this request in USD (from + /// `openhuman.billing.charged_amount_usd`). Zero when unavailable. + pub charged_amount_usd: f64, +} diff --git a/api/src/lib.rs b/api/src/lib.rs index c092b87..85fc087 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -57,12 +57,16 @@ //! [`tree::NodeLevel`], [`tree::TreeStatus`], …). //! - [`tool_memory`]: tool-scoped rule contracts ([`tool_memory::ToolMemoryRule`], …). //! - [`goals`]: the long-term goals document ([`goals::GoalsDoc`], [`goals::GoalItem`]). +//! - [`host`]: the **host seam** — [`host::MemoryHostConfig`], +//! [`host::EmbeddingProvider`], [`host::MemoryEventSink`], and the memory +//! config sections whose serde form is persisted in a host's `config.toml`. pub mod capabilities; pub mod chunks; pub mod error; pub mod goals; pub mod health; +pub mod host; pub mod null; pub mod provider; pub mod recall; diff --git a/core/Cargo.toml b/core/Cargo.toml new file mode 100644 index 0000000..d9e8f81 --- /dev/null +++ b/core/Cargo.toml @@ -0,0 +1,92 @@ +[package] +name = "tinymemory-core" +version = "0.1.0" +edition = "2021" +rust-version = "1.85" +license = "MIT" +description = "The engine-neutral memory subsystem: store, summary tree, sync pipelines, ingestion and recall" +repository = "https://github.com/tinyhumansai/tinymemory" +readme = "../README.md" + +[dependencies] +# The contract. `tinymemory-core` implements and consumes it; the host seam +# traits (config, event sink, embeddings, chat) live in `tinymemory_api::host`. +tinymemory-api = { path = "../api", version = "0.1.1" } +tinymemory = { path = "..", version = "0.1.0" } + +# The default embedded engine. `store/`, `tree/` and `sync/` drive it directly; +# `tinycortex-api` is a direct dependency because `tinycortex::memory` aliases +# back only `{error, traits, types}`. +tinycortex = { version = "0.1", features = ["obsidian", "persona", "sync"] } +tinycortex-api = { version = "0.1" } + +# Chat-model and embedding primitives used by the tree summarizer and the +# embedding factory. +tinyagents = { version = "2.1", features = ["sqlite"] } +anyhow = "1.0" +async-trait = "0.1" +# Only under `memory-git`; see the feature's comment. +git2 = { version = "0.21", optional = true, default-features = false, features = ["vendored-libgit2"] } +# `store/factories.rs` exposes a tiny health router for the embedded provider. +axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio", "query", "ws", "macros"] } +chrono = { version = "0.4", features = ["serde"] } +# `tinycortex/persona.rs` resolves the user's home directory for the obsidian +# vault default. +dirs = "5" +futures = "0.3" +log = "0.4" +parking_lot = "0.12" +# `tree/tree/registry.rs` salts a tree id; the sync readers talk HTTP to the +# provider APIs; the pipelines are `tracing`-instrumented. +rand = "0.8" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } +tracing = "0.1" +regex = "1.10" +rusqlite = { version = "=0.40.0", features = ["bundled"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2.0" +tokio = { version = "1", features = ["full"] } +url = "2" +uuid = { version = "1", features = ["v4"] } +walkdir = "2" + +# macOS address-book reader behind `people/address_book.rs`. Gated by the +# host's `contacts` feature, forwarded here. +[target.'cfg(target_os = "macos")'.dependencies] +objc2 = { version = "0.6", optional = true } +objc2-foundation = { version = "0.3", features = ["NSArray", "NSError", "NSObject", "NSString", "NSPredicate"], optional = true } +objc2-contacts = { version = "0.3.2", features = ["CNContact", "CNContactFetchRequest", "CNContactStore", "CNLabeledValue", "CNPhoneNumber"], optional = true } +block2 = { version = "0.6", optional = true } + +[dev-dependencies] +# `TestHostConfig` — the concrete `MemoryHostConfig` the extracted test suites +# build, since `Config` is a trait object and cannot be `Default`ed. +tinymemory-api = { path = "../api", version = "0.1.1", features = ["test-support"] } +tempfile = "3" +tokio = { version = "1", features = ["test-util"] } + +[features] +default = [] +# Git-backed diff snapshots and the wiki git mirror. Gated because it is what +# drags `git2` / `libgit2-sys` / `libz-sys` — a native build — into the graph. +# The host forwards its own `memory-git` feature to this one; when off, the +# `diff` domain's ops are stubbed rather than `#[cfg]`'d at each call site, so a +# diff simply never materialises. +# +# NOT in `default`, matching the host. The off-path builds because tinycortex +# makes the same carve-out: its `memory::diff::{types, source}` are serde-only +# and stay compiled, so only the git-touching half disappears. +memory-git = ["dep:git2", "tinycortex/git-diff", "tinycortex/wiki-git"] +# Exposes the crate's test helpers (`chat::test_override`, +# `chat::StaticChatProvider`, `tool_memory::test_helpers`) to *other* crates' +# tests. They were `#[cfg(test)]` before the extraction, when the host and the +# memory subsystem were one crate and that was enough; a downstream test +# harness cannot see `#[cfg(test)]` items across a crate boundary. +# +# Not in `default`: a production build must not carry them. +test-support = ["tinymemory-api/test-support"] + +# The macOS CNContactStore address-book seeding path. No-op off macOS. +contacts = ["dep:objc2", "dep:objc2-foundation", "dep:objc2-contacts", "dep:block2"] diff --git a/core/src/chat.rs b/core/src/chat.rs new file mode 100644 index 0000000..7afb4e5 --- /dev/null +++ b/core/src/chat.rs @@ -0,0 +1,300 @@ +//! Memory LLM adapter backed by the unified inference provider stack. +//! +//! Memory callers still want a tiny prompt surface: one system message, one +//! user message, and a string response. This module keeps that narrow contract +//! for the rest of the memory layer, but routes every production call through +//! `openhuman::inference::provider` so memory uses the same workload routing as +//! the rest of the app. + +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; + +use crate::chat_host::{create_chat_model_with_model_id, provider_for_role, UsageInfo}; +use crate::Config; +use tinyagents::harness::message::Message; +use tinyagents::harness::model::{ChatModel, ModelRequest}; + +/// One pair of prompt messages handed to the memory LLM backend. +#[derive(Debug, Clone)] +pub struct ChatPrompt { + pub system: String, + pub user: String, + pub temperature: f64, + pub kind: &'static str, + /// Optional output-token cap forwarded to the provider as `max_tokens`. + /// `None` leaves generation open-ended. Memory callers with a bounded + /// response (entity extraction) set a small value so credit-metered + /// providers don't reserve the model's full output window in their + /// balance pre-flight (TAURI-RUST-C62). + pub max_tokens: Option, +} + +/// Pluggable LLM surface used by the memory layer. +#[async_trait] +pub trait ChatProvider: Send + Sync { + fn name(&self) -> &str; + + async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result; + + async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result { + self.chat_for_json(prompt).await + } + + /// Like `chat_for_text`, but also surfaces the provider-reported + /// [`UsageInfo`] (real token counts + `charged_amount_usd`) when the + /// backing provider returns it. + /// + /// The default implementation simply runs `chat_for_text` and reports + /// `None` usage, so implementors (e.g. test doubles, external impls) + /// that don't thread usage keep compiling unchanged. The production + /// `InferenceChatProvider` overrides this to route through the + /// inference `Provider::chat` API, which already parses usage out of + /// the backend response. + async fn chat_for_text_with_usage( + &self, + prompt: &ChatPrompt, + ) -> Result<(String, Option)> { + let text = self.chat_for_text(prompt).await?; + Ok((text, None)) + } +} + +struct InferenceChatProvider { + inner: Arc>, + model_id: String, + display: String, +} + +impl InferenceChatProvider { + fn new(inner: Arc>, model_id: String) -> Self { + let display = format!("inference:{model_id}"); + Self { + inner, + model_id, + display, + } + } + + async fn run(&self, prompt: &ChatPrompt) -> Result { + let (text, _usage) = self.run_with_usage(prompt).await?; + Ok(text) + } + + /// Run the prompt through the inference `Provider::chat` API and return + /// both the text and the provider-reported usage. Memory historically + /// called `chat_with_history` (which returns only `String` and drops + /// the parsed usage); routing through `chat` instead lets us thread the + /// real token counts + `charged_amount_usd` into the sync audit log + /// (issue #3110) without re-deriving them from `body.len() / 4`. + async fn run_with_usage(&self, prompt: &ChatPrompt) -> Result<(String, Option)> { + log::debug!( + "[memory::chat] provider={} kind={} model={} sys_chars={} user_chars={}", + self.display, + prompt.kind, + self.model_id, + prompt.system.len(), + prompt.user.len() + ); + + // One system + one user turn — the crate model interface's native shape. + // Temperature and the output cap ride the request (the shared model is + // reused across memory prompts of differing temperature/budget), and the + // adapter honors both per-request values. + let mut request = ModelRequest::new(vec![ + Message::system(prompt.system.clone()), + Message::user(prompt.user.clone()), + ]) + .with_temperature(prompt.temperature); + if let Some(cap) = prompt.max_tokens { + request = request.with_max_tokens(cap); + } + + let response = self.inner.invoke(&(), request).await?; + + // Fail fast on a missing body rather than masking it as an empty + // string: an empty summary would still be ingested (and, post-#3110, + // counted against the run's real charge) as if it were valid output. + // The caller's fallback path (`fallback_summary`) is the correct + // recovery for a silent provider, and it only runs on `Err`. + let text = response.text(); + if text.is_empty() { + anyhow::bail!( + "inference provider '{}' returned no text for {} summarise request", + self.display, + prompt.kind + ); + } + // Recover the full host usage (real token counts + backend-charged USD + + // context window) the adapter round-tripped through the response (G1). + let usage = crate::chat_host::usage_from_response(&response); + + log::debug!( + "[memory::chat] provider={} kind={} response_chars={} usage_present={} input_tokens={} output_tokens={} charged_usd={}", + self.display, + prompt.kind, + text.len(), + usage.is_some(), + usage.as_ref().map(|u| u.input_tokens).unwrap_or(0), + usage.as_ref().map(|u| u.output_tokens).unwrap_or(0), + usage.as_ref().map(|u| u.charged_amount_usd).unwrap_or(0.0), + ); + + Ok((text, usage)) + } +} + +#[async_trait] +impl ChatProvider for InferenceChatProvider { + fn name(&self) -> &str { + &self.display + } + + async fn chat_for_json(&self, prompt: &ChatPrompt) -> Result { + self.run(prompt).await + } + + async fn chat_for_text(&self, prompt: &ChatPrompt) -> Result { + self.run(prompt).await + } + + async fn chat_for_text_with_usage( + &self, + prompt: &ChatPrompt, + ) -> Result<(String, Option)> { + self.run_with_usage(prompt).await + } +} + +#[cfg(any(test, feature = "test-support"))] +fn test_override_runtime() -> Option<(Arc, String)> { + test_override::current().map(|provider| (provider, "test:override".to_string())) +} + +#[cfg(not(any(test, feature = "test-support")))] +fn test_override_runtime() -> Option<(Arc, String)> { + None +} + +/// Build the memory LLM provider and return the resolved model id. +pub fn build_chat_runtime(config: &Config) -> Result<(Arc, String)> { + if let Some(runtime) = test_override_runtime() { + return Ok(runtime); + } + + // The managed summarization tier is fixed at `summarization-v1`, resolved + // inside `make_openhuman_backend` for the `summarization` role — so no + // per-caller `default_model` pre-routing is needed here. BYOK/local routes + // carry their own model in the provider string. + let resolved_provider = provider_for_role("summarization", config); + // Temperature is applied per-prompt via `ModelRequest::with_temperature` + // (each memory `ChatPrompt` carries its own), so the construction temperature + // is just a default the per-call value overrides. + let (model, model_id) = + create_chat_model_with_model_id("summarization", config, config.default_temperature())?; + + log::debug!( + "[memory::chat] built provider route={} model={}", + resolved_provider, + model_id + ); + + Ok(( + Arc::new(InferenceChatProvider::new(model, model_id.clone())), + model_id, + )) +} + +/// Build the memory LLM provider dictated by the inference workload routing. +pub fn build_chat_provider(config: &Config) -> Result> { + Ok(build_chat_runtime(config)?.0) +} + +#[cfg(any(test, feature = "test-support"))] +pub struct StaticChatProvider { + pub response: String, + pub calls: std::sync::atomic::AtomicUsize, +} + +#[cfg(any(test, feature = "test-support"))] +impl StaticChatProvider { + pub fn new(response: impl Into) -> Self { + Self { + response: response.into(), + calls: std::sync::atomic::AtomicUsize::new(0), + } + } +} + +#[cfg(any(test, feature = "test-support"))] +#[async_trait] +impl ChatProvider for StaticChatProvider { + fn name(&self) -> &str { + "test:static" + } + + async fn chat_for_json(&self, _prompt: &ChatPrompt) -> Result { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(self.response.clone()) + } +} + +#[cfg(any(test, feature = "test-support"))] +pub mod test_override { + use super::ChatProvider; + use std::sync::Arc; + + tokio::task_local! { + static OVERRIDE: Arc; + } + + pub fn current() -> Option> { + OVERRIDE.try_with(Arc::clone).ok() + } + + pub async fn with_provider(provider: Arc, fut: F) -> T + where + F: std::future::Future, + { + OVERRIDE.scope(provider, fut).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn static_chat_provider_returns_response_and_counts() { + let p = StaticChatProvider::new("hello"); + let prompt = ChatPrompt { + system: "sys".into(), + user: "u".into(), + temperature: 0.0, + kind: "test", + max_tokens: None, + }; + assert_eq!(p.chat_for_json(&prompt).await.unwrap(), "hello"); + assert_eq!(p.calls.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn chat_for_text_with_usage_default_impl_reports_no_usage() { + // A provider that doesn't override `chat_for_text_with_usage` + // (here the `chat_for_json`-only `StaticChatProvider`) must still + // return its text, with `None` usage — so summarise() falls back + // to the estimate rather than reporting a bogus zero charge. + let p = StaticChatProvider::new("summary text"); + let prompt = ChatPrompt { + system: "sys".into(), + user: "u".into(), + temperature: 0.0, + kind: "test", + max_tokens: None, + }; + let (text, usage) = p.chat_for_text_with_usage(&prompt).await.unwrap(); + assert_eq!(text, "summary text"); + assert!(usage.is_none()); + } +} diff --git a/core/src/chat_host.rs b/core/src/chat_host.rs new file mode 100644 index 0000000..c6edc7c --- /dev/null +++ b/core/src/chat_host.rs @@ -0,0 +1,152 @@ +//! [`ChatHost`] — chat-model *construction*, which the host owns. +//! +//! The summary-tree summariser and the memory chat helper run LLM turns. Which +//! provider answers a given role, which model id that resolves to, and what +//! credentials it uses are host routing policy — the same policy that serves +//! every other role in the application, not something a memory engine should +//! re-derive. +//! +//! # Why this trait is here and not in `tinymemory-api` +//! +//! It names `tinyagents::harness::model::ChatModel`, and the contract crate is +//! deliberately dependency-light — it must not pull in tinyagents. This crate +//! already depends on tinyagents, so it is the one place that can name both the +//! model trait and the config seam. The host implements it here. +//! +//! Reached through a process-global for the same reason as +//! [`crate::embedding_host`]; see that module for the rationale, and for why an +//! unwired host fails loudly rather than degrading. + +use std::sync::Arc; + +use parking_lot::RwLock; +use tinyagents::harness::model::{ChatModel, ModelResponse}; + +use crate::Config; + +pub use tinymemory_api::host::UsageInfo; + +/// Builds chat models on the core's behalf. +pub trait ChatHost: Send + Sync + std::fmt::Debug { + /// The provider slug that `role` currently routes to, e.g. `"cloud"`. + /// + /// Used for reporting and budget attribution, so it answers even when no + /// model can actually be constructed. + fn provider_for_role(&self, role: &str, config: &Config) -> String; + + /// Builds the chat model for `role`, returning it with its resolved model + /// id. + /// + /// # Errors + /// + /// Returns `Err` when no provider is configured for `role`, or when the + /// configured one cannot be constructed (missing credentials, unreachable + /// local runtime). + fn create_chat_model_with_model_id( + &self, + role: &str, + config: &Config, + temperature: f64, + ) -> Result<(Arc>, String), String>; + + /// Extract token accounting from a completed model response. + /// + /// The host's own usage metadata rides in the response's provider-specific + /// `raw` payload under a key only it knows, which is why this cannot be a + /// free function here. + fn usage_from_response(&self, response: &ModelResponse) -> Option; + + /// Whether summarisation can run right now, and a user-facing explanation. + /// + /// Local AI off is **not** a fault by itself — since #002 FR-007 + /// summarisation runs on the configured cloud provider in that case. Only + /// "no provider resolves at all" is bad. + fn summarizer_available(&self, config: &Config) -> (bool, &'static str); +} + +static HOST: RwLock>> = RwLock::new(None); + +const NOT_INSTALLED: &str = + "no ChatHost installed — the host must call memory::chat_host::set_chat_host during \ + startup wiring, before any summarisation runs"; + +/// Install the host's chat-model factory. Called once during startup wiring. +pub fn set_chat_host(host: Arc) { + *HOST.write() = Some(host); +} + +/// Remove any installed host. For tests. +pub fn clear_chat_host() { + *HOST.write() = None; +} + +/// The installed host, or `None` when nothing has been wired up. +#[must_use] +pub fn chat_host() -> Option> { + HOST.read().clone() +} + +/// The installed host. +/// +/// # Errors +/// +/// Returns `Err` when no host has been installed. +pub fn require_chat_host() -> Result, String> { + chat_host().ok_or_else(|| NOT_INSTALLED.to_string()) +} + +/// The provider slug `role` routes to, or `"unknown"` with no host installed. +/// +/// Unlike model construction this never fails: every caller is building a log +/// line or a status field, and an error there would be less useful than the +/// honest string. +#[must_use] +pub fn provider_for_role(role: &str, config: &Config) -> String { + chat_host().map_or_else( + || "unknown".to_string(), + |host| host.provider_for_role(role, config), + ) +} + +/// Builds the chat model for `role`. +/// +/// # Errors +/// +/// Returns `Err` when no host is installed, or when the host cannot build one. +pub fn create_chat_model_with_model_id( + role: &str, + config: &Config, + temperature: f64, +) -> anyhow::Result<(Arc>, String)> { + require_chat_host() + .and_then(|host| host.create_chat_model_with_model_id(role, config, temperature)) + .map_err(|error| anyhow::anyhow!(error)) +} + +/// Serialises tests that mutate inference-related process environment. See +/// [`crate::embedding_host::embedding_test_guard`] for why this is a separate +/// lock from the host's. +#[must_use = "bind the guard for the whole test (`let _guard = inference_test_guard();`); \ + dropping it straight away releases the lock and the test races"] +pub fn inference_test_guard() -> std::sync::MutexGuard<'static, ()> { + static GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); + GUARD + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Token accounting from a completed model response, or `None` when the +/// provider reported none or no host is installed. +#[must_use] +pub fn usage_from_response(response: &ModelResponse) -> Option { + chat_host()?.usage_from_response(response) +} + +/// Whether summarisation can run, and why. Reports unavailable when unwired. +#[must_use] +pub fn summarizer_available(config: &Config) -> (bool, &'static str) { + chat_host().map_or( + (false, "no chat host installed — summarisation cannot run"), + |host| host.summarizer_available(config), + ) +} diff --git a/core/src/composio_host.rs b/core/src/composio_host.rs new file mode 100644 index 0000000..3258901 --- /dev/null +++ b/core/src/composio_host.rs @@ -0,0 +1,153 @@ +//! [`ComposioHost`] — the Composio integration, as the memory sync layer sees it. +//! +//! The sync pipelines need three things from Composio: which connections are +//! active, the ability to execute a tool against one, and the direct-mode API +//! key. Everything else about the integration — OAuth, the backend session +//! token, per-toolkit allowlists, HMAC-verified trigger fan-out, and the choice +//! between backend-proxied and direct mode — is host concern. +//! +//! # Why the client itself stayed in the host +//! +//! `ComposioClientKind::Direct` wraps an `Arc`, +//! a host *agent tool*. There is no way to name that from here, and no reason +//! to: mode dispatch reads `config.composio.mode` and fails loud on a typo, +//! which is exactly the kind of policy the README's split assigns to the host. +//! +//! So this trait is deliberately **behavioural, not structural**. It hides +//! `ComposioClientKind` entirely — the core never learns that two modes exist, +//! and the three call sites that used to do +//! `create_composio_client(config)?` → `match kind` → call collapse to one +//! method each. +//! +//! The value types those methods return did move, to +//! [`tinymemory_api::host::composio`], because the pipelines read their fields +//! directly. +//! +//! # Unwired is an error +//! +//! Same reasoning as [`crate::embedding_host`]: a sync run that quietly saw +//! zero connections would look like "nothing to sync" rather than "not wired +//! up", and the difference would only surface as missing memory days later. + +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::RwLock; + +use crate::Config; + +pub use tinymemory_api::host::composio::{ + ComposioCapability, ComposioConnection, ComposioExecuteResponse, +}; + +/// The Composio operations the memory sync layer performs. +#[async_trait] +pub trait ComposioHost: Send + Sync + std::fmt::Debug { + /// Every connection the signed-in user has, active or not. + /// + /// Filtering to active ones is the caller's job — `ComposioConnection` + /// carries the status and treats an empty one as inactive, so a malformed + /// upstream row is never presented as connected. + /// + /// # Errors + /// + /// Returns `Err` when no client can be built (no backend session, bad mode + /// string) or the upstream call fails. + async fn list_connections(&self, config: &Config) -> Result, String>; + + /// Execute `tool` against a connection. + /// + /// # Errors + /// + /// Returns `Err` when no client can be built or the call fails. A provider + /// that answers with `successful: false` is **not** an error — that is + /// reported in the returned [`ComposioExecuteResponse`]. + async fn execute( + &self, + config: &Config, + tool: &str, + arguments: Option, + entity_id: &str, + connection_id: Option<&str>, + ) -> Result; + + /// The direct-mode Composio API key from the host's credential store, or + /// `None` when direct mode is not configured. + fn api_key(&self, config: &Config) -> Option; + + /// Whether *some* viable client resolves for the current config. + /// + /// The sync layer uses this as its "is the user signed in?" probe. It must + /// answer for **either** mode: direct-mode users typically have no backend + /// session token, and probing for one alone would falsely skip them. + fn is_available(&self, config: &Config) -> bool; +} + +static HOST: RwLock>> = RwLock::new(None); + +const NOT_INSTALLED: &str = + "no ComposioHost installed — the host must call memory::composio_host::set_composio_host \ + during startup wiring, before any sync runs"; + +/// Install the host's Composio integration. Called once during startup wiring. +pub fn set_composio_host(host: Arc) { + *HOST.write() = Some(host); +} + +/// Remove any installed host. For tests. +pub fn clear_composio_host() { + *HOST.write() = None; +} + +/// The installed host, or `None` when nothing has been wired up. +#[must_use] +pub fn composio_host() -> Option> { + HOST.read().clone() +} + +/// The installed host. +/// +/// # Errors +/// +/// Returns `Err` when no host has been installed. +pub fn require_composio_host() -> Result, String> { + composio_host().ok_or_else(|| NOT_INSTALLED.to_string()) +} + +/// Active-or-not connections for the signed-in user. +/// +/// # Errors +/// +/// Returns `Err` when no host is installed, or the upstream call fails. +pub async fn list_connections(config: &Config) -> Result, String> { + require_composio_host()?.list_connections(config).await +} + +/// Execute a Composio tool. +/// +/// # Errors +/// +/// Returns `Err` when no host is installed, or the call fails. +pub async fn execute( + config: &Config, + tool: &str, + arguments: Option, + entity_id: &str, + connection_id: Option<&str>, +) -> Result { + require_composio_host()? + .execute(config, tool, arguments, entity_id, connection_id) + .await +} + +/// The direct-mode API key, or `None` when unset or unwired. +#[must_use] +pub fn api_key(config: &Config) -> Option { + composio_host()?.api_key(config) +} + +/// Whether a viable Composio client resolves. `false` when unwired. +#[must_use] +pub fn is_available(config: &Config) -> bool { + composio_host().is_some_and(|host| host.is_available(config)) +} diff --git a/core/src/config_loader.rs b/core/src/config_loader.rs new file mode 100644 index 0000000..56e0d56 --- /dev/null +++ b/core/src/config_loader.rs @@ -0,0 +1,103 @@ +//! [`ConfigLoader`] — reading a fresh config, which only the host can do. +//! +//! [`crate::Config`] is a trait object: this crate can *read* a host config but +//! has no idea how one is produced — which file, which env overrides, which +//! migrations run on load. The background loops need a fresh one anyway, not +//! the snapshot they were spawned with: a mid-session settings change (a new +//! Composio API key, a toggled sync interval) has to take effect on the next +//! tick rather than at the next restart. +//! +//! # Two methods, and why the snapshot one matters +//! +//! [`ConfigLoader::load`] resolves the config the way startup would. +//! [`ConfigLoader::reload_snapshot`] re-reads *the same file the snapshot came +//! from*. The distinction is load-bearing: `load` follows the ambient +//! environment, so in a test process — or on a host with several workspaces — +//! it can land on a different workspace than the caller is working in. Anchor +//! to the snapshot whenever one is in hand. +//! +//! # Unwired is an error +//! +//! A loop that silently kept its stale snapshot would apply the user's settings +//! change never, and look like it was working. + +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::RwLock; + +use crate::Config; + +/// Produces host configs on the core's behalf. +#[async_trait] +pub trait ConfigLoader: Send + Sync + std::fmt::Debug { + /// Load the config the way startup does, following the ambient + /// environment. + /// + /// Returns a `Box`, not an `Arc`: callers that run a config *migration* + /// need `&mut` to write settings back, which an `Arc` cannot give. + /// Sharing one is a free `Arc::from` at the few sites that need it. + /// + /// # Errors + /// + /// Returns `Err` when the config cannot be read or times out. + async fn load(&self) -> Result, String>; + + /// Re-read the config from the same path `snapshot` was loaded from. + /// + /// # Errors + /// + /// Returns `Err` when the config cannot be read or times out. + async fn reload_snapshot(&self, snapshot: &Config) -> Result, String>; +} + +static LOADER: RwLock>> = RwLock::new(None); + +const NOT_INSTALLED: &str = + "no ConfigLoader installed — the host must call memory::config_loader::set_config_loader \ + during startup wiring, before any background loop runs"; + +/// Install the host's config loader. Called once during startup wiring. +pub fn set_config_loader(loader: Arc) { + *LOADER.write() = Some(loader); +} + +/// Remove any installed loader. For tests. +pub fn clear_config_loader() { + *LOADER.write() = None; +} + +/// The installed loader, or `None` when nothing has been wired up. +#[must_use] +pub fn config_loader() -> Option> { + LOADER.read().clone() +} + +/// Load a fresh config. +/// +/// # Errors +/// +/// Returns `Err` when no loader is installed, or the load fails. +pub async fn load_config_with_timeout() -> Result, String> { + let loader = config_loader().ok_or_else(|| NOT_INSTALLED.to_string())?; + loader.load().await +} + +/// Load a fresh config as a shareable handle, for callers that only read it. +/// +/// # Errors +/// +/// Returns `Err` when no loader is installed, or the load fails. +pub async fn load_config_arc() -> Result, String> { + Ok(Arc::from(load_config_with_timeout().await?)) +} + +/// Re-read the config `snapshot` came from. +/// +/// # Errors +/// +/// Returns `Err` when no loader is installed, or the load fails. +pub async fn reload_config_snapshot_with_timeout(snapshot: &Config) -> Result, String> { + let loader = config_loader().ok_or_else(|| NOT_INSTALLED.to_string())?; + loader.reload_snapshot(snapshot).await +} diff --git a/core/src/conversations/blocking.rs b/core/src/conversations/blocking.rs new file mode 100644 index 0000000..eda10aa --- /dev/null +++ b/core/src/conversations/blocking.rs @@ -0,0 +1,188 @@ +//! Async wrappers that run the conversation store's **blocking** operations on +//! tokio's blocking pool (#5156). +//! +//! Every `tinycortex::memory::conversations` entry point is synchronous, and +//! each one takes the process-global `CONVERSATION_STORE_LOCK` — a +//! `parking_lot::Mutex` — and then does fsync'd JSONL file IO while holding it. +//! Calling one directly from an `async fn` therefore parks a tokio **worker** +//! thread for the whole wait, and the wait is not short: +//! +//! * `threads.jsonl` is folded from scratch on nearly every operation +//! (`thread_index_unlocked`), and it grows by roughly two lines per appended +//! message and is never compacted — so the per-call fold cost grows with the +//! user's whole history; +//! * `append_message` writes two fsync'd appends under the lock, and +//! `update_message` reads and rewrites a thread's entire message log under it, +//! so a live streaming turn holds the lock repeatedly; +//! * `search_cross_thread_messages` reads every thread's transcript on a cold +//! index. +//! +//! Once more concurrent conversation operations than there are worker threads +//! are parked on that mutex, the runtime stops polling **anything** — including +//! the HTTP task that owes the client its response. That is how a create that +//! only needs one append blows the frontend's 30 s RPC budget: +//! `UnhandledRejection: Core RPC openhuman.threads_create_new timed out after +//! 30000ms` (Sentry TAURI-REACT-10, #5156). +//! +//! Moving each call onto the blocking pool keeps the lock wait off the async +//! workers. The work is just as serialized as before — the store's lock still +//! decides who writes when — but the executor stays live, so the RPC server +//! keeps answering while a slow conversation operation drains, and a queued +//! create completes as soon as the lock frees instead of after the client has +//! given up. +//! +//! Callers pass owned arguments because the closure must be `'static`. + +use std::path::PathBuf; + +use tinycortex::memory::conversations as store; + +use tinycortex::memory::conversations::{ + ConversationMessage, ConversationMessagePatch, ConversationPurgeStats, ConversationStore, + ConversationThread, CreateConversationThread, CrossThreadHit, +}; + +/// Run one blocking store call on the blocking pool. +/// +/// A `JoinError` here means the pool task panicked (or the runtime is shutting +/// down); surface it as a store error rather than propagating the panic into +/// the RPC dispatcher, which would turn a transient store fault into a 500-shaped +/// unknown failure. +async fn run(operation: &'static str, f: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + match tokio::task::spawn_blocking(f).await { + Ok(result) => result, + Err(error) => { + tracing::warn!( + operation, + error = %error, + "[conversations] blocking store task failed to join" + ); + Err(format!( + "conversation store {operation} task failed: {error}" + )) + } + } +} + +/// [`store::ensure_thread`] on the blocking pool. +pub async fn ensure_thread( + workspace_dir: PathBuf, + request: CreateConversationThread, +) -> Result { + run("ensure_thread", move || { + store::ensure_thread(workspace_dir, request) + }) + .await +} + +/// [`store::list_threads`] on the blocking pool. +pub async fn list_threads(workspace_dir: PathBuf) -> Result, String> { + run("list_threads", move || store::list_threads(workspace_dir)).await +} + +/// [`store::get_messages`] on the blocking pool. +pub async fn get_messages( + workspace_dir: PathBuf, + thread_id: String, +) -> Result, String> { + run("get_messages", move || { + store::get_messages(workspace_dir, &thread_id) + }) + .await +} + +/// [`store::append_message`] on the blocking pool. +pub async fn append_message( + workspace_dir: PathBuf, + thread_id: String, + message: ConversationMessage, +) -> Result { + run("append_message", move || { + store::append_message(workspace_dir, &thread_id, message) + }) + .await +} + +/// [`store::update_message`] on the blocking pool. +pub async fn update_message( + workspace_dir: PathBuf, + thread_id: String, + message_id: String, + patch: ConversationMessagePatch, +) -> Result { + run("update_message", move || { + store::update_message(workspace_dir, &thread_id, &message_id, patch) + }) + .await +} + +/// [`store::update_thread_title`] on the blocking pool. +pub async fn update_thread_title( + workspace_dir: PathBuf, + thread_id: String, + title: String, + updated_at: String, +) -> Result { + run("update_thread_title", move || { + store::update_thread_title(workspace_dir, &thread_id, &title, &updated_at) + }) + .await +} + +/// [`store::update_thread_labels`] on the blocking pool. +pub async fn update_thread_labels( + workspace_dir: PathBuf, + thread_id: String, + labels: Vec, + updated_at: String, +) -> Result { + run("update_thread_labels", move || { + store::update_thread_labels(workspace_dir, &thread_id, labels, &updated_at) + }) + .await +} + +/// [`store::delete_thread`] on the blocking pool. +pub async fn delete_thread( + workspace_dir: PathBuf, + thread_id: String, + deleted_at: String, +) -> Result { + run("delete_thread", move || { + store::delete_thread(workspace_dir, &thread_id, &deleted_at) + }) + .await +} + +/// [`store::purge_threads`] on the blocking pool. +pub async fn purge_threads(workspace_dir: PathBuf) -> Result { + run("purge_threads", move || store::purge_threads(workspace_dir)).await +} + +/// [`ConversationStore::search_cross_thread_messages`] on the blocking pool. +/// +/// The heaviest operation in the family: a cold inverted index reads every +/// thread's transcript before the search runs. +pub async fn search_cross_thread_messages( + workspace_dir: PathBuf, + query: String, + limit: usize, + exclude_thread_id: Option, +) -> Result, String> { + run("search_cross_thread_messages", move || { + ConversationStore::new(workspace_dir).search_cross_thread_messages( + &query, + limit, + exclude_thread_id.as_deref(), + ) + }) + .await +} + +#[cfg(test)] +#[path = "blocking_tests.rs"] +mod tests; diff --git a/core/src/conversations/blocking_tests.rs b/core/src/conversations/blocking_tests.rs new file mode 100644 index 0000000..239c5be --- /dev/null +++ b/core/src/conversations/blocking_tests.rs @@ -0,0 +1,300 @@ +//! Tests for the blocking-pool conversation-store wrappers (#5156). +//! +//! The property under test is that the wrappers are a faithful, `.await`-able +//! stand-in for the synchronous store: same results, same errors, and safe to +//! drive concurrently from several tasks on a runtime with a single async worker +//! — which is exactly the shape that used to starve (a sync call parks the +//! worker on the store's `parking_lot` mutex, so with every worker parked the +//! runtime stops polling the HTTP task that owes the client its response). + +use serde_json::json; +use tempfile::TempDir; + +use super::*; + +fn message(id: &str, content: &str) -> ConversationMessage { + ConversationMessage { + id: id.to_string(), + content: content.to_string(), + message_type: "text".to_string(), + extra_metadata: json!({}), + sender: "user".to_string(), + created_at: "2026-07-30T00:00:00Z".to_string(), + } +} + +fn create(id: &str) -> CreateConversationThread { + CreateConversationThread { + id: id.to_string(), + title: format!("Title {id}"), + created_at: "2026-07-30T00:00:00Z".to_string(), + parent_thread_id: None, + labels: None, + personality_id: None, + } +} + +#[tokio::test] +async fn create_append_read_round_trips_through_the_blocking_pool() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); + + let thread = ensure_thread(dir.clone(), create("thread-a")) + .await + .expect("ensure_thread"); + assert_eq!(thread.id, "thread-a"); + + append_message(dir.clone(), "thread-a".to_string(), message("m1", "hello")) + .await + .expect("append_message"); + + let messages = get_messages(dir.clone(), "thread-a".to_string()) + .await + .expect("get_messages"); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].content, "hello"); + + let threads = list_threads(dir.clone()).await.expect("list_threads"); + assert_eq!(threads.len(), 1); + assert_eq!(threads[0].message_count, 1); +} + +#[tokio::test] +async fn store_errors_surface_unchanged() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); + + // Appending to a thread that was never created is the store's own error, + // not a join failure — the wrapper must pass it through verbatim so the + // RPC layer's thread-scoped error mapping still recognises it. + let error = append_message(dir, "missing".to_string(), message("m1", "hello")) + .await + .expect_err("append to a missing thread must fail"); + assert!( + error.contains("not found"), + "expected the store's own error, got: {error}" + ); +} + +/// The starvation shape from #5156: several conversation operations in flight at +/// once on a runtime with a **single** async worker. Each one contends for the +/// store's process-global mutex, so with the calls made inline the single worker +/// is parked in `lock()` and nothing else on the runtime can be polled. Off the +/// blocking pool they all complete, and a plain cooperative task keeps being +/// polled while they do. +/// +/// The cooperative ticker alone would not prove that: its 64 yields can all +/// retire before any store task even reaches the lock, so it would stay green +/// against an implementation that ran the store inline. Determinism comes from +/// `probe` — a store closure *held open* for the whole window — plus a watchdog +/// that observes progress while it is held. +/// +/// The watchdog is a plain OS thread on purpose. A `tokio::time::timeout` cannot +/// bound this: if the store ran inline, the sole worker would be parked inside +/// the probe closure and the timeout future would never be polled either, so the +/// test would hang instead of failing. An OS thread cannot be starved by the +/// runtime, and it releases the probe on its way out so a starved runtime always +/// recovers far enough to report the failure. +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn concurrent_operations_complete_without_stalling_the_single_async_worker() { + use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + const TICK_TARGET: u32 = 64; + const WATCHDOG_BUDGET: Duration = Duration::from_secs(10); + + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); + + // A store call pinned open for as long as we hold `release`. `recv()` is a + // genuinely blocking wait — the same shape as `parking_lot::Mutex::lock`. + let probe_entered = Arc::new(AtomicBool::new(false)); + let (release, held) = std::sync::mpsc::channel::<()>(); + let probe = tokio::spawn(run("blocked_probe", { + let probe_entered = Arc::clone(&probe_entered); + move || { + probe_entered.store(true, Ordering::SeqCst); + let _ = held.recv(); + Ok::<(), String>(()) + } + })); + + // A cooperative task that only ever yields: it can make progress solely + // because no store call is holding the worker hostage. + let ticks = Arc::new(AtomicU32::new(0)); + let ticker = tokio::spawn({ + let ticks = Arc::clone(&ticks); + async move { + for _ in 0..TICK_TARGET { + tokio::task::yield_now().await; + ticks.fetch_add(1, Ordering::SeqCst); + } + } + }); + + let watchdog = std::thread::spawn({ + let probe_entered = Arc::clone(&probe_entered); + let ticks = Arc::clone(&ticks); + move || { + let deadline = Instant::now() + WATCHDOG_BUDGET; + // 1. The probe must actually be executing, or its "blocking window" + // means nothing. + while Instant::now() < deadline && !probe_entered.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(5)); + } + let entered = probe_entered.load(Ordering::SeqCst); + // 2. The ticker must then run to completion *while* that store call + // is still held. Inline execution parks the sole worker inside + // the closure, so this expires with ticks stuck at 0. + let mut observed = 0; + while Instant::now() < deadline { + observed = ticks.load(Ordering::SeqCst); + if observed >= TICK_TARGET { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + // Always release, even on failure: a starved runtime has to recover + // far enough for the assertions below to run and report. + let _ = release.send(()); + (entered, observed) + } + }); + + let mut handles = Vec::new(); + for idx in 0..8 { + let dir = dir.clone(); + handles.push(tokio::spawn(async move { + let thread = ensure_thread(dir.clone(), create(&format!("thread-{idx}"))) + .await + .expect("ensure_thread"); + append_message( + dir, + thread.id.clone(), + message(&format!("m{idx}"), "concurrent"), + ) + .await + .expect("append_message"); + thread.id + })); + } + + // The load-bearing assertions, both observed from outside the runtime. + let (probe_started, observed_ticks) = watchdog.join().expect("join watchdog"); + assert!( + probe_started, + "the blocking store probe never started — the pool is not running store work" + ); + assert_eq!( + observed_ticks, TICK_TARGET, + "the async worker must keep polling while a store call is blocked; \ + only {observed_ticks}/{TICK_TARGET} ticks retired within {WATCHDOG_BUDGET:?}" + ); + + ticker.await.expect("join ticker"); + probe + .await + .expect("join probe") + .expect("probe closure result"); + + let mut ids = Vec::new(); + for handle in handles { + ids.push(handle.await.expect("join create task")); + } + ids.sort(); + ids.dedup(); + assert_eq!( + ids.len(), + 8, + "every concurrent create must land its own thread" + ); + + let threads = list_threads(dir).await.expect("list_threads"); + assert_eq!(threads.len(), 8); + assert!(threads.iter().all(|thread| thread.message_count == 1)); +} + +#[tokio::test] +async fn title_labels_delete_and_purge_round_trip() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); + ensure_thread(dir.clone(), create("thread-a")) + .await + .expect("ensure_thread"); + + let retitled = update_thread_title( + dir.clone(), + "thread-a".to_string(), + "Renamed".to_string(), + "2026-07-30T00:01:00Z".to_string(), + ) + .await + .expect("update_thread_title"); + assert_eq!(retitled.title, "Renamed"); + + let relabelled = update_thread_labels( + dir.clone(), + "thread-a".to_string(), + vec!["general".to_string()], + "2026-07-30T00:02:00Z".to_string(), + ) + .await + .expect("update_thread_labels"); + assert_eq!(relabelled.labels, vec!["general".to_string()]); + assert_eq!(relabelled.title, "Renamed", "labels update preserves title"); + + assert!( + delete_thread( + dir.clone(), + "thread-a".to_string(), + "2026-07-30T00:03:00Z".to_string() + ) + .await + .expect("delete_thread"), + "deleting a live thread reports true" + ); + assert!(list_threads(dir.clone()).await.unwrap().is_empty()); + + ensure_thread(dir.clone(), create("thread-b")) + .await + .expect("ensure_thread"); + purge_threads(dir.clone()).await.expect("purge_threads"); + assert!(list_threads(dir).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn update_message_patches_metadata_and_search_finds_content() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); + ensure_thread(dir.clone(), create("thread-a")) + .await + .expect("ensure_thread"); + append_message( + dir.clone(), + "thread-a".to_string(), + message("m1", "quarterly roadmap review"), + ) + .await + .expect("append_message"); + + let patched = update_message( + dir.clone(), + "thread-a".to_string(), + "m1".to_string(), + ConversationMessagePatch { + extra_metadata: Some(json!({"pinned": true})), + }, + ) + .await + .expect("update_message"); + assert_eq!(patched.extra_metadata, json!({"pinned": true})); + + let hits = search_cross_thread_messages(dir, "roadmap".to_string(), 10, None) + .await + .expect("search_cross_thread_messages"); + assert!( + hits.iter().any(|hit| hit.thread_id == "thread-a"), + "cross-thread search must find the appended message, got: {hits:?}" + ); +} diff --git a/core/src/conversations/mod.rs b/core/src/conversations/mod.rs new file mode 100644 index 0000000..a6b5491 --- /dev/null +++ b/core/src/conversations/mod.rs @@ -0,0 +1,17 @@ +//! Host-side wiring for workspace-backed conversation thread/message storage. +//! +//! Conversations are stored as JSONL files under the workspace (thread metadata +//! append-only in `threads.jsonl`; each thread's messages in a dedicated JSONL +//! file). The store / inverted-index / tokenizer / types engine is the crate's +//! (a byte-identical port, incl. the D1 rank-before-materialize fix), and +//! consumers name `tinycortex::memory::conversations` directly — this module no +//! longer re-exports that surface under a second path. +//! +//! Host-retained: +//! - `bus` — the `core::bus` persistence subscriber that bridges typed channel +//! events onto the crate store (the crate abstracts the bus behind its own +//! `ConversationEventBus` trait; the host wires the real one). +//! - [`blocking`] — `spawn_blocking` wrappers around the store's synchronous +//! entry points. Request paths must use these, never the sync API (#5156). + +pub mod blocking; diff --git a/core/src/diff/mod.rs b/core/src/diff/mod.rs new file mode 100644 index 0000000..31b3f13 --- /dev/null +++ b/core/src/diff/mod.rs @@ -0,0 +1,71 @@ +//! Snapshot-based change tracking for memory sources. +//! +//! After each sync, this module captures what's in the chunk store for +//! that source, then diffs against previous snapshots to surface +//! additions, removals, and modifications — helping agents understand +//! how their world view has changed over time. +//! +//! Snapshots are built from already-ingested data in `mem_tree_chunks` +//! (not by re-calling source readers), making them free of API calls. +//! +//! Storage is a git repository at `/memory_diff/repo` (the diff +//! *ledger*): snapshots are commits, checkpoints are tags, read markers are +//! refs, and diffs are git tree diffs. `mem_tree_chunks` stays authoritative; +//! the ledger is a derived view used purely for change tracking. +//! +//! W7: the snapshot/diff/checkpoint/ledger engine is now +//! `tinycortex::memory::diff::DiffEngine` (a byte-identical port over the same +//! `/memory_diff/repo` git layout). This module is a thin host shim: +//! [`ops`] async-wraps the engine, [`source`] supplies the chunk-store item +//! seam (`DiffEngine`'s `SnapshotItemSource`), and `rpc`/`schemas`/`tools` +//! keep the RPC + agent surface. The wire types are the crate's, named directly +//! (`tinycortex::memory::diff::types`) rather than through a host re-export +//! module. +//! +//! Features: +//! - Per-source snapshots (auto after sync, or manual via RPC) +//! - Diff between any two snapshots +//! - Named checkpoints for cross-source "what changed since X" queries +//! - Agent tool for in-conversation diff queries + +//! ## The `memory-git` gate +//! +//! All of the above needs a git ledger, and libgit2 is one of the two most +//! expensive native builds left in the graph — so the behaviour sits behind +//! `memory-git` (default-OFF, product-ON), which also carries `git2` and +//! tinycortex's `git-diff`/`wiki-git`. Off, it sheds `git2` + `libgit2-sys` + +//! `libz-sys`, taking the kernel profile from 5 native builds to 3. +//! +//! **`types` stays ungated**, mirroring the carve-out on the tinycortex side: +//! it re-exports `serde`-only wire types that always-on callers name. The +//! subconscious memory profile renders `CrossSourceDiff` and `ChangeKind` into +//! prompts, and duplicating those in a stub would be two definitions of one +//! serde shape, free to drift apart. +//! +//! The three `ops` entry points always-on code calls are stubbed rather than +//! `#[cfg]`'d at each call site, so `memory::sources::sync` and the +//! subconscious profile need no feature awareness — a diff simply never +//! materialises. Registration sites get the opposite treatment: the schema +//! aggregators return empty vecs (the controllers become unknown-method) and +//! `MemoryDiffTool` is `#[cfg]`'d out at its one registration site in +//! `tools/ops.rs`, because a registered tool that always errors is worse than +//! an absent one — the model would keep choosing it and reporting the failure. + +#[cfg(feature = "memory-git")] +pub mod ops; + +#[cfg(not(feature = "memory-git"))] +mod stub; +#[cfg(not(feature = "memory-git"))] +pub use stub::ops; +#[cfg(feature = "memory-git")] +pub mod source; + +// Ungated in both builds, mirroring tinycortex's own carve-out: its +// `memory::diff::{types, source}` are serde-only wire types and stay compiled; +// only the git-touching `ledger`/`DiffEngine` half sits behind `git-diff`. A +// stub copy would be a second definition of one serde shape, free to drift. +pub use tinycortex::memory::diff::types::{ + ChangeKind, Checkpoint, CrossSourceDiff, DiffResult, DiffSummary, ItemChange, Snapshot, + SnapshotTrigger, +}; diff --git a/core/src/diff/ops.rs b/core/src/diff/ops.rs new file mode 100644 index 0000000..8c50e8b --- /dev/null +++ b/core/src/diff/ops.rs @@ -0,0 +1,575 @@ +//! Business logic for memory diff — thin host async wrappers over +//! `tinycortex::memory::diff::DiffEngine` (W7). +//! +//! The snapshot/diff/checkpoint/ledger engine is the crate's; the git ledger it +//! writes lives at the same `/memory_diff/repo` path with the same +//! libgit2 layout, so existing ledgers keep working byte-for-byte. `DiffEngine` +//! is synchronous and generic over a chunk-source seam, so each op here builds +//! the host [`ChunkStoreItemSource`] (which reads the authoritative +//! `mem_tree_chunks`) and drives the engine inside `spawn_blocking`, preserving +//! the host's `async` + `Result<_, String>` signatures, the `DomainEvent` +//! publishes, and the tracing that RPC/tools/sync/subconscious callers expect. + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::sources::types::MemorySourceEntry; +use crate::Config; + +use tinycortex::memory::diff::{DiffEngine, SourceDescriptor}; + +use super::source::ChunkStoreItemSource; +use tinycortex::memory::diff::types::*; + +/// A crate [`SourceDescriptor`] from a host source entry. +fn descriptor(source: &MemorySourceEntry) -> SourceDescriptor { + SourceDescriptor::new( + source.id.clone(), + source.kind.as_str().to_string(), + source.label.clone(), + ) +} + +/// Take a snapshot of the current chunk-store state for a source. +/// +/// Reads from `mem_tree_chunks` (already-ingested data) via the item-source +/// seam, groups by item, and commits one blob per item to the git ledger. +/// Returns the new [`Snapshot`] whose `id` is the commit SHA. +pub async fn take_snapshot( + source: &MemorySourceEntry, + config: &Config, + trigger: SnapshotTrigger, +) -> Result { + let workspace_dir = config.workspace_dir().clone(); + let config_clone = config.to_arc(); + let source_owned = source.clone(); + let desc = descriptor(source); + + let snapshot = tokio::task::spawn_blocking(move || -> anyhow::Result { + let items = ChunkStoreItemSource::single(config_clone, &source_owned); + let engine = DiffEngine::new(workspace_dir, items); + engine.take_snapshot(&desc, trigger) + }) + .await + .map_err(|e| format!("snapshot join error: {e}"))? + .map_err(|e: anyhow::Error| format!("take_snapshot: {e:#}"))?; + + tracing::debug!( + snapshot_id = %snapshot.id, + source_id = %source.id, + items = snapshot.item_count, + trigger = %snapshot.trigger.as_str(), + "[memory_diff] snapshot taken" + ); + + crate::events::publish(crate::events::MemoryEvent::DiffSnapshotTaken { + snapshot_id: snapshot.id.clone(), + source_id: source.id.clone(), + source_kind: source.kind.as_str().to_string(), + item_count: snapshot.item_count as usize, + trigger: snapshot.trigger.as_str().to_string(), + }); + + Ok(snapshot) +} + +/// Auto-snapshot hook called from `sync_source()` after a successful sync. +pub async fn auto_snapshot_after_sync( + source: &MemorySourceEntry, + config: &Config, +) -> Result { + take_snapshot(source, config, SnapshotTrigger::Auto).await +} + +/// List snapshots, newest first — for one source when `source_id` is `Some`, +/// across every source otherwise. +/// +/// Lifted verbatim out of `super::rpc::list_snapshots_rpc`, which had the +/// only copy of this query and returned it wrapped in an `RpcOutcome`. The +/// embedded memory driver's `MemoryDiff::snapshots` needs the same read without +/// the RPC envelope, and a second `Ledger::open` call site would make this +/// module no longer the only place that knows the ledger layout. +/// +/// An unknown `source_id` yields an empty vector rather than an error — the +/// ledger has no source registry to check against. +pub async fn list_snapshots( + config: &Config, + source_id: Option<&str>, + limit: u32, +) -> Result, String> { + let workspace_dir = config.workspace_dir().clone(); + let source_id = source_id.map(str::to_string); + + tokio::task::spawn_blocking(move || -> anyhow::Result> { + let ledger = tinycortex::memory::diff::Ledger::open(&workspace_dir)?; + ledger.list_snapshots(source_id.as_deref(), limit) + }) + .await + .map_err(|e| format!("list_snapshots join: {e}"))? + .map_err(|e: anyhow::Error| format!("list_snapshots: {e:#}")) +} + +/// Compute the diff between two snapshots of the same source. +pub async fn compute_diff( + config: &Config, + from_snapshot_id: Option<&str>, + to_snapshot_id: &str, + include_text_diff: bool, +) -> Result { + let workspace_dir = config.workspace_dir().clone(); + let config_clone = config.to_arc(); + let to_id = to_snapshot_id.to_string(); + let from_id = from_snapshot_id.map(|s| s.to_string()); + + tokio::task::spawn_blocking(move || -> anyhow::Result { + let engine = DiffEngine::new(workspace_dir, ChunkStoreItemSource::read_only(config_clone)); + engine.compute_diff(from_id.as_deref(), &to_id, include_text_diff) + }) + .await + .map_err(|e| format!("diff join: {e}"))? + .map_err(|e: anyhow::Error| format!("compute_diff: {e:#}")) +} + +/// Diff current state (latest snapshot) vs previous snapshot for a source. +pub async fn diff_since_last( + source: &MemorySourceEntry, + config: &Config, + include_text_diff: bool, +) -> Result { + let workspace_dir = config.workspace_dir().clone(); + let config_clone = config.to_arc(); + let source_id = source.id.clone(); + + tokio::task::spawn_blocking(move || -> anyhow::Result { + let engine = DiffEngine::new(workspace_dir, ChunkStoreItemSource::read_only(config_clone)); + engine.diff_since_last(&source_id, include_text_diff) + }) + .await + .map_err(|e| format!("diff_since_last join: {e}"))? + .map_err(|e: anyhow::Error| format!("diff_since_last: {e:#}")) +} + +/// Diff a source's latest snapshot against its read marker — i.e. everything +/// that changed since the agent last *read* this source's diff. +/// +/// When `commit` is true, the read marker (a git ref) is advanced to the head +/// snapshot after the diff is computed, so a subsequent call returns only newer +/// changes. This is the turn-to-turn primitive: read the world delta, then +/// acknowledge it as consumed. +pub async fn diff_since_read( + source: &MemorySourceEntry, + config: &Config, + include_text_diff: bool, + commit: bool, +) -> Result { + let workspace_dir = config.workspace_dir().clone(); + let config_clone = config.to_arc(); + let source_id = source.id.clone(); + + let diff = tokio::task::spawn_blocking(move || -> anyhow::Result { + let engine = DiffEngine::new(workspace_dir, ChunkStoreItemSource::read_only(config_clone)); + engine.diff_since_read(&source_id, include_text_diff, commit) + }) + .await + .map_err(|e| format!("diff_since_read join: {e}"))? + .map_err(|e: anyhow::Error| format!("diff_since_read: {e:#}"))?; + + if commit { + tracing::debug!( + source_id = %source.id, + snapshot_id = %diff.to_snapshot_id, + added = diff.summary.added, + modified = diff.summary.modified, + removed = diff.summary.removed, + "[memory_diff] read marker committed" + ); + } + + Ok(diff) +} + +/// Commit a read marker for one or more sources, advancing each to its +/// current head snapshot. When `source_ids` is `None`, marks all enabled +/// sources that have at least one snapshot. Returns the number of markers set. +pub async fn mark_read(config: &Config, source_ids: Option>) -> Result { + let target_ids: Vec = match source_ids { + Some(ids) => ids, + None => crate::sources::registry::list_sources() + .await + .map_err(|e| format!("list sources: {e}"))? + .into_iter() + .filter(|s| s.enabled) + .map(|s| s.id) + .collect(), + }; + + let workspace_dir = config.workspace_dir().clone(); + let config_clone = config.to_arc(); + let ids_for_blocking = target_ids.clone(); + + let (marked, snapshot_ids) = + tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, Vec)> { + let engine = + DiffEngine::new(workspace_dir, ChunkStoreItemSource::read_only(config_clone)); + // Gather the head snapshot ids that will be marked, for the event + // payload (the crate `mark_read` returns only a count). + let mut snapshot_ids = Vec::new(); + for sid in &ids_for_blocking { + if let Some(head) = engine.list_snapshots(Some(sid), 1)?.into_iter().next() { + snapshot_ids.push(head.id); + } + } + let marked = engine.mark_read(&ids_for_blocking)?; + Ok((marked, snapshot_ids)) + }) + .await + .map_err(|e| format!("mark_read join: {e}"))? + .map_err(|e: anyhow::Error| format!("mark_read: {e:#}"))?; + + tracing::debug!( + sources = marked, + "[memory_diff] mark_read committed read markers" + ); + + crate::events::publish(crate::events::MemoryEvent::DiffMarkedRead { + source_ids: target_ids, + snapshot_ids, + }); + + Ok(marked) +} + +/// Create a checkpoint (git tag at HEAD) grouping the latest snapshot per +/// enabled source. Sources lacking a snapshot are baselined first. +pub async fn create_checkpoint(label: &str, config: &Config) -> Result { + let sources = crate::sources::registry::list_sources() + .await + .map_err(|e| format!("list sources: {e}"))?; + let enabled: Vec = sources.into_iter().filter(|s| s.enabled).collect(); + + let workspace_dir = config.workspace_dir().clone(); + let config_clone = config.to_arc(); + let label_owned = label.to_string(); + + let checkpoint = tokio::task::spawn_blocking(move || -> anyhow::Result { + let descriptors: Vec = enabled.iter().map(descriptor).collect(); + let items = ChunkStoreItemSource::for_sources(config_clone, &enabled); + let engine = DiffEngine::new(workspace_dir, items); + engine.create_checkpoint(&label_owned, &descriptors) + }) + .await + .map_err(|e| format!("checkpoint persist join: {e}"))? + .map_err(|e: anyhow::Error| format!("create_checkpoint: {e:#}"))?; + + tracing::debug!( + checkpoint_id = %checkpoint.id, + snapshots = checkpoint.snapshot_ids.len(), + "[memory_diff] checkpoint created" + ); + + Ok(checkpoint) +} + +/// Compute a cross-source diff: everything that changed since a checkpoint. +pub async fn diff_since_checkpoint( + checkpoint_id: &str, + config: &Config, + include_text_diff: bool, +) -> Result { + let workspace_dir = config.workspace_dir().clone(); + let config_clone = config.to_arc(); + let ckpt_id = checkpoint_id.to_string(); + + tokio::task::spawn_blocking(move || -> anyhow::Result { + let engine = DiffEngine::new(workspace_dir, ChunkStoreItemSource::read_only(config_clone)); + engine.diff_since_checkpoint(&ckpt_id, include_text_diff) + }) + .await + .map_err(|e| format!("diff_since_checkpoint join: {e}"))? + .map_err(|e: anyhow::Error| format!("diff_since_checkpoint: {e:#}")) +} + +/// Delete checkpoint tags older than `older_than_days`. +/// +/// Snapshot commits are retained — git history *is* the ledger, and git's +/// delta compression keeps it compact — so cleanup only prunes named baselines. +/// Returns the number of checkpoints deleted. +pub async fn cleanup(config: &Config, older_than_days: u32) -> Result { + let workspace_dir = config.workspace_dir().clone(); + let config_clone = config.to_arc(); + + tokio::task::spawn_blocking(move || -> anyhow::Result { + let engine = DiffEngine::new(workspace_dir, ChunkStoreItemSource::read_only(config_clone)); + engine.cleanup(older_than_days) + }) + .await + .map_err(|e| format!("cleanup join: {e}"))? + .map_err(|e: anyhow::Error| format!("cleanup: {e:#}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use tinycortex::memory::diff::{Ledger, SnapshotMeta}; + + fn test_config() -> TestHostConfig { + crate::test_seams::init(); + let dir = tempfile::tempdir().unwrap(); + let mut config = TestHostConfig::default(); + config.workspace_dir = dir.path().to_path_buf(); + // Leak the tempdir so the path stays valid for the test's lifetime. + std::mem::forget(dir); + config + } + + fn folder_source(id: &str) -> MemorySourceEntry { + MemorySourceEntry { + id: id.into(), + kind: crate::sources::types::SourceKind::Folder, + label: "Docs".into(), + enabled: true, + toolkit: None, + connection_id: None, + path: Some("/tmp".into()), + glob: None, + url: None, + branch: None, + paths: Vec::new(), + query: None, + since_days: None, + max_items: None, + max_commits: None, + max_issues: None, + max_prs: None, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days: None, + } + } + + /// Seed a snapshot directly through the (crate) ledger, bypassing the chunk + /// store — exercises the host async wrappers over real ledger state. + fn seed( + config: &Config, + source_id: &str, + taken_at_ms: i64, + items: &[(&str, &str)], + ) -> Snapshot { + let ledger = Ledger::open(config.workspace_dir()).unwrap(); + let items: Vec<(String, String)> = items + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + ledger + .commit_snapshot( + &SnapshotMeta { + source_id: source_id.to_string(), + source_kind: "folder".to_string(), + label: "Docs".to_string(), + trigger: SnapshotTrigger::Auto, + }, + &items, + taken_at_ms, + ) + .unwrap() + } + + #[tokio::test] + async fn compute_diff_detects_added_modified_removed() { + let config = test_config(); + let from = seed( + &config, + "src_a", + 1000, + &[("a", "alpha"), ("b", "beta"), ("c", "gamma")], + ); + let to = seed( + &config, + "src_a", + 2000, + &[("a", "alpha"), ("b", "beta v2"), ("d", "delta")], + ); + + let diff = compute_diff(&config, Some(&from.id), &to.id, false) + .await + .unwrap(); + + assert_eq!(diff.summary.added, 1, "d added"); + assert_eq!(diff.summary.modified, 1, "b modified"); + assert_eq!(diff.summary.removed, 1, "c removed"); + assert_eq!(diff.summary.unchanged, 1, "a unchanged"); + + let kind_of = |id: &str| { + diff.changes + .iter() + .find(|c| c.item_id == id) + .map(|c| c.kind.clone()) + }; + assert_eq!(kind_of("d"), Some(ChangeKind::Added)); + assert_eq!(kind_of("b"), Some(ChangeKind::Modified)); + assert_eq!(kind_of("c"), Some(ChangeKind::Removed)); + assert_eq!(kind_of("a"), None, "unchanged items are not in changes"); + } + + #[tokio::test] + async fn compute_diff_against_none_marks_all_added() { + let config = test_config(); + let to = seed(&config, "src_a", 1000, &[("a", "x")]); + let diff = compute_diff(&config, None, &to.id, false).await.unwrap(); + assert_eq!(diff.summary.added, 1); + assert_eq!(diff.from_snapshot_id, None); + } + + #[tokio::test] + async fn compute_diff_rejects_cross_source() { + let config = test_config(); + let from = seed(&config, "src_a", 1000, &[("a", "x")]); + let to = seed(&config, "src_b", 2000, &[("b", "y")]); + let err = compute_diff(&config, Some(&from.id), &to.id, false) + .await + .unwrap_err(); + assert!(err.contains("cross-source"), "got: {err}"); + } + + #[tokio::test] + async fn compute_diff_text_diff_only_when_requested() { + let config = test_config(); + let from = seed(&config, "src_a", 1000, &[("a", "line one\nline two\n")]); + let to = seed( + &config, + "src_a", + 2000, + &[("a", "line one\nline TWO changed\n")], + ); + + let without = compute_diff(&config, Some(&from.id), &to.id, false) + .await + .unwrap(); + assert!(without.changes[0].text_diff.is_none()); + + let with = compute_diff(&config, Some(&from.id), &to.id, true) + .await + .unwrap(); + let td = with.changes[0] + .text_diff + .as_ref() + .expect("text diff present"); + assert!(td.contains("line TWO changed"), "got: {td}"); + } + + #[tokio::test] + async fn diff_since_last_handles_zero_one_two_snapshots() { + let config = test_config(); + let source = folder_source("src_a"); + + // 0 snapshots → error + assert!(diff_since_last(&source, &config, false).await.is_err()); + + // 1 snapshot → everything added (diff vs None) + seed(&config, "src_a", 1000, &[("a", "x")]); + let one = diff_since_last(&source, &config, false).await.unwrap(); + assert_eq!(one.summary.added, 1); + + // 2 snapshots → diff latest vs previous + seed(&config, "src_a", 2000, &[("a", "x"), ("b", "y")]); + let two = diff_since_last(&source, &config, false).await.unwrap(); + assert_eq!(two.summary.added, 1, "b is new in s2"); + assert_eq!(two.summary.unchanged, 1, "a unchanged"); + } + + #[tokio::test] + async fn diff_since_read_commits_marker_and_returns_only_new_changes() { + let config = test_config(); + let source = folder_source("src_a"); + + seed(&config, "src_a", 1000, &[("a", "x")]); + + // First read: no marker → full diff (a added), and commit advances marker. + let first = diff_since_read(&source, &config, false, true) + .await + .unwrap(); + assert_eq!(first.summary.added, 1); + + // Second read with no new snapshot: marker == head → nothing changed. + let second = diff_since_read(&source, &config, false, true) + .await + .unwrap(); + assert_eq!(second.summary.added, 0); + assert_eq!(second.summary.modified, 0); + assert_eq!(second.summary.removed, 0); + assert!(second.changes.is_empty()); + + // New snapshot then read: only the delta since the marker shows. + seed(&config, "src_a", 2000, &[("a", "x"), ("b", "y")]); + let third = diff_since_read(&source, &config, false, true) + .await + .unwrap(); + assert_eq!(third.summary.added, 1, "only b is new since last read"); + assert_eq!(third.summary.unchanged, 1); + } + + #[tokio::test] + async fn diff_since_read_without_commit_does_not_advance_marker() { + let config = test_config(); + let source = folder_source("src_a"); + seed(&config, "src_a", 1000, &[("a", "x")]); + + // Preview (commit=false) twice → both show the full diff. + let a = diff_since_read(&source, &config, false, false) + .await + .unwrap(); + let b = diff_since_read(&source, &config, false, false) + .await + .unwrap(); + assert_eq!(a.summary.added, 1); + assert_eq!(b.summary.added, 1, "marker was not advanced"); + } + + #[tokio::test] + async fn mark_read_advances_marker_for_explicit_sources() { + let config = test_config(); + let source = folder_source("src_a"); + seed(&config, "src_a", 1000, &[("a", "x")]); + + let marked = mark_read(&config, Some(vec!["src_a".to_string()])) + .await + .unwrap(); + assert_eq!(marked, 1); + + // After marking, a read shows no changes (marker already at head). + let diff = diff_since_read(&source, &config, false, false) + .await + .unwrap(); + assert_eq!(diff.summary.added, 0); + assert!(diff.changes.is_empty()); + } + + #[tokio::test] + async fn diff_since_checkpoint_aggregates_across_sources() { + let config = test_config(); + // Baseline snapshots for two sources, grouped into a checkpoint. + let a1 = seed(&config, "src_a", 1000, &[("a", "x")]); + let b1 = seed(&config, "src_b", 1000, &[("b", "y")]); + { + let ledger = Ledger::open(&config.workspace_dir).unwrap(); + ledger + .create_checkpoint("ckpt_1", "base", &[a1.id.clone(), b1.id.clone()], 1500) + .unwrap(); + } + + // src_a gets a new head with a modification; src_b unchanged. + seed(&config, "src_a", 2000, &[("a", "x v2")]); + + let cross = diff_since_checkpoint("ckpt_1", &config, false) + .await + .unwrap(); + assert_eq!(cross.summary.modified, 1, "src_a 'a' modified"); + assert_eq!( + cross.per_source.len(), + 1, + "only src_a changed; unchanged src_b is skipped" + ); + assert_eq!(cross.per_source[0].source_id, "src_a"); + } +} diff --git a/core/src/diff/source.rs b/core/src/diff/source.rs new file mode 100644 index 0000000..2baff36 --- /dev/null +++ b/core/src/diff/source.rs @@ -0,0 +1,203 @@ +//! The host implementation of the crate diff engine's chunk-source seam. +//! +//! `tinycortex::memory::diff::DiffEngine` is generic over a +//! [`SnapshotItemSource`]: during +//! `take_snapshot` (directly, and transitively from `create_checkpoint` for any +//! source lacking a baseline) it asks the source for a source's already-ingested +//! items rather than re-calling readers. In OpenHuman that data lives in +//! `mem_tree_chunks`, so [`ChunkStoreItemSource`] answers the seam by querying +//! the chunk store — the exact query the host `take_snapshot` used before the +//! engine was ported to the crate (group by item id, concatenate chunk bodies in +//! `seq_in_source` order, sort by item id). +//! +//! ## Why the adapter holds a prefix map +//! +//! The crate calls `items_for_source(source_id)` with the *logical* source id, +//! but the host chunk `source_id LIKE` prefix is kind-dependent — Composio +//! sources key their chunks by `:%`, not `mem_src::%`, and the +//! toolkit is not derivable from the logical id alone. The adapter is therefore +//! built from the full [`MemorySourceEntry`] list (which carries `toolkit`) and +//! resolves each id → prefix up front. + +use std::collections::HashMap; +use std::sync::Arc; + +use tinycortex::memory::diff::{extract_item_id, SnapshotItem, SnapshotItemSource}; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::sources::types::{MemorySourceEntry, SourceKind}; +use crate::Config; + +/// Host [`SnapshotItemSource`] backed by `mem_tree_chunks`. +/// +/// Construct with [`single`](Self::single) for the per-source `take_snapshot` +/// path, [`for_sources`](Self::for_sources) for `create_checkpoint` (which may +/// baseline several sources), or [`read_only`](Self::read_only) for operations +/// that never materialise items (diff/list/cleanup) and just need *some* source +/// to satisfy the engine's type parameter. +pub struct ChunkStoreItemSource { + config: Arc, + /// Logical source id → chunk `source_id LIKE` prefix. + prefixes: HashMap, +} + +impl ChunkStoreItemSource { + /// Adapter that can materialise items for any of `sources`. + pub fn for_sources(config: Arc, sources: &[MemorySourceEntry]) -> Self { + let prefixes = sources + .iter() + .map(|s| (s.id.clone(), source_id_prefix(s))) + .collect(); + Self { config, prefixes } + } + + /// Adapter scoped to a single source (the common `take_snapshot` path). + pub fn single(config: Arc, source: &MemorySourceEntry) -> Self { + let mut prefixes = HashMap::new(); + prefixes.insert(source.id.clone(), source_id_prefix(source)); + Self { config, prefixes } + } + + /// Adapter that never yields items — for read-only ops (`compute_diff`, + /// `diff_since_*`, `mark_read`, `diff_since_checkpoint`, `cleanup`) whose + /// engine calls only touch the ledger. `items_for_source` always returns + /// empty; it is never invoked on these paths. + pub fn read_only(config: Arc) -> Self { + Self { + config, + prefixes: HashMap::new(), + } + } +} + +impl SnapshotItemSource for ChunkStoreItemSource { + fn items_for_source(&self, source_id: &str) -> Vec { + let Some(prefix) = self.prefixes.get(source_id) else { + return Vec::new(); + }; + + let result = crate::store::chunks::store::with_connection(&*self.config, |conn| { + let mut stmt = conn.prepare( + "SELECT source_id, content \ + FROM mem_tree_chunks \ + WHERE source_id LIKE ?1 \ + ORDER BY source_id, seq_in_source", + )?; + + let mut groups: HashMap> = HashMap::new(); + let rows = stmt.query_map([prefix], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) + })?; + for row in rows { + let (composite_source_id, content) = row?; + let item_id = extract_item_id(&composite_source_id); + groups.entry(item_id).or_default().push(content); + } + + let mut items: Vec = groups + .into_iter() + .map(|(item_id, parts)| SnapshotItem { + item_id, + content: parts.join(""), + }) + .collect(); + items.sort_by(|a, b| a.item_id.cmp(&b.item_id)); + Ok(items) + }); + + match result { + Ok(items) => items, + Err(e) => { + // The crate seam has no error channel. A chunk-store read + // failure here yields an empty snapshot (every item reads as + // removed for that one diff) rather than a propagated error — + // but the ledger is a derived, rebuildable view, so the next + // successful snapshot restores the true state. Log loudly. + tracing::error!( + source_id = %source_id, + error = %format!("{e:#}"), + "[memory_diff] chunk item-source query failed; snapshot will see no items" + ); + Vec::new() + } + } + } +} + +/// Build the `source_id LIKE` prefix that matches chunks belonging to a source. +/// Mirrors `memory_sources::status::source_id_prefix`. +pub(crate) fn source_id_prefix(source: &MemorySourceEntry) -> String { + match source.kind { + SourceKind::Composio => source + .toolkit + .as_deref() + .map(|t| format!("{t}:%")) + .unwrap_or_else(|| "__no_toolkit__:%".to_string()), + _ => format!("mem_src:{}:%", source.id), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn folder_source(id: &str) -> MemorySourceEntry { + MemorySourceEntry { + id: id.into(), + kind: SourceKind::Folder, + label: "Docs".into(), + enabled: true, + toolkit: None, + connection_id: None, + path: Some("/tmp".into()), + glob: None, + url: None, + branch: None, + paths: Vec::new(), + query: None, + since_days: None, + max_items: None, + max_commits: None, + max_issues: None, + max_prs: None, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days: None, + } + } + + #[test] + fn source_id_prefix_folder() { + assert_eq!( + source_id_prefix(&folder_source("src_abc")), + "mem_src:src_abc:%" + ); + } + + #[test] + fn source_id_prefix_composio() { + let mut entry = folder_source("src_cmp"); + entry.kind = SourceKind::Composio; + entry.toolkit = Some("gmail".into()); + assert_eq!(source_id_prefix(&entry), "gmail:%"); + } + + #[test] + fn source_id_prefix_composio_without_toolkit() { + let mut entry = folder_source("src_cmp"); + entry.kind = SourceKind::Composio; + entry.toolkit = None; + assert_eq!(source_id_prefix(&entry), "__no_toolkit__:%"); + } + + #[test] + fn read_only_adapter_never_yields_items() { + let source = ChunkStoreItemSource::read_only( + std::sync::Arc::new(TestHostConfig::default()) as std::sync::Arc, + ); + assert!(source.items_for_source("anything").is_empty()); + } +} diff --git a/core/src/diff/stub.rs b/core/src/diff/stub.rs new file mode 100644 index 0000000..f5658c8 --- /dev/null +++ b/core/src/diff/stub.rs @@ -0,0 +1,64 @@ +//! The `memory-git`-disabled surface of `memory::diff`. +//! +//! Mirrors **functions only**. The wire types stay in [`super::types`] and are +//! compiled in both directions, so — unlike the `voice` stub, which had to +//! re-declare types living inside its gated tree — there is zero type +//! duplication here and nothing that can drift. +//! +//! Only the three entry points that always-on code reaches are mirrored: +//! +//! | Caller | Function | +//! | --- | --- | +//! | `memory::sources::sync` | `auto_snapshot_after_sync` | +//! | `subconscious::profiles::memory` | `diff_since_checkpoint`, `create_checkpoint` | +//! +//! Everything else in the real `ops` is reached only from inside this module's +//! own gated files, so it needs no mirror. If you add a cross-domain caller, +//! add its function here rather than `#[cfg]`-ing the call site — keeping +//! feature awareness out of always-on domains is the whole point of the stub. +//! +//! **These return `Err`, not `Ok`-with-empty.** An empty `CrossSourceDiff` +//! would say "your world did not change", which the subconscious profile would +//! faithfully act on; an error says "this build cannot tell you", which it +//! already knows how to log and skip. Failing closed matters more than being +//! quiet: the caller in `profiles/memory.rs` logs and moves on. + +use crate::sources::types::MemorySourceEntry; +use crate::Config; + +use tinycortex::memory::diff::types::{Checkpoint, CrossSourceDiff, Snapshot}; + +/// The message every disabled entry point returns. +/// +/// Names the feature, because the reader is a developer looking at a log line +/// from a slim build and the actionable fact is which gate to turn on. +const DISABLED: &str = "memory diff is disabled at compile time (built without the `memory-git` \ + feature); rebuild with `--features memory-git` for git-backed snapshots, \ + checkpoints and diffs"; + +/// Function mirrors of the real [`super::ops`]. +pub mod ops { + use super::*; + + /// See [`super::super::ops::auto_snapshot_after_sync`]. + pub async fn auto_snapshot_after_sync( + _source: &MemorySourceEntry, + _config: &Config, + ) -> Result { + Err(DISABLED.to_string()) + } + + /// See [`super::super::ops::create_checkpoint`]. + pub async fn create_checkpoint(_label: &str, _config: &Config) -> Result { + Err(DISABLED.to_string()) + } + + /// See [`super::super::ops::diff_since_checkpoint`]. + pub async fn diff_since_checkpoint( + _checkpoint_id: &str, + _config: &Config, + _include_text_diff: bool, + ) -> Result { + Err(DISABLED.to_string()) + } +} diff --git a/core/src/embedding_adapter.rs b/core/src/embedding_adapter.rs new file mode 100644 index 0000000..2507358 --- /dev/null +++ b/core/src/embedding_adapter.rs @@ -0,0 +1,61 @@ +//! [`TinyAgentsEmbeddingProvider`] — the adapter from tinyagents' own embedding +//! model trait onto the seam's [`EmbeddingProvider`]. +//! +//! It lives in this crate rather than in `tinymemory-api` because the contract +//! crate must stay dependency-light and cannot name `tinyagents`; and rather +//! than in the host because the tree's embedder factory — which is core code — +//! builds Ollama models directly and needs to wrap them. The host re-exports it +//! from `inference::embeddings`, so every existing path there keeps resolving +//! and keeps naming this one type. + +use async_trait::async_trait; +use tinyagents::harness::embeddings::EmbeddingModel; + +pub use tinymemory_api::host::{format_embedding_signature, EmbeddingProvider}; + +/// Compatibility adapter from the canonical tinyagents embedding model. +pub struct TinyAgentsEmbeddingProvider { + model: Box, +} + +impl TinyAgentsEmbeddingProvider { + pub fn new(model: impl EmbeddingModel + 'static) -> Self { + Self { + model: Box::new(model), + } + } + + pub fn boxed(model: impl EmbeddingModel + 'static) -> Box { + Box::new(Self::new(model)) + } +} + +#[async_trait] +impl EmbeddingProvider for TinyAgentsEmbeddingProvider { + fn name(&self) -> &str { + self.model.name() + } + + fn model_id(&self) -> &str { + self.model.model_id() + } + + fn dimensions(&self) -> usize { + self.model.dimensions() + } + + fn signature(&self) -> String { + self.model.signature() + } + + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + let owned = texts + .iter() + .map(|text| (*text).to_owned()) + .collect::>(); + self.model + .embed(&owned) + .await + .map_err(|error| anyhow::anyhow!(error)) + } +} diff --git a/core/src/embedding_host.rs b/core/src/embedding_host.rs new file mode 100644 index 0000000..c5847ad --- /dev/null +++ b/core/src/embedding_host.rs @@ -0,0 +1,163 @@ +//! The process-global [`EmbeddingHost`], and the accessors the extracted code +//! calls in place of the host's `inference::embeddings` factory functions. +//! +//! Mirrors [`crate::events`]'s shape — see that module for why provider +//! construction is reached through a global rather than threaded as a +//! parameter. +//! +//! # Unwired is an error here, unlike the event sink +//! +//! [`crate::events::publish`] drops events when no sink is installed, because +//! the work being announced already happened. Embedding construction is the +//! opposite: silently returning "no embedder" would write vectors into the +//! wrong space or downgrade a semantic query to lexical-only, and neither +//! failure is visible until a later search returns the wrong answer. So the +//! accessors here fail loudly, and callers propagate. + +use std::sync::Arc; + +use parking_lot::RwLock; + +pub use tinymemory_api::host::EmbeddingHost; + +static HOST: RwLock>> = RwLock::new(None); + +/// The message every accessor fails with before a host wires itself up. +const NOT_INSTALLED: &str = + "no EmbeddingHost installed — the host must call memory::embedding_host::set_embedding_host \ + during startup wiring, before any memory work begins"; + +/// Install the host's embedding factory. Called once during startup wiring. +/// Calling it again replaces it, which is what test harnesses want between +/// cases. +pub fn set_embedding_host(host: Arc) { + *HOST.write() = Some(host); +} + +/// Remove any installed host. For tests. +pub fn clear_embedding_host() { + *HOST.write() = None; +} + +/// The installed host, or `None` when nothing has been wired up. +#[must_use] +pub fn embedding_host() -> Option> { + HOST.read().clone() +} + +/// The installed host. +/// +/// # Errors +/// +/// Returns `Err` when no host has been installed. +pub fn require_embedding_host() -> Result, String> { + embedding_host().ok_or_else(|| NOT_INSTALLED.to_string()) +} + +/// The host's default embedding provider — the managed cloud embedder. +/// +/// # Errors +/// +/// Returns `Err` when no [`EmbeddingHost`] has been installed. +pub fn default_embedding_provider( +) -> Result, String> { + Ok(require_embedding_host()?.default_embedding_provider()) +} + +/// Serialises tests that mutate embedding-related process environment. +/// +/// The host has its own guard over the same variables (`inference::local:: +/// inference_test_guard`). They are deliberately *different* locks: each crate's +/// tests link into their own binary and therefore their own process, so a shared +/// lock would buy nothing and would mean the contract crate owning a mutex for +/// the host's benefit. +pub fn embedding_test_guard() -> std::sync::MutexGuard<'static, ()> { + static GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); + GUARD + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// A stub [`EmbeddingHost`] for tests. +/// +/// Several tests assert on the cloud-fallback tuple, which the core no longer +/// owns — the managed model id and dimensionality are the host's to state, so +/// with no host installed there is nothing true to assert. Installing this +/// gives those tests a known answer without reaching for the real provider +/// stack. +#[cfg(test)] +#[derive(Debug, Clone, Copy)] +pub(crate) struct TestEmbeddingHost; + +#[cfg(test)] +impl TestEmbeddingHost { + /// The model id [`Self`] reports as the managed cloud default. + pub(crate) const CLOUD_MODEL: &'static str = "test-cloud-embed"; + /// The dimensionality [`Self::CLOUD_MODEL`] emits. + pub(crate) const CLOUD_DIMENSIONS: usize = 1024; + + /// Install this stub as the process-global embedding host. + pub(crate) fn install() { + set_embedding_host(Arc::new(Self)); + } +} + +#[cfg(test)] +impl EmbeddingHost for TestEmbeddingHost { + fn resolve_api_key(&self, _provider: &str) -> Option { + None + } + + fn ollama_base_url(&self) -> String { + std::env::var("OPENHUMAN_OLLAMA_BASE_URL") + .unwrap_or_else(|_| "http://127.0.0.1:11434".to_string()) + } + + fn default_embedding_provider(&self) -> Arc { + Arc::new(tinymemory_api::host::NoopEmbedding) + } + + fn create_embedding_provider_with_credentials( + &self, + _provider: &str, + _model: &str, + _dims: usize, + _api_key: &str, + _custom_endpoint: Option<&str>, + ) -> Result, String> { + Ok(Box::new(tinymemory_api::host::NoopEmbedding)) + } + + fn model_supports_dimensions(&self, model: &str) -> bool { + // Mirrors the host's rule rather than answering `true`: the tests that + // reach this are about the *ladder's* reaction to a non-reducible + // model, so a stub that says everything is reducible would make them + // pass without exercising anything. + model.starts_with("text-embedding-3-") + } + + fn cloud_embedding_provider( + &self, + _model: &str, + _dims: usize, + ) -> Result, String> { + Ok(Box::new(tinymemory_api::host::NoopEmbedding)) + } + + fn default_cloud_embedding_model(&self) -> &str { + Self::CLOUD_MODEL + } + + fn default_cloud_embedding_dimensions(&self) -> usize { + Self::CLOUD_DIMENSIONS + } + + fn ollama_embedding_provider( + &self, + _base_url: &str, + _model: &str, + _dims: usize, + ) -> Result, String> { + Ok(Box::new(tinymemory_api::host::NoopEmbedding)) + } +} diff --git a/core/src/events.rs b/core/src/events.rs new file mode 100644 index 0000000..696c722 --- /dev/null +++ b/core/src/events.rs @@ -0,0 +1,94 @@ +//! The process-global [`MemoryEventSink`], and the `publish` the extracted code +//! calls in place of the host's bus. +//! +//! # Why a global +//! +//! The publish sites are scattered across ingestion, the summary tree, the sync +//! pipelines and the store — deep inside call stacks that already thread a +//! config, a store handle and a cancellation token. Threading a fourth +//! parameter through all of them to reach a sink would be a large, mechanical, +//! reviewer-hostile diff for no gain, and it is exactly the shape the host's own +//! `BUS` static had before the extraction. This mirrors that shape rather than +//! inventing a new one. +//! +//! # Default is silence, not a panic +//! +//! Before a host installs a sink — in unit tests, in the standalone engine +//! build, during early startup — [`publish`] drops the event. An event bus that +//! panicked or errored when unwired would turn every emit site into an error +//! path, and none of the call sites have anything useful to do with that error: +//! the work they are reporting on has already happened. + +use std::sync::Arc; + +use parking_lot::RwLock; + +pub use tinymemory_api::host::{ + EmbeddingHealthReason, MemoryEvent, MemoryEventSink, NoopEventSink, +}; + +static SINK: RwLock>> = RwLock::new(None); + +/// Install the host's event sink. Called once during startup wiring, before any +/// memory work begins. Calling it again replaces the sink, which is what test +/// harnesses want between cases. +pub fn set_event_sink(sink: Arc) { + *SINK.write() = Some(sink); +} + +/// Remove any installed sink, returning to silent-drop behaviour. For tests. +pub fn clear_event_sink() { + *SINK.write() = None; +} + +/// The installed sink, or `None` when no host has wired one up. +#[must_use] +pub fn event_sink() -> Option> { + SINK.read().clone() +} + +/// Announce a memory-domain event to the host. A no-op when no sink is +/// installed. +pub fn publish(event: MemoryEvent) { + let sink = SINK.read().clone(); + if let Some(sink) = sink { + sink.publish(event); + } else { + log::trace!("[memory:events] dropped event with no sink installed: {event:?}"); + } +} + +/// A [`MemoryEventSink`] that records what it was given, for tests. +/// +/// Several tests used to assert on the host's web-channel broadcast, because +/// before the extraction the publish went straight onto that channel. The +/// decision to publish is core behaviour; the wire format is the host's. So the +/// tests kept the half they are actually about — *did the transition publish, +/// and did it publish exactly once* — and assert it here instead. +#[cfg(test)] +#[derive(Debug, Default)] +pub(crate) struct RecordingSink { + events: parking_lot::Mutex>, +} + +#[cfg(test)] +impl RecordingSink { + /// Install a fresh recorder and return it. Replaces any existing sink. + pub(crate) fn install() -> Arc { + let sink = Arc::new(Self::default()); + set_event_sink(Arc::clone(&sink) as Arc); + sink + } + + /// Take everything recorded so far, leaving the recorder empty. + pub(crate) fn drain(&self) -> Vec { + std::mem::take(&mut *self.events.lock()) + } +} + +#[cfg(test)] +impl MemoryEventSink for RecordingSink { + fn publish(&self, event: MemoryEvent) { + self.events.lock().push(event); + } +} diff --git a/core/src/global.rs b/core/src/global.rs new file mode 100644 index 0000000..6fd07e5 --- /dev/null +++ b/core/src/global.rs @@ -0,0 +1,394 @@ +//! Process-global memory client singleton. +//! +//! One `MemoryClient` (and its background ingestion-queue worker) lives for the +//! entire core process. Every subsystem — RPC handlers, node runtime, screen +//! intelligence, CLI — shares this single instance so the worker is never +//! prematurely dropped. +//! +//! # Usage +//! +//! ```ignore +//! // At startup (core server, CLI, etc.) +//! memory::global::init(workspace_dir)?; +//! +//! // Anywhere that needs to write/read memory: +//! let client = memory::global::client()?; +//! client.put_doc(input).await?; +//! ``` + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock, RwLock}; + +use crate::store::{MemoryClient, MemoryClientRef}; + +#[derive(Clone)] +struct GlobalMemoryClient { + workspace_dir: PathBuf, + client: MemoryClientRef, +} + +type GlobalClientSlot = RwLock>; + +/// The process-global memory client slot. +static GLOBAL_CLIENT: OnceLock = OnceLock::new(); + +fn global_slot() -> &'static GlobalClientSlot { + GLOBAL_CLIENT.get_or_init(GlobalClientSlot::default) +} + +/// Initialise or re-bind the global memory client from a workspace directory. +/// +/// Safe to call multiple times. Calls for the same workspace return the +/// existing client; calls for a different workspace replace the global handle +/// so a post-login active-user switch does not keep writing to the pre-login +/// workspace. +pub fn init(workspace_dir: PathBuf) -> Result { + init_in_slot(global_slot(), workspace_dir) +} + +fn init_in_slot( + slot: &GlobalClientSlot, + workspace_dir: PathBuf, +) -> Result { + if let Some(existing) = slot + .read() + .map_err(|e| format!("[memory:global] read lock poisoned: {e}"))? + .as_ref() + { + if existing.workspace_dir == workspace_dir { + log::debug!("[memory:global] already initialised for current workspace"); + return Ok(Arc::clone(&existing.client)); + } + } + + // Reuse the per-workspace cache before constructing anything. A desktop + // active-user switch A -> B -> A lands here with the global slot pointing + // at B, and building a *second* client for A would put two ingestion + // workers over A's SQLite file — duplicate graph extraction and duplicate + // embedding work — while any `MemoryBinding` cached for A still held the + // first one. `client_for_workspace` writes into the same map, so the two + // resolution paths converge on one client per workspace. + if let Some(cached) = cached_client(&workspace_dir)? { + log::debug!( + "[memory:global] reusing cached workspace client for {}", + workspace_dir.display() + ); + let mut guard = slot + .write() + .map_err(|e| format!("[memory:global] write lock poisoned: {e}"))?; + *guard = Some(GlobalMemoryClient { + workspace_dir, + client: Arc::clone(&cached), + }); + return Ok(cached); + } + + log::info!( + "[memory:global] initialising global MemoryClient workspace={}", + workspace_dir.display() + ); + let client = match MemoryClient::from_workspace_dir(workspace_dir.clone()) { + Ok(client) => Arc::new(client), + Err(error) => { + let mut guard = slot + .write() + .map_err(|e| format!("[memory:global] write lock poisoned: {e}"))?; + if guard + .as_ref() + .is_some_and(|existing| existing.workspace_dir != workspace_dir) + { + log::warn!( + "[memory:global] clearing stale MemoryClient after failed rebind to {}", + workspace_dir.display() + ); + *guard = None; + } + return Err(error); + } + }; + + let mut guard = slot + .write() + .map_err(|e| format!("[memory:global] write lock poisoned: {e}"))?; + if let Some(existing) = guard.as_ref() { + if existing.workspace_dir == workspace_dir { + let client = Arc::clone(&existing.client); + cache_client(&workspace_dir, &client)?; + return Ok(client); + } + + log::info!( + "[memory:global] rebinding MemoryClient workspace {} -> {}", + existing.workspace_dir.display(), + workspace_dir.display() + ); + } + + // Publish into the shared cache under the same client the global slot is + // about to hold, so a later `client_for_workspace(workspace)` — or a return + // to this workspace after a switch — reuses it rather than building a + // second engine over the same store. + let client = cache_client(&workspace_dir, &client)?; + + *guard = Some(GlobalMemoryClient { + workspace_dir, + client: Arc::clone(&client), + }); + Ok(client) +} + +/// Initialise using the default `~/.openhuman/workspace` directory. +/// +/// **TEST-ONLY.** Production code must call [`init`] with the real workspace +/// directory at startup wiring. If this function ran first in production it +/// would pin the singleton to `~/.openhuman/workspace`, causing every +/// subsequent `init(custom_workspace)` to silently no-op and return the wrong +/// handle (`OnceLock::set` is one-shot). +/// +/// The host resolves this path through `config::default_root_openhuman_dir`, +/// which this crate cannot see; the home-directory lookup is reproduced here +/// rather than added to the config seam for a test-only helper. +#[cfg(test)] +pub fn init_default() -> Result { + let workspace_dir = dirs::home_dir() + .ok_or_else(|| "Could not find home directory".to_string())? + .join(".openhuman") + .join("workspace"); + init(workspace_dir) +} + +/// Returns the global memory client. +/// +/// Returns `Err` if [`init`] has not yet been called. There is **no** lazy +/// fallback: a fallback would pin the global to `~/.openhuman/workspace` on +/// the first stray call (test, early RPC, etc.). The explicit init/rebind path +/// keeps workspace ownership visible at startup and after login. +/// +/// Callers that can tolerate "not yet ready" should use +/// [`client_if_ready`] instead. +pub fn client() -> Result { + client_from(global_slot()) +} + +/// Implementation backing [`client`] — extracted so unit tests can pass a +/// freshly-constructed local slot and assert the uninitialised-error +/// contract without racing the process-global singleton. +fn client_from(slot: &GlobalClientSlot) -> Result { + slot.read() + .map_err(|e| format!("[memory:global] read lock poisoned: {e}"))? + .as_ref() + .map(|entry| Arc::clone(&entry.client)) + .ok_or_else(|| { + "memory global accessed before init — call init(workspace) at startup".to_string() + }) +} + +/// The workspace the process-global client is currently bound to, or `None` +/// when [`init`] has not run yet. +/// +/// Exists so `memory::ops::guard::active_memory_guard` can resolve *the same* +/// workspace `memory::ops::helpers::active_memory_client` in the host +/// would, in the pre-boot case where there is no ambient `CoreContext` to ask. +/// Reading the workspace rather than the client keeps the two resolutions +/// answering about the same store instead of drifting onto whatever +/// `Config::load_or_init` happens to say. +pub fn active_workspace_dir() -> Option { + global_slot() + .read() + .ok()? + .as_ref() + .map(|entry| entry.workspace_dir.clone()) +} + +/// Per-workspace client cache used by [`client_for_workspace`]. +/// +/// A *map*, not a slot, for the same reason +/// the host's `memory::binding` caches bindings in a map: a subsystem +/// driver is resolved per workspace and must never be handed another +/// workspace's handle. +static WORKSPACE_CLIENTS: OnceLock>> = OnceLock::new(); + +/// The cached client for `workspace_dir`, if one has already been built by +/// either resolution path ([`init`] or [`client_for_workspace`]). +fn cached_client(workspace_dir: &Path) -> Result, String> { + Ok(WORKSPACE_CLIENTS + .get_or_init(Default::default) + .read() + .map_err(|e| format!("[memory:global] workspace cache read lock poisoned: {e}"))? + .get(workspace_dir) + .map(Arc::clone)) +} + +/// Publish `client` as *the* client for `workspace_dir`, returning whichever +/// client wins. +/// +/// A racing caller may have inserted first; theirs wins, so the "one ingestion +/// worker per workspace" property holds even when two paths construct +/// concurrently. Callers must use the returned handle, not the one they passed. +fn cache_client(workspace_dir: &Path, client: &MemoryClientRef) -> Result { + let mut guard = WORKSPACE_CLIENTS + .get_or_init(Default::default) + .write() + .map_err(|e| format!("[memory:global] workspace cache write lock poisoned: {e}"))?; + let entry = guard + .entry(workspace_dir.to_path_buf()) + .or_insert_with(|| Arc::clone(client)); + Ok(Arc::clone(entry)) +} + +/// The `MemoryClient` for `workspace_dir`, **reusing the process-global client +/// when it already owns that workspace**. +/// +/// Exists for the embedded memory driver +/// (the host's `memory::driver::embedded`), which is constructed +/// synchronously at bind time and must resolve its client lazily on the first +/// contract call. +/// +/// The reuse check is load-bearing, not an optimisation: [`MemoryClient`] owns +/// a `UnifiedMemory` handle *and* spawns a background ingestion worker, so two +/// clients over one workspace means two workers doing duplicate graph +/// extraction and duplicate embedding work against the same SQLite file. +/// +/// # Errors +/// +/// Lock poisoning, or any failure constructing a fresh +/// [`MemoryClient::from_workspace_dir`] (directory creation, store open). +pub fn client_for_workspace(workspace_dir: &Path) -> Result { + if let Some(existing) = global_slot() + .read() + .map_err(|e| format!("[memory:global] read lock poisoned: {e}"))? + .as_ref() + { + if existing.workspace_dir == workspace_dir { + // Record it under the workspace too. Without this the global's + // client is invisible to the cache, so a switch away and back + // rebuilds a second client for this workspace while a binding + // cached here still holds the first. + return cache_client(workspace_dir, &existing.client); + } + } + + if let Some(existing) = cached_client(workspace_dir)? { + return Ok(existing); + } + + log::info!( + "[memory:global] building workspace-scoped MemoryClient workspace={}", + workspace_dir.display() + ); + let client: MemoryClientRef = Arc::new(MemoryClient::from_workspace_dir( + workspace_dir.to_path_buf(), + )?); + + cache_client(workspace_dir, &client) +} + +/// Returns the global client if already initialised, without lazy init. +pub fn client_if_ready() -> Option { + global_slot() + .read() + .ok()? + .as_ref() + .map(|entry| Arc::clone(&entry.client)) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + /// All tests that touch `GLOBAL_CLIENT` must contend with process-wide + /// state. We tolerate both branches so test ordering doesn't flake the + /// suite. + #[tokio::test] + async fn client_if_ready_is_some_after_init_or_remains_none() { + crate::test_seams::init(); + let before = client_if_ready(); + let tmp = TempDir::new().unwrap(); + let _ = init(tmp.path().join("ws")); + let after = client_if_ready(); + if before.is_some() { + assert!(after.is_some(), "if global was set, it must remain set"); + } else { + // First setter wins; if our init succeeded it's set now. + assert!(after.is_some()); + } + } + + #[tokio::test] + async fn init_returns_existing_client_when_already_set() { + crate::test_seams::init(); + let slot = GlobalClientSlot::default(); + let tmp = TempDir::new().unwrap(); + let workspace = tmp.path().join("ws"); + + let first = init_in_slot(&slot, workspace.clone()).unwrap(); + let second = init_in_slot(&slot, workspace).unwrap(); + + assert!(Arc::ptr_eq(&first, &second)); + } + + #[tokio::test] + async fn init_rebinds_client_when_workspace_changes() { + crate::test_seams::init(); + let slot = GlobalClientSlot::default(); + let tmp = TempDir::new().unwrap(); + + let first = init_in_slot(&slot, tmp.path().join("ws-a")).unwrap(); + let second = init_in_slot(&slot, tmp.path().join("ws-b")).unwrap(); + let current = client_from(&slot).unwrap(); + + assert!(!Arc::ptr_eq(&first, &second)); + assert!(Arc::ptr_eq(&second, ¤t)); + } + + #[tokio::test] + async fn init_clears_existing_client_when_rebind_workspace_cannot_initialise() { + crate::test_seams::init(); + let slot = GlobalClientSlot::default(); + let tmp = TempDir::new().unwrap(); + + let _first = init_in_slot(&slot, tmp.path().join("ws-a")).unwrap(); + let file_path = tmp.path().join("not-a-directory"); + std::fs::write(&file_path, b"not a workspace").unwrap(); + + let err = match init_in_slot(&slot, file_path) { + Ok(_) => panic!("rebind to a file path must fail"), + Err(err) => err, + }; + + assert!(err.contains("Create workspace dir")); + assert!(client_from(&slot).is_err()); + } + + #[tokio::test] + async fn client_returns_a_handle_after_explicit_init() { + crate::test_seams::init(); + // Bind TempDir at test scope so its directory outlives the global + // client — the singleton holds the path and may be used later in + // this test binary. + let tmp = TempDir::new().unwrap(); + // Explicit init: client() no longer lazily initialises. + let _ = client_if_ready().or_else(|| init(tmp.path().join("ws")).ok()); + let c = client().expect("global client should be available after init"); + let _arc: Arc = c; + } + + #[tokio::test] + async fn client_errs_clearly_when_not_initialised() { + crate::test_seams::init(); + // Use a fresh local `OnceLock` rather than the process-global one: + // other tests may have already called `init()` on the singleton, so + // an `is_none`-gated check on `GLOBAL_CLIENT` would race / silently + // skip. `client_from` lets us assert the contract deterministically. + let local = GlobalClientSlot::default(); + match client_from(&local) { + Ok(_) => panic!("client_from(empty) must error"), + Err(err) => assert!( + err.contains("init"), + "error should mention init contract, got: {err}" + ), + } + } +} diff --git a/core/src/ingest_pipeline.rs b/core/src/ingest_pipeline.rs new file mode 100644 index 0000000..f3fcd2e --- /dev/null +++ b/core/src/ingest_pipeline.rs @@ -0,0 +1,190 @@ +//! Product shell over tinycortex on-demand ingestion. + +use anyhow::Result; + +use crate::store::chunks::store::RawRef; +use crate::Config; +use tinycortex::memory::ingest::canonicalize::{ + chat::{self, ChatBatch}, + document::{self, DocumentInput}, + email::{self, EmailThread}, + CanonicalisedSource, +}; + +pub use tinycortex::memory::ingest::IngestSummary as IngestResult; + +pub async fn ingest_chat( + config: &Config, + source_id: &str, + owner: &str, + tags: Vec, + batch: ChatBatch, +) -> Result { + let canonical = + chat::canonicalise(source_id, owner, &tags, batch.clone()).map_err(anyhow::Error::msg)?; + let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); + let result = tinycortex::memory::ingest::ingest_chat( + &memory, source_id, owner, tags, batch, &sink, &scoring, + ) + .await?; + publish_canonicalized(source_id, canonical.as_ref(), &result); + Ok(result) +} + +pub async fn ingest_email( + config: &Config, + source_id: &str, + owner: &str, + tags: Vec, + thread: EmailThread, +) -> Result { + let canonical = + email::canonicalise(source_id, owner, &tags, thread.clone()).map_err(anyhow::Error::msg)?; + let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); + let result = tinycortex::memory::ingest::ingest_email( + &memory, source_id, owner, tags, thread, &sink, &scoring, + ) + .await?; + publish_canonicalized(source_id, canonical.as_ref(), &result); + Ok(result) +} + +pub async fn ingest_email_with_raw_refs( + config: &Config, + source_id: &str, + owner: &str, + tags: Vec, + thread: EmailThread, + raw_refs: Vec, +) -> Result { + let canonical = + email::canonicalise(source_id, owner, &tags, thread.clone()).map_err(anyhow::Error::msg)?; + let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); + let result = tinycortex::memory::ingest::ingest_email_with_raw_refs( + &memory, source_id, owner, tags, thread, raw_refs, &sink, &scoring, + ) + .await?; + publish_canonicalized(source_id, canonical.as_ref(), &result); + Ok(result) +} + +pub async fn ingest_document( + config: &Config, + source_id: &str, + owner: &str, + tags: Vec, + doc: DocumentInput, +) -> Result { + ingest_document_with_scope(config, source_id, owner, tags, doc, None).await +} + +pub async fn ingest_document_with_scope( + config: &Config, + source_id: &str, + owner: &str, + tags: Vec, + doc: DocumentInput, + path_scope: Option, +) -> Result { + ingest_document_versioned(config, source_id, owner, tags, doc, path_scope, None).await +} + +pub async fn ingest_document_versioned( + config: &Config, + source_id: &str, + owner: &str, + tags: Vec, + doc: DocumentInput, + path_scope: Option, + version_ms: Option, +) -> Result { + let canonical = + document::canonicalise(source_id, owner, &tags, doc.clone(), path_scope.clone()) + .map_err(anyhow::Error::msg)?; + let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); + let result = tinycortex::memory::ingest::ingest_document_versioned( + &memory, source_id, owner, tags, doc, path_scope, version_ms, &sink, &scoring, + ) + .await?; + publish_canonicalized(source_id, canonical.as_ref(), &result); + Ok(result) +} + +fn publish_canonicalized( + source_id: &str, + canonical: Option<&CanonicalisedSource>, + result: &IngestResult, +) { + let Some(canonical) = canonical else { + return; + }; + let source_kind = canonical.metadata.source_kind.as_str(); + let body_preview = if matches!(source_kind, "email" | "document") { + utf8_suffix(&canonical.markdown, 2048) + } else { + utf8_prefix(&canonical.markdown, 2048) + }; + crate::events::publish(crate::events::MemoryEvent::DocumentCanonicalized { + source_id: source_id.into(), + source_kind: canonical.metadata.source_kind.as_str().into(), + chunks_written: result.chunks_written, + chunk_ids: result.chunk_ids.clone(), + canonicalized_at: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(), + body_preview: Some(body_preview), + }); +} + +fn utf8_suffix(value: &str, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value.to_owned(); + } + let target = value.len().saturating_sub(max_bytes); + let start = value + .char_indices() + .map(|(index, _)| index) + .find(|index| *index >= target) + .unwrap_or(value.len()); + value[start..].to_owned() +} + +fn utf8_prefix(value: &str, max_bytes: usize) -> String { + let end = value + .char_indices() + .map(|(index, _)| index) + .take_while(|index| *index <= max_bytes) + .last() + .unwrap_or(0); + let end = if value.len() <= max_bytes { + value.len() + } else if end == 0 { + 0 + } else { + end + }; + value[..end].to_string() +} + +#[cfg(test)] +mod tests { + use super::{utf8_prefix, utf8_suffix}; + + #[test] + fn preview_keeps_short_text() { + assert_eq!(utf8_prefix("hello", 2048), "hello"); + } + + #[test] + fn preview_respects_utf8_byte_boundary() { + assert_eq!(utf8_prefix("aéb", 2), "a"); + assert_eq!(utf8_prefix("éb", 2), "é"); + } + + #[test] + fn suffix_preview_preserves_trailing_utf8() { + assert_eq!(utf8_suffix("aéb", 2), "b"); + assert_eq!(utf8_suffix("aéb", 3), "éb"); + } +} diff --git a/core/src/ingestion/README.md b/core/src/ingestion/README.md new file mode 100644 index 0000000..7924901 --- /dev/null +++ b/core/src/ingestion/README.md @@ -0,0 +1,18 @@ +# Memory ingestion + +Pipeline that turns raw document text into chunks plus extracted entities and relations, then upserts everything into `UnifiedMemory`. Runs synchronously when callers need the result (`MemoryClient::ingest_doc`) and as a background worker for fire-and-forget submissions (`MemoryClient::put_doc`). + +## Files + +- **`mod.rs`** — adds `ingest_document` and `extract_graph` to `UnifiedMemory`, plus the internal `upsert_graph_relations` helper. Re-exports the public types and the queue / state surface. +- **`types.rs`** — public ingestion API: `MemoryIngestionRequest` / `MemoryIngestionResult` / `MemoryIngestionConfig`, `ExtractionMode` (sentence vs chunk), `ExtractedEntity` / `ExtractedRelation`, `DEFAULT_MEMORY_EXTRACTION_MODEL`. Crate-internal intermediates (`RawEntity`, `RawRelation`, `ExtractionUnit`, `ExtractionAccumulator`, `ParsedIngestion`) live here too. +- **`parse.rs`** — `parse_document` pipeline: chunking, header / metadata enrichment, alias resolution, regex- and rule-driven extraction. Produces a `ParsedIngestion`. +- **`regex.rs`** — lazily-initialised regexes (email headers, named emails, graph facts, ownership, preferences, action items, recipients, spatial relations, dates, person names) plus `sanitize_entity_name`, `sanitize_fact_text`, `classify_entity`. +- **`rules.rs`** — semantic validation rules for graph predicates (allowed head/tail entity types) and the `ExtractionAccumulator` impl that gates `add_entity` / `add_relation` on those rules. +- **`queue.rs`** — `IngestionQueue` (cloneable submit handle) plus `IngestionJob` and the background worker started via `start_worker_with_state`. The worker shares an `IngestionState` with synchronous callers so all ingestion serialises through the same singleton lock. +- **`state.rs`** — `IngestionState` / `IngestionStatusSnapshot`: queue depth, in-flight metadata, last-completed status, and the `tokio::sync::Mutex` that enforces single-threaded extraction (the local LLM path can't be re-entered safely). +- **`tests.rs`** — pipeline coverage exercising `parse_document`, regex extraction, and `UnifiedMemory::ingest_document` end-to-end. + +## How it fits + +`MemoryClient` owns the singleton `IngestionQueue` and forwards to it from `put_doc` (background) or `ingest_doc` (synchronous, behind the same lock). Every ingestion run publishes `MemoryIngestionStarted` / `MemoryIngestionCompleted` events on the global event bus so the UI status pill and `openhuman.memory_ingestion_status` RPC stay in sync. Output rows feed `UnifiedMemory`'s `memory_docs`, `vector_chunks`, and `graph_namespace` tables. diff --git a/core/src/ingestion/mod.rs b/core/src/ingestion/mod.rs new file mode 100644 index 0000000..9ee1fb8 --- /dev/null +++ b/core/src/ingestion/mod.rs @@ -0,0 +1,127 @@ +//! Document ingestion and knowledge extraction for the OpenHuman memory system. +//! +//! This module provides the pipeline for taking raw unstructured text and +//! transforming it into structured memory. The process includes: +//! 1. **Chunking**: Splitting the document into manageable pieces. +//! 2. **Structured Extraction**: Using regex-based rules to identify known patterns +//! (e.g., email headers, specific project labels). +//! 3. **Heuristic Extraction**: Using rule-based parsing to identify entities +//! and their relationships. +//! 4. **Aggregation**: Resolving aliases, merging duplicates, and normalizing names. +//! 5. **Persistence**: Upserting the document, text chunks, and graph relations into +//! the memory store. + +pub mod queue; +pub mod state; + +pub use queue::{IngestionJob, IngestionQueue, DEFAULT_QUEUE_CAPACITY}; +pub use state::{IngestionState, IngestionStatusSnapshot}; +pub use tinycortex::memory::ingest::{ + ExtractedEntity, ExtractedRelation, ExtractionMode, MemoryIngestionConfig, + MemoryIngestionRequest, MemoryIngestionResult, DEFAULT_MEMORY_EXTRACTION_MODEL, +}; + +use serde_json::json; + +use crate::store::types::NamespaceDocumentInput; +use crate::store::UnifiedMemory; + +impl UnifiedMemory { + /// Run the full ingestion pipeline for a document: parse + chunk + extract + /// entities/relations, upsert the document row + vector chunks, and write + /// the extracted relations into the namespace graph. + pub async fn ingest_document( + &self, + request: MemoryIngestionRequest, + ) -> Result { + let (enriched_input, mut extraction) = + tinycortex::memory::ingest::extract_enriched_document( + &request.document, + &request.config, + ); + let namespace = Self::sanitize_namespace(&enriched_input.namespace); + let document_id = self.upsert_document(enriched_input).await?; + + self.upsert_graph_relations(&namespace, &document_id, &extraction, &request.config) + .await?; + extraction.document_id = document_id; + extraction.namespace = namespace; + Ok(extraction) + } + + /// Extract entities/relations and write them to the graph for a document + /// that has already been stored via `upsert_document`. + /// + /// This avoids the redundant second upsert that would happen if the + /// background ingestion queue called `ingest_document` on an already- + /// persisted document. + pub async fn extract_graph( + &self, + document_id: &str, + document: &NamespaceDocumentInput, + config: &MemoryIngestionConfig, + ) -> Result { + let (_enriched, mut extraction) = + tinycortex::memory::ingest::extract_enriched_document(document, config); + let namespace = Self::sanitize_namespace(&document.namespace); + + self.upsert_graph_relations(&namespace, document_id, &extraction, config) + .await?; + extraction.document_id = document_id.to_string(); + extraction.namespace = namespace; + Ok(extraction) + } + + /// Clear existing relations for the document then upsert all extracted + /// relations into the namespace graph. + async fn upsert_graph_relations( + &self, + namespace: &str, + document_id: &str, + extraction: &MemoryIngestionResult, + config: &MemoryIngestionConfig, + ) -> Result<(), String> { + self.graph_remove_document_namespace(namespace, document_id) + .await?; + + for relation in &extraction.relations { + let chunk_ids = relation + .chunk_ids + .iter() + .filter_map(|chunk_id| chunk_id.strip_prefix("chunk:")) + .map(|chunk_index| format!("{document_id}:{chunk_index}")) + .collect::>(); + + let attrs = json!({ + "source": "ingestion", + "model_name": config.model_name, + "extraction_mode": config.extraction_mode.as_str(), + "confidence": relation.confidence, + "evidence_count": relation.evidence_count, + "order_index": relation.order_index, + "document_id": document_id, + "document_ids": [document_id], + "chunk_ids": chunk_ids, + "entity_types": { + "subject": relation.subject_type, + "object": relation.object_type, + }, + "metadata": relation.metadata, + }); + + self.graph_upsert_namespace( + namespace, + &relation.subject, + &relation.predicate, + &relation.object, + &attrs, + ) + .await?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/core/src/ingestion/queue.rs b/core/src/ingestion/queue.rs new file mode 100644 index 0000000..14e682b --- /dev/null +++ b/core/src/ingestion/queue.rs @@ -0,0 +1,420 @@ +//! # Background Ingestion Queue +//! +//! Processes documents through the entity/relation extraction pipeline on a +//! dedicated worker thread. This ensures that `doc_put` callers never block +//! on the heavier parsing and graph-write path. +//! +//! The queue uses a bounded `tokio::sync::mpsc` channel +//! ([`DEFAULT_QUEUE_CAPACITY`]) to decouple document submission from the +//! actual extraction process. Producers call [`IngestionQueue::submit`], +//! which is non-blocking; when the buffer is full the job is dropped with a +//! warn-level log so a runaway producer cannot grow the queue without bound +//! and exhaust process memory. + +use std::sync::Arc; +use std::time::Instant; + +use tokio::sync::mpsc; + +use super::state::IngestionState; +use super::MemoryIngestionConfig; +use crate::store::{NamespaceDocumentInput, UnifiedMemory}; + +/// Default capacity of the ingestion job channel. +/// +/// Producers (`put_doc`, `store_skill_sync`) push jobs into this channel +/// without blocking; the worker drains them one-at-a-time under the +/// `IngestionState` singleton lock because the local extraction LLM cannot +/// run concurrently. A buggy or compromised producer can submit jobs much +/// faster than the worker drains them, so the channel must enforce an +/// explicit cap or the queue grows without bound and exhausts process +/// memory (each [`IngestionJob`] holds an owned document body). +/// +/// 512 is a deliberate middle ground: it absorbs reasonable bulk-import +/// bursts (e.g. backfilling a Notion workspace or a long Slack history) +/// without letting a runaway loop balloon RSS — at typical document sizes +/// of 1–100 KB the in-flight buffer caps below ~50 MB. +pub const DEFAULT_QUEUE_CAPACITY: usize = 512; + +/// A job submitted to the ingestion worker. +/// +/// Contains all the necessary information to process a document for graph +/// extraction, including the document content itself and the configuration +/// for the extraction process. +#[derive(Debug, Clone)] +pub struct IngestionJob { + /// The document that was already stored via `upsert_document`. + pub document: NamespaceDocumentInput, + /// The document ID returned by `upsert_document`. + pub document_id: String, + /// Configuration for the extraction process (e.g., model name, thresholds). + pub config: MemoryIngestionConfig, +} + +/// Handle used by callers to submit ingestion jobs. +/// +/// This is a thin wrapper around a bounded `tokio::sync::mpsc::Sender` and +/// can be cloned freely to be shared across multiple producers. The bound +/// (see [`DEFAULT_QUEUE_CAPACITY`]) protects the core from runaway +/// producers; once the buffer is full, [`Self::submit`] returns `false` +/// instead of blocking or growing the queue. +#[derive(Clone)] +pub struct IngestionQueue { + /// Sender half of the bounded job queue channel. + tx: mpsc::Sender, + /// Shared state — singleton lock, queue depth, status snapshot. + state: IngestionState, + /// The actual channel capacity this queue was created with. Stored so + /// backpressure logs always reflect the real configured size rather than + /// the `DEFAULT_QUEUE_CAPACITY` constant (which may differ for test + /// queues or future callers of `start_worker_with_capacity`). + capacity: usize, +} + +impl IngestionQueue { + /// Submit a document for background graph extraction. Returns immediately. + /// + /// # Arguments + /// + /// * `job` - The [`IngestionJob`] to be processed. + /// + /// # Returns + /// + /// Returns `true` if the job was successfully enqueued, `false` if the + /// queue is full (capacity reached) or the worker has shut down (e.g., + /// during application termination). In both drop cases the job is not + /// persisted into the extraction pipeline — the underlying document + /// upsert that the caller already performed is unaffected. The queue + /// depth counter is restored before returning so the + /// `memory_ingestion_status` RPC stays accurate. + pub fn submit(&self, job: IngestionJob) -> bool { + self.state.enqueue(); + match self.tx.try_send(job) { + Ok(()) => true, + Err(mpsc::error::TrySendError::Full(dropped)) => { + // Channel is at capacity — log loudly so observability can + // surface the drop, then undo the enqueue bump so the queue + // depth gauge does not drift upward forever under sustained + // overflow. Include the stable `document_id` so the warn + // line is the breadcrumb back to the upserted document + // whose graph-extraction follow-up was skipped. + self.state.dequeue(); + log::warn!( + "[memory:ingestion_queue] dropping job: queue at capacity (cap={}) doc_id={} namespace={} title={}", + self.capacity, + dropped.document_id, + dropped.document.namespace, + dropped.document.title, + ); + false + } + Err(mpsc::error::TrySendError::Closed(dropped)) => { + // Worker is gone — same accounting as the full case, but a + // different reason worth distinguishing in logs because it + // means the entire pipeline is dead, not just over-pressure. + self.state.dequeue(); + log::warn!( + "[memory:ingestion_queue] dropping job: worker channel closed (shutdown?) doc_id={} namespace={} title={}", + dropped.document_id, + dropped.document.namespace, + dropped.document.title, + ); + false + } + } + } + + /// Returns a clone of the shared ingestion state. Use this to drive the + /// status RPC or to share the singleton lock with synchronous ingest + /// paths that bypass the queue. + pub fn state(&self) -> IngestionState { + self.state.clone() + } + + /// Build a queue handle from a raw sender, state, and capacity. Test-only. + #[cfg(test)] + fn from_parts(tx: mpsc::Sender, state: IngestionState, capacity: usize) -> Self { + Self { + tx, + state, + capacity, + } + } +} + +/// Start the background ingestion worker. +/// +/// # Arguments +/// +/// * `memory` - An `Arc` to the [`UnifiedMemory`] instance used for extraction. +/// +/// # Returns +/// +/// Returns an [`IngestionQueue`] handle that can be cloned and shared with +/// any number of producers. The worker runs on a dedicated tokio task, +/// processing jobs sequentially so ingestion work stays serialized. +pub fn start_worker(memory: Arc) -> IngestionQueue { + let state = IngestionState::new(); + start_worker_with_state(memory, state) +} + +/// Start a worker bound to a caller-supplied [`IngestionState`]. Useful when +/// the synchronous ingest path needs to share the same singleton lock and +/// snapshot as the queue worker. Uses [`DEFAULT_QUEUE_CAPACITY`]. +pub fn start_worker_with_state( + memory: Arc, + state: IngestionState, +) -> IngestionQueue { + start_worker_with_capacity(memory, state, DEFAULT_QUEUE_CAPACITY) +} + +/// Start a worker with an explicit channel capacity. Exposed so unit tests +/// can drive the at-capacity drop path deterministically without faking a +/// slow worker. +/// +/// # Panics +/// +/// Panics if `capacity == 0`. `tokio::sync::mpsc::channel` itself panics on +/// a zero buffer, but the message is cryptic; the explicit guard here turns +/// the misuse into a clear, grep-friendly assertion at the call site. +pub(crate) fn start_worker_with_capacity( + memory: Arc, + state: IngestionState, + capacity: usize, +) -> IngestionQueue { + assert!( + capacity > 0, + "ingestion queue capacity must be greater than zero" + ); + let (tx, rx) = mpsc::channel::(capacity); + + tokio::spawn(ingestion_worker(memory, rx, state.clone())); + + log::info!("[memory:ingestion_queue] background worker started capacity={capacity}"); + IngestionQueue { + tx, + state, + capacity, + } +} + +/// The main worker loop for background document ingestion. +/// +/// This function runs as a long-lived tokio task, waiting for jobs to arrive +/// on the receiver channel and processing them one by one. +/// +/// # Arguments +/// +/// * `memory` - The [`UnifiedMemory`] instance. +/// * `rx` - The receiver half of the job queue channel. +async fn ingestion_worker( + memory: Arc, + mut rx: mpsc::Receiver, + state: IngestionState, +) { + log::debug!("[memory:ingestion_queue] worker loop entered"); + + // Continuously receive and process jobs until the channel is closed. + while let Some(job) = rx.recv().await { + let title = job.document.title.clone(); + let namespace = job.document.namespace.clone(); + let document_id = job.document_id.clone(); + + log::debug!( + "[memory:ingestion_queue] processing job: namespace={namespace}, \ + doc_id={document_id}, title={title}", + ); + + // Acquire the singleton lock so only one ingestion runs at a time + // (covers both queue worker and synchronous callers sharing this + // state). Decrement the pending-queue counter only after we hold the + // lock — while we're blocked waiting on it the job is still queued. + let _guard = state.acquire().await; + state.dequeue(); + + let queue_depth = state.snapshot().queue_depth; + state.mark_running(&document_id, &title, &namespace); + crate::events::publish(crate::events::MemoryEvent::IngestionStarted { + document_id: document_id.clone(), + title: title.clone(), + namespace: namespace.clone(), + queue_depth, + }); + + let started = Instant::now(); + let success = match memory + .extract_graph(&document_id, &job.document, &job.config) + .await + { + Ok(result) => { + log::info!( + "[memory:ingestion_queue] extracted namespace={namespace} \ + doc_id={document_id} title={title} \ + — entities={}, relations={}, chunks={}", + result.entity_count, + result.relation_count, + result.chunk_count, + ); + true + } + Err(e) => { + crate::observability::report_error( + &e, + "memory", + "ingestion_extract", + &[ + ("namespace", namespace.as_str()), + ("doc_id", document_id.as_str()), + ], + ); + false + } + }; + + let elapsed_ms = started.elapsed().as_millis() as u64; + let completed_at_ms = chrono::Utc::now().timestamp_millis(); + state.mark_completed(&document_id, success, completed_at_ms); + crate::events::publish(crate::events::MemoryEvent::IngestionCompleted { + document_id, + namespace, + success, + elapsed_ms, + queue_depth: state.snapshot().queue_depth, + }); + } + + log::info!("[memory:ingestion_queue] worker shut down (channel closed)"); +} + +#[cfg(test)] +mod tests { + //! Channel-bound tests. These build an [`IngestionQueue`] from a raw + //! `mpsc::channel` without spawning a worker — that lets the suite drive + //! the at-capacity and channel-closed branches deterministically without + //! standing up a real `UnifiedMemory` or contending with a draining task. + use super::*; + + use serde_json::json; + + fn fixture_job(title: &str) -> IngestionJob { + IngestionJob { + document_id: format!("doc-{title}"), + document: NamespaceDocumentInput { + namespace: "skill-test".to_string(), + key: title.to_string(), + title: title.to_string(), + content: "body".to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: Vec::new(), + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }, + config: MemoryIngestionConfig::default(), + } + } + + #[tokio::test] + async fn submit_succeeds_until_capacity_then_drops() { + let state = IngestionState::new(); + let (tx, _rx) = mpsc::channel::(2); + let queue = IngestionQueue::from_parts(tx, state.clone(), 2); + + assert!(queue.submit(fixture_job("a")), "first submit must enqueue"); + assert!(queue.submit(fixture_job("b")), "second submit must enqueue"); + + // Channel is now full. tokio's bounded mpsc reserves one slot per + // permit, so capacity=2 means at most two pending; the third must be + // rejected with `false`. + assert!( + !queue.submit(fixture_job("c")), + "submit at capacity must return false (drop)" + ); + + // queue_depth must reflect only the accepted jobs — the drop path + // is required to decrement so the status RPC does not drift upward. + assert_eq!( + state.snapshot().queue_depth, + 2, + "queue_depth must roll back on overflow drop" + ); + } + + #[tokio::test] + async fn submit_recovers_after_drain() { + let state = IngestionState::new(); + let (tx, mut rx) = mpsc::channel::(1); + let queue = IngestionQueue::from_parts(tx, state.clone(), 1); + + assert!(queue.submit(fixture_job("first"))); + assert!( + !queue.submit(fixture_job("over")), + "second submit at cap=1 must drop" + ); + + // Drain the receiver to free a slot. + let pulled = rx.try_recv().expect("first job must be readable"); + assert_eq!(pulled.document.title, "first"); + // Mirror the worker's accounting (queue depth -> dequeue) so the + // post-drain snapshot does not look like a leftover queued job. + state.dequeue(); + + assert!( + queue.submit(fixture_job("after-drain")), + "submit after drain must enqueue" + ); + assert_eq!(state.snapshot().queue_depth, 1); + } + + #[tokio::test] + async fn submit_after_worker_gone_returns_false() { + let state = IngestionState::new(); + let (tx, rx) = mpsc::channel::(4); + drop(rx); // simulate worker task exiting and dropping its receiver + let queue = IngestionQueue::from_parts(tx, state.clone(), 4); + + assert!( + !queue.submit(fixture_job("orphan")), + "submit must return false once the receiver is dropped" + ); + assert_eq!( + state.snapshot().queue_depth, + 0, + "channel-closed drop path must roll the depth counter back" + ); + } + + #[test] + fn default_queue_capacity_is_bounded_and_reasonable() { + // Guardrail so future changes don't accidentally regress to an + // arbitrarily large default (or `usize::MAX`) without thinking about + // the producer-side memory bound. + const _: () = assert!(DEFAULT_QUEUE_CAPACITY > 0); + const _: () = assert!( + DEFAULT_QUEUE_CAPACITY <= 8 * 1024, + "default capacity is the memory ceiling under sustained overflow — keep it tight" + ); + } + + /// Zero capacity would otherwise panic from inside + /// `tokio::sync::mpsc::channel` with a cryptic Tokio-internal message + /// (`mpsc bounded channel requires buffer > 0`) — the explicit guard in + /// [`start_worker_with_capacity`] turns that into a clear, grep-friendly + /// assertion at the call site so misuse fails fast with an actionable + /// message instead of looking like a Tokio bug. + #[tokio::test] + #[should_panic(expected = "ingestion queue capacity must be greater than zero")] + async fn start_worker_rejects_zero_capacity() { + use tempfile::TempDir; + use tinymemory_api::host::NoopEmbedding; + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + // Panic must surface from our own assert, not from the Tokio + // channel constructor on the line after — that's the contract this + // test pins. + let _ = start_worker_with_capacity(Arc::new(memory), IngestionState::new(), 0); + } +} diff --git a/core/src/ingestion/state.rs b/core/src/ingestion/state.rs new file mode 100644 index 0000000..560733c --- /dev/null +++ b/core/src/ingestion/state.rs @@ -0,0 +1,237 @@ +//! Shared state + singleton lock for memory ingestion. +//! +//! Memory ingestion runs the local extraction LLM and must not run more than +//! once concurrently — otherwise multiple jobs contend for the same local AI +//! and either thrash or fail. [`IngestionState`] enforces the singleton via +//! [`tokio::sync::Mutex`] and exposes a snapshot suitable for the +//! `openhuman.memory_ingestion_status` RPC. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use parking_lot::RwLock; +use serde::Serialize; +use tokio::sync::Mutex; + +/// Snapshot of ingestion state, surfaced over RPC. +#[derive(Debug, Clone, Default, Serialize)] +pub struct IngestionStatusSnapshot { + /// Whether an ingestion job is currently running. + pub running: bool, + /// Document id of the in-flight job, if any. + pub current_document_id: Option, + /// Document title of the in-flight job, if any (best-effort). + pub current_title: Option, + /// Namespace of the in-flight job, if any. + pub current_namespace: Option, + /// Number of jobs waiting in the queue (not counting the running one). + pub queue_depth: usize, + /// Unix-ms timestamp of when the most recent job completed. + pub last_completed_at: Option, + /// Document id of the most recent completed job. + pub last_document_id: Option, + /// Whether the most recent job succeeded. + pub last_success: Option, +} + +/// Shared ingestion state + singleton lock. Cheap to clone. +#[derive(Clone)] +pub struct IngestionState { + inner: Arc, +} + +struct IngestionStateInner { + /// Singleton lock — held while a job is running. + run_lock: Mutex<()>, + /// Queue depth — bumped on submit, decremented when the worker pulls a job. + queue_depth: AtomicUsize, + /// Snapshot for status RPC. + snapshot: RwLock, +} + +impl Default for IngestionState { + fn default() -> Self { + Self::new() + } +} + +impl IngestionState { + /// Create a fresh state with empty snapshot and zero queue depth. + pub fn new() -> Self { + Self { + inner: Arc::new(IngestionStateInner { + run_lock: Mutex::new(()), + queue_depth: AtomicUsize::new(0), + snapshot: RwLock::new(IngestionStatusSnapshot::default()), + }), + } + } + + /// Bump the pending-queue depth (call on `submit`). + pub fn enqueue(&self) { + self.inner.queue_depth.fetch_add(1, Ordering::SeqCst); + } + + /// Decrement pending-queue depth (call when the worker has pulled a job + /// off the channel and is about to acquire the run lock). + pub fn dequeue(&self) { + self.inner.queue_depth.fetch_sub(1, Ordering::SeqCst); + } + + /// Acquire the singleton run lock. Holders run ingestion serialised; any + /// other caller blocks until the holder drops the guard. + pub async fn acquire(&self) -> tokio::sync::MutexGuard<'_, ()> { + self.inner.run_lock.lock().await + } + + /// Mark a job as in-flight in the snapshot. Caller must already hold + /// [`Self::acquire`]. + pub fn mark_running(&self, document_id: &str, title: &str, namespace: &str) { + let mut snap = self.inner.snapshot.write(); + snap.running = true; + snap.current_document_id = Some(document_id.to_string()); + snap.current_title = Some(title.to_string()); + snap.current_namespace = Some(namespace.to_string()); + } + + /// Mark the in-flight job as finished. + pub fn mark_completed(&self, document_id: &str, success: bool, completed_at_ms: i64) { + let mut snap = self.inner.snapshot.write(); + snap.running = false; + snap.current_document_id = None; + snap.current_title = None; + snap.current_namespace = None; + snap.last_completed_at = Some(completed_at_ms); + snap.last_document_id = Some(document_id.to_string()); + snap.last_success = Some(success); + } + + /// Returns a clone of the current snapshot. Includes live queue depth. + pub fn snapshot(&self) -> IngestionStatusSnapshot { + let mut snap = self.inner.snapshot.read().clone(); + snap.queue_depth = self.inner.queue_depth.load(Ordering::SeqCst); + snap + } + + /// Reset the queue depth counter and running snapshot to idle. + /// + /// Neutralises residue from background ingestion workers that outlived a + /// prior test's lock scope. Call at the start of each test body that + /// asserts exact `queue_depth` or `running` state. + /// + /// Preserves `last_completed_at`, `last_document_id`, and `last_success` + /// so tests that assert completion history still work. + #[cfg(any(test, feature = "test-support"))] + pub fn reset_for_test(&self) { + self.inner.queue_depth.store(0, Ordering::SeqCst); + let mut snap = self.inner.snapshot.write(); + snap.running = false; + snap.current_document_id = None; + snap.current_title = None; + snap.current_namespace = None; + // Preserve last_completed_at, last_document_id, last_success so + // tests that assert completion history still work. + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use tokio::time::{sleep, Duration}; + + #[tokio::test] + async fn singleton_serialises_concurrent_acquires() { + let state = IngestionState::new(); + let counter = Arc::new(parking_lot::Mutex::new(0u32)); + let max_concurrent = Arc::new(parking_lot::Mutex::new(0u32)); + + let mut handles = Vec::new(); + for _ in 0..4 { + let state = state.clone(); + let counter = Arc::clone(&counter); + let max_concurrent = Arc::clone(&max_concurrent); + handles.push(tokio::spawn(async move { + let _g = state.acquire().await; + let now = { + let mut c = counter.lock(); + *c += 1; + *c + }; + { + let mut m = max_concurrent.lock(); + if now > *m { + *m = now; + } + } + sleep(Duration::from_millis(20)).await; + *counter.lock() -= 1; + })); + } + + for h in handles { + h.await.unwrap(); + } + + assert_eq!(*max_concurrent.lock(), 1, "ingestion must be singleton"); + } + + #[test] + fn snapshot_reports_running_and_queue_depth() { + let state = IngestionState::new(); + state.enqueue(); + state.enqueue(); + let snap = state.snapshot(); + assert_eq!(snap.queue_depth, 2); + assert!(!snap.running); + + state.dequeue(); + state.mark_running("doc-1", "title", "ns"); + let snap = state.snapshot(); + assert_eq!(snap.queue_depth, 1); + assert!(snap.running); + assert_eq!(snap.current_document_id.as_deref(), Some("doc-1")); + + state.mark_completed("doc-1", true, 12345); + let snap = state.snapshot(); + assert!(!snap.running); + assert_eq!(snap.last_document_id.as_deref(), Some("doc-1")); + assert_eq!(snap.last_success, Some(true)); + assert_eq!(snap.last_completed_at, Some(12345)); + } + + #[test] + fn reset_for_test_clears_queue_depth_and_running_state() { + let state = IngestionState::new(); + state.enqueue(); + state.enqueue(); + state.mark_running("doc-x", "title", "ns"); + + state.reset_for_test(); + + let snap = state.snapshot(); + assert_eq!(snap.queue_depth, 0, "queue_depth must be zero after reset"); + assert!(!snap.running, "running must be false after reset"); + assert!(snap.current_document_id.is_none()); + } + + #[test] + fn reset_for_test_preserves_completion_history() { + let state = IngestionState::new(); + state.enqueue(); + state.mark_running("doc-y", "title", "ns"); + state.mark_completed("doc-y", true, 99999); + state.dequeue(); + + state.reset_for_test(); + + let snap = state.snapshot(); + assert_eq!(snap.queue_depth, 0); + assert_eq!( + snap.last_document_id.as_deref(), + Some("doc-y"), + "completion history should survive reset" + ); + assert_eq!(snap.last_success, Some(true)); + } +} diff --git a/core/src/ingestion/tests.rs b/core/src/ingestion/tests.rs new file mode 100644 index 0000000..a668902 --- /dev/null +++ b/core/src/ingestion/tests.rs @@ -0,0 +1,214 @@ +//! Tests for the ingestion pipeline — `parse_document`, regex extraction, +//! and `UnifiedMemory::ingest_document` end-to-end. + +use std::sync::Arc; + +use serde_json::json; +use tempfile::TempDir; + +use crate::store::{NamespaceDocumentInput, UnifiedMemory}; +use crate::{MemoryIngestionConfig, MemoryIngestionRequest}; +use tinymemory_api::host::NoopEmbedding; + +/// Test config for the heuristic-only ingestion pipeline. +fn ci_safe_config() -> MemoryIngestionConfig { + MemoryIngestionConfig::default() +} + +fn fixture(path: &str) -> String { + let base = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + std::fs::read_to_string( + base.join("tests") + .join("fixtures") + .join("ingestion") + .join(path), + ) + .expect("fixture should load") +} + +#[tokio::test] +async fn gmail_fixture_ingestion_recovers_required_signals() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let result = memory + .ingest_document(MemoryIngestionRequest { + document: NamespaceDocumentInput { + namespace: "skill-gmail".to_string(), + key: "gmail-thread-memory-integration".to_string(), + title: "Memory integration plan for OpenHuman desktop".to_string(), + content: fixture("gmail_thread_example.txt"), + source_type: "gmail".to_string(), + priority: "high".to_string(), + tags: Vec::new(), + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }, + config: ci_safe_config(), + }) + .await + .unwrap(); + + assert!(result + .entities + .iter() + .any(|entity| entity.name == "SANIL JAIN")); + assert!(result + .entities + .iter() + .any(|entity| entity.name == "RAVI KULKARNI")); + assert!(result + .entities + .iter() + .any(|entity| entity.name == "ASHA MEHTA")); + assert!(result + .entities + .iter() + .any(|entity| entity.name == "OPENHUMAN")); + assert!(result + .relations + .iter() + .any(|relation| relation.subject == "OPENHUMAN" + && relation.predicate == "USES" + && relation.object.contains("JSON-RPC"))); + assert!(result + .relations + .iter() + .any(|relation| relation.subject == "RAVI KULKARNI" && relation.predicate == "OWNS")); + assert!(result.preference_count >= 1); + assert!(result.decision_count >= 1); + + let context = memory + .query_namespace_context_data("skill-gmail", "who owns the rust memory api alignment", 5) + .await + .unwrap(); + assert!(context + .hits + .iter() + .flat_map(|hit| hit.supporting_relations.iter()) + .any(|relation| relation.subject == "RAVI KULKARNI" && relation.predicate == "OWNS")); + + let recall = memory + .recall_namespace_context_data("skill-gmail", 5) + .await + .unwrap(); + assert!(!recall.context_text.is_empty()); + assert!(recall + .hits + .iter() + .any(|hit| hit.content.contains("OpenHuman") || hit.content.contains("JSON-RPC"))); + assert!(recall + .hits + .iter() + .any(|hit| !hit.supporting_relations.is_empty())); + + let memories = memory + .recall_namespace_memories("skill-gmail", 5) + .await + .unwrap(); + assert!(memories.iter().any(|hit| hit.content.contains("JSON-RPC"))); + assert!(memories + .iter() + .any(|hit| matches!(hit.kind, crate::store::MemoryItemKind::Document))); + assert!(memories + .iter() + .any(|hit| !hit.supporting_relations.is_empty())); +} + +#[tokio::test] +async fn notion_fixture_ingestion_recovers_required_signals() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let result = memory + .ingest_document(MemoryIngestionRequest { + document: NamespaceDocumentInput { + namespace: "skill-notion".to_string(), + key: "notion-roadmap-memory-layer".to_string(), + title: "OpenHuman Memory Layer Roadmap".to_string(), + content: fixture("notion_page_example.txt"), + source_type: "notion".to_string(), + priority: "high".to_string(), + tags: Vec::new(), + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }, + config: ci_safe_config(), + }) + .await + .unwrap(); + + assert!(result + .entities + .iter() + .any(|entity| entity.name == "OPENHUMAN")); + assert!(result + .entities + .iter() + .any(|entity| entity.name == "SANIL JAIN")); + assert!(result + .relations + .iter() + .any(|relation| relation.subject == "OPENHUMAN" + && relation.predicate == "USES" + && relation.object.contains("JSON-RPC"))); + assert!(result + .relations + .iter() + .any(|relation| relation.subject == "CORE CONTRACT LOCKED" + && relation.predicate == "HAS_DEADLINE")); + assert!(result + .relations + .iter() + .any(|relation| relation.subject == "SANIL JAIN" && relation.predicate == "PREFERS")); + assert!(result.preference_count >= 1); + assert!(result.decision_count >= 1); + + let graph_rows = memory + .graph_query_namespace("skill-notion", Some("OPENHUMAN"), Some("USES")) + .await + .unwrap(); + assert!(!graph_rows.is_empty()); + + let context = memory + .query_namespace_context_data( + "skill-notion", + "who prefers core-first delivery over ui-first delivery", + 5, + ) + .await + .unwrap(); + assert!(context + .hits + .iter() + .flat_map(|hit| hit.supporting_relations.iter()) + .any(|relation| relation.subject == "SANIL JAIN" && relation.predicate == "PREFERS")); + + let recall = memory + .recall_namespace_context_data("skill-notion", 5) + .await + .unwrap(); + assert!(!recall.context_text.is_empty()); + assert!(recall + .hits + .iter() + .any(|hit| hit.content.contains("OpenHuman"))); + + let memories = memory + .recall_namespace_memories("skill-notion", 5) + .await + .unwrap(); + assert!(memories + .iter() + .any(|hit| hit.content.contains("OpenHuman") || hit.content.contains("core-first"))); + assert!(memories + .iter() + .any(|hit| matches!(hit.kind, crate::store::MemoryItemKind::Document))); + assert!(memories + .iter() + .any(|hit| !hit.supporting_relations.is_empty())); +} diff --git a/core/src/learning_candidate.rs b/core/src/learning_candidate.rs new file mode 100644 index 0000000..de8838b --- /dev/null +++ b/core/src/learning_candidate.rs @@ -0,0 +1,349 @@ +//! Learning candidate buffer — Phase 1 of issue #566. +//! +//! Defines the taxonomy types ([`FacetClass`], [`CueFamily`], [`EvidenceRef`]), +//! the unit-of-work [`LearningCandidate`], and a thread-safe ring-buffer +//! [`Buffer`] that collects candidates emitted by producers (Phase 2) before +//! they are consumed by the stability detector (Phase 3). +//! +//! The buffer is bounded: when full it evicts the oldest entry (FIFO overflow). +//! A global singleton is exposed via [`global()`]; individual tests may +//! construct their own [`Buffer`] with `Buffer::new(capacity)`. + +use std::collections::VecDeque; +use std::sync::OnceLock; + +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; + +// ── Taxonomy ──────────────────────────────────────────────────────────────── + +/// Six-class taxonomy of what the cache can hold. +/// +/// Keys are stored with a class prefix, e.g. `style/verbosity` or +/// `tooling/package_manager`. The class determines the half-life and +/// class budget used by the stability detector (Phase 3). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FacetClass { + /// Communication style preferences — verbosity, formality, code format. + Style, + /// Stable biographical facts — timezone, name, language, role. + Identity, + /// Developer toolchain preferences — package manager, editor, OS, language. + Tooling, + /// Hard user vetoes — things the user has explicitly rejected or forbidden. + Veto, + /// Active user goals or ongoing projects. + Goal, + /// Preferred communication channel or platform. + Channel, +} + +/// How a candidate signal was produced — determines the weight multiplier +/// applied in the stability formula. +/// +/// Higher-weight families contribute more strongly per evidence item. +/// The weights here are the canonical values from the Phase 1 plan: +/// `Explicit=1.0`, `Structural=0.9`, `Behavioral=0.7`, `Recurrence=0.6`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CueFamily { + /// Direct declaration of intent by the user (highest weight — 1.0). + /// + /// Examples: "I prefer pnpm", "my timezone is PST", "always use terse replies". + Explicit, + /// Inferred from structured file or provider metadata (weight 0.9). + /// + /// Examples: `package.json#packageManager`, Gmail display name, Slack workspace. + Structural, + /// Inferred by heuristics or LLM from observed behaviour (weight 0.7). + /// + /// Examples: rolling edit-window ratio, correction-repeat signal, reflection hook output. + Behavioral, + /// Materialized from recurrence statistics in the memory tree (weight 0.6). + /// + /// Examples: tree-topic hotness, source_weight per channel. + Recurrence, +} + +impl CueFamily { + /// Weight multiplier for this cue family in the stability formula. + /// + /// Phase 1 canonical values (matches the plan): + /// `Explicit=1.0`, `Structural=0.9`, `Behavioral=0.7`, `Recurrence=0.6`. + pub fn weight(self) -> f64 { + match self { + CueFamily::Explicit => 1.0, + CueFamily::Structural => 0.9, + CueFamily::Behavioral => 0.7, + CueFamily::Recurrence => 0.6, + } + } +} + +// ── Evidence reference ─────────────────────────────────────────────────────── + +/// Where a candidate's evidence points. Defined in the contract crate — the +/// memory store persists it, so both sides must name one type. See +/// [`tinymemory_api::host::EvidenceRef`]. +pub use tinymemory_api::host::EvidenceRef; + +// ── Learning candidate ─────────────────────────────────────────────────────── + +/// A single unit of learning evidence emitted by a producer and queued in the +/// [`Buffer`]. +/// +/// Each candidate asserts a specific `(class, key, value)` triple alongside +/// the evidence that backs it. The stability detector (Phase 3) aggregates +/// competing candidates for the same `(class, key)` pair and resolves them +/// into a single cache entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LearningCandidate { + /// Which facet class this evidence touches. + pub class: FacetClass, + /// Canonical slug key within the class, e.g. `"verbosity"`, `"package_manager"`. + /// + /// Convention: `snake_case`, lowercase, no class prefix (the class carries that). + pub key: String, + /// Canonical value string, e.g. `"terse"`, `"pnpm"`, `"UTC+5:30"`. + pub value: String, + /// How this candidate was produced. + pub cue_family: CueFamily, + /// Pointer to the backing evidence in the memory substrate. + pub evidence: EvidenceRef, + /// Source-provided confidence hint, `0.0..=1.0`. + /// + /// This is an initial hint; the stability detector will reweight it using + /// the cue-family weight and recency decay. + pub initial_confidence: f64, + /// When this candidate was observed, as seconds since the Unix epoch. + pub observed_at: f64, +} + +// ── Buffer ─────────────────────────────────────────────────────────────────── + +/// Thread-safe, bounded ring-buffer of [`LearningCandidate`] items. +/// +/// Backed by a `parking_lot::Mutex>`. When full +/// the oldest entry is evicted to make room (FIFO overflow). This keeps +/// memory bounded and naturally prioritises recent evidence. +/// +/// The global singleton has a default capacity of 1024. Tests should +/// construct their own buffer via [`Buffer::new`]. +pub struct Buffer { + inner: Mutex>, + capacity: usize, +} + +impl Buffer { + /// Create a new buffer with the given capacity. + /// + /// `capacity` must be ≥ 1. A capacity of zero would make every `push` + /// a no-op; callers should use a non-zero value. + pub fn new(capacity: usize) -> Self { + let cap = capacity.max(1); + Self { + inner: Mutex::new(VecDeque::with_capacity(cap)), + capacity: cap, + } + } + + /// Push a candidate onto the buffer. + /// + /// If the buffer is already at capacity, the oldest entry is evicted first + /// (FIFO overflow). This ensures the buffer always reflects the most recent + /// evidence. + pub fn push(&self, candidate: LearningCandidate) { + let mut guard = self.inner.lock(); + if guard.len() >= self.capacity { + guard.pop_front(); // evict oldest + } + guard.push_back(candidate); + } + + /// Drain all candidates from the buffer and return them in FIFO order. + /// + /// After this call the buffer is empty. + pub fn drain(&self) -> Vec { + let mut guard = self.inner.lock(); + guard.drain(..).collect() + } + + /// Clone all candidates without removing them. + /// + /// Useful for inspection or debugging. + pub fn peek(&self) -> Vec { + let guard = self.inner.lock(); + guard.iter().cloned().collect() + } + + /// Current number of candidates in the buffer. + pub fn len(&self) -> usize { + self.inner.lock().len() + } + + /// Returns `true` when the buffer holds no candidates. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Maximum number of candidates the buffer will hold. + pub fn capacity(&self) -> usize { + self.capacity + } +} + +// ── Global singleton ───────────────────────────────────────────────────────── + +static GLOBAL_BUFFER: OnceLock = OnceLock::new(); + +/// Return the global [`Buffer`] singleton. +/// +/// Initialised on first call with a default capacity of 1024. All producers +/// push into this buffer; the stability detector drains it. +pub fn global() -> &'static Buffer { + GLOBAL_BUFFER.get_or_init(|| Buffer::new(1024)) +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() + } + + fn make_candidate(value: &str) -> LearningCandidate { + LearningCandidate { + class: FacetClass::Style, + key: "verbosity".into(), + value: value.into(), + cue_family: CueFamily::Explicit, + evidence: EvidenceRef::Episodic { episodic_id: 1 }, + initial_confidence: 0.8, + observed_at: now_secs(), + } + } + + #[test] + fn push_then_drain_preserves_fifo_order() { + let buf = Buffer::new(10); + buf.push(make_candidate("a")); + buf.push(make_candidate("b")); + buf.push(make_candidate("c")); + + let drained = buf.drain(); + assert_eq!(drained.len(), 3); + assert_eq!(drained[0].value, "a"); + assert_eq!(drained[1].value, "b"); + assert_eq!(drained[2].value, "c"); + } + + #[test] + fn drain_empties_the_buffer() { + let buf = Buffer::new(10); + buf.push(make_candidate("x")); + buf.push(make_candidate("y")); + assert_eq!(buf.len(), 2); + + let _ = buf.drain(); + assert_eq!(buf.len(), 0); + assert!(buf.is_empty()); + } + + #[test] + fn bounded_capacity_evicts_oldest() { + let buf = Buffer::new(3); + buf.push(make_candidate("first")); + buf.push(make_candidate("second")); + buf.push(make_candidate("third")); + // Buffer is full — next push evicts "first" + buf.push(make_candidate("fourth")); + + assert_eq!(buf.len(), 3); + let items = buf.drain(); + assert_eq!(items[0].value, "second"); + assert_eq!(items[1].value, "third"); + assert_eq!(items[2].value, "fourth"); + } + + #[test] + fn peek_does_not_remove() { + let buf = Buffer::new(10); + buf.push(make_candidate("p")); + buf.push(make_candidate("q")); + + let peeked = buf.peek(); + assert_eq!(peeked.len(), 2); + // Buffer still holds the items + assert_eq!(buf.len(), 2); + + let drained = buf.drain(); + assert_eq!(drained[0].value, "p"); + assert_eq!(drained[1].value, "q"); + } + + #[test] + fn cue_family_weight_values() { + assert_eq!(CueFamily::Explicit.weight(), 1.0); + assert_eq!(CueFamily::Structural.weight(), 0.9); + assert_eq!(CueFamily::Behavioral.weight(), 0.7); + assert_eq!(CueFamily::Recurrence.weight(), 0.6); + } + + #[test] + fn roundtrip_serde_evidence_ref() { + let cases: Vec = vec![ + EvidenceRef::Episodic { episodic_id: 42 }, + EvidenceRef::EpisodicWindow { + from_id: 10, + to_id: 20, + }, + EvidenceRef::SourceSummary { + summary_id: "sum-abc".into(), + }, + EvidenceRef::TreeTopic { + topic_id: "topic-xyz".into(), + }, + EvidenceRef::DocumentChunk { + source_id: "notion:page1".into(), + chunk_id: "chunk-001".into(), + }, + EvidenceRef::EmailMessage { + source_id: "gmail:user@example.com".into(), + message_id: "".into(), + }, + EvidenceRef::Provider { + toolkit: "gmail".into(), + connection_id: "conn-1".into(), + field: "display_name".into(), + }, + EvidenceRef::ToolCall { + tool_name: "write_file".into(), + episodic_id: 99, + }, + EvidenceRef::TreeSourceWeight { + window_label: "2026-W18".into(), + }, + ]; + + for ev in &cases { + let json = serde_json::to_string(ev).expect("serialize failed"); + let back: EvidenceRef = serde_json::from_str(&json).expect("deserialize failed"); + assert_eq!(ev, &back, "round-trip failed for variant: {json}"); + } + } + + #[test] + fn global_returns_same_instance_across_calls() { + let a = global() as *const Buffer; + let b = global() as *const Buffer; + assert_eq!(a, b, "global() must return the same static instance"); + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs new file mode 100644 index 0000000..1935dcd --- /dev/null +++ b/core/src/lib.rs @@ -0,0 +1,106 @@ +//! `tinymemory-core` — the engine-neutral memory subsystem, extracted from +//! OpenHuman's `src/openhuman/memory/`. +//! +//! This crate owns the *substance* of a memory subsystem: the SQLite/vector +//! store, the markdown summary tree, the provider sync pipelines, ingestion, +//! recall/query/search, the ingest queue, conversations, people, goals and the +//! tool-memory rules. It is host-neutral: nothing here names an OpenHuman +//! type. +//! +//! What deliberately stays in the host (see the repository README's split): +//! the RPC surface, agent tools, security policy and the taint/scope guard, +//! credentials, schedulers, the event bus, and config mapping. The host +//! supplies those through the seam traits in [`tinymemory_api::host`]. + +/// The host's configuration, as this crate sees it. +/// +/// This is the load-bearing trick of the whole extraction. Before the move, +/// every function in this crate took `config: &crate::Config` +/// — a concrete host struct. Aliasing `Config` to the *trait object* means those +/// signatures read `config: &Config` exactly as they did before, and the host's +/// concrete `Config` unsize-coerces at each of the ~550 call sites on the other +/// side of the seam with no edit at all. +/// +/// What did change inside this crate: field reads became method calls +/// (`config.workspace_dir()` → `config.workspace_dir()`), by-value `Config` +/// parameters became `Arc`, and `TestHostConfig::default()` in tests became +/// [`tinymemory_api::host::test_support::TestHostConfig`], which cannot be built +/// from a trait object. +/// +/// See [`tinymemory_api::host::MemoryHostConfig`] for the accessor surface and +/// why its return types are shaped the way they are. +pub type Config = dyn tinymemory_api::host::MemoryHostConfig; + +pub mod chat; +pub mod chat_host; +pub mod composio_host; +pub mod config_loader; +pub mod conversations; +pub mod diff; +pub mod embedding_adapter; +pub mod embedding_host; +pub mod events; +pub mod global; +pub mod ingest_pipeline; +pub mod ingestion; +pub mod learning_candidate; +pub mod nlp_host; +pub mod observability; +pub mod people; +pub mod preferences; +pub mod queue; +pub mod remember; +pub mod rpc_models; +pub mod scheduler_gate; +pub mod search; +pub mod shutdown; +pub mod source_scope; +pub mod sources; +pub mod store; +pub mod sync; +pub mod sync_events; +pub mod test_env_lock; +#[cfg(test)] +pub(crate) mod test_seams; +pub mod thread_context; +pub mod tinycortex; +pub mod tool_memory; +pub mod traits; +pub mod tree; +pub mod tree_policy; +pub mod tree_source; +pub mod util; + +// The host seam, re-exported so downstream code takes one dependency. These are +// the *only* types this crate accepts from its host. +pub use tinymemory_api::host::{ + format_embedding_signature, ComposioMode, EmbeddingProvider, MemoryEvent, MemoryEventSink, + MemoryHostConfig, NoopEmbedding, NoopEventSink, COMPOSIO_MODE_BACKEND, COMPOSIO_MODE_DIRECT, + DEFAULT_MEMORY_SYNC_INTERVAL_SECS, +}; + +/// The default OpenHuman root directory, `~/.openhuman`. +/// +/// The host resolves this through `config::default_root_openhuman_dir`, which +/// this crate cannot see. Reproduced here rather than added to the config seam +/// because the two callers only need it as a last-resort fallback when no +/// workspace was supplied. +/// +/// # Errors +/// +/// Returns `Err` when the home directory cannot be determined. +pub fn default_openhuman_dir() -> Result { + dirs::home_dir() + .ok_or_else(|| "Could not find home directory".to_string()) + .map(|home| home.join(".openhuman")) +} + +pub use ingestion::{ + ExtractedEntity, ExtractedRelation, ExtractionMode, IngestionJob, IngestionQueue, + IngestionState, IngestionStatusSnapshot, MemoryIngestionConfig, MemoryIngestionRequest, + MemoryIngestionResult, DEFAULT_MEMORY_EXTRACTION_MODEL, +}; +pub use rpc_models::*; +pub use store::types::NamespaceDocumentInput; +pub use store::{MemoryClient, UnifiedMemory}; +pub use traits::{Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts}; diff --git a/core/src/nlp_host.rs b/core/src/nlp_host.rs new file mode 100644 index 0000000..5364efc --- /dev/null +++ b/core/src/nlp_host.rs @@ -0,0 +1,68 @@ +//! [`NlpHost`] — spaCy entity extraction, run by the host. +//! +//! The summary tree extracts entities from a query so it can match them against +//! the entities indexed on chunks. spaCy gives it far better recall than the +//! regex fallback, but running spaCy means provisioning a Python toolchain, +//! downloading a model and supervising a server process — none of which belongs +//! in a memory engine. +//! +//! So the host owns the runtime and this trait is the one call the core makes +//! into it. The wire types are in [`tinymemory_api::host`], shared by both +//! sides. +//! +//! # Unwired falls back, it does not fail +//! +//! Unlike the embedding and Composio seams, an absent [`NlpHost`] is benign: +//! the caller already has a regex extractor for exactly this case (spaCy +//! disabled in config, model not provisioned yet, extraction erroring). Losing +//! recall is the documented degraded mode; failing the query would be worse. + +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::RwLock; + +use crate::Config; + +pub use tinymemory_api::host::{SpacyEntity, SpacyResponse}; + +/// Runs spaCy extraction on the core's behalf. +#[async_trait] +pub trait NlpHost: Send + Sync + std::fmt::Debug { + /// Extract entities and noun chunks from `text`. + /// + /// # Errors + /// + /// Returns `Err` when the runtime is disabled, not provisioned, or the + /// request fails. Callers fall back to regex extraction. + async fn extract_spacy(&self, config: &Config, text: &str) -> Result; +} + +static HOST: RwLock>> = RwLock::new(None); + +/// Install the host's NLP runtime. Called once during startup wiring. +pub fn set_nlp_host(host: Arc) { + *HOST.write() = Some(host); +} + +/// Remove any installed host. For tests. +pub fn clear_nlp_host() { + *HOST.write() = None; +} + +/// The installed host, or `None` when nothing has been wired up. +#[must_use] +pub fn nlp_host() -> Option> { + HOST.read().clone() +} + +/// Extract entities from `text`. +/// +/// # Errors +/// +/// Returns `Err` when no host is installed or extraction fails — in both cases +/// the caller should fall back to regex extraction. +pub async fn extract_spacy(config: &Config, text: &str) -> Result { + let host = nlp_host().ok_or_else(|| "no NlpHost installed".to_string())?; + host.extract_spacy(config, text).await +} diff --git a/core/src/observability.rs b/core/src/observability.rs new file mode 100644 index 0000000..04b65b5 --- /dev/null +++ b/core/src/observability.rs @@ -0,0 +1,71 @@ +//! The process-global [`ErrorReporter`], and the `report_error` / +//! `report_error_or_expected` the extracted code calls in place of the host's +//! `core::observability`. +//! +//! Same global shape and same rationale as [`crate::events`] — including the +//! default. With no reporter installed the report is dropped after a local log +//! line, because every call site here is already handling the failure it is +//! reporting; telemetry is the side effect, not the recovery. + +use std::sync::Arc; + +use parking_lot::RwLock; + +pub use tinymemory_api::host::ErrorReporter; + +static REPORTER: RwLock>> = RwLock::new(None); + +/// Install the host's error reporter. Called once during startup wiring. +pub fn set_error_reporter(reporter: Arc) { + *REPORTER.write() = Some(reporter); +} + +/// Remove any installed reporter. For tests. +pub fn clear_error_reporter() { + *REPORTER.write() = None; +} + +/// The installed reporter, or `None` when no host has wired one up. +#[must_use] +pub fn error_reporter() -> Option> { + REPORTER.read().clone() +} + +/// Report `error` as a defect. A no-op beyond logging when nothing is installed. +/// +/// Generic over `Display` exactly like the host's own `report_error`, and +/// rendered with `{:#}` so an `anyhow::Error` carries its full context chain +/// across the seam rather than just its outermost message. +pub fn report_error( + error: &E, + domain: &str, + operation: &str, + tags: &[(&str, &str)], +) { + let rendered = format!("{error:#}"); + match error_reporter() { + Some(reporter) => reporter.report_error(&rendered, domain, operation, tags), + None => log::debug!( + "[memory:observability] dropped report (no reporter installed) \ + domain={domain} operation={operation}: {rendered}" + ), + } +} + +/// Report `error`, letting the host classify defect vs expected failure. A +/// no-op beyond logging when nothing is installed. +pub fn report_error_or_expected( + error: &E, + domain: &str, + operation: &str, + tags: &[(&str, &str)], +) { + let rendered = format!("{error:#}"); + match error_reporter() { + Some(reporter) => reporter.report_error_or_expected(&rendered, domain, operation, tags), + None => log::debug!( + "[memory:observability] dropped classified report (no reporter installed) \ + domain={domain} operation={operation}: {rendered}" + ), + } +} diff --git a/core/src/people/README.md b/core/src/people/README.md new file mode 100644 index 0000000..e9142b2 --- /dev/null +++ b/core/src/people/README.md @@ -0,0 +1,85 @@ +# people + +Contact resolution + relationship scoring (the "A5" module). Maps any of three handle kinds — iMessage handle, email, or display name — to a single stable `PersonId`, and ranks known people by a deterministic composite score (recency × frequency × reciprocity × depth) derived from observed interaction rows. Backed by its own SQLite database (people / handle aliases / interactions). Can seed itself from the macOS system Address Book (`CNContactStore`). Intentionally self-contained — per its module docstring it has no dependency on `life_capture`, `chronicle`, `nudges`, or UI; downstream integration is left to later slices. + +## Responsibilities + +- Canonicalize handles (lowercase/trim emails + email-style iMessage handles; whitespace-collapse display names) so the same person resolves consistently across case and spacing. +- Deterministically resolve a `Handle` to an existing `PersonId`, or mint a new `Person` skeleton on first sight (`create_if_missing`). +- Link handles together (`link`) so an email + phone + display name can be attached to one person — without ever *auto*-merging distinct identities that share only a display name or an unverified handle. +- Record interactions and aggregate them into a per-person composite score plus an explainable component breakdown. +- Rank all known people by score for `people.list`. +- Seed the store from the macOS Address Book, distinguishing "permission denied" from "no contacts". +- Persist people, handle aliases, and interactions in a dedicated SQLite DB with idempotent migrations. + +## Key files + +| File | Role | +| --- | --- | +| `src/openhuman/memory/people/mod.rs` | Export-focused. Declares submodules and re-exports `all_people_controller_schemas` / `all_people_registered_controllers`. | +| `src/openhuman/memory/people/types.rs` | Domain types: `PersonId`, `Handle` (with `canonicalize` / `as_key`), `Person`, `Interaction`, `ScoreComponents`, `AddressBookContact`. | +| `src/openhuman/memory/people/resolver.rs` | `HandleResolver` — `resolve`, `resolve_or_create(_with_status)`, `link`, `seed_from_address_book`. The deterministic handle→PersonId logic + cross-source merge-safety contract. | +| `src/openhuman/memory/people/scorer.rs` | Pure `score(interactions, now) -> ScoreComponents`. Recency half-life, frequency window/cap, reciprocity balance, depth cap as module constants. | +| `src/openhuman/memory/people/store.rs` | SQLite-backed `PeopleStore` (`Arc>`) + rebindable process-global accessor (`init_from_workspace` / `get`). CRUD, lookup, interaction read/write, batched interaction fetch. | +| `src/openhuman/memory/people/address_book.rs` | `ContactsSource` trait + `SystemContactsSource` (macOS `CNContactStore` FFI via objc2) and non-mac stub; `MockContactsSource` for tests; `AddressBookError`. | +| `src/openhuman/memory/people/rpc.rs` | Domain RPC handlers (`handle_list`, `handle_resolve`, `handle_score`, `handle_refresh_address_book`) returning `RpcOutcome`; callable directly in tests with a constructed `PeopleStore`. | +| `src/openhuman/memory/people/schemas.rs` | Controller schemas + param-parsing adapter handlers that fetch the global store and delegate to `rpc.rs`. | +| `src/openhuman/memory/people/migrations.rs` | Idempotent migration runner (bookkeeping table `_people_migrations`, per-migration transaction). | +| `src/openhuman/memory/people/migrations/0001_init.sql` | Schema: `people`, `handle_aliases`, `interactions` + indexes. | +| `src/openhuman/memory/people/tests.rs` | Cross-file integration tests for the domain. | + +## Public surface + +- Types: `PersonId`, `Handle` (`IMessage` / `Email` / `DisplayName`), `Person`, `Interaction`, `ScoreComponents`, `AddressBookContact`. +- `HandleResolver::{resolve, resolve_or_create, resolve_or_create_with_status, link, seed_from_address_book}`. +- `scorer::score` + tunable constants `RECENCY_HALF_LIFE_DAYS`, `FREQUENCY_WINDOW_DAYS`, `FREQUENCY_CAP`, `DEPTH_CAP_CHARS`. +- `store::{PeopleStore, init, get}` and `ConnHandle`. +- `address_book::{ContactsSource, SystemContactsSource, read, read_with, AddressBookError}`. +- `mod.rs` re-exports `all_people_controller_schemas` / `all_people_registered_controllers` for the controller registry. + +## RPC / controllers + +Registered via the controller registry (wired in `src/core/all.rs`). Four controllers in the `people` namespace: + +| Method | Inputs | Output | +| --- | --- | --- | +| `people.list` | `limit?` (default 100, capped at 500) | `people[]` ranked by score desc — each with `person_id`, `display_name?`, `primary_email?`, `primary_phone?`, `handles[]`, `score`, `components`, `interaction_count`. | +| `people.resolve` | `kind` (`imessage`/`email`/`display_name`), `value`, `create_if_missing?` | `person_id?` (null when unknown and not creating), `created`. | +| `people.score` | `person_id` (UUID) | `person_id`, `score`, `components`, `interaction_count`. Errors if person not found. | +| `people.refresh_address_book` | — | `seeded`, `skipped`, `permission_denied`. | + +`score` / composite is `recency * frequency * reciprocity * depth`, each clamped to `[0,1]`. + +## Persistence + +Dedicated SQLite DB managed by `PeopleStore` (open via `open_at(path)` or `open_in_memory()`; migrations run on open). Three tables (see `0001_init.sql`): + +- `people` — one row per resolved person (uuid id, display name, primary email/phone, timestamps). +- `handle_aliases` — `(kind, value)` primary key → `person_id` (FK, `ON DELETE CASCADE`); `value` is the canonicalized form. This table *is* the resolver index. +- `interactions` — `(person_id, ts, is_outbound, length)` rows the scorer aggregates; indexed by `(person_id, ts DESC)` and `ts DESC`. + +Migrations are tracked in `_people_migrations` and applied idempotently in a transaction. The store is exposed process-globally through a workspace-tagged `RwLock>` slot (`get` from controller handlers). `store::init_from_workspace(workspace_dir)` seeds it, opening `/people/people.db`; it is called at core boot (`src/core/jsonrpc.rs`, alongside `memory::global` and `whatsapp_data::global`) and again on active-user switch (`credentials::ops`, `app_state::ops`), where a **different** workspace rebinds the store — mirroring `memory::global` so people never keeps writing the pre-login workspace. Same-workspace calls are a no-op. Tests construct stores directly with `open_in_memory`. + +## Dependencies + +- the host's `core::all::{ControllerFuture, RegisteredController}` — controller registry types, used by the RPC surface that stayed in the host. +- the host's `core::{ControllerSchema, FieldSchema, TypeSchema}` — controller schema definitions, used by the RPC surface that stayed in the host. +- `crate::rpc::RpcOutcome` — standard RPC result envelope (`RpcOutcome`). +- External crates: `rusqlite` (storage), `tokio` (async + `spawn_blocking` for sync SQL, `Mutex`), `chrono` (timestamps/scoring), `uuid` (`PersonId`), `serde`; on macOS, `block2` / `objc2` / `objc2-contacts` / `objc2-foundation` for the `CNContactStore` FFI in `address_book.rs`. The global store slot uses `std::sync::{OnceLock, RwLock}`. + +Notably it depends on **no other `openhuman` domain** — consistent with its "self-contained" docstring. + +## Used by + +- `src/core/all.rs` — registers the people controllers and schemas, and routes the `"people"` namespace. +- `src/openhuman/memory/store/` — reuses `people::types::{Person, PersonId, Handle}` (e.g. `Person` aliased as `Contact` in `kinds.rs`, and in `traits.rs`). + +## Notes / gotchas + +- **Cross-source merge safety (issue #1538):** two identities that share only a display name or only an unverified handle from different sources are **never** auto-merged. Merging only happens via explicit `link()`. Resolver tests lock this contract in. +- **Idempotent seeding:** `seed_from_address_book` re-runs as a no-op for already-known handles; on `PermissionDenied` it writes nothing (no partial state). The "primary" link target is first email, else first phone, else display name. +- **macOS Address Book FFI must not run on the main thread** — `CNContactStore` access requests deadlock there; `request_access` blocks on a completion-handler channel. Non-mac builds return an empty contact list. +- **Scoring constants are module-level**, not config-driven yet — kept fixed so tests stay stable; the docstring notes they can move to config later without breaking the API. +- **Composite is a product:** any zero component (e.g. a one-sided conversation → reciprocity 0) zeroes the whole score. +- **SQL runs on `spawn_blocking`:** the connection is sync `rusqlite` behind `Arc>`; `JoinError`s from blocking tasks are mapped into a synthetic `rusqlite` IO error. +- **Tests bypass the global store:** they construct `PeopleStore::open_in_memory()` and call `rpc::*` / `HandleResolver` directly rather than going through the schema adapters (which require the workspace-seeded global). diff --git a/core/src/people/address_book.rs b/core/src/people/address_book.rs new file mode 100644 index 0000000..d32973e --- /dev/null +++ b/core/src/people/address_book.rs @@ -0,0 +1,382 @@ +//! macOS Address Book read via `CNContactStore`. +//! +//! Uses the documented Contacts framework API (`CNContactStore`) which: +//! - Triggers the TCC Contacts permission prompt (sandboxed builds work correctly). +//! - Returns a structured error for "permission denied" so callers can distinguish +//! that case from "no contacts". +//! +//! A trait (`ContactsSource`) provides a mockable seam so unit tests can inject a +//! canned list or a permission-denied error without any FFI calls. +//! +//! On non-mac platforms `read()` returns an empty vec (stub path). + +use crate::people::types::AddressBookContact; + +/// Result type distinguishing permission errors from other failures. +#[derive(Debug, PartialEq)] +pub enum AddressBookError { + /// The user denied or restricted Contacts access. + PermissionDenied, + /// Any other error (typically returned as a descriptive string). + Other(String), +} + +impl std::fmt::Display for AddressBookError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AddressBookError::PermissionDenied => { + write!( + f, + "contacts access denied — grant access in System Settings > Privacy > Contacts" + ) + } + AddressBookError::Other(s) => write!(f, "{s}"), + } + } +} + +/// Mockable seam for contact fetching. The real impl calls CNContactStore; +/// tests inject a `MockContactsSource`. +pub trait ContactsSource: Send + Sync { + fn fetch_contacts(&self) -> Result, AddressBookError>; +} + +/// Real implementation backed by CNContactStore (macOS only). +/// On non-mac this is an empty struct whose `fetch_contacts` always returns `Ok(vec![])`. +pub struct SystemContactsSource; + +impl ContactsSource for SystemContactsSource { + fn fetch_contacts(&self) -> Result, AddressBookError> { + imp::fetch_via_cn_contact_store() + } +} + +/// Fetch all contacts using the provided `ContactsSource`. +/// +/// Errors are logged at `warn` level and surfaced to the caller so RPC +/// handlers can distinguish "permission denied" from "no contacts found". +pub fn read_with(source: &dyn ContactsSource) -> Result, AddressBookError> { + match source.fetch_contacts() { + Ok(v) => { + tracing::debug!("[people::address_book] fetched {} contacts", v.len()); + Ok(v) + } + Err(AddressBookError::PermissionDenied) => { + tracing::warn!( + "[people::address_book] contacts access denied — \ + grant access in System Settings > Privacy > Contacts" + ); + Err(AddressBookError::PermissionDenied) + } + Err(AddressBookError::Other(ref e)) => { + tracing::warn!("[people::address_book] fetch error: {e}"); + Err(AddressBookError::Other(e.clone())) + } + } +} + +/// Convenience wrapper using the real `SystemContactsSource`. +pub fn read() -> Result, AddressBookError> { + read_with(&SystemContactsSource) +} + +// ── macOS implementation ────────────────────────────────────────────────────── +// +// Gated on `contacts` as well as the target: the four objc2 crates this needs +// are exclusive to this module, so a slim macOS build sheds the whole cohort. + +#[cfg(all(target_os = "macos", feature = "contacts"))] +mod imp { + use super::{AddressBookContact, AddressBookError}; + + use block2::RcBlock; + use core::ptr::NonNull; + use objc2::runtime::Bool; + use objc2::runtime::ProtocolObject; + use objc2::AnyThread as _; + use objc2_contacts::{ + CNAuthorizationStatus, CNContact, CNContactFetchRequest, CNContactStore, CNEntityType, + }; + use objc2_foundation::{NSArray, NSError, NSString}; + use std::sync::{Arc, Mutex}; + + // CNKeyDescriptor is a protocol; NSString conforms to it. + // We build the keys array as NSArray>. + use objc2_contacts::CNKeyDescriptor; + + /// Build the keys array used for CNContactFetchRequest. + /// + /// # Safety + /// NSString::from_str is safe; casting to ProtocolObject is safe because + /// `NSString: CNKeyDescriptor` (confirmed by the objc2-contacts bindings). + unsafe fn make_keys_array() -> objc2::rc::Retained>> + { + let given = NSString::from_str("givenName"); + let family = NSString::from_str("familyName"); + let emails = NSString::from_str("emailAddresses"); + let phones = NSString::from_str("phoneNumbers"); + + // NSString conforms to CNKeyDescriptor, so we can cast the refs. + let refs: &[&ProtocolObject] = &[ + ProtocolObject::from_ref(&*given), + ProtocolObject::from_ref(&*family), + ProtocolObject::from_ref(&*emails), + ProtocolObject::from_ref(&*phones), + ]; + NSArray::from_slice(refs) + } + + /// Request contacts access from TCC. Blocks on the calling thread until + /// the completion handler fires. Must not be called from the main thread + /// on macOS (CNContactStore will deadlock). + fn request_access(store: &CNContactStore) -> Result<(), AddressBookError> { + unsafe { + let status = CNContactStore::authorizationStatusForEntityType(CNEntityType::Contacts); + match status { + CNAuthorizationStatus::Authorized | CNAuthorizationStatus::Limited => { + tracing::debug!("[people::address_book] contacts access already authorized"); + return Ok(()); + } + CNAuthorizationStatus::Denied | CNAuthorizationStatus::Restricted => { + return Err(AddressBookError::PermissionDenied); + } + _ => { + tracing::debug!( + "[people::address_book] requesting contacts access (status={status:?})" + ); + } + } + + let (tx, rx) = std::sync::mpsc::channel::>(); + let tx = Arc::new(Mutex::new(Some(tx))); + let tx_clone = Arc::clone(&tx); + + let block = RcBlock::new(move |granted: Bool, _error: *mut NSError| { + let mut slot = tx_clone.lock().unwrap(); + if let Some(sender) = slot.take() { + let result = if granted.as_bool() { + Ok(()) + } else { + Err(AddressBookError::PermissionDenied) + }; + let _ = sender.send(result); + } + }); + + store.requestAccessForEntityType_completionHandler(CNEntityType::Contacts, &block); + + rx.recv().map_err(|_| { + AddressBookError::Other("contacts permission callback never fired".into()) + })? + } + } + + pub fn fetch_via_cn_contact_store() -> Result, AddressBookError> { + tracing::debug!("[people::address_book] fetch_via_cn_contact_store entry"); + unsafe { + let store = CNContactStore::new(); + request_access(&store)?; + + let keys_array = make_keys_array(); + let request = CNContactFetchRequest::initWithKeysToFetch( + CNContactFetchRequest::alloc(), + &keys_array, + ); + + let mut contacts: Vec = Vec::new(); + + // We use a raw pointer to the vec inside the block so that we can + // push from within the block. The block runs synchronously within + // enumerateContactsWithFetchRequest (it blocks until done), so the + // pointer is valid throughout. + let contacts_ptr: *mut Vec = &mut contacts; + + let block = RcBlock::new( + move |contact_nn: NonNull, _stop: NonNull| { + let contact: &CNContact = contact_nn.as_ref(); + + let given = contact.givenName().to_string(); + let family = contact.familyName().to_string(); + let full = { + let g = given.trim(); + let f = family.trim(); + match (g.is_empty(), f.is_empty()) { + (true, true) => None, + (false, true) => Some(g.to_string()), + (true, false) => Some(f.to_string()), + (false, false) => Some(format!("{g} {f}")), + } + }; + + let emails: Vec = { + let arr = contact.emailAddresses(); + let mut v = Vec::new(); + for i in 0..arr.len() { + let lv = arr.objectAtIndex(i); + // CNLabeledValue.value() → Retained + let email = lv.value().to_string(); + let trimmed = email.trim().to_string(); + if !trimmed.is_empty() { + v.push(trimmed); + } + } + v + }; + + let phones: Vec = { + let arr = contact.phoneNumbers(); + let mut v = Vec::new(); + for i in 0..arr.len() { + let lv = arr.objectAtIndex(i); + // CNLabeledValue.value() → Retained + let num = lv.value().stringValue().to_string(); + let trimmed = num.trim().to_string(); + if !trimmed.is_empty() { + v.push(trimmed); + } + } + v + }; + + if full.is_none() && emails.is_empty() && phones.is_empty() { + return; + } + + (*contacts_ptr).push(AddressBookContact { + display_name: full, + emails, + phones, + }); + }, + ); + + let mut error: Option> = None; + let ok = store.enumerateContactsWithFetchRequest_error_usingBlock( + &request, + Some(&mut error), + &block, + ); + if !ok { + let msg = error + .map(|e| e.localizedDescription().to_string()) + .unwrap_or_else(|| "unknown error from CNContactStore".into()); + return Err(AddressBookError::Other(msg)); + } + + tracing::debug!( + "[people::address_book] enumerated {} contacts", + contacts.len() + ); + Ok(contacts) + } + } +} + +// ── stub: non-macOS, or macOS with `contacts` compiled out ─────────────────── +// +// Pre-dates the gate — it already existed for Linux/Windows. Widening its cfg +// is the whole off-state: `read()`, `read_with()`, `AddressBookError` and +// `SystemContactsSource` stay compiled everywhere, so the `people` RPC surface +// is identical and an address-book refresh seeds nothing rather than failing. + +#[cfg(not(all(target_os = "macos", feature = "contacts")))] +mod imp { + use super::{AddressBookContact, AddressBookError}; + + pub fn fetch_via_cn_contact_store() -> Result, AddressBookError> { + Ok(vec![]) + } +} + +// ── tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +pub mod tests { + use super::*; + + /// Test double that returns a canned list without any FFI calls. + pub struct MockContactsSource { + pub result: Result, AddressBookError>, + } + + impl MockContactsSource { + pub fn ok(contacts: Vec) -> Self { + Self { + result: Ok(contacts), + } + } + + pub fn permission_denied() -> Self { + Self { + result: Err(AddressBookError::PermissionDenied), + } + } + } + + impl ContactsSource for MockContactsSource { + fn fetch_contacts(&self) -> Result, AddressBookError> { + match &self.result { + Ok(v) => Ok(v.clone()), + Err(AddressBookError::PermissionDenied) => Err(AddressBookError::PermissionDenied), + Err(AddressBookError::Other(s)) => Err(AddressBookError::Other(s.clone())), + } + } + } + + fn mk_contact(name: &str, email: &str) -> AddressBookContact { + AddressBookContact { + display_name: Some(name.into()), + emails: vec![email.into()], + phones: vec![], + } + } + + #[test] + fn mock_source_returns_canned_contacts() { + let source = MockContactsSource::ok(vec![ + mk_contact("Alice", "alice@example.com"), + mk_contact("Bob", "bob@example.com"), + ]); + let result = read_with(&source).unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].display_name.as_deref(), Some("Alice")); + assert_eq!(result[1].emails[0], "bob@example.com"); + } + + #[test] + fn mock_source_permission_denied_is_distinguished() { + let source = MockContactsSource::permission_denied(); + let err = read_with(&source).unwrap_err(); + assert_eq!(err, AddressBookError::PermissionDenied); + } + + #[test] + fn system_source_non_mac_returns_empty() { + // Mirrors the `imp` cfgs above: the stub is what compiles whenever the + // real CNContactStore path is absent, whether by target or by gate. + #[cfg(not(all(target_os = "macos", feature = "contacts")))] + { + let source = SystemContactsSource; + let result = read_with(&source).unwrap(); + assert!(result.is_empty()); + } + #[cfg(all(target_os = "macos", feature = "contacts"))] + { + // TCC state is environment-dependent; just verify no panic. + let source = SystemContactsSource; + let _ = read_with(&source); + } + } + + #[test] + fn contact_with_no_fields_is_excluded_by_mock() { + let source = MockContactsSource::ok(vec![AddressBookContact { + display_name: Some("Sarah Lee".into()), + emails: vec![], + phones: vec!["+1 555 000 0001".into()], + }]); + let result = read_with(&source).unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].phones[0], "+1 555 000 0001"); + } +} diff --git a/core/src/people/migrations.rs b/core/src/people/migrations.rs new file mode 100644 index 0000000..57d153e --- /dev/null +++ b/core/src/people/migrations.rs @@ -0,0 +1,93 @@ +//! SQLite migrations for the people module. Mirrors the life_capture +//! migration style: idempotent, per-migration transaction, recorded in a +//! dedicated bookkeeping table. + +use rusqlite::{Connection, Result}; + +const MIGRATIONS: &[(&str, &str)] = &[("0001_init", include_str!("migrations/0001_init.sql"))]; + +pub fn run(conn: &Connection) -> Result<()> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS _people_migrations ( + name TEXT PRIMARY KEY, + applied_at INTEGER NOT NULL + )", + )?; + + for (name, sql) in MIGRATIONS { + let already: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM _people_migrations WHERE name = ?1)", + rusqlite::params![name], + |row| row.get(0), + )?; + if already { + continue; + } + + conn.execute_batch("BEGIN")?; + let result = (|| -> Result<()> { + conn.execute_batch(sql)?; + conn.execute( + "INSERT INTO _people_migrations(name, applied_at) \ + VALUES (?1, CAST(strftime('%s','now') AS INTEGER))", + rusqlite::params![name], + )?; + Ok(()) + })(); + match result { + Ok(()) => conn.execute_batch("COMMIT")?, + Err(e) => { + let _ = conn.execute_batch("ROLLBACK"); + return Err(e); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fresh() -> Connection { + Connection::open_in_memory().unwrap() + } + + #[test] + fn migrations_create_expected_tables() { + let conn = fresh(); + run(&conn).unwrap(); + let mut stmt = conn + .prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + .unwrap(); + let names: Vec = stmt + .query_map([], |row| row.get(0)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + for expected in [ + "people", + "handle_aliases", + "interactions", + "_people_migrations", + ] { + assert!( + names.iter().any(|n| n == expected), + "missing {expected}: {names:?}" + ); + } + } + + #[test] + fn migrations_are_idempotent() { + let conn = fresh(); + run(&conn).unwrap(); + run(&conn).unwrap(); + let count: i64 = conn + .query_row("SELECT count(*) FROM _people_migrations", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(count, MIGRATIONS.len() as i64); + } +} diff --git a/core/src/people/migrations/0001_init.sql b/core/src/people/migrations/0001_init.sql new file mode 100644 index 0000000..ee692b9 --- /dev/null +++ b/core/src/people/migrations/0001_init.sql @@ -0,0 +1,37 @@ +-- People module schema. +-- +-- `people` holds one row per resolved person. `handle_aliases` holds all +-- known (kind, canonical_value) handles that map to that person; the +-- resolver is a lookup on `(kind, value)` → `person_id`. +-- +-- `interactions` records observed exchanges for scoring. Single-user v1; +-- each row is attributed to (local-user, person_id). + +CREATE TABLE IF NOT EXISTS people ( + id TEXT PRIMARY KEY, -- uuid + display_name TEXT, + primary_email TEXT, + primary_phone TEXT, + created_at INTEGER NOT NULL, -- unix seconds + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS handle_aliases ( + kind TEXT NOT NULL, -- 'imessage' | 'email' | 'display_name' + value TEXT NOT NULL, -- canonicalized (lowercase / trimmed) + person_id TEXT NOT NULL REFERENCES people(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL, + PRIMARY KEY (kind, value) +); + +CREATE INDEX IF NOT EXISTS handle_aliases_person_idx ON handle_aliases(person_id); + +CREATE TABLE IF NOT EXISTS interactions ( + person_id TEXT NOT NULL REFERENCES people(id) ON DELETE CASCADE, + ts INTEGER NOT NULL, -- unix seconds + is_outbound INTEGER NOT NULL, -- 1 = user sent, 0 = received + length INTEGER NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS interactions_person_idx ON interactions(person_id, ts DESC); +CREATE INDEX IF NOT EXISTS interactions_ts_idx ON interactions(ts DESC); diff --git a/core/src/people/mod.rs b/core/src/people/mod.rs new file mode 100644 index 0000000..a0ee412 --- /dev/null +++ b/core/src/people/mod.rs @@ -0,0 +1,18 @@ +//! People: contact resolution + scoring. +//! +//! A5 module. Deterministic resolver maps (imessage handle | email | display +//! name) to a stable `PersonId`. Scoring blends recency × frequency × +//! reciprocity × depth from interaction rows into a ranked `people.list`. +//! +//! Intentionally self-contained: no dependency on `life_capture`, +//! `chronicle`, `nudges`, or UI. Integration happens in later slices. + +pub mod address_book; +pub mod migrations; +pub mod resolver; +pub mod scorer; +pub mod store; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/core/src/people/resolver.rs b/core/src/people/resolver.rs new file mode 100644 index 0000000..bb53512 --- /dev/null +++ b/core/src/people/resolver.rs @@ -0,0 +1,527 @@ +//! HandleResolver — deterministic mapping (Handle) → PersonId. +//! +//! Given the same store contents, resolving the same handle twice returns +//! the same `PersonId`. If the handle is unknown and `create_if_missing` +//! is set, the resolver mints a new `PersonId`, inserts a `Person` skeleton +//! with the handle attached, and returns the new id. +//! +//! `seed_from_address_book` wires the `address_book` read path into the +//! resolver so that contacts from the system address book are pre-populated +//! as `Person` rows (and their handles are registered for future resolution). + +use chrono::Utc; + +use crate::people::address_book::{self, AddressBookError, ContactsSource}; +use crate::people::store::PeopleStore; +use crate::people::types::{Handle, Person, PersonId}; + +pub struct HandleResolver<'a> { + store: &'a PeopleStore, +} + +impl<'a> HandleResolver<'a> { + pub fn new(store: &'a PeopleStore) -> Self { + Self { store } + } + + /// Look up the person for a handle. Returns `None` if unknown. + pub async fn resolve(&self, handle: &Handle) -> Result, String> { + let canonical = handle.canonicalize(); + self.store + .lookup(&canonical) + .await + .map_err(|e| format!("lookup: {e}")) + } + + /// Look up or mint. Display-name / email fields on the newly-minted + /// `Person` are populated from the handle itself so the UI has + /// something to render before any enrichment runs. + pub async fn resolve_or_create(&self, handle: &Handle) -> Result { + self.resolve_or_create_with_status(handle) + .await + .map(|(id, _created)| id) + } + + pub async fn resolve_or_create_with_status( + &self, + handle: &Handle, + ) -> Result<(PersonId, bool), String> { + let canonical = handle.canonicalize(); + let id = PersonId::new(); + let (display_name, primary_email, primary_phone) = match &canonical { + Handle::DisplayName(s) => (Some(s.clone()), None, None), + Handle::Email(s) => (None, Some(s.clone()), None), + Handle::IMessage(s) => { + if s.contains('@') { + (None, Some(s.clone()), None) + } else { + (None, None, Some(s.clone())) + } + } + }; + let now = Utc::now(); + let person = Person { + id, + display_name, + primary_email, + primary_phone, + handles: vec![canonical.clone()], + created_at: now, + updated_at: now, + }; + self.store + .resolve_or_insert_person(&person, &canonical) + .await + .map_err(|e| format!("resolve_or_insert_person: {e}")) + } + + /// Merge: attach `other` as an alias on the person `primary` resolves to. + /// Useful for the sync path that learns "this email and this phone + /// belong to the same contact". + pub async fn link(&self, primary: &Handle, other: Handle) -> Result { + let pid = self.resolve_or_create(primary).await?; + let other = other.canonicalize(); + self.store + .add_alias(pid, other) + .await + .map_err(|e| format!("add_alias: {e}"))?; + Ok(pid) + } + + /// Seed the people store from the system address book. + /// + /// For each contact returned by `source`: + /// - Pick the first email or phone as the "primary" handle and look it + /// up or mint a `PersonId`. + /// - Link any additional emails / phones as aliases on the same person. + /// - If only a display name is present, mint via display name. + /// + /// Contacts that produce no handles at all are skipped. This is + /// idempotent: re-running on the same contact list is a no-op because + ///`lookup` finds existing handle rows. + /// + /// Returns `(seeded, skipped)` counts, and propagates `AddressBookError` + /// to let callers distinguish permission-denied from other failures. + pub async fn seed_from_address_book( + &self, + source: &dyn ContactsSource, + ) -> Result<(usize, usize), AddressBookError> { + let contacts = address_book::read_with(source)?; + let mut seeded = 0usize; + let mut skipped = 0usize; + + for c in contacts { + // Build a flat list of all handles for this contact. + let mut handles: Vec = Vec::new(); + for email in &c.emails { + let trimmed = email.trim(); + if !trimmed.is_empty() { + handles.push(Handle::Email(trimmed.to_string())); + } + } + for phone in &c.phones { + let trimmed = phone.trim(); + if !trimmed.is_empty() { + handles.push(Handle::IMessage(trimmed.to_string())); + } + } + if let Some(ref name) = c.display_name { + let trimmed = name.trim(); + if !trimmed.is_empty() { + handles.push(Handle::DisplayName(trimmed.to_string())); + } + } + + if handles.is_empty() { + skipped += 1; + continue; + } + + // The "primary" handle is the first email if present, otherwise + // the first phone, otherwise the display name. This gives the + // most stable link target for future interactions. + let primary = handles[0].clone(); + + // mint or look up the primary handle + match self.resolve_or_create(&primary).await { + Err(e) => { + tracing::warn!( + "[people::resolver] seed_from_address_book: failed to upsert primary handle {:?}: {e}", + primary.as_key() + ); + skipped += 1; + continue; + } + Ok(pid) => { + // link all additional handles as aliases + for alias in handles.into_iter().skip(1) { + if let Err(e) = self.store.add_alias(pid, alias.canonicalize()).await { + tracing::warn!( + "[people::resolver] seed_from_address_book: add_alias failed: {e}" + ); + } + } + seeded += 1; + } + } + } + + tracing::debug!( + "[people::resolver] seed_from_address_book done: seeded={seeded} skipped={skipped}" + ); + Ok((seeded, skipped)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::people::address_book::tests::MockContactsSource; + use crate::people::types::AddressBookContact; + + #[tokio::test] + async fn resolve_returns_none_for_unknown_handle() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + let got = r.resolve(&Handle::Email("x@y.z".into())).await.unwrap(); + assert!(got.is_none()); + } + + #[tokio::test] + async fn resolve_or_create_is_deterministic_across_case_and_whitespace() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + let a = r + .resolve_or_create(&Handle::Email("Sarah@Example.COM".into())) + .await + .unwrap(); + let b = r + .resolve_or_create(&Handle::Email(" sarah@example.com ".into())) + .await + .unwrap(); + assert_eq!(a, b, "canonicalization must collapse case+whitespace"); + } + + #[tokio::test] + async fn concurrent_resolve_or_create_returns_one_database_id() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + let handles: Vec<_> = (0..16) + .map(|_| Handle::Email("Race@Example.COM".into())) + .collect(); + + let ids = futures::future::join_all(handles.iter().map(|h| r.resolve_or_create(h))).await; + let first = ids[0].as_ref().unwrap(); + for id in &ids { + assert_eq!(id.as_ref().unwrap(), first); + } + + let people = s.list().await.unwrap(); + assert_eq!(people.len(), 1); + assert_eq!(people[0].id, *first); + } + + #[tokio::test] + async fn same_email_different_display_name_resolve_same_id() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + let via_email = r + .resolve_or_create(&Handle::Email("a@b.c".into())) + .await + .unwrap(); + // Linking a display name to the same email must not mint a second id. + let via_linked = r + .link( + &Handle::Email("a@b.c".into()), + Handle::DisplayName("Alice".into()), + ) + .await + .unwrap(); + assert_eq!(via_email, via_linked); + // And now resolving the display name returns the same id. + let via_name = r + .resolve(&Handle::DisplayName("Alice".into())) + .await + .unwrap(); + assert_eq!(via_name, Some(via_email)); + } + + #[tokio::test] + async fn distinct_handles_without_linking_produce_distinct_ids() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + let a = r + .resolve_or_create(&Handle::Email("a@b.c".into())) + .await + .unwrap(); + let b = r + .resolve_or_create(&Handle::Email("x@y.z".into())) + .await + .unwrap(); + assert_ne!(a, b); + } + + #[tokio::test] + async fn seed_from_address_book_populates_store() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + let source = MockContactsSource::ok(vec![ + AddressBookContact { + display_name: Some("Alice Smith".into()), + emails: vec!["alice@example.com".into()], + phones: vec!["+1 555 000 0001".into()], + }, + AddressBookContact { + display_name: Some("Bob Jones".into()), + emails: vec!["bob@example.com".into()], + phones: vec![], + }, + ]); + + let (seeded, skipped) = r.seed_from_address_book(&source).await.unwrap(); + assert_eq!(seeded, 2, "both contacts should be seeded"); + assert_eq!(skipped, 0); + + // Alice is resolvable by email + let alice_id = r + .resolve(&Handle::Email("alice@example.com".into())) + .await + .unwrap(); + assert!(alice_id.is_some(), "alice must be resolvable after seed"); + + // Alice is also resolvable by phone (linked as alias) + let alice_via_phone = r + .resolve(&Handle::IMessage("+1 555 000 0001".into())) + .await + .unwrap(); + assert_eq!( + alice_id, alice_via_phone, + "email and phone must resolve to same person" + ); + + // Bob is resolvable + let bob_id = r + .resolve(&Handle::Email("bob@example.com".into())) + .await + .unwrap(); + assert!(bob_id.is_some()); + assert_ne!(alice_id, bob_id, "distinct contacts must have distinct ids"); + } + + #[tokio::test] + async fn seed_from_address_book_permission_denied_is_propagated() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + let source = MockContactsSource::permission_denied(); + let err = r.seed_from_address_book(&source).await.unwrap_err(); + assert_eq!(err, AddressBookError::PermissionDenied); + + // Store must still be empty — no partial writes. + let people = s.list().await.unwrap(); + assert!( + people.is_empty(), + "no people should be inserted on permission denied" + ); + } + + #[tokio::test] + async fn seed_is_idempotent() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + let source = MockContactsSource::ok(vec![AddressBookContact { + display_name: Some("Carol".into()), + emails: vec!["carol@example.com".into()], + phones: vec![], + }]); + + let (s1, _) = r.seed_from_address_book(&source).await.unwrap(); + let (s2, _) = r.seed_from_address_book(&source).await.unwrap(); + assert_eq!(s1, 1); + assert_eq!(s2, 1, "second seed call should still report 1 (upsert)"); + + // Only one person in store. + let people = s.list().await.unwrap(); + assert_eq!(people.len(), 1, "idempotent — must not duplicate"); + } + + #[tokio::test] + async fn contact_with_only_display_name_is_seeded() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + let source = MockContactsSource::ok(vec![AddressBookContact { + display_name: Some("No Email Person".into()), + emails: vec![], + phones: vec![], + }]); + let (seeded, skipped) = r.seed_from_address_book(&source).await.unwrap(); + assert_eq!(seeded, 1); + assert_eq!(skipped, 0); + } + + #[tokio::test] + async fn contact_with_no_fields_is_skipped() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + let source = MockContactsSource::ok(vec![AddressBookContact { + display_name: None, + emails: vec![], + phones: vec![], + }]); + let (seeded, skipped) = r.seed_from_address_book(&source).await.unwrap(); + assert_eq!(seeded, 0); + assert_eq!(skipped, 1); + } + + // ── Cross-source merge safety tests (issue#1538) ────────────────────────── + // + // The people resolver must NOT silently merge two distinct identities that + // happen to share only a display name or only an unverified handle from + // different sources. These tests lock in the "ambiguous cross-source" + // contract: two handles from unrelated sources remain distinct unless + // explicitly linked via `link()`. + + /// Two contacts that share only a display name (no email or phone overlap) + /// must NOT be merged — they may be homonymous individuals. + #[tokio::test] + async fn same_display_name_from_different_sources_does_not_merge() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + // Source A — email-backed identity + let id_a = r + .resolve_or_create(&Handle::Email("alice@company-a.com".into())) + .await + .unwrap(); + r.link( + &Handle::Email("alice@company-a.com".into()), + Handle::DisplayName("Alice Smith".into()), + ) + .await + .unwrap(); + + // Source B — different email; the same display name surfaces again, + // but as a *separate* DisplayName-backed mint (NOT linked to either + // email). This is the actual collision scenario: two ingestion paths + // both encounter "Alice Smith" without any cross-source identifier. + let id_b = r + .resolve_or_create(&Handle::Email("alice@company-b.com".into())) + .await + .unwrap(); + // The display-name resolver must already pin to id_a (linked above), + // so a second mint of the same DisplayName does NOT spawn a third + // identity — but crucially it also does NOT silently merge id_b into id_a. + let id_name_again = r + .resolve_or_create(&Handle::DisplayName("Alice Smith".into())) + .await + .unwrap(); + + // The two email-backed identities must be distinct. + assert_ne!( + id_a, id_b, + "two email handles with identical display names must not be merged without explicit link" + ); + + // The repeated DisplayName mint resolves to the linked identity (id_a), + // NOT to id_b. If display names auto-merged, id_b would have collapsed + // into id_a; if they minted fresh on every call, this would be a third id. + assert_eq!( + id_name_again, id_a, + "repeated DisplayName mint should resolve to the existing linked identity" + ); + assert_ne!( + id_name_again, id_b, + "DisplayName collision must not silently merge id_b into id_a" + ); + + // Resolving the display name returns the ONE identity that was explicitly linked. + let via_name = r + .resolve(&Handle::DisplayName("Alice Smith".into())) + .await + .unwrap(); + assert_eq!( + via_name, + Some(id_a), + "display name resolves to the explicitly linked identity" + ); + + // company-b Alice is still addressable by email only. + let via_b_email = r + .resolve(&Handle::Email("alice@company-b.com".into())) + .await + .unwrap(); + assert_eq!(via_b_email, Some(id_b)); + } + + /// Minting the same email handle from two logically distinct call sites + /// must always collapse to one `PersonId` (idempotent mint). This is the + /// safe side of cross-source: we never mint duplicates for an identical + /// canonical handle. + #[tokio::test] + async fn same_email_from_two_sources_collapses_to_one_person() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + // Simulate two different ingestion paths (gmail vs slack) that both + // surface the same email address. + let from_gmail = r + .resolve_or_create(&Handle::Email("shared@example.com".into())) + .await + .unwrap(); + let from_slack = r + .resolve_or_create(&Handle::Email("shared@example.com".into())) + .await + .unwrap(); + + assert_eq!( + from_gmail, from_slack, + "identical canonical email from two ingestion paths must resolve to one PersonId" + ); + + // Exactly one person in the store. + let people = s.list().await.unwrap(); + assert_eq!( + people.len(), + 1, + "no duplicate person rows must exist for the same canonical email" + ); + } + + /// An iMessage phone handle from one source and an email from a different + /// source for the SAME real person must stay distinct until explicitly linked. + /// Memory must not unsafely merge the same person's identities across sources + /// (issue#1538). + #[tokio::test] + async fn phone_and_email_from_different_sources_are_not_merged_without_link() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + // iMessage source sees only a phone. + let id_phone = r + .resolve_or_create(&Handle::IMessage("+15550001234".into())) + .await + .unwrap(); + + // Gmail source sees only an email. + let id_email = r + .resolve_or_create(&Handle::Email("sam@example.com".into())) + .await + .unwrap(); + + // Without an explicit link these are separate identities. This is the + // contract under test — cross-source handles for the same real person + // must NOT auto-merge. Asserting post-link merge semantics is out of + // scope: link()'s exact propagation rule (does the email handle + // afterwards canonically resolve to the phone PersonId, or remain + // independent with only the link table updated?) is a separate + // behavior tested in store_tests.rs. + assert_ne!( + id_phone, id_email, + "phone and email from unrelated sources must not be auto-merged" + ); + } +} diff --git a/core/src/people/scorer.rs b/core/src/people/scorer.rs new file mode 100644 index 0000000..dc9745f --- /dev/null +++ b/core/src/people/scorer.rs @@ -0,0 +1,210 @@ +//! Scoring: recency × frequency × reciprocity × depth. +//! +//! Each component is deterministic given the same interaction list + `now` +//! timestamp, and each is clamped to `[0,1]`. The composite is the product; +//! clamping the product is redundant but kept for defense-in-depth. +//! +//! Weights (half-life / caps) are module constants so tests are stable. +//! They can move to config later without breaking the API. + +use chrono::{DateTime, Utc}; + +use crate::people::types::{Interaction, ScoreComponents}; + +/// Recency half-life in days. An interaction this many days old contributes +/// 0.5 to the recency signal; older interactions decay exponentially. +pub const RECENCY_HALF_LIFE_DAYS: f32 = 14.0; + +/// Frequency is measured within this rolling window (days). Only interactions +/// more recent than `now - FREQUENCY_WINDOW_DAYS` count toward frequency. +pub const FREQUENCY_WINDOW_DAYS: u32 = 30; + +/// Frequency saturates at this many interactions inside `FREQUENCY_WINDOW_DAYS`. +/// 50+ qualifying interactions yields frequency = 1.0. +pub const FREQUENCY_CAP: f32 = 50.0; + +/// Depth saturates when the mean message length reaches this many chars. +pub const DEPTH_CAP_CHARS: f32 = 500.0; + +/// Compute component scores for a person given their interaction list. +/// `now` is passed in so tests can fix time. +pub fn score(interactions: &[Interaction], now: DateTime) -> ScoreComponents { + if interactions.is_empty() { + return ScoreComponents { + recency: 0.0, + frequency: 0.0, + reciprocity: 0.0, + depth: 0.0, + score: 0.0, + }; + } + + // Recency: highest-signal (= most recent) interaction drives the score. + let newest = interactions.iter().map(|i| i.ts).max().unwrap_or(now); + let age_days = ((now - newest).num_seconds() as f32 / 86_400.0).max(0.0); + let recency = (-(age_days * 2f32.ln() / RECENCY_HALF_LIFE_DAYS)) + .exp() + .clamp(0.0, 1.0); + + // Frequency: count within the rolling window, saturated at FREQUENCY_CAP. + // Using a window (rather than total-ever) prevents an old burst of + // messages from inflating the score of a now-silent contact. + let window_cutoff = now - chrono::Duration::days(FREQUENCY_WINDOW_DAYS as i64); + let window_count = interactions + .iter() + .filter(|i| i.ts >= window_cutoff) + .count() as f32; + let frequency = (window_count / FREQUENCY_CAP).clamp(0.0, 1.0); + + // Reciprocity: balance of outbound vs inbound — perfect balance = 1.0, + // all-one-direction = 0.0. Uses all interactions (not windowed) so that + // the long-term pattern is captured even when recent volume is low. + let (out_n, in_n) = interactions.iter().fold((0u32, 0u32), |(o, i), x| { + if x.is_outbound { + (o + 1, i) + } else { + (o, i + 1) + } + }); + let reciprocity = if out_n + in_n == 0 { + 0.0 + } else { + let o = out_n as f32; + let i = in_n as f32; + let min = o.min(i); + let max = o.max(i); + (min / max).clamp(0.0, 1.0) + }; + + // Depth: mean interaction length, saturated at DEPTH_CAP_CHARS. + let count = interactions.len() as f32; + let total_len: u64 = interactions.iter().map(|x| x.length as u64).sum(); + let mean_len = total_len as f32 / count.max(1.0); + let depth = (mean_len / DEPTH_CAP_CHARS).clamp(0.0, 1.0); + + let composite = (recency * frequency * reciprocity * depth).clamp(0.0, 1.0); + + ScoreComponents { + recency, + frequency, + reciprocity, + depth, + score: composite, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::people::types::PersonId; + use chrono::Duration; + + fn mk(ts: DateTime, outbound: bool, length: u32) -> Interaction { + Interaction { + person_id: PersonId::new(), + ts, + is_outbound: outbound, + length, + } + } + + #[test] + fn empty_interactions_score_zero() { + let s = score(&[], Utc::now()); + assert_eq!(s.score, 0.0); + assert_eq!(s.recency, 0.0); + assert_eq!(s.frequency, 0.0); + } + + #[test] + fn recency_half_life_matches_config() { + let now = Utc::now(); + let half_ago = now - Duration::days(RECENCY_HALF_LIFE_DAYS as i64); + let s = score(&[mk(half_ago, true, 100)], now); + // Half-life point → recency ≈ 0.5 (allow small float slack). + assert!((s.recency - 0.5).abs() < 0.05, "got {}", s.recency); + } + + #[test] + fn all_components_clamped_to_unit_interval() { + let now = Utc::now(); + let interactions: Vec = (0..200) + .map(|i| mk(now - Duration::hours(i), i % 2 == 0, 10_000)) + .collect(); + let s = score(&interactions, now); + for c in [s.recency, s.frequency, s.reciprocity, s.depth, s.score] { + assert!((0.0..=1.0).contains(&c), "component out of range: {c}"); + } + // 200 interactions all within a few days → window_count ≥ FREQUENCY_CAP + assert_eq!(s.frequency, 1.0); + assert_eq!(s.depth, 1.0); + } + + #[test] + fn one_sided_conversation_has_zero_reciprocity() { + let now = Utc::now(); + let v: Vec<_> = (0..5) + .map(|i| mk(now - Duration::hours(i), true, 100)) + .collect(); + let s = score(&v, now); + assert_eq!(s.reciprocity, 0.0); + assert_eq!( + s.score, 0.0, + "composite must be zero when any factor is zero" + ); + } + + #[test] + fn deterministic_given_same_inputs() { + let now = Utc::now(); + let v = vec![ + mk(now - Duration::days(1), true, 100), + mk(now - Duration::days(2), false, 150), + mk(now - Duration::days(3), true, 200), + ]; + let a = score(&v, now); + let b = score(&v, now); + assert_eq!(a.score, b.score); + assert_eq!(a.recency, b.recency); + } + + #[test] + fn old_burst_does_not_inflate_frequency_score() { + // 100 interactions from 90 days ago (outside FREQUENCY_WINDOW_DAYS=30) + // should contribute 0 to frequency; 1 interaction today should give + // 1/FREQUENCY_CAP. + let now = Utc::now(); + let mut v: Vec = (0..100) + .map(|i| mk(now - Duration::days(90 + i), true, 100)) + .collect(); + // Add one recent interaction to avoid zero reciprocity forcing score=0 + v.push(mk(now - Duration::hours(1), false, 100)); + let s = score(&v, now); + // Only 1 interaction falls within the 30-day window. + let expected_frequency = 1.0 / FREQUENCY_CAP; + assert!( + (s.frequency - expected_frequency).abs() < 0.001, + "frequency should be {expected_frequency}, got {}", + s.frequency + ); + } + + #[test] + fn interactions_exactly_at_window_boundary_are_included() { + let now = Utc::now(); + // Interaction exactly FREQUENCY_WINDOW_DAYS ago — should be included + // (boundary is inclusive via >=). + let boundary = now - Duration::days(FREQUENCY_WINDOW_DAYS as i64); + let v = vec![ + mk(boundary, true, 100), + mk(now - Duration::hours(1), false, 100), + ]; + let s = score(&v, now); + let expected = 2.0 / FREQUENCY_CAP; + assert!( + (s.frequency - expected).abs() < 0.001, + "expected {expected} got {}", + s.frequency + ); + } +} diff --git a/core/src/people/store.rs b/core/src/people/store.rs new file mode 100644 index 0000000..2ceec52 --- /dev/null +++ b/core/src/people/store.rs @@ -0,0 +1,653 @@ +//! SQLite-backed store for people + handle aliases + interactions. +//! +//! Connection is wrapped in `Arc>` so handlers and tests +//! can share ownership across tokio tasks; operations are synchronous and +//! fast (all single-row CRUD or small aggregates). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock, RwLock}; + +use chrono::{DateTime, TimeZone, Utc}; +use rusqlite::{params, Connection, OptionalExtension, Result as SqlResult}; +use tokio::sync::Mutex; + +use crate::people::migrations; +use crate::people::types::{Handle, Interaction, Person, PersonId}; + +pub type ConnHandle = Arc>; +type PersonRow = ( + String, + Option, + Option, + Option, + i64, + i64, +); + +/// Process-global handle to the `PeopleStore`, tagged with the workspace it is +/// bound to. Controller handlers are free functions with no `&self`, so they +/// fetch the store via `get()`. Seeded at core boot and re-bound on active-user +/// switch via [`init_from_workspace`]. Absent at test time unless a test seeds +/// it; most tests construct stores directly with `open_in_memory`. +#[derive(Clone)] +struct GlobalPeopleStore { + workspace_dir: PathBuf, + store: Arc, +} + +type GlobalStoreSlot = RwLock>; + +static GLOBAL: OnceLock = OnceLock::new(); + +fn global_slot() -> &'static GlobalStoreSlot { + GLOBAL.get_or_init(GlobalStoreSlot::default) +} + +/// Initialise or re-bind the process-global people store from a workspace +/// directory, opening `/people/people.db` (schema migrations run on +/// open). +/// +/// Mirrors [`crate::global::init`]: safe to call repeatedly. +/// A call for the **same** workspace returns the existing store; a call for a +/// **different** workspace replaces the global handle so a post-login +/// active-user switch (or `restart_core_process`, which restarts the embedded +/// core in the same Tauri process) does not keep people controllers/tools +/// reading and writing the pre-login (or a previous user's) workspace. +/// +/// Wired into core boot (`src/core/jsonrpc.rs`) and the active-user rebind +/// sites (`credentials::ops`, `app_state::ops`) alongside `memory::global`. +/// Without the boot seed every people controller / `people_*` tool fails with +/// "people store not initialised" (Sentry TAURI-RUST-8NM); without the rebind +/// they'd write the wrong workspace after login (#4378). +pub fn init_from_workspace(workspace_dir: &Path) -> Result, String> { + let slot = global_slot(); + if let Some(existing) = slot + .read() + .map_err(|e| format!("[people:store] read lock poisoned: {e}"))? + .as_ref() + { + if existing.workspace_dir == workspace_dir { + log::debug!("[people:store] already initialised for current workspace"); + return Ok(Arc::clone(&existing.store)); + } + } + + let db_path = workspace_dir.join("people").join("people.db"); + let store = Arc::new( + PeopleStore::open_at(&db_path).map_err(|e| format!("people store open failed: {e}"))?, + ); + + let mut guard = slot + .write() + .map_err(|e| format!("[people:store] write lock poisoned: {e}"))?; + // Re-check under the write lock: a concurrent caller may have seeded the + // same workspace while we were opening — reuse theirs. A different-workspace + // entry is replaced (rebind). + if let Some(existing) = guard.as_ref() { + if existing.workspace_dir == workspace_dir { + return Ok(Arc::clone(&existing.store)); + } + } + log::info!( + "[people:store] bound store workspace={}", + workspace_dir.display() + ); + *guard = Some(GlobalPeopleStore { + workspace_dir: workspace_dir.to_path_buf(), + store: Arc::clone(&store), + }); + Ok(store) +} + +pub fn get() -> Result, &'static str> { + global_slot() + .read() + .ok() + .and_then(|guard| guard.as_ref().map(|entry| Arc::clone(&entry.store))) + .ok_or("people store not initialised — core startup hasn't completed") +} + +/// Per-workspace store cache keyed by workspace dir. Backs [`for_workspace`], +/// the context-scoped accessor (the host's context-scoped `CoreContext::people`). +/// Distinct from the single `GLOBAL` slot above (which tracks the one +/// active-user workspace for the legacy free-function handlers): this map lets +/// multiple workspaces' stores coexist in one process, which is what per-context +/// isolation (Phase 3) needs. +static STORES: OnceLock>>> = + OnceLock::new(); + +/// Open (or return the cached) people store for a specific workspace dir. Unlike +/// [`get`], this is not tied to the single active-user global — two different +/// workspaces resolve to two isolated stores, and the same workspace always +/// resolves to the same cached `Arc`. Opening `/people/people.db` +/// runs schema migrations. +pub fn for_workspace(workspace_dir: &Path) -> Result, String> { + let cache = STORES.get_or_init(Default::default); + if let Some(store) = cache + .read() + .map_err(|e| format!("[people:store] cache read lock poisoned: {e}"))? + .get(workspace_dir) + { + return Ok(Arc::clone(store)); + } + + let db_path = workspace_dir.join("people").join("people.db"); + let store = Arc::new( + PeopleStore::open_at(&db_path).map_err(|e| format!("people store open failed: {e}"))?, + ); + + let mut guard = cache + .write() + .map_err(|e| format!("[people:store] cache write lock poisoned: {e}"))?; + // Re-check under the write lock: a concurrent caller may have opened the + // same workspace while we were opening — reuse theirs so callers always + // share one store per workspace. + let entry = guard + .entry(workspace_dir.to_path_buf()) + .or_insert_with(|| Arc::clone(&store)); + Ok(Arc::clone(entry)) +} + +pub struct PeopleStore { + pub conn: ConnHandle, +} + +impl PeopleStore { + pub fn open_in_memory() -> SqlResult { + let conn = Connection::open_in_memory()?; + migrations::run(&conn)?; + Ok(Self { + conn: Arc::new(Mutex::new(conn)), + }) + } + + pub fn open_at(path: &std::path::Path) -> SqlResult { + if let Some(parent) = path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let conn = Connection::open(path)?; + migrations::run(&conn)?; + Ok(Self { + conn: Arc::new(Mutex::new(conn)), + }) + } + + /// Insert a new person and its initial set of handles, atomically. + pub async fn insert_person(&self, person: &Person, handles: &[Handle]) -> SqlResult<()> { + let conn = self.conn.clone(); + let person = person.clone(); + let handles: Vec = handles.iter().map(|h| h.canonicalize()).collect(); + tokio::task::spawn_blocking(move || { + let mut guard = conn.blocking_lock(); + let tx = guard.transaction()?; + tx.execute( + "INSERT INTO people(id, display_name, primary_email, primary_phone, created_at, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + person.id.to_string(), + person.display_name, + person.primary_email, + person.primary_phone, + person.created_at.timestamp(), + person.updated_at.timestamp(), + ], + )?; + for h in &handles { + let (kind, value) = h.as_key(); + tx.execute( + "INSERT OR IGNORE INTO handle_aliases(kind, value, person_id, created_at) \ + VALUES (?1, ?2, ?3, CAST(strftime('%s','now') AS INTEGER))", + params![kind, value, person.id.to_string()], + )?; + } + tx.commit() + }) + .await + .map_err(|e| rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ))? + } + + /// Resolve an existing canonical handle or insert a new person and alias + /// under one connection lock. Returns the database-authoritative id plus + /// whether this call created the row. + pub async fn resolve_or_insert_person( + &self, + person: &Person, + handle: &Handle, + ) -> SqlResult<(PersonId, bool)> { + let conn = self.conn.clone(); + let person = person.clone(); + let handle = handle.canonicalize(); + tokio::task::spawn_blocking(move || -> SqlResult<(PersonId, bool)> { + let mut guard = conn.blocking_lock(); + let tx = guard.transaction()?; + let (kind, value) = handle.as_key(); + let existing: Option = tx + .query_row( + "SELECT person_id FROM handle_aliases WHERE kind = ?1 AND value = ?2", + params![kind, value], + |row| row.get(0), + ) + .optional()?; + if let Some(id) = existing { + let id = uuid::Uuid::parse_str(&id) + .map(PersonId) + .map_err(|e| rusqlite::Error::InvalidColumnName(e.to_string()))?; + return Ok((id, false)); + } + + tx.execute( + "INSERT INTO people(id, display_name, primary_email, primary_phone, created_at, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + person.id.to_string(), + person.display_name, + person.primary_email, + person.primary_phone, + person.created_at.timestamp(), + person.updated_at.timestamp(), + ], + )?; + tx.execute( + "INSERT INTO handle_aliases(kind, value, person_id, created_at) \ + VALUES (?1, ?2, ?3, CAST(strftime('%s','now') AS INTEGER))", + params![kind, value, person.id.to_string()], + )?; + tx.commit()?; + Ok((person.id, true)) + }) + .await + .map_err(|e| { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ) + })? + } + + /// Attach a handle alias to an existing person. Idempotent via + /// `INSERT OR IGNORE` on `(kind, value)`. + pub async fn add_alias(&self, person_id: PersonId, handle: Handle) -> SqlResult<()> { + let conn = self.conn.clone(); + let handle = handle.canonicalize(); + tokio::task::spawn_blocking(move || { + let guard = conn.blocking_lock(); + let (kind, value) = handle.as_key(); + guard.execute( + "INSERT OR IGNORE INTO handle_aliases(kind, value, person_id, created_at) \ + VALUES (?1, ?2, ?3, CAST(strftime('%s','now') AS INTEGER))", + params![kind, value, person_id.to_string()], + )?; + Ok(()) + }) + .await + .map_err(|e| { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ) + })? + } + + /// Resolve a canonicalized handle to a `PersonId`, or `None` if unknown. + pub async fn lookup(&self, handle: &Handle) -> SqlResult> { + let conn = self.conn.clone(); + let handle = handle.canonicalize(); + tokio::task::spawn_blocking(move || { + let guard = conn.blocking_lock(); + let (kind, value) = handle.as_key(); + let id: Option = guard + .query_row( + "SELECT person_id FROM handle_aliases WHERE kind = ?1 AND value = ?2", + params![kind, value], + |row| row.get(0), + ) + .optional()?; + Ok(id.and_then(|s| uuid::Uuid::parse_str(&s).ok().map(PersonId))) + }) + .await + .map_err(|e| { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ) + })? + } + + /// Load a person and all their aliases. + pub async fn get(&self, person_id: PersonId) -> SqlResult> { + let conn = self.conn.clone(); + tokio::task::spawn_blocking(move || -> SqlResult> { + let guard = conn.blocking_lock(); + let row: Option = + guard + .query_row( + "SELECT id, display_name, primary_email, primary_phone, created_at, updated_at \ + FROM people WHERE id = ?1", + params![person_id.to_string()], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?)), + ) + .optional()?; + let Some((id_str, display_name, primary_email, primary_phone, created, updated)) = row + else { + return Ok(None); + }; + let id = uuid::Uuid::parse_str(&id_str) + .map(PersonId) + .map_err(|e| rusqlite::Error::InvalidColumnName(e.to_string()))?; + let handles = load_handles(&guard, &id)?; + Ok(Some(Person { + id, + display_name, + primary_email, + primary_phone, + handles, + created_at: ts_to_dt(created), + updated_at: ts_to_dt(updated), + })) + }) + .await + .map_err(|e| rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ))? + } + + /// List all people (unordered — scorer applies ranking separately). + pub async fn list(&self) -> SqlResult> { + let conn = self.conn.clone(); + tokio::task::spawn_blocking(move || -> SqlResult> { + let guard = conn.blocking_lock(); + let mut stmt = guard.prepare( + "SELECT id, display_name, primary_email, primary_phone, created_at, updated_at \ + FROM people ORDER BY display_name", + )?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, Option>(1)?, + r.get::<_, Option>(2)?, + r.get::<_, Option>(3)?, + r.get::<_, i64>(4)?, + r.get::<_, i64>(5)?, + )) + })?; + let mut out = Vec::new(); + for r in rows { + let (id_str, display_name, primary_email, primary_phone, created, updated) = r?; + let id = uuid::Uuid::parse_str(&id_str) + .map(PersonId) + .map_err(|e| rusqlite::Error::InvalidColumnName(e.to_string()))?; + let handles = load_handles(&guard, &id)?; + out.push(Person { + id, + display_name, + primary_email, + primary_phone, + handles, + created_at: ts_to_dt(created), + updated_at: ts_to_dt(updated), + }); + } + Ok(out) + }) + .await + .map_err(|e| { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ) + })? + } + + /// Record a single interaction. + pub async fn record_interaction(&self, i: Interaction) -> SqlResult<()> { + let conn = self.conn.clone(); + tokio::task::spawn_blocking(move || { + let guard = conn.blocking_lock(); + guard.execute( + "INSERT INTO interactions(person_id, ts, is_outbound, length) \ + VALUES (?1, ?2, ?3, ?4)", + params![ + i.person_id.to_string(), + i.ts.timestamp(), + if i.is_outbound { 1_i64 } else { 0_i64 }, + i.length as i64, + ], + )?; + Ok(()) + }) + .await + .map_err(|e| { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ) + })? + } + + /// Fetch all interactions for a person, newest first. + pub async fn interactions_for(&self, person_id: PersonId) -> SqlResult> { + let conn = self.conn.clone(); + tokio::task::spawn_blocking(move || -> SqlResult> { + let guard = conn.blocking_lock(); + let mut stmt = guard.prepare( + "SELECT ts, is_outbound, length FROM interactions \ + WHERE person_id = ?1 ORDER BY ts DESC", + )?; + let rows = stmt.query_map(params![person_id.to_string()], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, i64>(1)?, + r.get::<_, i64>(2)?, + )) + })?; + let mut out = Vec::new(); + for r in rows { + let (ts, is_out, length) = r?; + out.push(Interaction { + person_id, + ts: ts_to_dt(ts), + is_outbound: is_out != 0, + length: length.max(0) as u32, + }); + } + Ok(out) + }) + .await + .map_err(|e| { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ) + })? + } + + /// Fetch interactions for several people in one query, keyed by person id. + pub async fn batch_interactions_for( + &self, + person_ids: &[PersonId], + ) -> SqlResult>> { + if person_ids.is_empty() { + return Ok(HashMap::new()); + } + let conn = self.conn.clone(); + let ids: Vec = person_ids.to_vec(); + tokio::task::spawn_blocking(move || -> SqlResult>> { + let guard = conn.blocking_lock(); + let placeholders = std::iter::repeat_n("?", ids.len()) + .collect::>() + .join(","); + let sql = format!( + "SELECT person_id, ts, is_outbound, length FROM interactions \ + WHERE person_id IN ({placeholders}) ORDER BY person_id, ts DESC" + ); + let id_strings: Vec = ids.iter().map(ToString::to_string).collect(); + let mut stmt = guard.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(id_strings.iter()), |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, i64>(1)?, + r.get::<_, i64>(2)?, + r.get::<_, i64>(3)?, + )) + })?; + let mut out: HashMap> = HashMap::new(); + for r in rows { + let (id_str, ts, is_out, length) = r?; + let person_id = uuid::Uuid::parse_str(&id_str) + .map(PersonId) + .map_err(|e| rusqlite::Error::InvalidColumnName(e.to_string()))?; + out.entry(person_id).or_default().push(Interaction { + person_id, + ts: ts_to_dt(ts), + is_outbound: is_out != 0, + length: length.max(0) as u32, + }); + } + Ok(out) + }) + .await + .map_err(|e| { + rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ffi::ErrorCode::SystemIoFailure, + extended_code: 0, + }, + Some(e.to_string()), + ) + })? + } +} + +fn load_handles(conn: &Connection, id: &PersonId) -> SqlResult> { + let mut stmt = conn.prepare( + "SELECT kind, value FROM handle_aliases WHERE person_id = ?1 ORDER BY kind, value", + )?; + let rows = stmt.query_map(params![id.to_string()], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)) + })?; + let mut out = Vec::new(); + for r in rows { + let (kind, value) = r?; + let h = match kind.as_str() { + "imessage" => Handle::IMessage(value), + "email" => Handle::Email(value), + "display_name" => Handle::DisplayName(value), + other => { + return Err(rusqlite::Error::InvalidColumnName(format!( + "unknown handle kind: {other}" + ))); + } + }; + out.push(h); + } + Ok(out) +} + +fn ts_to_dt(ts: i64) -> DateTime { + Utc.timestamp_opt(ts, 0) + .single() + .unwrap_or_else(|| Utc.timestamp_opt(0, 0).unwrap()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn insert_list_and_lookup_round_trip() { + let s = PeopleStore::open_in_memory().unwrap(); + let now = Utc::now(); + let p = Person { + id: PersonId::new(), + display_name: Some("Sarah Lee".into()), + primary_email: Some("sarah@example.com".into()), + primary_phone: None, + handles: vec![], + created_at: now, + updated_at: now, + }; + s.insert_person( + &p, + &[ + Handle::Email("Sarah@Example.com".into()), + Handle::DisplayName("Sarah Lee".into()), + ], + ) + .await + .unwrap(); + + let got = s + .lookup(&Handle::Email("sarah@example.com".into())) + .await + .unwrap(); + assert_eq!(got, Some(p.id)); + + let list = s.list().await.unwrap(); + assert_eq!(list.len(), 1); + assert_eq!(list[0].handles.len(), 2); + } + + #[tokio::test] + async fn interactions_round_trip() { + let s = PeopleStore::open_in_memory().unwrap(); + let now = Utc::now(); + let pid = PersonId::new(); + let p = Person { + id: pid, + display_name: Some("X".into()), + primary_email: None, + primary_phone: None, + handles: vec![], + created_at: now, + updated_at: now, + }; + s.insert_person(&p, &[]).await.unwrap(); + s.record_interaction(Interaction { + person_id: pid, + ts: now, + is_outbound: true, + length: 100, + }) + .await + .unwrap(); + s.record_interaction(Interaction { + person_id: pid, + ts: now, + is_outbound: false, + length: 50, + }) + .await + .unwrap(); + let ints = s.interactions_for(pid).await.unwrap(); + assert_eq!(ints.len(), 2); + } +} diff --git a/core/src/people/tests.rs b/core/src/people/tests.rs new file mode 100644 index 0000000..fc91e19 --- /dev/null +++ b/core/src/people/tests.rs @@ -0,0 +1,96 @@ +//! Cross-file integration tests for the people domain. + +use std::sync::Arc; + +use chrono::Utc; + +#[cfg(not(target_os = "macos"))] +use crate::people::address_book; +use crate::people::resolver::HandleResolver; +use crate::people::store::PeopleStore; +use crate::people::types::{Handle, PersonId}; + +#[tokio::test] +async fn resolver_and_store_cooperate_across_handle_kinds() { + let s = PeopleStore::open_in_memory().unwrap(); + let r = HandleResolver::new(&s); + + // Email mints. + let id = r + .resolve_or_create(&Handle::Email("a@b.c".into())) + .await + .unwrap(); + // iMessage handle linked to same person. + let id2 = r + .link( + &Handle::Email("a@b.c".into()), + Handle::IMessage("+15551234".into()), + ) + .await + .unwrap(); + assert_eq!(id, id2); + + // Resolving by the linked iMessage handle returns the same id. + let via_imsg = r + .resolve(&Handle::IMessage("+15551234".into())) + .await + .unwrap(); + assert_eq!(via_imsg, Some(id)); +} + +#[cfg(not(target_os = "macos"))] +#[test] +fn address_book_is_empty_on_non_mac() { + assert!(address_book::read().unwrap().is_empty()); +} + +/// Regression for Sentry TAURI-RUST-8NM (store never seeded → `get()` always +/// errored) and its #4378 follow-up (store stayed bound to the pre-login +/// workspace after an active-user switch). Verify `init_from_workspace` seeds +/// the global + creates the on-disk db, is an idempotent no-op for the same +/// workspace, and **rebinds** to a different workspace like `memory::global`. +/// +/// Serialised (not `#[tokio::test]` parallel) because it mutates the +/// process-global store slot other people tests may observe via `get()`. +#[test] +fn init_from_workspace_seeds_and_rebinds_global_store() { + use crate::people::store; + + let ws_a = tempfile::tempdir().unwrap(); + let store_a = store::init_from_workspace(ws_a.path()).unwrap(); + assert!( + ws_a.path().join("people").join("people.db").exists(), + "seed must create /people/people.db" + ); + + // Previously-dead global is now reachable — the 8NM fix. + let via_global = store::get().expect("people store reachable after seed"); + assert!(Arc::ptr_eq(&store_a, &via_global)); + + // Same workspace → idempotent no-op, returns the same instance. + let again = store::init_from_workspace(ws_a.path()).unwrap(); + assert!(Arc::ptr_eq(&store_a, &again)); + + // Different workspace (active-user switch) → rebind to a new store. #4378. + let ws_b = tempfile::tempdir().unwrap(); + let store_b = store::init_from_workspace(ws_b.path()).unwrap(); + assert!( + !Arc::ptr_eq(&store_a, &store_b), + "a new workspace must rebind to a fresh store, not reuse the old one" + ); + let after_switch = store::get().expect("people store reachable after rebind"); + assert!( + Arc::ptr_eq(&store_b, &after_switch), + "get() must return the rebound (workspace B) store after a switch" + ); +} + +#[test] +fn person_id_uuid_format() { + let id = PersonId::new(); + // Round-trips through a string. + let s = id.to_string(); + let parsed: uuid::Uuid = s.parse().unwrap(); + assert_eq!(parsed, id.0); + let _now = Utc::now(); +} diff --git a/core/src/people/types.rs b/core/src/people/types.rs new file mode 100644 index 0000000..34a0ec7 --- /dev/null +++ b/core/src/people/types.rs @@ -0,0 +1,159 @@ +//! Core types for the people domain. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Canonical, stable identifier for a person across handles. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct PersonId(pub Uuid); + +impl PersonId { + pub fn new() -> Self { + Self(Uuid::new_v4()) + } +} + +impl Default for PersonId { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for PersonId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// A handle is an opaque label by which the user or a source knows a person. +/// `IMessage(h)` is an iMessage chat handle (phone in E.164, or apple id +/// email). `Email(e)` and `DisplayName(n)` are the other two kinds the A5 +/// resolver accepts. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum Handle { + IMessage(String), + Email(String), + DisplayName(String), +} + +impl Handle { + /// Return a canonical, case-folded, whitespace-trimmed form used both + /// for storage and for the resolver lookup key. Emails are lowercased; + /// iMessage handles strip surrounding whitespace and lowercase email- + /// style handles; display names are whitespace-collapsed and trimmed. + pub fn canonicalize(&self) -> Handle { + match self { + Handle::IMessage(s) => { + let t = s.trim(); + // An apple id email handle ("foo@bar.com") is treated the + // same regardless of case; phone-style handles ("+1…") have + // no case. Lowercasing is safe for both. + Handle::IMessage(t.to_lowercase()) + } + Handle::Email(s) => Handle::Email(s.trim().to_lowercase()), + Handle::DisplayName(s) => { + let collapsed: String = s.split_whitespace().collect::>().join(" "); + Handle::DisplayName(collapsed) + } + } + } + + /// `(kind, value)` tuple suitable for use as a SQL key. + pub fn as_key(&self) -> (&'static str, &str) { + match self { + Handle::IMessage(s) => ("imessage", s.as_str()), + Handle::Email(s) => ("email", s.as_str()), + Handle::DisplayName(s) => ("display_name", s.as_str()), + } + } +} + +/// Stored representation of a person plus display metadata. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Person { + pub id: PersonId, + pub display_name: Option, + pub primary_email: Option, + pub primary_phone: Option, + pub handles: Vec, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +/// A single interaction observed with a person. The scorer aggregates +/// these. `is_outbound = true` means the user sent it; that's what drives +/// reciprocity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Interaction { + pub person_id: PersonId, + pub ts: DateTime, + pub is_outbound: bool, + /// Token or character count used as a proxy for "depth". Clamped in + /// scoring; callers may pass e.g. message body length. + pub length: u32, +} + +/// Per-component breakdown of a person-score in `[0,1]`. Exposed so that +/// callers (UI, nudge engine) can explain ranking. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct ScoreComponents { + pub recency: f32, + pub frequency: f32, + pub reciprocity: f32, + pub depth: f32, + /// Final composite score. `recency * frequency * reciprocity * depth`, + /// clamped to `[0,1]`. + pub score: f32, +} + +/// Lightweight row returned from the macOS Address Book. We keep this a +/// plain data struct so `address_book::read()` can return the same shape +/// on every OS (empty on non-mac). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AddressBookContact { + pub display_name: Option, + pub emails: Vec, + pub phones: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn handle_canonicalize_lowercases_emails_and_imessage() { + assert_eq!( + Handle::Email(" Foo@Example.COM ".into()).canonicalize(), + Handle::Email("foo@example.com".into()) + ); + assert_eq!( + Handle::IMessage("+1 (555) 123".into()).canonicalize(), + Handle::IMessage("+1 (555) 123".into()) + ); + assert_eq!( + Handle::IMessage(" Foo@Bar.com ".into()).canonicalize(), + Handle::IMessage("foo@bar.com".into()) + ); + } + + #[test] + fn handle_canonicalize_collapses_display_name_whitespace() { + assert_eq!( + Handle::DisplayName(" Sarah Lee ".into()).canonicalize(), + Handle::DisplayName("Sarah Lee".into()) + ); + } + + #[test] + fn handle_as_key_returns_correct_kind() { + assert_eq!(Handle::Email("a@b.c".into()).as_key(), ("email", "a@b.c")); + assert_eq!(Handle::IMessage("+1".into()).as_key(), ("imessage", "+1")); + assert_eq!( + Handle::DisplayName("X".into()).as_key(), + ("display_name", "X") + ); + } +} diff --git a/core/src/preferences.rs b/core/src/preferences.rs new file mode 100644 index 0000000..31f4c68 --- /dev/null +++ b/core/src/preferences.rs @@ -0,0 +1,174 @@ +//! Two-lane explicit user preferences — namespaces + read helpers. +//! +//! Preferences written by the `save_preference` tool live in one of two +//! namespaces depending on their relevance scope: +//! +//! - [`USER_PREF_GENERAL_NAMESPACE`] — always-on; injected into the system +//! prompt at thread start (Lane A). +//! - [`USER_PREF_SITUATIONAL_NAMESPACE`] — topic-scoped; recalled per-turn by +//! semantic similarity to the user's message (Lane B). +//! +//! Keeping the namespace constants and read helpers here (rather than in the +//! tool module) lets the write path, the system-prompt builder, and the +//! per-turn recall path all share one definition. + +use std::sync::Arc; + +use super::Memory; + +/// Always-on preferences — injected into the system prompt every thread. +pub const USER_PREF_GENERAL_NAMESPACE: &str = "user_pref_general"; + +/// Topic-scoped preferences — recalled per query against the user's message. +pub const USER_PREF_SITUATIONAL_NAMESPACE: &str = "user_pref_situational"; + +/// Default cap on general preferences injected into the system prompt. Keeps +/// the always-on block bounded so it can't blow a small model's context window +/// (see the legacy `gpt-4` 8K overflow). +pub const STANDING_PREFS_LIMIT: usize = 10; + +/// Load the latest-`limit` general preferences as plain-language strings, +/// newest-first (by `updated_at`). This is the Lane-A system-prompt block. +/// +/// `list()` returns entries ordered newest-first but with `content` set to the +/// title (= topic key), so the body value is fetched via `get()`. +pub async fn load_general_preferences(memory: &Arc, limit: usize) -> Vec { + let entries = memory + .list(Some(USER_PREF_GENERAL_NAMESPACE), None, None) + .await + .unwrap_or_default(); + + let mut out = Vec::new(); + for entry in entries.into_iter().take(limit) { + if let Ok(Some(full)) = memory.get(USER_PREF_GENERAL_NAMESPACE, &entry.key).await { + let value = full.content.trim(); + if !value.is_empty() { + out.push(value.to_string()); + } + } + } + out +} + +/// Top-K situational preferences to recall per turn (Lane B). +pub const SITUATIONAL_RECALL_LIMIT: usize = 5; + +/// Minimum query↔preference vector similarity for a situational preference to be +/// injected. Below this the current message isn't considered relevant to the +/// preference, so nothing is injected (the "unrelated query → no block" +/// behaviour). Tunable against live data. +pub const SITUATIONAL_MIN_SIMILARITY: f64 = 0.35; + +/// Recall situational preferences semantically relevant to `query` (Lane B). +/// +/// Returns only preferences whose vector similarity to the message clears +/// [`SITUATIONAL_MIN_SIMILARITY`], so an unrelated message yields an empty list +/// (and no injected block). Uses the model-aware embedding recall, so a stale +/// embedding-model signature is excluded rather than mis-scored. +pub async fn recall_situational_preferences(memory: &Arc, query: &str) -> Vec { + if query.trim().is_empty() { + return Vec::new(); + } + memory + .recall_relevant_by_vector( + USER_PREF_SITUATIONAL_NAMESPACE, + query, + SITUATIONAL_RECALL_LIMIT, + SITUATIONAL_MIN_SIMILARITY, + ) + .await + .unwrap_or_default() + .into_iter() + .map(|(_topic, value)| value) + .collect() +} + +/// Minimum similarity for an existing preference to be flagged as a possible +/// contradiction of a newly-saved one. Higher than the Lane-B recall floor — we +/// only surface genuinely-close matches as contradiction candidates. Tunable. +pub const CONTRADICTION_SIMILARITY: f64 = 0.6; + +/// Find existing preferences (across both lanes) semantically close to `value`, +/// excluding `exclude_topic` (the just-saved one). Returns `(topic, value)` +/// pairs so the chat agent — which captured the preference in the first place — +/// can resolve a contradiction itself: overwrite the conflicting topic or remove +/// it. No separate model call; the conversation affirms it. +pub async fn recall_related_preferences( + memory: &Arc, + value: &str, + exclude_topic: &str, + limit: usize, +) -> Vec<(String, String)> { + if value.trim().is_empty() { + return Vec::new(); + } + let mut out = Vec::new(); + // `limit` is a global cap across *both* lanes, not per-namespace — spend a + // shared budget so the total surfaced for one contradiction check can never + // exceed what the caller asked for. + let mut remaining = limit; + for ns in [USER_PREF_GENERAL_NAMESPACE, USER_PREF_SITUATIONAL_NAMESPACE] { + if remaining == 0 { + break; + } + if let Ok(hits) = memory + .recall_relevant_by_vector(ns, value, remaining, CONTRADICTION_SIMILARITY) + .await + { + for (topic, val) in hits { + if topic != exclude_topic { + out.push((topic, val)); + remaining = remaining.saturating_sub(1); + if remaining == 0 { + break; + } + } + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::UnifiedMemory; + use crate::MemoryCategory; + use tempfile::TempDir; + use tinymemory_api::host::NoopEmbedding; + + #[tokio::test] + async fn load_general_preferences_returns_values_newest_first_capped() { + let tmp = TempDir::new().unwrap(); + let mem: Arc = + Arc::new(UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap()); + + mem.store( + USER_PREF_GENERAL_NAMESPACE, + "reply_language", + "Reply in British English.", + MemoryCategory::Core, + None, + ) + .await + .unwrap(); + mem.store( + USER_PREF_GENERAL_NAMESPACE, + "tone", + "Be terse.", + MemoryCategory::Core, + None, + ) + .await + .unwrap(); + + let general = load_general_preferences(&mem, 10).await; + // Returns the values (bodies), not the topic keys. + assert!(general.iter().any(|v| v.contains("British English"))); + assert!(general.iter().any(|v| v.contains("Be terse"))); + assert!(!general.iter().any(|v| v == "reply_language")); + + // The limit caps the block. + assert_eq!(load_general_preferences(&mem, 1).await.len(), 1); + } +} diff --git a/core/src/queue/README.md b/core/src/queue/README.md new file mode 100644 index 0000000..9f055cf --- /dev/null +++ b/core/src/queue/README.md @@ -0,0 +1,37 @@ +# Memory tree — jobs + +Async job pipeline driving extraction, scoring, summarisation, and digesting off the ingest hot path. Replaces the previous synchronous `append_leaf → cascade_seal → LLM summarise` chain with a SQLite-backed queue (`mem_tree_jobs`) and a worker pool. Producers commit side-effect + follow-up job atomically inside one transaction via `enqueue_tx`. + +## Pipeline shape + +```text +ingest::persist → enqueues `extract_chunk` +worker pool (3 tasks): + extract_chunk → LLM extraction → admission → enqueue `append_buffer` + `topic_route` + append_buffer → push to L0 → enqueue `seal` if gate met + seal → seal one level → enqueue parent seal if cascading + topic_route → match topics → enqueue per-topic `append_buffer` + digest_daily → call `tree_global::digest::end_of_day_digest` + flush_stale → enqueue seals for time-stale buffers +scheduler (1 task) → daily wall-clock tick → `digest_daily(yesterday)` + `flush_stale(today)` +``` + +## Public surface + +- `pub fn enqueue` / `enqueue_tx` / `claim_next` / `mark_done` / `mark_failed` / `recover_stale_locks` / `get_job` / `count_by_status` / `count_total` — `store.rs` — queue persistence. +- `pub fn start` / `wake_workers` — `worker.rs` — spawn the worker pool (idempotent) and notify idle workers. +- `pub fn trigger_digest` / `backfill_missing_digests` — `scheduler.rs` — manual digest enqueues. +- `pub fn drain_until_idle` — `testing.rs` — deterministic test runner that processes all eligible jobs. +- `pub enum JobKind` / `JobStatus` / `pub struct Job` / `NewJob` / payload structs (`ExtractChunkPayload`, `AppendBufferPayload`, `SealPayload`, `TopicRoutePayload`, `DigestDailyPayload`, `FlushStalePayload`) and `NodeRef` / `AppendTarget` — `types.rs`. +- `pub const DEFAULT_LOCK_DURATION_MS` — `store.rs` — claim lease window (5 min). + +## Files + +- `mod.rs` — module surface and re-exports. +- `types.rs` — `JobKind`, `JobStatus`, payload structs, `NewJob` builders. Each payload owns its `dedupe_key()` so duplicates in flight are silently suppressed. +- `store.rs` — SQLite persistence: `INSERT OR IGNORE` + partial unique index on `dedupe_key WHERE status IN ('ready','running')` for at-most-one-active dedupe; `claim_next` is a single `UPDATE ... RETURNING`; `mark_done`/`mark_failed` are claim-token gated to make stale-worker settlements no-ops. +- `worker.rs` — three worker tasks plus startup `recover_stale_locks` and a 3-permit semaphore around LLM-bound jobs. Calls into `crate::scheduler_gate::wait_for_capacity()` before claiming so Throttled / Paused modes back off without holding DB leases. +- `scheduler.rs` — daily tick at UTC 00:05 that enqueues `digest_daily(yesterday)` + `flush_stale(today)`; `trigger_digest` and `backfill_missing_digests` are manual catch-up helpers. +- `testing.rs` — `drain_until_idle` for tests that need the pipeline to settle synchronously. + +Per-`JobKind` dispatch (the former `handlers/` module) was deleted at the W4 flip: `worker::run_once` now delegates claim → dispatch → settle to `tinycortex::memory::queue::run_once` through ``tinymemory_core::tinycortex::HostQueueDelegates``, which bridges each heavy step back to the host `memory_tree`/score/embed engine. diff --git a/core/src/queue/mod.rs b/core/src/queue/mod.rs new file mode 100644 index 0000000..5fd1035 --- /dev/null +++ b/core/src/queue/mod.rs @@ -0,0 +1,52 @@ +//! Async job pipeline for memory-tree work. +//! +//! Replaces the previous synchronous `append_leaf → cascade_seal → LLM +//! summarise` chain on the ingest hot path with a SQLite-backed job queue +//! and a worker pool. The shape is: +//! +//! ```text +//! ingest::persist +//! └── writes chunk row (lifecycle = pending_extraction) +//! enqueues `extract_chunk` +//! +//! worker pool (3 tasks) ──► claims jobs by kind: +//! extract_chunk → LLM extraction → admission decision → enqueue append_buffer +//! append_buffer → push to L0 → enqueue seal if gate met → enqueue topic_route +//! seal → seal one level → enqueue parent seal if cascading +//! topic_route → match topics → enqueue per-topic append_buffer +//! digest_daily → call tree_global::digest::end_of_day_digest +//! flush_stale → enqueue seals for time-stale buffers +//! +//! scheduler (1 task) ──► daily wall-clock tick: +//! enqueues digest_daily(yesterday) + flush_stale(today) +//! ``` +//! +//! All persistence lives in the same `chunks.db` as `mem_tree_chunks` so a +//! producer can insert its side-effect and its follow-up job in one tx. +//! See [`store::enqueue_tx`] for the in-tx producer entry point. +//! +//! This queue used to live under `openhuman::memory::jobs`; it now has a +//! dedicated top-level home (`openhuman::memory::queue`) because it is an +//! execution/runtime concern rather than a leaf of the memory policy API. + +mod ops; +pub mod scheduler; +pub mod store; +pub mod testing; +pub mod types; +pub(crate) mod worker; + +pub use ops::{ + backfill_in_progress, ensure_reembed_backfill, requeue_failed_after_provider_change, + set_backfill_in_progress, +}; +pub use store::{ + claim_next, count_by_status, count_total, enqueue, enqueue_tx, get_job, mark_deferred, + mark_done, mark_failed, recover_stale_locks, DEFAULT_LOCK_DURATION_MS, +}; +pub use testing::drain_until_idle; +pub use types::{ + AppendBufferPayload, AppendTarget, ExtractChunkPayload, FlushStalePayload, Job, JobKind, + JobOutcome, JobStatus, NewJob, NodeRef, SealPayload, +}; +pub use worker::{start, wake_workers}; diff --git a/core/src/queue/ops.rs b/core/src/queue/ops.rs new file mode 100644 index 0000000..56468b1 --- /dev/null +++ b/core/src/queue/ops.rs @@ -0,0 +1,201 @@ +//! Memory-queue operations: backfill-progress signalling, the re-embed +//! backfill switch-path trigger, and the provider-change failed-job un-park. +//! +//! Split out of `mod.rs` so the module root stays export-focused. Public paths +//! are preserved via re-exports in [`super`], so callers keep using +//! `crate::queue::`. + +/// Mark whether a re-embed backfill currently has pending work. +pub fn set_backfill_in_progress(v: bool) { + tinycortex::memory::queue::set_backfill_in_progress(v); +} + +/// True while a re-embed backfill chain still has rows to process. The +/// #1365 absence-reasoning consumer checks this before treating an empty +/// semantic-recall result as "no memory exists". +pub fn backfill_in_progress() -> bool { + tinycortex::memory::queue::backfill_in_progress() +} + +/// #1574 §4: ensure a re-embed backfill chain exists for the **current** +/// active signature, if (and only if) there is uncovered work. +/// +/// This is the switch-path trigger: call it after the embedder config +/// changes (a new signature → every prior row is missing at it). The §7 +/// migration is one-shot (`user_version`-gated) so it does NOT fire on a +/// later model switch — without this, switching silently blinds prior +/// memory. Standalone (own connection); the §7 migration keeps its own +/// in-tx enqueue (atomic with the copy). Idempotent + non-fatal: the +/// per-signature dedupe key means at most one chain per space, and a +/// covered space enqueues nothing. Errors are logged, never propagated — +/// a failed enqueue must not fail the user's settings save. +pub fn ensure_reembed_backfill(config: &crate::Config) { + let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); + let delegates = crate::tinycortex::HostQueueDelegates::new(config.to_arc()); + if let Err(error) = tinycortex::memory::queue::ensure_reembed_backfill(&memory, &delegates) { + log::warn!("[memory::jobs] ensure_reembed_backfill failed: {error:#}"); + } +} + +/// #5324: un-park terminally-`failed` jobs after the user changes their +/// embedding provider or supplies a key. +/// +/// The motivating case is `budget_exhausted`: once the managed embedding +/// budget is spent, every embed job fails as `Unrecoverable` and is parked +/// forever. `Unrecoverable` is meant to read as "cannot succeed *without user +/// action*", not "will never be retried" — so the moment the user takes that +/// action (points embeddings at local Ollama, or pastes a BYO key) the parked +/// jobs must get a fresh attempt budget instead of waiting for the user to +/// find the "Retry failed" button in Memory Tree settings. +/// +/// Deliberately requeues **all** failed jobs, not just the budget-exhausted +/// ones: scoping by `failure_code` would need a new filtered query in the +/// vendored `tinycortex` crate, and the cost of the wider net is bounded — a +/// job that fails for an unrelated reason (dim mismatch, empty input) simply +/// fails once more and re-parks, and this only runs on an explicit user +/// config change, never on a timer or on login. +/// +/// Non-fatal to the settings save by design, but NOT silent: on a store +/// failure this returns `Err` rather than a `0` that reads identically to +/// "nothing to requeue". The caller keeps the save successful and surfaces the +/// recovery failure in its RPC outcome, so a queue that stayed parked is never +/// presented to the user as remediated. `Ok(n)` is the number of jobs flipped +/// back to `ready` (`Ok(0)` = nothing was parked). +pub fn requeue_failed_after_provider_change(config: &crate::Config) -> Result { + // Entry record (see AGENTS.md "Debug logging"): state-transition op, so log + // entry + every branch + outcome. Prefix matches this module's sibling + // `ensure_reembed_backfill` (`[memory::jobs]`) — a stable, grep-friendly + // domain prefix. + log::debug!("[memory::jobs] provider change: evaluating parked failed jobs for requeue"); + match super::store::requeue_failed(config) { + Ok(0) => { + log::debug!("[memory::jobs] provider change: no failed jobs to requeue"); + Ok(0) + } + Ok(requeued) => { + log::info!( + "[memory::jobs] provider change: requeued {requeued} failed job(s) for a fresh attempt" + ); + // Wake the worker pool so the un-parked jobs are picked up + // promptly rather than at the next scheduled flush window. + super::wake_workers(); + Ok(requeued) + } + Err(error) => { + log::warn!("[memory::jobs] provider change: requeue_failed failed: {error:#}"); + Err(format!("{error:#}")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tree::health::{FailureCode, PipelineFailure}; + use tempfile::TempDir; + use tinymemory_api::host::test_support::TestHostConfig; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + /// Nothing parked ⇒ nothing to un-park. Must not error, and must not wake + /// the worker pool for no reason. + #[test] + fn requeue_after_provider_change_is_zero_on_an_empty_queue() { + let (_tmp, cfg) = test_config(); + assert_eq!(requeue_failed_after_provider_change(&cfg).unwrap(), 0); + } + + /// The #5324 case: jobs parked as `budget_exhausted` (unrecoverable, so + /// the periodic transient-requeue deliberately leaves them alone) must be + /// flipped back to `ready` when the user changes their embedding provider. + #[tokio::test] + async fn requeue_after_provider_change_unparks_budget_exhausted_jobs() { + use crate::queue::store; + use crate::queue::types::{FlushStalePayload, JobStatus, NewJob}; + + let (_tmp, cfg) = test_config(); + let new_job = NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-05", 3).unwrap(); + let id = store::enqueue(&cfg, &new_job) + .unwrap() + .expect("enqueue job"); + let job = store::get_job(&cfg, &id).unwrap().expect("job exists"); + + // Park it exactly the way an exhausted managed budget does. + let failure = PipelineFailure::new(FailureCode::BudgetExhausted); + assert!( + failure.is_unrecoverable(), + "precondition: parked, not retried" + ); + store::mark_failed_typed(&cfg, &job, "Insufficient budget", Some(&failure)).unwrap(); + assert_eq!( + store::count_by_status(&cfg, JobStatus::Failed).unwrap(), + 1, + "precondition: the job is parked" + ); + assert_eq!( + store::count_failed_unrecoverable(&cfg).unwrap(), + 1, + "precondition: parked as unrecoverable, so periodic retry skips it" + ); + + assert_eq!(requeue_failed_after_provider_change(&cfg).unwrap(), 1); + + assert_eq!( + store::count_by_status(&cfg, JobStatus::Ready).unwrap(), + 1, + "the job must be retryable again after the user fixes their provider" + ); + assert_eq!(store::count_by_status(&cfg, JobStatus::Failed).unwrap(), 0); + } + + /// Idempotent: calling it again once the queue is drained of failures is a + /// no-op, so re-saving settings repeatedly cannot spam the worker pool. + #[tokio::test] + async fn requeue_after_provider_change_is_idempotent() { + use crate::queue::store; + use crate::queue::types::{FlushStalePayload, NewJob}; + + let (_tmp, cfg) = test_config(); + let new_job = NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-05", 3).unwrap(); + let id = store::enqueue(&cfg, &new_job) + .unwrap() + .expect("enqueue job"); + let job = store::get_job(&cfg, &id).unwrap().expect("job exists"); + store::mark_failed_typed( + &cfg, + &job, + "Insufficient budget", + Some(&PipelineFailure::new(FailureCode::BudgetExhausted)), + ) + .unwrap(); + + assert_eq!(requeue_failed_after_provider_change(&cfg).unwrap(), 1); + assert_eq!(requeue_failed_after_provider_change(&cfg).unwrap(), 0); + } + + /// CodeRabbit (#5324): a store failure must SURFACE as `Err`, not collapse + /// into a `0` that reads identically to "nothing to requeue" and makes a + /// still-parked queue look remediated. + #[test] + fn requeue_after_provider_change_surfaces_store_errors() { + let tmp = TempDir::new().unwrap(); + // Point workspace_dir at a regular file, so the queue DB underneath it + // cannot be opened (ENOTDIR). The failure must propagate to the caller. + let as_file = tmp.path().join("workspace-is-a-file"); + std::fs::write(&as_file, b"not a directory").unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = as_file; + + let out = requeue_failed_after_provider_change(&cfg); + assert!( + out.is_err(), + "a store failure must surface as Err, not a misleading Ok(0): {out:?}" + ); + } +} diff --git a/core/src/queue/scheduler.rs b/core/src/queue/scheduler.rs new file mode 100644 index 0000000..aa01e88 --- /dev/null +++ b/core/src/queue/scheduler.rs @@ -0,0 +1,127 @@ +//! Wall-clock scheduler that periodically enqueues a `JobKind::FlushStale` +//! so low-volume source-tree L0 buffers seal promptly. +//! +//! The daily global-digest loop was removed along with the global tree — +//! source trees plus the entity index are the substrate, so there is no +//! cross-source digest to enqueue. Only the stale-buffer flush remains. + +use std::sync::Arc; +use std::time::Duration; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::Config; + +static STARTED: std::sync::Once = std::sync::Once::new(); + +/// Start the periodic flush_stale scheduler. Takes the full `Config` so the +/// enqueues match the same workspace + LLM settings the workers see — not +/// `TestHostConfig::default()`. +pub fn start(config: Arc) { + STARTED.call_once(|| { + // Periodic flush_stale loop (every 3 h) so L0 buffers seal + // promptly even for low-volume sources. + let cfg = config.to_arc(); + tokio::spawn(async move { + // Fire once on startup so new installs & restarts don't wait + // up to 3 h for the first seal window. + retry_transient_failures(&*cfg); + enqueue_flush_stale(&*cfg); + loop { + tokio::time::sleep(Duration::from_secs(3 * 60 * 60)).await; + retry_transient_failures(&*cfg); + enqueue_flush_stale(&*cfg); + } + }); + }); +} + +/// Self-heal the pipeline before each flush window: requeue jobs that +/// failed for transient reasons (network blips, timeouts, SQLITE_BUSY) +/// so chunks never sit unprocessed until the next manual sync. +/// Unrecoverable failures stay parked — see +/// [`store::requeue_transient_failed`]. +fn retry_transient_failures(config: &Config) { + let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); + match tinycortex::memory::queue::scheduler::self_heal(&memory) { + Ok(0) => {} + Ok(n) => { + log::info!("[memory::jobs] periodic retry requeued {n} transient-failed job(s)"); + super::worker::wake_workers(); + } + Err(err) => { + log::warn!("[memory::jobs] periodic transient-failure retry failed: {err:#}"); + } + } +} + +/// Enqueue one stale-buffer flush job for the current 3-hour UTC window and +/// wake the workers, reporting whether a job was actually created. +/// +/// `Ok(false)` means the window already had one — the enqueue is deduped on +/// `(date, hour_block)`, so this is the normal repeat-call outcome and not a +/// failure. +/// +/// `pub(crate)` (it was private) so the embedded memory driver's +/// `MemoryMaintenance::consolidate` can drive the **same** path the periodic +/// scheduler drives. That matters: the flush job fans out into per-tree `Seal` +/// jobs whose label strategy is derived per tree by the queue worker +/// (`TreeFactory::from_tree(&tree).label_strategy(...)`). The alternative entry +/// point, `tree::tree::flush::flush_stale_buffers_default`, takes **one** +/// `LabelStrategy` for every tree, which no production caller uses and which +/// would apply one tree kind's labelling to all of them. +pub fn enqueue_flush_stale_job(config: &Config) -> Result { + let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); + match tinycortex::memory::queue::scheduler::enqueue_flush_stale(&memory) { + Ok(Some(_)) => { + super::worker::wake_workers(); + Ok(true) + } + Ok(None) => Ok(false), + Err(err) => Err(format!("{err:#}")), + } +} + +fn enqueue_flush_stale(config: &Config) { + if let Err(err) = enqueue_flush_stale_job(config) { + log::warn!("[memory::jobs] periodic flush_stale enqueue failed: {err}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::queue::store::{claim_next, count_by_status, DEFAULT_LOCK_DURATION_MS}; + use crate::queue::types::{FlushStalePayload, JobKind, JobStatus}; + use tempfile::TempDir; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + (tmp, cfg) + } + + #[test] + fn enqueue_flush_stale_enqueues_at_most_one_job_per_current_block() { + let (_tmp, cfg) = test_config(); + enqueue_flush_stale(&cfg); + enqueue_flush_stale(&cfg); + + assert_eq!( + count_by_status(&cfg, JobStatus::Ready).unwrap(), + 1, + "second enqueue in same 3h block should be dedupe-suppressed" + ); + + let claimed = claim_next(&cfg, DEFAULT_LOCK_DURATION_MS).unwrap().unwrap(); + assert_eq!(claimed.kind, JobKind::FlushStale); + let payload: FlushStalePayload = serde_json::from_str(&claimed.payload_json).unwrap(); + assert_eq!(payload.max_age_secs, None); + } +} diff --git a/core/src/queue/store.rs b/core/src/queue/store.rs new file mode 100644 index 0000000..2ad8d55 --- /dev/null +++ b/core/src/queue/store.rs @@ -0,0 +1,90 @@ +//! Product Config adapters over tinycortex's SQLite queue store. + +use anyhow::Result; +use rusqlite::Transaction; + +use crate::tree::health::PipelineFailure; +use crate::Config; + +use super::types::{Job, JobFailure, JobStatus, NewJob}; +use crate::tinycortex::engine_config; + +pub use tinycortex::memory::queue::DEFAULT_LOCK_DURATION_MS; + +pub fn enqueue(config: &Config, job: &NewJob) -> Result> { + tinycortex::memory::queue::enqueue(&engine_config(config), job) +} + +pub fn enqueue_tx(tx: &Transaction<'_>, job: &NewJob) -> Result> { + tinycortex::memory::queue::enqueue_tx(tx, job) +} + +pub fn claim_next(config: &Config, lock_duration_ms: i64) -> Result> { + tinycortex::memory::queue::claim_next(&engine_config(config), lock_duration_ms) +} + +pub fn mark_done(config: &Config, job: &Job) -> Result<()> { + tinycortex::memory::queue::mark_done(&engine_config(config), job) +} + +pub fn mark_failed(config: &Config, job: &Job, error: &str) -> Result<()> { + tinycortex::memory::queue::mark_failed(&engine_config(config), job, error) +} + +pub fn mark_failed_typed( + config: &Config, + job: &Job, + error: &str, + failure: Option<&PipelineFailure>, +) -> Result<()> { + let failure = failure.map(|failure| JobFailure { + code: failure.code.as_str(), + class: failure.class.as_str(), + }); + tinycortex::memory::queue::mark_failed_typed( + &engine_config(config), + job, + error, + failure.as_ref(), + ) +} + +pub fn mark_deferred(config: &Config, job: &Job, until_ms: i64, reason: &str) -> Result<()> { + tinycortex::memory::queue::mark_deferred(&engine_config(config), job, until_ms, reason) +} + +pub fn recover_stale_locks(config: &Config) -> Result { + tinycortex::memory::queue::recover_stale_locks(&engine_config(config)) +} + +pub fn requeue_failed(config: &Config) -> Result { + tinycortex::memory::queue::requeue_failed(&engine_config(config)) +} + +pub fn requeue_transient_failed(config: &Config) -> Result { + tinycortex::memory::queue::requeue_transient_failed(&engine_config(config)) +} + +pub fn release_running_locks(config: &Config) -> Result { + tinycortex::memory::queue::release_running_locks(&engine_config(config)) +} + +pub fn count_by_status(config: &Config, status: JobStatus) -> Result { + tinycortex::memory::queue::count_by_status(&engine_config(config), status) +} + +pub fn count_failed_unrecoverable(config: &Config) -> Result { + tinycortex::memory::queue::count_failed_unrecoverable(&engine_config(config)) +} + +pub fn count_total(config: &Config) -> Result { + tinycortex::memory::queue::count_total(&engine_config(config)) +} + +pub fn retry_all_failed(config: &Config) -> Result { + tinycortex::memory::queue::retry_all_failed(&engine_config(config)) +} + +pub fn get_job(config: &Config, id: &str) -> Result> { + tinycortex::memory::queue::get_job(&engine_config(config), id) +} diff --git a/core/src/queue/testing.rs b/core/src/queue/testing.rs new file mode 100644 index 0000000..d1a9ea1 --- /dev/null +++ b/core/src/queue/testing.rs @@ -0,0 +1,40 @@ +//! Test helpers for the jobs runtime — not used in production code paths. + +use anyhow::Result; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::Config; + +/// Deterministically run queued memory-tree jobs until no immediately +/// claimable work remains. Intended for tests that need the async pipeline +/// to settle without spawning background tasks. +pub async fn drain_until_idle(config: &Config) -> Result<()> { + loop { + if !super::worker::run_once(config).await? { + break; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + #[tokio::test] + async fn drain_until_idle_is_noop_when_queue_is_empty() { + let (_tmp, cfg) = test_config(); + drain_until_idle(&cfg).await.unwrap(); + } +} diff --git a/core/src/queue/types.rs b/core/src/queue/types.rs new file mode 100644 index 0000000..fc87750 --- /dev/null +++ b/core/src/queue/types.rs @@ -0,0 +1,7 @@ +//! Queue wire types owned by tinycortex. + +pub use tinycortex::memory::queue::{ + AppendBufferPayload, AppendTarget, ExtractChunkPayload, FlushStalePayload, Job, JobFailure, + JobKind, JobOutcome, JobStatus, NewJob, NodeRef, ReembedBackfillPayload, SealDocumentPayload, + SealPayload, +}; diff --git a/core/src/queue/worker.rs b/core/src/queue/worker.rs new file mode 100644 index 0000000..811c540 --- /dev/null +++ b/core/src/queue/worker.rs @@ -0,0 +1,1083 @@ +//! Worker pool: drives the crate queue engine (W4 flip). Each `run_once` +//! delegates claim → dispatch → settle to `tinycortex::memory::queue::run_once` +//! via [`crate::tinycortex::HostQueueDelegates`]; the legacy host +//! `handlers` engine that used to own dispatch was deleted at the flip. +//! +//! Concurrency control for LLM-bound work is delegated to +//! [`crate::scheduler_gate`] — its global single-slot +//! semaphore (`LlmPermit`) is the one source of truth across this +//! worker, voice cleanup, autocomplete, triage, and reflection. The +//! worker itself just calls `wait_for_capacity()`; non-LLM jobs +//! (`AppendBuffer`, `FlushStale`) run without acquiring a permit. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use anyhow::Result; +use tokio::sync::Notify; + +// Test-only: the tests below build a `TestHostConfig` and call +// `MemoryHostConfig` methods on it directly. Production code in this module +// goes through `crate::Config`, so neither name is needed in a non-test build. +#[cfg(test)] +use tinymemory_api::host::{test_support::TestHostConfig, MemoryHostConfig}; + +use crate::Config; +// W4 flip: `run_once` now delegates claim/dispatch/settle to the crate, so the +// legacy `handlers`, per-job settle (`mark_*`/`scrub_for_log`), and claim +// helpers are gone from this module. Only startup lock recovery + the loop's +// storage-degraded signalling remain host. +use crate::queue::store::{recover_stale_locks, release_running_locks}; +use crate::tree::health::{clear_storage_degraded, mark_storage_degraded, FailureCode}; + +/// Number of concurrent job-worker tasks. Each worker claims one job +/// at a time via `claim_next` (atomic UPDATE under SQLite WAL with +/// `locked_until_ms` + status='running'), so multiple workers +/// parallelize independent jobs without double-claim risk. +/// +/// On cloud backends, LLM-bound jobs drop the global LLM permit +/// after claim (see `run_once`) so all 4 workers can run cloud +/// extract/summarise calls in parallel. +/// +/// On local backends, the single global LLM slot still serialises +/// Ollama calls for laptop-RAM safety. Note that `wait_for_capacity` +/// is acquired **before** `claim_next`, so non-LLM jobs (AppendBuffer, +/// FlushStale, TopicRoute) also block on the gate when an LLM job +/// holds the permit — they only run in parallel with each other while +/// no LLM job is in flight. Bumping `WORKER_COUNT` therefore helps +/// throughput most when local LLM calls are sparse. +const WORKER_COUNT: usize = 4; +const POLL_INTERVAL: Duration = Duration::from_secs(5); + +static WORKER_NOTIFY: OnceLock> = OnceLock::new(); +static STARTED: std::sync::Once = std::sync::Once::new(); + +/// Process-wide latch so a `SQLITE_CORRUPT` flood is reported to Sentry **once**, +/// not on every poll from every worker. Set on the first malformed-image +/// detection; cleared after a recovery attempt settles (quarantine+rebuild or a +/// quick_check that now passes) so a genuinely-new, later corruption can still +/// page once. Without this, 4 workers polling a wedged DB re-page ~1/sec +/// (Sentry TAURI-RUST-E93: 1,633 events in ~17 min from one host). +static CORRUPT_REPORTED: AtomicBool = AtomicBool::new(false); + +/// Process-wide latch so a persistent host-filesystem failure (EIO/ENOSPC/ +/// EROFS on the memory_tree dir/DB path) is reported to Sentry **once**, not on +/// every poll from every worker. Set on the first host-I/O failure; cleared on +/// the next successful claim (storage recovered) so a genuinely-new, later +/// failure can still page once. Without this, 4 workers re-polling a dead disk +/// flood the dashboard (Sentry CORE-RUST-19J: ~10k events in ~50 min from one +/// Raspberry Pi with a failing SD card). +static STORAGE_IO_REPORTED: AtomicBool = AtomicBool::new(false); + +/// Notify any idle workers so they re-poll immediately instead of waiting +/// out `POLL_INTERVAL`. Cheap no-op before [`start`] has run. +pub fn wake_workers() { + if let Some(notify) = WORKER_NOTIFY.get() { + notify.notify_waiters(); + } +} + +/// Start the worker pool + daily scheduler. Takes the full `Config` so +/// each spawned task sees the user's actual settings (LLM endpoints, +/// embedder model, timeouts) — not `TestHostConfig::default()`. Without this, +/// workers fall back to inert/regex-only behavior regardless of what's +/// in `config.toml`, defeating the entire async pipeline. +/// +/// Idempotent (`Once`-guarded) so repeat calls during bootstrap are +/// safe no-ops after the first. +pub fn start(config: Arc) { + STARTED.call_once(|| { + let notify = WORKER_NOTIFY + .get_or_init(|| Arc::new(Notify::new())) + .clone(); + if let Err(err) = recover_stale_locks(&*config) { + log::warn!("[memory::jobs] recover_stale_locks failed at startup: {err:#}"); + } + + // Release in-flight locks on graceful shutdown so a clean restart + // re-claims the work immediately instead of waiting out the lease + // (which surfaced as a stale-lock recovery warn on every launch). + // Hard kills still fall back to lease-expiry recovery at startup + // (bug-report-2026-05-26 I2). + let shutdown_cfg = config.to_arc(); + crate::shutdown::register(move || { + // NOTE: `shutdown::register` is bound `F: Fn() -> Fut`, so this + // closure may be invoked more than once; each call must hand the + // returned future its own owned `Config`. Moving `shutdown_cfg` + // in directly is `E0507` (cannot move out of an `Fn` closure), so + // the per-call clone is required, not redundant. + let cfg = shutdown_cfg.clone(); + async move { + match release_running_locks(&*cfg) { + Ok(n) if n > 0 => { + log::info!( + "[memory::jobs] released {n} in-flight job lock(s) on graceful shutdown" + ); + } + Ok(_) => {} + Err(err) => { + log::warn!( + "[memory::jobs] failed to release job locks on shutdown: {err:#}" + ); + } + } + } + }); + + for idx in 0..WORKER_COUNT { + let notify = notify.clone(); + let cfg = config.to_arc(); + tokio::spawn(async move { + loop { + match run_once(&*cfg).await { + Ok(processed) => { + // A successful claim proves the memory_tree DB + // opened, so the host filesystem is healthy again. + // Clear any prior host-I/O degradation so the status + // banner self-heals and a genuinely-new later failure + // can page once more. Guarded on the latch swap so the + // clear + info log fire only on the recovery edge, not + // every poll. + if STORAGE_IO_REPORTED.swap(false, Ordering::Relaxed) { + clear_storage_degraded(); + log::info!( + "[memory::jobs] worker {idx} storage recovered; \ + cleared host-I/O degraded flag" + ); + } + if processed { + continue; + } + tokio::select! { + _ = notify.notified() => {} + _ = tokio::time::sleep(POLL_INTERVAL) => {} + } + } + Err(err) => { + // SQLite `BUSY` / `LOCKED` is transient write-lock + // contention (multiple workers + the scheduler + + // ingest producers all write the same DB). The + // configured `busy_timeout` already retries + // inside rusqlite; if we still see it here, the + // right answer is to back off and re-poll — not + // to page Sentry. The next loop iteration will + // try `claim_next` again and almost always + // succeed. See OPENHUMAN-TAURI-BP. + if is_sqlite_busy(&err) { + log::warn!( + "[memory::jobs] worker {idx} hit SQLite busy/locked, \ + backing off 1s: {err:#}" + ); + tokio::time::sleep(Duration::from_secs(1)).await; + } else if is_sqlite_io_transient(&err) { + // I/O errors (IOERR_TRUNCATE 1546, the `-shm` family + // 4618/4874/5386, IN_PAGE 8714, CANTOPEN 14) or circuit + // breaker open — transient + // filesystem / WAL condition. Back off 30 s and let the + // connection cache try a fresh open on next poll. These + // are NOT reported to Sentry (they are transient and were + // flooding ~19K events/4 days, see #2206). + log::warn!( + "[memory::jobs] worker {idx} hit transient I/O error, \ + backing off 30s: {err:#}" + ); + tokio::time::sleep(Duration::from_secs(30)).await; + } else if is_sqlite_disk_full(&err) { + // SQLITE_FULL (code 13): the host disk is full. + // A claim UPDATE cannot succeed until the user + // frees space — this is persistent, not + // transient, so re-polling every second and + // paging Sentry on each failure floods the + // dashboard (TAURI-RUST-4R8: ~95k events, one + // user) for a condition only the user can + // clear. Back off long and stay silent; the + // `ready` rows resume when space returns and + // `notify` still wakes us on new enqueues. + log::warn!( + "[memory::jobs] worker {idx} hit SQLITE_FULL (disk full), \ + backing off 300s without reporting: {err:#}" + ); + tokio::time::sleep(Duration::from_secs(300)).await; + } else if is_sqlite_corrupt(&err) { + // SQLITE_CORRUPT (code 11): the on-disk mem_tree + // image is malformed. Unlike busy/io-transient/ + // disk-full, this NEVER clears on its own — the + // claim UPDATE fails forever, so re-polling every + // second and paging Sentry each time turns one + // unrecoverable file into a flood (TAURI-RUST-E93: + // 1,633 events in ~17 min, one host). Report once, + // drive quarantine+rebuild recovery (factored into + // `recover_corrupt_db_once` so it is unit-testable + // without spinning the live loop), then back off + // long so a failed recovery never re-floods. + // `notify` still wakes us on new enqueues once the + // rebuild succeeds. + recover_corrupt_db_once(idx, &err, &*cfg); + tokio::time::sleep(Duration::from_secs(300)).await; + } else if is_host_io_error(&err) { + // Persistent host-filesystem failure (EIO 5 / + // ENOSPC 28 / EROFS 30) creating or opening the + // memory_tree dir/DB — e.g. a failing or + // disconnected SD card, or a volume the kernel + // remounted read-only. Like SQLITE_FULL/CORRUPT + // this is a persistent host condition only the + // user can clear (reseat/replace/free storage), + // so re-polling every second and paging Sentry on + // each failure floods the dashboard (CORE-RUST-19J: + // ~10k events in ~50 min from one Raspberry Pi). + // + // Resolution, not just suppression: mark the + // memory_tree degraded with `StorageUnavailable` + // so the status panel shows the user an actionable + // "check your disk" banner — they own the only + // lever. Report ONCE (process-wide latch) for dev + // telemetry, then back off 300s and stay silent. + // Jobs stay `ready` and resume when storage + // returns; the degraded flag + latch clear on the + // next successful claim. + mark_storage_degraded(FailureCode::StorageUnavailable); + if !STORAGE_IO_REPORTED.swap(true, Ordering::Relaxed) { + crate::observability::report_error( + &err, + "memory", + "tree_jobs_worker_host_io", + &[("worker_idx", &idx.to_string())], + ); + } + log::warn!( + "[memory::jobs] worker {idx} hit host filesystem I/O error \ + (EIO/ENOSPC/EROFS — failing or read-only storage), \ + backing off 300s: {err:#}" + ); + tokio::time::sleep(Duration::from_secs(300)).await; + } else { + crate::observability::report_error( + &err, + "memory", + "tree_jobs_worker", + &[("worker_idx", &idx.to_string())], + ); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + } + }); + } + + super::scheduler::start(config); + }); +} + +/// Claim and run a single job. Returns `true` when work was processed, +/// `false` when no eligible row was available. +pub async fn run_once(config: &Config) -> Result { + // Cooperative throttle BEFORE claiming, so memory queue work still yields to + // voice/autocomplete/triage under load (Throttled/Paused modes), exactly as + // the legacy pool did. Held across the single crate step below; returns + // immediately in Aggressive/Normal so idle desktops pay zero cost. + let _gate_permit = crate::scheduler_gate::wait_for_capacity().await; + + // W4 flip: TinyCortex now owns claim → dispatch → settle. `queue::run_once` + // claims one `mem_tree_jobs` row (the same table host producers enqueue + // into — identical schema, parity P4) and runs it through the crate's + // `handle_job` + `HostQueueDelegates`, which bridge each heavy step + // (score/admit, buffer push, seal, seal-document, re-embed) back to the host + // `memory_tree`/score/embed engine, then settles the row itself. The crate's + // single-slot LLM gate serialises llm-bound jobs; the legacy per-job + // local/cloud permit routing and the extract-batch coalescing are + // intentionally dropped here (perf, not correctness — W4 follow-up). + let mc = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); + let delegates = crate::tinycortex::HostQueueDelegates::new(config.to_arc()); + tinycortex::memory::queue::run_once(&mc, &delegates).await +} + +/// Classify whether an error is a transient I/O failure that should be +/// silently backed off without a Sentry report (#2206). +/// +/// Covers: +/// - `SQLITE_IOERR_TRUNCATE` (1546): WAL truncation failed — usually a +/// transient filesystem hiccup. +/// - WAL `-shm` family — `SHMOPEN` (4618, the macOS cold-start failure), +/// `SHMSIZE` (4874), `SHMMAP` (5386): shared-memory side-file temporarily +/// unavailable. (4874 is SHMSIZE, not SHMMAP — the real SHMMAP is 5386.) +/// - `SQLITE_IOERR_IN_PAGE` (8714): mmap-page I/O fault. +/// - `SQLITE_CANTOPEN` / `CannotOpen` (14): DB file temporarily inaccessible. +/// - Text fallback: circuit breaker message, or rusqlite phrases that don't +/// downcast cleanly after multiple `.context()` layers. +fn is_sqlite_io_transient(err: &anyhow::Error) -> bool { + if let Some(rusqlite::Error::SqliteFailure(f, _)) = err.downcast_ref::() { + // 14 CANTOPEN, 1546 TRUNCATE, 4618 SHMOPEN, 4874 SHMSIZE, 5386 SHMMAP, + // 8714 IN_PAGE — the WAL `-shm` cold-start family (4874 is SHMSIZE, not + // SHMMAP; the real SHMMAP is 5386). + if matches!(f.extended_code, 14 | 1546 | 4618 | 4874 | 5386 | 8714) { + return true; + } + if f.code == rusqlite::ErrorCode::CannotOpen { + return true; + } + } + // Text fallback for errors wrapped under `.context()` layers or + // emitted as plain `anyhow!` strings (e.g. circuit breaker message). + let msg = format!("{err:#}").to_ascii_lowercase(); + msg.contains("circuit breaker open") + || msg.contains("disk i/o error") + || msg.contains("unable to open database file") + || msg.contains("xshmmap") + || msg.contains("truncate file") +} + +/// Classify whether an error from `run_once` is a transient SQLite +/// write-lock contention (`SQLITE_BUSY` or `SQLITE_LOCKED`). +/// +/// The configured `busy_timeout` already absorbs short waits inside +/// rusqlite; this helper catches the residual case where the busy +/// handler exhausts and the error bubbles up. Treated as a soft signal: +/// the worker logs a warning and re-polls on the next loop iteration +/// rather than escalating to Sentry. +fn is_sqlite_busy(err: &anyhow::Error) -> bool { + if let Some(rusqlite::Error::SqliteFailure(sqlite_err, _)) = + err.downcast_ref::() + { + return matches!( + sqlite_err.code, + rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked + ); + } + // Fallback for chained/wrapped errors: the rusqlite `Error` may sit + // a few `context()` layers deep. anyhow's alternate `Display` + // joins every cause with ": ", so the SQLite-rendered text is + // searchable in the flattened chain. Match the two well-known + // phrases SQLite emits for these codes. + let msg = format!("{err:#}").to_ascii_lowercase(); + msg.contains("database is locked") || msg.contains("database table is locked") +} + +/// Classify whether an error from `claim_next` is a `SQLITE_FULL` disk-full +/// condition (primary code `DiskFull`, extended 13). +/// +/// Unlike `SQLITE_BUSY`/`LOCKED` or the transient I/O family, a full disk is a +/// **persistent** host condition: the claim `UPDATE` cannot succeed until the +/// user frees space. Re-polling every second and paging Sentry on each failure +/// turns one unrecoverable condition into a flood (Sentry TAURI-RUST-4R8: +/// ~95k events from a single user). The worker backs off long and stays +/// silent; the rows stay `ready` and resume when space returns. +/// +/// Matching on the `DiskFull` error code is rusqlite-version-stable. The text +/// fallback covers the case where the error was flattened to a plain `anyhow!` +/// string across `.context()` layers — rusqlite renders `SQLITE_FULL` as +/// `"database or disk is full: Error code 13: Insertion failed because +/// database is full"`, so anchor on either canonical fragment. +fn is_sqlite_disk_full(err: &anyhow::Error) -> bool { + if let Some(rusqlite::Error::SqliteFailure(sqlite_err, _)) = + err.downcast_ref::() + { + if sqlite_err.code == rusqlite::ErrorCode::DiskFull { + return true; + } + } + let msg = format!("{err:#}").to_ascii_lowercase(); + msg.contains("database or disk is full") + || msg.contains("insertion failed because database is full") +} + +/// Classify whether an error from `claim_next` is a `SQLITE_CORRUPT` malformed- +/// image condition (primary code `DatabaseCorrupt`, code 11) or the closely- +/// related `NotADatabase` (code 26 — the header itself is unreadable). +/// +/// Unlike `SQLITE_BUSY`/`LOCKED`, the transient I/O family, or `SQLITE_FULL`, +/// a malformed image is **persistent on-disk damage**: the claim `UPDATE` can +/// never succeed, so re-polling every second and paging Sentry on each failure +/// turns one corrupt file into an infinite flood (Sentry TAURI-RUST-E93: +/// ~1.6k events in ~17 min from a single host). The worker reports once, drives +/// a quarantine+rebuild recovery (`recover_corrupt_db`), and backs off long. +/// +/// Matching on the error code is rusqlite-version-stable. The text fallback +/// covers the case where the rusqlite error was flattened to a plain `anyhow!` +/// string across `.context()` layers — SQLite renders these as "database disk +/// image is malformed" (code 11) and "file is not a database" (code 26). +fn is_sqlite_corrupt(err: &anyhow::Error) -> bool { + if let Some(rusqlite::Error::SqliteFailure(sqlite_err, _)) = + err.downcast_ref::() + { + if matches!( + sqlite_err.code, + rusqlite::ErrorCode::DatabaseCorrupt | rusqlite::ErrorCode::NotADatabase + ) { + return true; + } + } + let msg = format!("{err:#}").to_ascii_lowercase(); + msg.contains("database disk image is malformed") || msg.contains("file is not a database") +} + +/// Classify whether an error is a **persistent host-filesystem failure** — +/// `std::fs::create_dir_all` / file open returning an OS-level I/O error on the +/// memory_tree path. Matches the three persistent, user-only-fixable POSIX +/// codes: +/// - **EIO `5`** — the block device can't service I/O (failing/disconnected SD +/// card or USB drive). This is the CORE-RUST-19J signal: +/// `"Failed to create memory_tree dir: …: Input/output error (os error 5)"`. +/// - **ENOSPC `28`** — no space left at the *filesystem* layer (distinct from +/// `SQLITE_FULL`, which is the SQLite-write code handled separately). +/// - **EROFS `30`** — read-only filesystem; Linux remounts a failing SD card +/// read-only, so this is the common next stage of the same Pi failure. +/// +/// Unlike the SQLite busy/transient family, these are persistent host +/// conditions the app has no lever to fix: re-polling every second and paging +/// Sentry on each failure turns one dead disk into a flood. The worker backs +/// off long, surfaces a `StorageUnavailable` degradation to the user, and stays +/// silent after a single report. +/// +/// Matching on `raw_os_error()` is platform-stable. The text fallback covers +/// the case where the `io::Error` was flattened to a plain `anyhow!` string +/// across `.context()` layers (anyhow renders `"… (os error N)"`); it anchors +/// on the unambiguous os-error number, not the loose phrase, so a network +/// "input/output" string can't false-positive. A non-OS error has +/// `raw_os_error() == None` and matches neither path. +fn is_host_io_error(err: &anyhow::Error) -> bool { + if let Some(io_err) = err.downcast_ref::() { + if matches!(io_err.raw_os_error(), Some(5) | Some(28) | Some(30)) { + return true; + } + } + let msg = format!("{err:#}").to_ascii_lowercase(); + msg.contains("(os error 5)") || msg.contains("(os error 28)") || msg.contains("(os error 30)") +} + +/// Handle a confirmed `SQLITE_CORRUPT` failure from the worker loop: report it +/// to Sentry **once** (process-wide [`CORRUPT_REPORTED`] latch, not per-poll +/// across the workers) and drive the quarantine+rebuild recovery in +/// [`recover_corrupt_db`](crate::store::chunks::store::recover_corrupt_db). +/// +/// Factored out of [`start`]'s error arm so the report-once + recovery decision +/// logic is unit-testable without spinning the live worker loop. The caller +/// applies the long backoff after this returns. +fn recover_corrupt_db_once(idx: usize, err: &anyhow::Error, config: &Config) { + if !CORRUPT_REPORTED.swap(true, Ordering::Relaxed) { + crate::observability::report_error( + err, + "memory", + "tree_jobs_worker_corrupt", + &[("worker_idx", &idx.to_string())], + ); + } + log::error!( + "[memory::jobs] worker {idx} hit SQLITE_CORRUPT (malformed DB image), \ + attempting quarantine + rebuild recovery: {err:#}" + ); + match crate::store::chunks::store::recover_corrupt_db(config) { + Ok(true) => { + log::warn!( + "[memory::jobs] worker {idx} quarantined corrupt mem_tree DB and rebuilt \ + empty schema; queue will resume" + ); + // Recovery settled — allow a future, genuinely-new corruption to + // page once. + CORRUPT_REPORTED.store(false, Ordering::Relaxed); + } + Ok(false) => { + log::info!( + "[memory::jobs] worker {idx} corruption recovery: quick_check now passes, \ + no quarantine needed" + ); + CORRUPT_REPORTED.store(false, Ordering::Relaxed); + } + Err(rec_err) => { + log::error!( + "[memory::jobs] worker {idx} corruption recovery FAILED, retrying after \ + backoff: {rec_err:#}" + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::queue::store::{count_by_status, enqueue, get_job}; + use crate::queue::types::{ + FlushStalePayload, JobKind, JobStatus, NewJob, ReembedBackfillPayload, + }; + use crate::store::chunks::store::{ + tree_active_signature, upsert_chunks, upsert_staged_chunks_tx, with_connection, + }; + use crate::store::chunks::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::store::content as content_store; + use chrono::{TimeZone, Utc}; + use tempfile::TempDir; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + (tmp, cfg) + } + + /// Raw `rusqlite::Error::SqliteFailure` with the `DatabaseBusy` code + /// is what surfaces when the `busy_timeout` is exhausted on a write. + #[test] + fn is_sqlite_busy_matches_database_busy_code() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DatabaseBusy, + extended_code: 5, // SQLITE_BUSY + }, + Some("database is locked".into()), + ); + let err = anyhow::Error::from(raw); + assert!(is_sqlite_busy(&err)); + } + + /// `SQLITE_LOCKED` is the per-table flavour (e.g. shared cache); same + /// classification — transient, retry. + #[test] + fn is_sqlite_busy_matches_database_locked_code() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DatabaseLocked, + extended_code: 6, // SQLITE_LOCKED + }, + Some("database table is locked".into()), + ); + let err = anyhow::Error::from(raw); + assert!(is_sqlite_busy(&err)); + } + + /// When the rusqlite error is buried under `.context(...)` layers + /// (as happens when `with_connection` wraps the closure result), + /// the downcast still finds it. Regression guard: don't rely on + /// matching the top-level error type. + #[test] + fn is_sqlite_busy_matches_through_context_layers() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DatabaseBusy, + extended_code: 5, + }, + Some("database is locked".into()), + ); + let wrapped: anyhow::Error = anyhow::Error::from(raw) + .context("Failed to claim next mem_tree_jobs row") + .context("with_connection closure failed"); + assert!(is_sqlite_busy(&wrapped)); + } + + /// Fallback text-match: if the rusqlite error has been re-rendered + /// into a plain `anyhow!` (no downcast available), the "database is + /// locked" phrase still triggers the busy classification. + #[test] + fn is_sqlite_busy_text_fallback() { + let err = anyhow::anyhow!("Failed to claim next mem_tree_jobs row: database is locked"); + assert!(is_sqlite_busy(&err)); + } + + /// Non-busy SQLite failures (e.g. UNIQUE constraint) must NOT be + /// reclassified — those are real bugs worth reporting. + #[test] + fn is_sqlite_busy_does_not_match_constraint_violation() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::ConstraintViolation, + extended_code: 19, + }, + Some("UNIQUE constraint failed: mem_tree_jobs.dedupe_key".into()), + ); + let err = anyhow::Error::from(raw); + assert!(!is_sqlite_busy(&err)); + } + + /// Generic non-SQLite errors must not be reclassified as busy. + #[test] + fn is_sqlite_busy_does_not_match_unrelated_errors() { + let err = anyhow::anyhow!("upstream returned 500: internal server error"); + assert!(!is_sqlite_busy(&err)); + } + + // ── is_sqlite_io_transient tests (#2206) ───────────────────────────── + + /// SQLITE_IOERR_TRUNCATE (extended code 1546) must be classified as + /// transient so the worker backs off without hitting Sentry. + #[test] + fn is_sqlite_io_transient_matches_ioerr_truncate() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::SystemIoFailure, + extended_code: 1546, // SQLITE_IOERR_TRUNCATE + }, + Some("disk I/O error".into()), + ); + assert!(is_sqlite_io_transient(&anyhow::Error::from(raw))); + } + + /// The WAL `-shm` family must classify as transient via the NUMERIC arm + /// (the message deliberately avoids the text-fallback phrases). 4618 + /// SHMOPEN is the macOS cold-start failure; 4874 is SHMSIZE; 5386 is the + /// real SHMMAP; 8714 is IN_PAGE. + #[test] + fn is_sqlite_io_transient_matches_shm_family() { + for ext in [4618, 4874, 5386, 8714] { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::SystemIoFailure, + extended_code: ext, + }, + Some("sqlite extended io failure".into()), + ); + assert!( + is_sqlite_io_transient(&anyhow::Error::from(raw)), + "extended_code {ext} must classify as transient (numeric arm)" + ); + } + } + + /// SQLITE_CANTOPEN (code CannotOpen, extended code 14) must be + /// classified as transient — temporary inability to open the file. + #[test] + fn is_sqlite_io_transient_matches_cantopen() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::CannotOpen, + extended_code: 14, // SQLITE_CANTOPEN + }, + Some("unable to open database file".into()), + ); + assert!(is_sqlite_io_transient(&anyhow::Error::from(raw))); + } + + /// The circuit breaker error message produced by `get_or_init_connection` + /// must be classified as transient via the text fallback. + #[test] + fn is_sqlite_io_transient_text_fallback() { + let err = anyhow::anyhow!("memory_tree_db circuit breaker open: too many init failures"); + assert!(is_sqlite_io_transient(&err)); + } + + /// UNIQUE constraint violation must NOT be reclassified as a transient + /// I/O error — those are genuine bugs. + #[test] + fn is_sqlite_io_transient_negative_constraint_violation() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::ConstraintViolation, + extended_code: 19, + }, + Some("UNIQUE constraint failed: mem_tree_jobs.dedupe_key".into()), + ); + assert!(!is_sqlite_io_transient(&anyhow::Error::from(raw))); + } + + // ── is_sqlite_disk_full tests (#3909 / Sentry TAURI-RUST-4R8) ───────── + + /// `SQLITE_FULL` (primary code `DiskFull`, extended 13) is the disk-full + /// signal from `claim_next`; it must classify so the worker backs off + /// long instead of paging Sentry every second. + #[test] + fn is_sqlite_disk_full_matches_disk_full_code() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DiskFull, + extended_code: 13, + }, + Some("database or disk is full".into()), + ); + assert!(is_sqlite_disk_full(&anyhow::Error::from(raw))); + } + + /// The rusqlite error sits a few `.context()` layers deep when it bubbles + /// out of `claim_next` → `with_connection`; the downcast must still find + /// the `DiskFull` code. + #[test] + fn is_sqlite_disk_full_matches_through_context_layers() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DiskFull, + extended_code: 13, + }, + Some("database or disk is full".into()), + ); + let wrapped = anyhow::Error::from(raw) + .context("Failed to claim next mem_tree_jobs row") + .context("with_connection closure failed"); + assert!(is_sqlite_disk_full(&wrapped)); + } + + /// Text fallback: the exact flattened Sentry string (TAURI-RUST-4R8) is + /// classified even when no rusqlite error is available to downcast (the + /// canonical phrase is mid-string, not a suffix). + #[test] + fn is_sqlite_disk_full_text_fallback() { + let err = anyhow::anyhow!( + "Failed to claim next mem_tree_jobs row: database or disk is full: \ + Error code 13: Insertion failed because database is full" + ); + assert!(is_sqlite_disk_full(&err)); + } + + /// Busy/locked, constraint violations, and unrelated errors must NOT be + /// swallowed as disk-full — those still warrant their own handling / + /// Sentry escalation. + #[test] + fn is_sqlite_disk_full_does_not_match_other_errors() { + let busy = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DatabaseBusy, + extended_code: 5, + }, + Some("database is locked".into()), + ); + assert!(!is_sqlite_disk_full(&anyhow::Error::from(busy))); + + let constraint = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::ConstraintViolation, + extended_code: 19, + }, + Some("UNIQUE constraint failed: mem_tree_jobs.dedupe_key".into()), + ); + assert!(!is_sqlite_disk_full(&anyhow::Error::from(constraint))); + + assert!(!is_sqlite_disk_full(&anyhow::anyhow!( + "upstream returned 500: internal server error" + ))); + } + + // ── is_sqlite_corrupt tests (#4048 / Sentry TAURI-RUST-E93) ────────────── + + /// `SQLITE_CORRUPT` (primary code `DatabaseCorrupt`, code 11) is the + /// malformed-image signal from `claim_next`; it must classify so the worker + /// quarantines + rebuilds instead of paging Sentry every second. + #[test] + fn is_sqlite_corrupt_matches_database_corrupt_code() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DatabaseCorrupt, + extended_code: 11, + }, + Some("database disk image is malformed".into()), + ); + assert!(is_sqlite_corrupt(&anyhow::Error::from(raw))); + } + + /// `SQLITE_NOTADB` (code `NotADatabase`, 26 — header unreadable) is the + /// same broad on-disk-damage class and must classify too. + #[test] + fn is_sqlite_corrupt_matches_not_a_database_code() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::NotADatabase, + extended_code: 26, + }, + Some("file is not a database".into()), + ); + assert!(is_sqlite_corrupt(&anyhow::Error::from(raw))); + } + + /// The rusqlite error sits a few `.context()` layers deep when it bubbles + /// out of `claim_next` → `with_connection`; the downcast must still find + /// the `DatabaseCorrupt` code. + #[test] + fn is_sqlite_corrupt_matches_through_context_layers() { + let raw = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DatabaseCorrupt, + extended_code: 11, + }, + Some("database disk image is malformed".into()), + ); + let wrapped = anyhow::Error::from(raw) + .context("Failed to claim next mem_tree_jobs row") + .context("with_connection closure failed"); + assert!(is_sqlite_corrupt(&wrapped)); + } + + /// Text fallback: the exact flattened Sentry string (TAURI-RUST-E93) must + /// classify even when no rusqlite error is available to downcast. + #[test] + fn is_sqlite_corrupt_text_fallback() { + let err = anyhow::anyhow!( + "Failed to claim next mem_tree_jobs row: database disk image is malformed: \ + Error code 11: The database disk image is malformed" + ); + assert!(is_sqlite_corrupt(&err)); + } + + /// Busy/locked, disk-full, constraint violations, and unrelated errors must + /// NOT be swallowed as corruption — quarantining on those would destroy a + /// perfectly good DB. + #[test] + fn is_sqlite_corrupt_does_not_match_other_errors() { + let busy = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DatabaseBusy, + extended_code: 5, + }, + Some("database is locked".into()), + ); + assert!(!is_sqlite_corrupt(&anyhow::Error::from(busy))); + + let disk_full = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DiskFull, + extended_code: 13, + }, + Some("database or disk is full".into()), + ); + assert!(!is_sqlite_corrupt(&anyhow::Error::from(disk_full))); + + let constraint = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::ConstraintViolation, + extended_code: 19, + }, + Some("UNIQUE constraint failed: mem_tree_jobs.dedupe_key".into()), + ); + assert!(!is_sqlite_corrupt(&anyhow::Error::from(constraint))); + + assert!(!is_sqlite_corrupt(&anyhow::anyhow!( + "upstream returned 500: internal server error" + ))); + } + + // ── is_host_io_error tests (CORE-RUST-19J) ─────────────────────────────── + + /// EIO (`os error 5`) is the CORE-RUST-19J signal: `create_dir_all` on a + /// failing/disconnected SD card. Must classify so the worker surfaces a + /// `StorageUnavailable` degradation + backs off long instead of paging + /// Sentry every second. + #[test] + fn is_host_io_error_matches_eio() { + let err = anyhow::Error::from(std::io::Error::from_raw_os_error(5)); + assert!(is_host_io_error(&err)); + } + + /// ENOSPC (28, filesystem-level out-of-space on `create_dir`) and EROFS (30, + /// kernel-remounted-read-only — the common next stage of a dying SD card) + /// are the same persistent, user-only-fixable host condition. + #[test] + fn is_host_io_error_matches_enospc_and_erofs() { + for code in [28, 30] { + let err = anyhow::Error::from(std::io::Error::from_raw_os_error(code)); + assert!( + is_host_io_error(&err), + "os error {code} must classify as host I/O" + ); + } + } + + /// The production shape: the `io::Error` bubbles out of `open_and_init` + /// wrapped in `.with_context("Failed to create memory_tree dir: …")` then + /// the `with_connection` layer. The downcast must still find it through the + /// anyhow context chain (regression guard: don't rely on the top-level type). + #[test] + fn is_host_io_error_matches_through_context_layers() { + let wrapped = anyhow::Error::from(std::io::Error::from_raw_os_error(5)) + .context("Failed to create memory_tree dir: /home/x/.openhuman-workspace/workspace/memory_tree") + .context("with_connection closure failed"); + assert!(is_host_io_error(&wrapped)); + } + + /// Text fallback: when no `io::Error` is available to downcast (flattened to + /// a plain `anyhow!` string), the exact flattened CORE-RUST-19J message is + /// still classified via the os-error-number anchor. + #[test] + fn is_host_io_error_text_fallback() { + let err = anyhow::anyhow!( + "Failed to create memory_tree dir: /home/x/.openhuman-workspace/workspace/memory_tree: \ + Input/output error (os error 5)" + ); + assert!(is_host_io_error(&err)); + } + + /// Permission-denied (13), not-found (2), a SQLite disk-full failure (its + /// own arm), and unrelated errors must NOT be swallowed as host I/O — those + /// are real bugs / handled elsewhere and must keep reporting. + #[test] + fn is_host_io_error_does_not_match_other_errors() { + // EACCES — a genuine permission bug, not failing hardware. + assert!(!is_host_io_error(&anyhow::Error::from( + std::io::Error::from_raw_os_error(13) + ))); + // ENOENT. + assert!(!is_host_io_error(&anyhow::Error::from( + std::io::Error::from_raw_os_error(2) + ))); + // SQLITE_FULL stays in is_sqlite_disk_full's arm, not here. + let disk_full = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code: rusqlite::ErrorCode::DiskFull, + extended_code: 13, + }, + Some("database or disk is full".into()), + ); + assert!(!is_host_io_error(&anyhow::Error::from(disk_full))); + // Unrelated. + assert!(!is_host_io_error(&anyhow::anyhow!( + "upstream returned 500: internal server error" + ))); + } + + /// The worker's corruption arm must quarantine a malformed image and rebuild + /// an empty, queryable schema so the queue resumes — exercising the + /// report-once + recover path the live loop runs. + #[tokio::test] + async fn recover_corrupt_db_once_quarantines_and_rebuilds() { + let (_tmp, cfg) = test_config(); + // Lay down a malformed `chunks.db` (garbage header) at the canonical path. + let db_path = cfg.workspace_dir.join("memory_tree").join("chunks.db"); + std::fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + std::fs::write(&db_path, b"not a sqlite database, just garbage bytes").unwrap(); + + let err = anyhow::anyhow!( + "Failed to claim next mem_tree_jobs row: database disk image is malformed" + ); + recover_corrupt_db_once(0, &err, &cfg); + + // Corrupt bytes are preserved alongside (never silently dropped) ... + let quarantined = std::fs::read_dir(db_path.parent().unwrap()) + .unwrap() + .filter_map(|e| e.ok()) + .any(|e| { + e.file_name() + .to_string_lossy() + .contains("chunks.db.corrupt-") + }); + assert!( + quarantined, + "corrupt image must be quarantined, not deleted" + ); + + // ... and the rebuilt queue DB is healthy and empty. + let processed = run_once(&cfg).await.unwrap(); + assert!(!processed, "rebuilt queue starts empty"); + } + + #[tokio::test] + async fn wake_workers_is_noop_before_start() { + wake_workers(); + } + + #[tokio::test] + async fn run_once_returns_false_when_queue_is_empty() { + let (_tmp, cfg) = test_config(); + let processed = run_once(&cfg).await.unwrap(); + assert!(!processed); + } + + #[tokio::test] + async fn run_once_claims_and_completes_a_flush_stale_job() { + let (_tmp, cfg) = test_config(); + let new_job = NewJob::flush_stale(&FlushStalePayload::default(), "2026-05-24", 3).unwrap(); + let id = enqueue(&cfg, &new_job).unwrap().expect("enqueue job"); + + let processed = run_once(&cfg).await.unwrap(); + assert!(processed); + + let job = get_job(&cfg, &id).unwrap().expect("job should still exist"); + assert_eq!(job.kind.as_str(), "flush_stale"); + assert_eq!(job.status, JobStatus::Done); + assert_eq!(count_by_status(&cfg, JobStatus::Done).unwrap(), 1); + assert!(job.completed_at_ms.is_some()); + assert!(job.locked_until_ms.is_none()); + } + + #[tokio::test] + async fn run_once_reschedules_reembed_backfill_jobs_that_defer() { + let (_tmp, mut cfg) = test_config(); + // Deliberate "none" opt-out → InertEmbedder (zero vectors, no network) + // so the backfill has work and Defers; this test pins the worker's + // defer-reschedule path, not embed quality. + cfg.embeddings_provider = Some("none".to_string()); + let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + let chunk = Chunk { + id: chunk_id(SourceKind::Chat, "slack:#eng", 0, "reembed-worker-seed"), + content: "memory content about the phoenix migration project".into(), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: "slack:#eng".into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: vec![], + source_ref: Some(SourceRef::new("slack://x")), + path_scope: None, + }, + token_count: 12, + seq_in_source: 0, + created_at: ts, + partial_message: false, + }; + upsert_chunks(&cfg, std::slice::from_ref(&chunk)).unwrap(); + let content_root = cfg.memory_tree_content_root(); + std::fs::create_dir_all(&content_root).unwrap(); + let staged = content_store::stage_chunks(&content_root, &[chunk]).unwrap(); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + upsert_staged_chunks_tx(&tx, &staged)?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let signature = tree_active_signature(&cfg); + let new_job = NewJob::reembed_backfill(&ReembedBackfillPayload { + signature: signature.clone(), + }) + .unwrap(); + let id = enqueue(&cfg, &new_job) + .unwrap() + .expect("enqueue backfill job"); + + // The TinyCortex LLM gate is process-global, so a parallel libtest can + // briefly own its single permit. In that case `run_once` legitimately + // defers this row for 50 ms with `llm concurrency gate busy` before the + // re-embed handler is reached. Retry that transient gate deferral so + // this test continues to pin the handler's own defer/reschedule path. + let mut job = None; + for _ in 0..20 { + let processed = run_once(&cfg).await.unwrap(); + assert!(processed); + let current = get_job(&cfg, &id).unwrap().expect("job should still exist"); + if current + .last_error + .as_deref() + .is_some_and(|reason| reason.contains("re-embed backfill")) + { + job = Some(current); + break; + } + assert_eq!( + current.last_error.as_deref(), + Some("llm concurrency gate busy"), + "unexpected defer reason before re-embed handler" + ); + tokio::time::sleep(Duration::from_millis(60)).await; + } + let job = job.expect("re-embed handler should run after transient gate contention"); + assert_eq!(job.kind, JobKind::ReembedBackfill); + assert_eq!(job.status, JobStatus::Ready); + assert_eq!( + job.attempts, 0, + "defer should revert the claim attempt bump" + ); + assert!(job.started_at_ms.is_none()); + assert!(job.locked_until_ms.is_none()); + assert!(job.completed_at_ms.is_none()); + assert!( + job.available_at_ms > Utc::now().timestamp_millis(), + "deferred job should be rescheduled into the future" + ); + let defer_reason = job.last_error.as_deref().unwrap_or(""); + assert!( + defer_reason.contains("re-embed backfill") + || defer_reason.contains("llm concurrency gate busy"), + "defer reason should identify the backfill or the shared gate: {defer_reason:?}" + ); + assert_eq!(count_by_status(&cfg, JobStatus::Ready).unwrap(), 1); + } +} diff --git a/core/src/remember.rs b/core/src/remember.rs new file mode 100644 index 0000000..13ebfde --- /dev/null +++ b/core/src/remember.rs @@ -0,0 +1,27 @@ +//! High-level memory capture / remember orchestration. +//! +//! `memory_store` owns persistence, `memory_sync` owns upstream pulls, and +//! `memory_tree` owns summarisation / traversal mechanics. This module is where +//! the `memory` domain decides how an incoming "remember this" request should be +//! classified before delegating to those backends. + +use serde::{Deserialize, Serialize}; + +/// Origin of a remember request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RememberSourceKind { + ChatHistory, + UploadedData, + LlmThought, +} + +impl RememberSourceKind { + pub fn as_str(self) -> &'static str { + match self { + Self::ChatHistory => "chat_history", + Self::UploadedData => "uploaded_data", + Self::LlmThought => "llm_thought", + } + } +} diff --git a/core/src/rpc_models.rs b/core/src/rpc_models.rs new file mode 100644 index 0000000..1b4552b --- /dev/null +++ b/core/src/rpc_models.rs @@ -0,0 +1,595 @@ +//! RPC data models for the OpenHuman memory system. +//! +//! This module defines the request and response structures used by the JSON-RPC +//! interface to interact with the memory system. These models ensure type-safe +//! communication between the frontend/client and the Rust backend. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Standard error structure for API responses. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ApiError { + /// A machine-readable error code. + pub code: String, + /// A human-readable error message. + pub message: String, + /// Optional additional error details. + #[serde(skip_serializing_if = "Option::is_none")] + pub details: Option, +} + +/// Pagination metadata for list-based responses. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PaginationMeta { + /// Maximum number of items requested. + pub limit: usize, + /// Number of items skipped. + pub offset: usize, + /// Total number of items available in the backend. + pub count: usize, +} + +/// General metadata included in all API envelopes. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ApiMeta { + /// Unique identifier for the request. + pub request_id: String, + /// Time taken to process the request in seconds. + #[serde(skip_serializing_if = "Option::is_none")] + pub latency_seconds: Option, + /// Whether the response was served from a cache. + #[serde(skip_serializing_if = "Option::is_none")] + pub cached: Option, + /// Optional counts of various items (e.g., by category). + #[serde(skip_serializing_if = "Option::is_none")] + pub counts: Option>, + /// Optional pagination information. + #[serde(skip_serializing_if = "Option::is_none")] + pub pagination: Option, +} + +/// Generic envelope for all API responses. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ApiEnvelope { + /// The actual payload of the response. + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, + /// Error information if the request failed. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Metadata about the request and response. + pub meta: ApiMeta, +} + +/// An empty request body for methods that don't require parameters. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EmptyRequest {} + +/// Request to create a new conversation thread. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CreateConversationThreadRequest { + #[serde(default)] + pub labels: Option>, + #[serde(default)] + pub personality_id: Option, +} + +/// Request payload for `openhuman.memory_init`. +/// +/// `jwt_token` is accepted for backward compatibility but **not used** — memory +/// is local-only (SQLite). Remote/cloud memory sync is a future consideration. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemoryInitRequest { + /// Optional token, currently ignored as memory is local-only. + #[serde(default)] + pub jwt_token: Option, +} + +/// Response payload for `openhuman.memory_init`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryInitResponse { + /// Whether the memory system was successfully initialized. + pub initialized: bool, + /// The root workspace directory. + pub workspace_dir: String, + /// The specific directory where memory data is stored. + pub memory_dir: String, +} + +/// Summary information for a workspace-backed conversation thread. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConversationThreadSummary { + pub id: String, + pub title: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub chat_id: Option, + pub is_active: bool, + pub message_count: usize, + pub last_message_at: String, + pub created_at: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_thread_id: Option, + #[serde(default)] + pub labels: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub personality_id: Option, +} + +/// A single persisted conversation message. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConversationMessageRecord { + pub id: String, + pub content: String, + #[serde(rename = "type")] + pub message_type: String, + #[serde(default)] + pub extra_metadata: serde_json::Value, + pub sender: String, + pub created_at: String, +} + +/// Request to create or update a thread in workspace storage. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UpsertConversationThreadRequest { + pub id: String, + pub title: String, + pub created_at: String, + #[serde(default)] + pub parent_thread_id: Option, + #[serde(default)] + pub labels: Option>, + #[serde(default)] + pub personality_id: Option, +} + +/// Request to update labels for a conversation thread. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UpdateConversationThreadLabelsRequest { + pub thread_id: String, + pub labels: Vec, +} + +/// Request to set a user-specified title on a conversation thread. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UpdateConversationThreadTitleRequest { + pub thread_id: String, + pub title: String, +} + +/// Response payload for thread list operations. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConversationThreadsListResponse { + pub threads: Vec, + pub count: usize, +} + +/// Request to fetch messages for a specific thread. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ConversationMessagesRequest { + pub thread_id: String, +} + +/// Response payload for message list operations. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConversationMessagesResponse { + pub messages: Vec, + pub count: usize, +} + +/// Request to append a message to a thread. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AppendConversationMessageRequest { + pub thread_id: String, + pub message: ConversationMessageRecord, +} + +/// Request to generate or refresh a thread title after the first exchange. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GenerateConversationThreadTitleRequest { + pub thread_id: String, + #[serde(default)] + pub assistant_message: Option, +} + +/// Request to patch a persisted message. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UpdateConversationMessageRequest { + pub thread_id: String, + pub message_id: String, + #[serde(default)] + pub extra_metadata: Option, +} + +/// Request to delete a thread and its message log. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteConversationThreadRequest { + pub thread_id: String, + pub deleted_at: String, +} + +/// Response payload for single-thread deletion. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DeleteConversationThreadResponse { + pub deleted: bool, +} + +/// Response payload for purging all workspace-backed conversations. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PurgeConversationThreadsResponse { + pub messages_deleted: usize, + pub agent_threads_deleted: usize, + pub agent_messages_deleted: usize, +} + +/// Request payload for `openhuman.list_documents`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ListDocumentsRequest { + /// Optional namespace filter. + #[serde(default)] + pub namespace: Option, +} + +/// Summary information for a document in memory. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryDocumentSummary { + /// Unique identifier for the document. + pub document_id: String, + /// Namespace the document belongs to. + pub namespace: String, + /// Lookup key for the document. + pub key: String, + /// Human-readable title. + pub title: String, + /// Type of the source (e.g., "file", "web", "note"). + pub source_type: String, + /// Ingestion priority. + pub priority: String, + /// Creation timestamp (Unix epoch). + pub created_at: f64, + /// Last update timestamp (Unix epoch). + pub updated_at: f64, +} + +/// Response payload for `openhuman.list_documents`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListDocumentsResponse { + /// The namespace used for filtering. + #[serde(default)] + pub namespace: Option, + /// The list of document summaries. + pub documents: Vec, + /// Total number of documents found. + pub count: usize, +} + +/// Response payload for `openhuman.list_namespaces`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListNamespacesResponse { + /// List of available namespace names. + pub namespaces: Vec, + /// Total number of namespaces. + pub count: usize, +} + +/// Request payload for `openhuman.delete_document`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeleteDocumentRequest { + /// Namespace containing the document. + pub namespace: String, + /// ID of the document to delete. + pub document_id: String, +} + +/// Response payload for `openhuman.delete_document`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeleteDocumentResponse { + /// Status message of the operation. + pub status: String, + /// Namespace of the document. + pub namespace: String, + /// ID of the deleted document. + pub document_id: String, + /// Whether the deletion was successful. + pub deleted: bool, +} + +/// Request payload for `openhuman.query_namespace`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct QueryNamespaceRequest { + /// Namespace to query. + pub namespace: String, + /// Natural language query or search term. + pub query: String, + /// Whether to include reference citations in the response. + #[serde(default)] + pub include_references: Option, + /// Optional filter to specific document IDs. + #[serde(default)] + pub document_ids: Option>, + /// Maximum number of results to return. + #[serde(default)] + pub limit: Option, + /// Alias for limit, specifying max number of chunks. + #[serde(default)] + pub max_chunks: Option, +} + +impl QueryNamespaceRequest { + /// Resolves the effective limit from `max_chunks`, `limit`, or a default value. + pub fn resolved_limit(&self) -> u32 { + self.max_chunks.or(self.limit).unwrap_or(10) + } +} + +/// Response payload for `openhuman.query_namespace`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryNamespaceResponse { + /// Retrieved context including entities, relations, and chunks. + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, + /// A formatted message suitable for inclusion in an LLM prompt. + #[serde(skip_serializing_if = "Option::is_none")] + pub llm_context_message: Option, +} + +/// Request payload for `openhuman.recall_context`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecallContextRequest { + /// Namespace to recall from. + pub namespace: String, + /// Whether to include references. + #[serde(default)] + pub include_references: Option, + /// Maximum number of results. + #[serde(default)] + pub limit: Option, + /// Maximum number of chunks. + #[serde(default)] + pub max_chunks: Option, +} + +impl RecallContextRequest { + /// Resolves the effective limit. + pub fn resolved_limit(&self) -> u32 { + self.max_chunks.or(self.limit).unwrap_or(10) + } +} + +/// Response payload for `openhuman.recall_context`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecallContextResponse { + /// Retrieved context. + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, + /// Formatted LLM message. + #[serde(skip_serializing_if = "Option::is_none")] + pub llm_context_message: Option, +} + +/// Request payload for `openhuman.recall_memories`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecallMemoriesRequest { + /// Namespace to recall from. + pub namespace: String, + /// Minimum retention score (0.0 to 1.0). + #[serde(default)] + pub min_retention: Option, + /// Temporal filter (Unix epoch). + #[serde(default)] + pub as_of: Option, + /// Maximum results. + #[serde(default)] + pub limit: Option, + /// Alias for limit. + #[serde(default)] + pub max_chunks: Option, + /// Alias for limit (top K results). + #[serde(default)] + pub top_k: Option, +} + +impl RecallMemoriesRequest { + /// Resolves the effective limit checking `top_k`, `max_chunks`, and `limit`. + pub fn resolved_limit(&self) -> u32 { + self.top_k.or(self.max_chunks).or(self.limit).unwrap_or(10) + } +} + +/// Represents an entity retrieved from memory. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRetrievalEntity { + /// Unique identifier for the entity. + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Name of the entity. + pub name: String, + /// Type of the entity (e.g., "Person", "Place"). + #[serde(skip_serializing_if = "Option::is_none")] + pub entity_type: Option, + /// Retrieval relevance score. + #[serde(skip_serializing_if = "Option::is_none")] + pub score: Option, + /// Additional arbitrary metadata. + #[serde(default)] + pub metadata: serde_json::Value, +} + +/// Represents a relationship between two entities. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRetrievalRelation { + /// The subject entity. + pub subject: String, + /// The relationship type (predicate). + pub predicate: String, + /// The object entity. + pub object: String, + /// Relevance score. + #[serde(skip_serializing_if = "Option::is_none")] + pub score: Option, + /// Number of times this relation was evidenced. + #[serde(skip_serializing_if = "Option::is_none")] + pub evidence_count: Option, + /// Additional metadata. + #[serde(default)] + pub metadata: serde_json::Value, +} + +/// Represents a text chunk retrieved from memory. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRetrievalChunk { + /// ID of the chunk. + #[serde(skip_serializing_if = "Option::is_none")] + pub chunk_id: Option, + /// ID of the parent document. + #[serde(skip_serializing_if = "Option::is_none")] + pub document_id: Option, + /// The text content of the chunk. + pub content: String, + /// Relevance score. + pub score: f64, + /// Additional metadata. + #[serde(default)] + pub metadata: serde_json::Value, + /// Creation timestamp. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_at: Option, + /// Last update timestamp. + #[serde(skip_serializing_if = "Option::is_none")] + pub updated_at: Option, +} + +/// Container for all retrieved memory components. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRetrievalContext { + /// List of entities found. + pub entities: Vec, + /// List of relations between entities. + pub relations: Vec, + /// List of raw text chunks. + pub chunks: Vec, +} + +/// A specific item recalled from memory. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRecallItem { + /// Type of memory item (e.g., "fact", "observation"). + #[serde(rename = "type")] + pub kind: String, + /// Unique ID of the item. + pub id: String, + /// Text content of the memory. + pub content: String, + /// Relevance score. + pub score: f64, + /// Retention strength (0.0 to 1.0). + #[serde(skip_serializing_if = "Option::is_none")] + pub retention: Option, + /// Timestamp of last access. + #[serde(skip_serializing_if = "Option::is_none")] + pub last_accessed_at: Option, + /// Total number of times this memory was accessed. + #[serde(skip_serializing_if = "Option::is_none")] + pub access_count: Option, + /// How many days the memory has remained stable. + #[serde(skip_serializing_if = "Option::is_none")] + pub stability_days: Option, +} + +/// Response payload for `openhuman.recall_memories`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecallMemoriesResponse { + /// List of recalled memory items. + pub memories: Vec, +} + +/// Request payload for `openhuman.list_memory_files`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ListMemoryFilesRequest { + /// Directory path relative to the memory root. + #[serde(default = "default_memory_relative_dir")] + pub relative_dir: String, +} + +/// Response payload for `openhuman.list_memory_files`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListMemoryFilesResponse { + /// The directory listed. + pub relative_dir: String, + /// List of filenames. + pub files: Vec, + /// Total count of files. + pub count: usize, +} + +/// Request payload for `openhuman.read_memory_file`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReadMemoryFileRequest { + /// Path to the file relative to the memory root. + pub relative_path: String, +} + +/// Response payload for `openhuman.read_memory_file`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReadMemoryFileResponse { + /// The path of the file read. + pub relative_path: String, + /// Full content of the file. + pub content: String, +} + +/// Request payload for `openhuman.write_memory_file`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WriteMemoryFileRequest { + /// Path to write to relative to the memory root. + pub relative_path: String, + /// Content to write. + pub content: String, +} + +/// Response payload for `openhuman.write_memory_file`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WriteMemoryFileResponse { + /// The path of the file written. + pub relative_path: String, + /// Whether the write was successful. + pub written: bool, + /// Number of bytes written. + pub bytes_written: usize, +} + +/// Default directory for memory operations. Empty string means the memory +/// root itself (`/memory`); the file-based memory RPCs resolve all +/// relative paths under that directory. +pub(crate) fn default_memory_relative_dir() -> String { + String::new() +} + +#[cfg(test)] +#[path = "rpc_models_tests.rs"] +mod tests; diff --git a/core/src/rpc_models_tests.rs b/core/src/rpc_models_tests.rs new file mode 100644 index 0000000..78e4d6d --- /dev/null +++ b/core/src/rpc_models_tests.rs @@ -0,0 +1,239 @@ +//! Unit tests for the memory RPC request/response models, covering +//! deserialization compatibility and limit-resolution helpers. + +use super::*; +use serde_json::json; + +#[test] +fn recall_memories_request_accepts_compatibility_noop_params() { + let request: RecallMemoriesRequest = serde_json::from_value(json!({ + "namespace": "team", + "top_k": 7, + "min_retention": 0.8, + "as_of": 1700000000.0 + })) + .expect("compatibility params should deserialize"); + + assert_eq!(request.namespace, "team"); + assert_eq!(request.top_k, Some(7)); + assert_eq!(request.min_retention, Some(0.8)); + assert_eq!(request.as_of, Some(1_700_000_000.0)); +} + +#[test] +fn recall_memories_request_limit_resolution_ignores_compatibility_noop_params() { + let request: RecallMemoriesRequest = serde_json::from_value(json!({ + "namespace": "team", + "limit": 3, + "min_retention": 0.5, + "as_of": 1700000000.0 + })) + .expect("request should deserialize"); + + assert_eq!(request.resolved_limit(), 3); +} + +// ── resolved_limit priorities ───────────────────────────────── + +#[test] +fn recall_memories_resolved_limit_prefers_top_k_over_max_chunks_and_limit() { + let req = RecallMemoriesRequest { + namespace: "n".into(), + min_retention: None, + as_of: None, + limit: Some(5), + max_chunks: Some(7), + top_k: Some(9), + }; + assert_eq!(req.resolved_limit(), 9); +} + +#[test] +fn recall_memories_resolved_limit_falls_back_to_max_chunks_then_limit_then_default() { + let without_top_k = RecallMemoriesRequest { + namespace: "n".into(), + min_retention: None, + as_of: None, + limit: Some(5), + max_chunks: Some(7), + top_k: None, + }; + assert_eq!(without_top_k.resolved_limit(), 7); + + let limit_only = RecallMemoriesRequest { + namespace: "n".into(), + min_retention: None, + as_of: None, + limit: Some(5), + max_chunks: None, + top_k: None, + }; + assert_eq!(limit_only.resolved_limit(), 5); + + let none = RecallMemoriesRequest { + namespace: "n".into(), + min_retention: None, + as_of: None, + limit: None, + max_chunks: None, + top_k: None, + }; + assert_eq!(none.resolved_limit(), 10); +} + +#[test] +fn query_namespace_resolved_limit_prefers_max_chunks_then_limit_then_default() { + let req = QueryNamespaceRequest { + namespace: "n".into(), + query: "q".into(), + include_references: None, + document_ids: None, + limit: Some(3), + max_chunks: Some(9), + }; + assert_eq!(req.resolved_limit(), 9); + + let req_limit_only = QueryNamespaceRequest { + namespace: "n".into(), + query: "q".into(), + include_references: None, + document_ids: None, + limit: Some(3), + max_chunks: None, + }; + assert_eq!(req_limit_only.resolved_limit(), 3); + + let req_none = QueryNamespaceRequest { + namespace: "n".into(), + query: "q".into(), + include_references: None, + document_ids: None, + limit: None, + max_chunks: None, + }; + assert_eq!(req_none.resolved_limit(), 10); +} + +#[test] +fn recall_context_resolved_limit_prefers_max_chunks_then_limit_then_default() { + let req = RecallContextRequest { + namespace: "n".into(), + include_references: None, + limit: Some(3), + max_chunks: Some(9), + }; + assert_eq!(req.resolved_limit(), 9); + + let req_limit_only = RecallContextRequest { + namespace: "n".into(), + include_references: None, + limit: Some(3), + max_chunks: None, + }; + assert_eq!(req_limit_only.resolved_limit(), 3); + + let req_none = RecallContextRequest { + namespace: "n".into(), + include_references: None, + limit: None, + max_chunks: None, + }; + assert_eq!(req_none.resolved_limit(), 10); +} + +// ── deny_unknown_fields enforcement ─────────────────────────── + +#[test] +fn query_namespace_request_rejects_unknown_fields() { + let err = serde_json::from_value::(json!({ + "namespace": "n", + "query": "q", + "bogus": 1 + })) + .unwrap_err(); + assert!(err.to_string().contains("bogus")); +} + +#[test] +fn recall_context_request_rejects_unknown_fields() { + let err = serde_json::from_value::(json!({ + "namespace": "n", + "bogus": true + })) + .unwrap_err(); + assert!(err.to_string().contains("bogus")); +} + +#[test] +fn empty_request_rejects_any_field() { + let err = serde_json::from_value::(json!({"x": 1})).unwrap_err(); + assert!(err.to_string().contains("x")); + serde_json::from_value::(json!({})).unwrap(); +} + +// ── MemoryInitRequest tolerates backwards-compatible jwt_token ──── + +#[test] +fn memory_init_request_jwt_token_is_optional_and_ignored() { + let without: MemoryInitRequest = serde_json::from_value(json!({})).unwrap(); + assert_eq!(without.jwt_token, None); + let with: MemoryInitRequest = serde_json::from_value(json!({"jwt_token": "abc"})).unwrap(); + assert_eq!(with.jwt_token.as_deref(), Some("abc")); +} + +// ── ApiError / ApiMeta / ApiEnvelope round-trip ────────────── + +#[test] +fn api_error_round_trips_with_optional_details() { + let err = ApiError { + code: "E".into(), + message: "boom".into(), + details: Some(json!({"why": "reason"})), + }; + let s = serde_json::to_string(&err).unwrap(); + let back: ApiError = serde_json::from_str(&s).unwrap(); + assert_eq!(back.code, "E"); + assert_eq!(back.message, "boom"); + assert!(back.details.is_some()); +} + +#[test] +fn api_error_without_details_omits_field_when_serialized() { + let err = ApiError { + code: "E".into(), + message: "boom".into(), + details: None, + }; + let s = serde_json::to_string(&err).unwrap(); + assert!(!s.contains("details"), "got: {s}"); +} + +#[test] +fn api_envelope_round_trip_preserves_data_and_meta() { + let env = ApiEnvelope:: { + data: Some(42), + error: None, + meta: ApiMeta { + request_id: "r1".into(), + latency_seconds: Some(0.5), + cached: Some(false), + counts: None, + pagination: Some(PaginationMeta { + limit: 10, + offset: 0, + count: 1, + }), + }, + }; + let s = serde_json::to_string(&env).unwrap(); + let back: ApiEnvelope = serde_json::from_str(&s).unwrap(); + assert_eq!(back.data, Some(42)); + assert!(back.error.is_none()); + assert_eq!(back.meta.pagination.unwrap().count, 1); +} + +#[test] +fn default_memory_relative_dir_is_memory() { + // Empty string == the memory root itself (`/memory`). + assert_eq!(crate::rpc_models::default_memory_relative_dir(), ""); +} diff --git a/core/src/scheduler_gate.rs b/core/src/scheduler_gate.rs new file mode 100644 index 0000000..2b2452a --- /dev/null +++ b/core/src/scheduler_gate.rs @@ -0,0 +1,97 @@ +//! [`SchedulerGate`] — the host's background-work throttle, as the core sees it. +//! +//! The memory subsystem runs three kinds of unattended work: the ingest-queue +//! workers, the periodic Composio sync, and the workspace watcher. All three +//! must back off when the host says background AI work is not welcome right now +//! — the user turned it off, the machine is on battery, or nobody is signed in. +//! +//! That decision is host policy and it is global: the same gate throttles cron, +//! the subconscious and the agent harness. The core only asks. +//! +//! # Why the trait is here and not in `tinymemory-api` +//! +//! [`SchedulerGate::resume_notify`] hands back a `tokio::sync::Notify`, and the +//! contract crate must not depend on an async runtime. This crate already does. +//! +//! # The permit is opaque on purpose +//! +//! [`SchedulerGate::wait_for_capacity`] returns a `Box` rather than +//! the host's concrete `LlmPermit`. Callers bind it and hold it for the +//! duration of the LLM-bound work; releasing it is `Drop`, which works exactly +//! the same through the box. Naming the concrete type would drag the host's +//! semaphore into the contract for no gain. +//! +//! # Unwired means "no throttling", not "blocked" +//! +//! With no gate installed — unit tests, the standalone engine build — the +//! policy reads [`Policy::Normal`] and `wait_for_capacity` returns immediately. +//! Failing closed here would deadlock every worker in every test that has not +//! wired a host up, and the gate is an optimisation, not a correctness barrier. + +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::RwLock; +use tokio::sync::Notify; + +pub use tinymemory_api::host::{PauseReason, Policy}; + +/// The host's view of whether background AI work should run right now. +#[async_trait] +pub trait SchedulerGate: Send + Sync + std::fmt::Debug { + /// The current scheduling tier. + fn current_policy(&self) -> Policy; + + /// A handle that is notified whenever the gate leaves a paused state, so a + /// sleeping loop can wake immediately instead of waiting out its tick. + fn resume_notify(&self) -> Arc; + + /// Wait until an LLM-bound slot is free, returning a permit to hold for the + /// duration of the call. `None` when the caller should proceed ungated. + async fn wait_for_capacity(&self) -> Option>; +} + +static GATE: RwLock>> = RwLock::new(None); + +/// Install the host's scheduler gate. Called once during startup wiring. +pub fn set_scheduler_gate(gate: Arc) { + *GATE.write() = Some(gate); +} + +/// Remove any installed gate, returning to ungated behaviour. For tests. +pub fn clear_scheduler_gate() { + *GATE.write() = None; +} + +/// The installed gate, or `None` when nothing has been wired up. +#[must_use] +pub fn scheduler_gate() -> Option> { + GATE.read().clone() +} + +/// The current scheduling tier, or [`Policy::Normal`] when ungated. +#[must_use] +pub fn current_policy() -> Policy { + scheduler_gate().map_or(Policy::Normal, |gate| gate.current_policy()) +} + +/// The resume handle. When ungated this is a `Notify` nobody ever fires, so a +/// `select!` on it simply never takes that arm. +#[must_use] +pub fn resume_notify() -> Arc { + match scheduler_gate() { + Some(gate) => gate.resume_notify(), + None => { + static IDLE: std::sync::OnceLock> = std::sync::OnceLock::new(); + Arc::clone(IDLE.get_or_init(|| Arc::new(Notify::new()))) + } + } +} + +/// Wait for an LLM-bound slot. Returns immediately when ungated. +pub async fn wait_for_capacity() -> Option> { + match scheduler_gate() { + Some(gate) => gate.wait_for_capacity().await, + None => None, + } +} diff --git a/core/src/search/mod.rs b/core/src/search/mod.rs new file mode 100644 index 0000000..d28c4f1 --- /dev/null +++ b/core/src/search/mod.rs @@ -0,0 +1,8 @@ +//! Consolidated memory search & retrieval module. +//! +//! All agent-facing retrieval tools, vector search infrastructure, and +//! scoring algorithms are accessible from here. Lower layers (`memory_store`, +//! `memory_tree`) provide persistence and tree traversal; this module composes +//! them into tools the agent can invoke. + +// ── Public re-exports ─────────────────────────────────────────────────────── diff --git a/core/src/shutdown.rs b/core/src/shutdown.rs new file mode 100644 index 0000000..553d0b5 --- /dev/null +++ b/core/src/shutdown.rs @@ -0,0 +1,66 @@ +//! [`ShutdownHost`] — registering work that must run before the process exits. +//! +//! The ingest-queue workers hold database leases while a job runs. On a clean +//! shutdown they release them, so the next launch re-claims the work +//! immediately instead of waiting out the lease — which otherwise surfaces as a +//! stale-lock recovery warning on every start. A hard kill still falls back to +//! lease expiry. +//! +//! Ordering shutdown across every subsystem is the host's job, so the core +//! hands it a hook rather than owning a lifecycle of its own. +//! +//! # Unwired means the hook never runs +//! +//! Which is the hard-kill path, and already handled: leases expire and startup +//! recovery reclaims them. So registering without a host installed logs and +//! moves on rather than failing. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use parking_lot::RwLock; + +/// A hook the host awaits during shutdown. Boxed because the host stores a +/// heterogeneous list of them. +/// +/// `Fn`, not `FnOnce`: the host may invoke it more than once, so each call must +/// own whatever state it needs. +pub type ShutdownHook = + Box Pin + Send>> + Send + Sync + 'static>; + +/// Accepts shutdown hooks from the core. +pub trait ShutdownHost: Send + Sync + std::fmt::Debug { + /// Register `hook` to be awaited during shutdown. + fn register(&self, hook: ShutdownHook); +} + +static HOST: RwLock>> = RwLock::new(None); + +/// Install the host's shutdown registry. Called once during startup wiring. +pub fn set_shutdown_host(host: Arc) { + *HOST.write() = Some(host); +} + +/// Remove any installed host. For tests. +pub fn clear_shutdown_host() { + *HOST.write() = None; +} + +/// Register a hook to run before the process exits. +/// +/// A no-op beyond logging when no host is installed — see the module docs. +pub fn register(hook: F) +where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future + Send + 'static, +{ + let host = HOST.read().clone(); + match host { + Some(host) => host.register(Box::new(move || Box::pin(hook()))), + None => log::debug!( + "[memory:shutdown] hook dropped — no shutdown host installed; \ + leases will be reclaimed by expiry at next startup instead" + ), + } +} diff --git a/core/src/source_scope.rs b/core/src/source_scope.rs new file mode 100644 index 0000000..d137bc9 --- /dev/null +++ b/core/src/source_scope.rs @@ -0,0 +1,212 @@ +//! Ambient per-turn allowlist of memory-source scopes an agent may recall from. +//! +//! Agent profiles can restrict which memory sources a flavour recalls (the +//! `AgentProfile::memory_sources` allowlist). Threading that allowlist through +//! every memory tool and the deep `select_trees` retrieval layer would touch +//! dozens of call sites, so — mirroring [`thread_context`] — the channel sets a +//! [`tokio::task_local`] around the agent turn and the source-tree retrieval +//! reads it. +//! +//! Semantics: +//! - `None` scope (outside any [`with_source_scope`], or `with_source_scope(None, …)`) +//! means **unrestricted** — every source tree is visible. This is the default +//! for profile-less cron, sub-agents, the CLI, and any profile that left +//! `memory_sources` unset. +//! - `Some(set)` restricts recall to source trees whose `scope` string is in the +//! set. An empty set surfaces nothing (the profile selected no sources). +//! +//! The allowlist entries are matched against tree `scope` strings — the same +//! identifiers the `memory_tree_query_source` tool accepts as `source_id`. +//! +//! [`thread_context`]: crate::thread_context +//! +//! ```ignore +//! use crate::source_scope::{with_source_scope, current_source_scope}; +//! +//! with_source_scope(Some(vec!["slack:#eng".into()]), async { +//! assert!(current_source_scope().unwrap().contains("slack:#eng")); +//! }).await; +//! ``` + +use std::collections::HashSet; +use std::future::Future; + +tokio::task_local! { + static SOURCE_SCOPE: Option>; +} + +/// Normalize a raw allowlist into the task-local representation. Trims entries +/// and drops empties. `None` → unrestricted; `Some(vec)` → restricted (an empty +/// vec stays `Some(empty)` = "no sources"). +fn normalize(allowlist: Option>) -> Option> { + allowlist.map(|items| { + items + .into_iter() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect::>() + }) +} + +/// Run `fut` with `allowlist` available to any descendant call to +/// [`current_source_scope`]. `None` leaves recall unrestricted. +pub async fn with_source_scope(allowlist: Option>, fut: F) -> T +where + F: Future, +{ + let value = normalize(allowlist); + log::debug!( + "[memory:source_scope] entering scope: {}", + match &value { + None => "unrestricted".to_string(), + Some(set) => format!("{} source(s)", set.len()), + } + ); + SOURCE_SCOPE.scope(value, fut).await +} + +/// Return the ambient source-scope allowlist set by an enclosing +/// [`with_source_scope`], or `None` (unrestricted) when called outside one. +pub fn current_source_scope() -> Option> { + SOURCE_SCOPE.try_with(|v| v.clone()).ok().flatten() +} + +/// Whether `scope` is recallable under the ambient allowlist. `true` when there +/// is no active scope (unrestricted) or when the scope is explicitly allowed. +pub fn scope_allowed(scope: &str) -> bool { + match current_source_scope() { + None => true, + Some(set) => set.contains(scope), + } +} + +/// The tag every memory-source–ingested chunk carries (set by +/// `memory_sources::sync` and the github reader). Used as the discriminator so +/// the chunk-level gate only touches memory-SOURCE chunks and never working / +/// conversation / internal chunks. +const MEMORY_SOURCE_TAG: &str = "memory_sources"; + +/// Whether a memory-store chunk is recallable under the ambient allowlist, +/// given its `tags` and `source_id`. +/// +/// Fail-open for everything that is NOT a memory-source chunk: a chunk without +/// the `memory_sources` tag (working memory, conversation transcripts, internal +/// chunks) always passes. A tagged memory-source chunk passes iff its source +/// identifier is allowed — matched flexibly against either the raw `source_id` +/// (Composio / channel scopes like `slack:#eng`) or the registry id extracted +/// from a `mem_src::` composite (reader-based sources). `None` scope +/// is unrestricted. +pub fn chunk_source_allowed(tags: &[String], source_id: &str) -> bool { + match current_source_scope() { + None => true, + Some(set) => chunk_source_allowed_in(&set, tags, source_id), + } +} + +/// Pure form of [`chunk_source_allowed`] against an explicit allowlist `set`, +/// for callers that already hold the scope (e.g. `list_chunks`, which captures +/// it on the async side and filters DB rows before applying the row limit so a +/// disallowed-source-heavy prefix can't starve permitted rows). +pub fn chunk_source_allowed_in(set: &HashSet, tags: &[String], source_id: &str) -> bool { + let is_memory_source = tags.iter().any(|t| t == MEMORY_SOURCE_TAG); + if !is_memory_source { + return true; + } + if set.contains(source_id) { + return true; + } + crate::sync_events::extract_mem_src_id(source_id).is_some_and(|id| set.contains(id)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn unrestricted_outside_scope() { + assert!(current_source_scope().is_none()); + assert!(scope_allowed("anything")); + } + + #[tokio::test] + async fn restricts_to_allowlisted_scopes() { + with_source_scope( + Some(vec!["slack:#eng".into(), " gmail:me ".into()]), + async { + let set = current_source_scope().expect("scope set"); + assert_eq!(set.len(), 2); + assert!(scope_allowed("slack:#eng")); + assert!(scope_allowed("gmail:me")); // trimmed + assert!(!scope_allowed("notion:team")); + }, + ) + .await; + // Must not leak past the scope. + assert!(current_source_scope().is_none()); + assert!(scope_allowed("notion:team")); + } + + #[tokio::test] + async fn empty_allowlist_blocks_everything() { + with_source_scope(Some(vec![]), async { + assert!(current_source_scope().is_some()); + assert!(!scope_allowed("slack:#eng")); + }) + .await; + } + + #[tokio::test] + async fn explicit_none_is_unrestricted() { + with_source_scope(None, async { + assert!(current_source_scope().is_none()); + assert!(scope_allowed("slack:#eng")); + }) + .await; + } + + #[tokio::test] + async fn chunk_gate_passes_non_source_chunks_and_gates_tagged_ones() { + let src_tags = vec!["memory_sources".to_string(), "document".to_string()]; + let other_tags = vec!["conversation".to_string()]; + + with_source_scope( + Some(vec!["slack:#eng".into(), "src-rss-42".into()]), + async { + // Non-source chunk (no memory_sources tag) always passes. + assert!(chunk_source_allowed(&other_tags, "thr_123:user")); + // Composio/channel source chunk: raw source_id == scope. + assert!(chunk_source_allowed(&src_tags, "slack:#eng")); + assert!(!chunk_source_allowed(&src_tags, "gmail:alice")); + // Reader-based composite: extracted registry id matches. + assert!(chunk_source_allowed( + &src_tags, + "mem_src:src-rss-42:https://example.com/item-7" + )); + assert!(!chunk_source_allowed( + &src_tags, + "mem_src:src-folder-9:/notes/a.md" + )); + }, + ) + .await; + } + + #[tokio::test] + async fn chunk_gate_unrestricted_without_scope() { + let src_tags = vec!["memory_sources".to_string()]; + // Outside any scope, even tagged source chunks pass. + assert!(chunk_source_allowed(&src_tags, "gmail:alice")); + } + + #[tokio::test] + async fn chunk_gate_empty_allowlist_blocks_tagged_sources_only() { + let src_tags = vec!["memory_sources".to_string()]; + let other_tags: Vec = vec![]; + with_source_scope(Some(vec![]), async { + assert!(!chunk_source_allowed(&src_tags, "slack:#eng")); + // Non-source chunks still pass even under an empty allowlist. + assert!(chunk_source_allowed(&other_tags, "thr_1:user")); + }) + .await; + } +} diff --git a/core/src/sources/README.md b/core/src/sources/README.md new file mode 100644 index 0000000..15c3f50 --- /dev/null +++ b/core/src/sources/README.md @@ -0,0 +1,107 @@ +# memory_sources + +Registry of data connectors that feed memory. This domain owns the **"what feeds my memory"** question: a typed registry of sources (Composio OAuth connections, local folders, GitHub repos, RSS feeds, Twitter queries, web pages) persisted in `config.toml` under `[[memory_sources]]`. It provides CRUD for source entries, a `SourceReader` trait with per-kind reader implementations that list items and read individual item content, manual sync orchestration that ingests reader output into the memory pipeline, per-source sync status, and the `openhuman.memory_sources_*` RPC surface. It does **not** own sync scheduling or the ingestion engine itself — `memory_sync` / `memory` do that; this module only defines connectors, reads from them, and dispatches sync work to the right backend. + +## Responsibilities + +- CRUD for `MemorySourceEntry` records (add/get/list/update/remove) persisted in `Config.memory_sources`. +- Validate kind-specific required fields at add/update time. +- Provide a uniform `SourceReader` trait with one implementation per `SourceKind`, plus a `reader_for(kind)` dispatcher. +- List readable items and read individual item content from each source. +- Trigger a manual sync per source: Composio sources delegate to `memory_sync::composio`; reader-backed kinds (folder/github/rss/web) walk items and ingest each via `memory::ingest_pipeline::ingest_document`; Twitter is a placeholder. +- Emit sync progress as `MemorySyncStageChanged` events tagged with `connection_id = Some(source.id)`. +- Compute per-source sync status (chunks synced/pending, last-chunk timestamp, freshness label) by querying `mem_tree_chunks`. +- Reconcile active Composio connections into the registry at boot / on list, so freshly-connected integrations appear as sources without a restart. +- Auto-upsert a Composio source on OAuth connection creation (called from `memory_sync::composio::bus`). + +## Key files + +| File | Role | +| --- | --- | +| `src/openhuman/memory/sources/mod.rs` | Module docstring + `pub mod` decls + re-exports (registry CRUD, schemas, core types). Export-focused. | +| `src/openhuman/memory/sources/types.rs` | Core types: `SourceKind`, `MemorySourceEntry` (flattened kind-specific `Option` fields) with `validate()`, `SourceItem`, `ContentType`, `SourceContent`. | +| `src/openhuman/memory/sources/registry.rs` | CRUD over `Config.memory_sources` via the config load/save cycle; `MemorySourcePatch` partial-update payload; `upsert_composio_source` for auto-registration. | +| `src/openhuman/memory/sources/rpc.rs` | RPC handler impls returning `RpcOutcome` (request/response structs for list/get/add/update/remove/list_items/read_item/sync/status_list). `list_rpc` lazily reconciles Composio sources. | +| `src/openhuman/memory/sources/schemas.rs` | Controller-registry schemas + `handle_*` fns delegating to `rpc.rs`; `all_controller_schemas` / `all_registered_controllers`. | +| `src/openhuman/memory/sources/sync.rs` | Per-source sync orchestration. Spawns background task, dispatches by kind, ingests reader output, emits stage events. | +| `src/openhuman/memory/sources/status.rs` | `SourceStatus`, `FreshnessLabel`, `source_status` / `status_list` — queries `mem_tree_chunks` by source-id prefix. | +| `src/openhuman/memory/sources/reconcile.rs` | `ensure_composio_sources` — scans active Composio sync targets and upserts them as sources. | +| `src/openhuman/memory/sources/readers/mod.rs` | `SourceReader` async trait + `reader_for(kind)` dispatcher. | +| `src/openhuman/memory/sources/readers/composio.rs` | `ComposioReader` — returns the connection as a single item; `read_item` is a non-op placeholder (sync is provider-driven). | +| `src/openhuman/memory/sources/readers/folder.rs` | `FolderReader` — glob over a local dir (default `**/*.md`, 10 MB cap), reads file content with path-traversal guard. | +| `src/openhuman/memory/sources/readers/github.rs` | `GithubReader` — pulls project activity (commits/issues/PRs) via `gh` CLI or public REST fallback. | +| `src/openhuman/memory/sources/readers/rss.rs` | `RssReader` — RSS/Atom feed items. | +| `src/openhuman/memory/sources/readers/twitter.rs` | `TwitterReader` — Twitter query reader (sync placeholder pending credentials). | +| `src/openhuman/memory/sources/readers/web_page.rs` | `WebPageReader` — fetches a web page, optional CSS selector. | + +## Public surface + +Re-exported from `mod.rs`: + +- **registry**: `add_source`, `get_source`, `list_sources`, `list_enabled_by_kind`, `remove_source`, `update_source`, `upsert_composio_source`, `MemorySourcePatch`. +- **schemas**: `all_memory_sources_controller_schemas`, `all_memory_sources_registered_controllers`. +- **types**: `ContentType`, `MemorySourceEntry`, `SourceContent`, `SourceItem`, `SourceKind`. + +Reader trait `SourceReader` and `reader_for` are public under `readers`; sync/status/reconcile entry points (`sync::sync_source`, `status::status_list` / `source_status`, `reconcile::ensure_composio_sources`) are public within the module path. + +## RPC / controllers + +Namespace `memory_sources` (`openhuman.memory_sources_*`). Nine controllers, each schema/handler pair defined in `schemas.rs` and delegating to `rpc.rs`: + +| Function | Description | +| --- | --- | +| `list` | List all configured sources (lazily reconciles Composio first). | +| `get` | Get one source by `id`. | +| `add` | Add a source; kind-specific fields are flat on the request. | +| `update` | Partial update of a source. | +| `remove` | Remove a source by `id`. | +| `list_items` | List readable items from a source via its reader. | +| `read_item` | Read one item's content. | +| `sync` | Queue a manual sync (returns immediately; progress via events). | +| `status_list` | Per-source sync status (chunks, freshness, last-chunk ts). | + +Wired into the registry via `core/all.rs` (`all_memory_sources_registered_controllers` / `all_memory_sources_controller_schemas`). + +## Agent tools + +None. This module exposes no agent tools (`tools.rs` does not exist). + +## Events + +No `bus.rs` / `EventHandler` of its own. It **publishes** `DomainEvent::MemorySyncStageChanged` indirectly via `memory::sync::emit_sync_stage` during `sync_source` (stages: Requested, Fetching, Stored, Ingesting, Completed, Failed), tagged `connection_id = Some(source.id)`. The reverse direction — auto-registering a Composio source on connection-created — is driven by `memory_sync::composio::bus`, which calls this module's `upsert_composio_source`. + +## Persistence + +- **Source registry**: persisted in `Config.memory_sources` (`config/schema/types.rs`), serialized as `[[memory_sources]]` in `config.toml`. All mutations reload the live config, apply, and `config.save()` atomically. +- **No dedicated `store.rs`.** Sync status is *read* (not written) from the memory store: `status.rs` queries `mem_tree_chunks` (via `memory_store::chunks::store::with_connection`) using a `source_id LIKE` prefix — `mem_src:{source.id}:%` for reader kinds, `{toolkit}:%` for Composio. Chunks themselves are written by the `memory` ingest pipeline, not here. + +## Dependencies + +- `openhuman::config` (`Config`, `config::rpc::load_config_with_timeout`) — the registry's backing store; readers/sync receive `&Config`. +- `openhuman::memory::ingest_pipeline::ingest_document` — ingests reader-backed source items into memory. +- `openhuman::memory::sync_events` (`emit_sync_stage`, `MemorySyncStage`, `MemorySyncTrigger`) — sync-progress event emission. +- `openhuman::memory::sync::composio` — Composio sync delegate (`run_connection_sync`, `scan_active_sync_targets`, `SyncReason`). +- `tinycortex::memory::ingest::canonicalize::document::DocumentInput` — document shape for ingestion. +- `openhuman::memory::store::chunks::store::with_connection` — SQLite access for status queries against `mem_tree_chunks`. +- `core::all` (`ControllerFuture`, `RegisteredController`) and `core` schema types (`ControllerSchema`, `FieldSchema`, `TypeSchema`) — controller registry wiring. +- `rpc::RpcOutcome` — RPC return contract. +- External: `glob`, `async_trait`, `uuid`, `chrono`, `serde`/`serde_json`, `schemars`, `toml`; `gh` CLI (optional) for the GitHub reader. + +## Used by + +- `core/all.rs`, `core/jsonrpc.rs` — registers the `memory_sources` controllers/schemas. +- `openhuman/mod.rs` — declares the domain module. +- `config/schema/types.rs` — `Config.memory_sources: Vec` field (the persisted store). +- `memory_sync::composio::bus` / `memory_sync::composio::mod` — calls `upsert_composio_source` to auto-register a source on connection creation. +- `composio::ops` — references gmail memory-source cleanup targets (`gmail_memory_sources_for_connection`). + +## Notes / gotchas + +- `MemorySourceEntry` is a single flat struct: all kind-specific fields are `Option`/`Vec` and the `kind` discriminator decides which are required, enforced only by `validate()` (not the type system). RPC `add`/`update` mirror this flat shape. +- `list_rpc` performs a lazy Composio reconciliation on every list call so newly-connected integrations show up immediately (the connection-created hook only fires on OAuth handoff, not on first launch after a prior connect). +- `sync_source` returns `Ok(())` as soon as work is queued; it spawns a nested `tokio::spawn` so a panic in the sync task surfaces as a `tracing::error!` rather than a dropped join handle. Actual completion/failure arrives only via `MemorySyncStageChanged` events. +- Composio sync does not ingest item-by-item — it delegates wholesale to `memory_sync::composio::run_connection_sync`. The `ComposioReader::read_item` body is an explanatory placeholder, never a real fetch. +- Twitter sync is intentionally unimplemented: `sync_source` returns an error for `TwitterQuery` ("Twitter sync not yet configured"). +- Status freshness thresholds: ≤30 s → `Active`, ≤5 min → `Recent`, else / no chunk → `Idle`. `status.rs` surfaces real SQL errors (so a broken DB isn't reported as a healthy zero-row state), but `status_list` degrades a per-source failure to an `Idle` zero-row entry rather than failing the whole call. +- Composio chunk-count matching is by `toolkit` prefix only (`{toolkit}:%`), so distinct connections sharing a toolkit (e.g. two Gmail accounts) are not disambiguated in status counts. +- `FolderReader` caps files at 10 MB on both list and read, and canonicalizes paths to deny traversal outside the configured base. diff --git a/core/src/sources/mod.rs b/core/src/sources/mod.rs new file mode 100644 index 0000000..c8dc477 --- /dev/null +++ b/core/src/sources/mod.rs @@ -0,0 +1,31 @@ +//! Memory sources — registry of data connectors that feed memory. +//! +//! This domain owns the **what feeds my memory** question: a typed +//! registry of sources (Composio OAuth connections, local folders, +//! GitHub repos, RSS feeds, Twitter queries, web pages) persisted +//! in `config.toml` under `[[memory_sources]]`. +//! +//! It provides: +//! - CRUD for source entries (add/remove/list/get/update) +//! - A `SourceReader` trait with per-kind reader implementations +//! that can list items and read individual item content +//! - RPC surface (`openhuman.memory_sources_*`) +//! +//! `memory_sync` consumes from this registry to decide what to sync +//! and when. This module does not own sync scheduling or ingestion — +//! it only defines connectors and reads from them. + +pub mod readers; +pub mod reconcile; +pub mod registry; +pub mod status; +pub mod sync; +pub mod types; + +pub use registry::{ + add_source, apply_all_in, apply_kind_defaults, decode_memory_sources, get_source, + list_enabled_by_kind, list_sources, memory_sync_defaults_for_toolkit, + remove_composio_source_by_connection_id, remove_source, update_source, upsert_composio_source, + MemorySourcePatch, +}; +pub use types::{ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind}; diff --git a/core/src/sources/readers/composio.rs b/core/src/sources/readers/composio.rs new file mode 100644 index 0000000..3bdc6e1 --- /dev/null +++ b/core/src/sources/readers/composio.rs @@ -0,0 +1,110 @@ +//! Composio source reader — delegates to the existing composio sync layer. +//! +//! For Composio sources, `list_items` returns the sync targets and +//! `read_item` is not meaningful (sync is provider-driven, not +//! item-by-item). The reader exists so the registry can uniformly +//! query all source kinds. + +use async_trait::async_trait; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::sources::types::{ + ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, +}; +use crate::Config; + +use super::SourceReader; + +pub struct ComposioReader; + +#[async_trait] +impl SourceReader for ComposioReader { + fn kind(&self) -> SourceKind { + SourceKind::Composio + } + + async fn list_items( + &self, + source: &MemorySourceEntry, + _config: &Config, + ) -> Result, String> { + let toolkit = source.toolkit.as_deref().unwrap_or("unknown"); + let connection_id = source.connection_id.as_deref().unwrap_or("unknown"); + + tracing::debug!( + toolkit = %toolkit, + connection_id = %connection_id, + "[memory_sources:composio] list_items" + ); + + Ok(vec![SourceItem { + id: connection_id.to_string(), + title: format!("{toolkit} connection"), + updated_at_ms: None, + }]) + } + + async fn read_item( + &self, + source: &MemorySourceEntry, + item_id: &str, + _config: &Config, + ) -> Result { + let toolkit = source.toolkit.as_deref().unwrap_or("unknown"); + Ok(SourceContent { + id: item_id.to_string(), + title: format!("{toolkit} sync data"), + body: format!( + "Composio {toolkit} data is synced via the provider sync pipeline, not read item-by-item." + ), + content_type: ContentType::Plaintext, + metadata: serde_json::json!({ + "toolkit": toolkit, + "connection_id": source.connection_id, + }), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sources::types::MemorySourceEntry; + + fn test_source() -> MemorySourceEntry { + MemorySourceEntry { + id: "src_1".into(), + kind: SourceKind::Composio, + label: "Gmail".into(), + enabled: true, + toolkit: Some("gmail".into()), + connection_id: Some("cmp_123".into()), + path: None, + glob: None, + url: None, + branch: None, + paths: Vec::new(), + query: None, + since_days: None, + max_items: None, + max_commits: None, + max_issues: None, + max_prs: None, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days: None, + } + } + + #[tokio::test] + async fn list_items_returns_connection_as_item() { + let reader = ComposioReader; + let config = TestHostConfig::default(); + let items = reader.list_items(&test_source(), &config).await.unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].id, "cmp_123"); + } +} diff --git a/core/src/sources/readers/conversation.rs b/core/src/sources/readers/conversation.rs new file mode 100644 index 0000000..7b18c40 --- /dev/null +++ b/core/src/sources/readers/conversation.rs @@ -0,0 +1,46 @@ +//! Product `Config` adapter for the tinycortex conversation reader. + +use async_trait::async_trait; + +use crate::sources::readers::SourceReader; +use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; +use crate::Config; + +pub struct ConversationReader; + +#[async_trait] +impl SourceReader for ConversationReader { + fn kind(&self) -> SourceKind { + SourceKind::Conversation + } + + async fn list_items( + &self, + source: &MemorySourceEntry, + config: &Config, + ) -> Result, String> { + tinycortex::memory::sources::SourceReader::list_items( + &tinycortex::memory::sources::readers::conversation::ConversationReader, + source, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } + + async fn read_item( + &self, + source: &MemorySourceEntry, + item_id: &str, + config: &Config, + ) -> Result { + tinycortex::memory::sources::SourceReader::read_item( + &tinycortex::memory::sources::readers::conversation::ConversationReader, + source, + item_id, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } +} diff --git a/core/src/sources/readers/folder.rs b/core/src/sources/readers/folder.rs new file mode 100644 index 0000000..31922d5 --- /dev/null +++ b/core/src/sources/readers/folder.rs @@ -0,0 +1,46 @@ +//! Product `Config` adapter for the tinycortex folder reader. + +use async_trait::async_trait; + +use crate::sources::readers::SourceReader; +use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; +use crate::Config; + +pub struct FolderReader; + +#[async_trait] +impl SourceReader for FolderReader { + fn kind(&self) -> SourceKind { + SourceKind::Folder + } + + async fn list_items( + &self, + source: &MemorySourceEntry, + config: &Config, + ) -> Result, String> { + tinycortex::memory::sources::SourceReader::list_items( + &tinycortex::memory::sources::readers::folder::FolderReader, + source, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } + + async fn read_item( + &self, + source: &MemorySourceEntry, + item_id: &str, + config: &Config, + ) -> Result { + tinycortex::memory::sources::SourceReader::read_item( + &tinycortex::memory::sources::readers::folder::FolderReader, + source, + item_id, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } +} diff --git a/core/src/sources/readers/github.rs b/core/src/sources/readers/github.rs new file mode 100644 index 0000000..91d464d --- /dev/null +++ b/core/src/sources/readers/github.rs @@ -0,0 +1,54 @@ +//! Product `Config` adapter for the tinycortex GitHub repo reader. +//! +//! The reader itself — commit/issue/PR fetching over `gh`, `git`, and the +//! public REST API — lives in the engine. This module keeps the host-side +//! `SourceReader` shape (`&Config`, `Result<_, String>`) that the sources RPC +//! surface and the sync runner are written against, and re-exports the two +//! coordinate helpers `sources::sync` derives its scopes from. + +use async_trait::async_trait; + +use crate::sources::readers::SourceReader; +use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; +use crate::Config; + +pub use tinycortex::memory::sources::readers::github::{repo_archive_source_id, repo_chunk_scope}; + +pub struct GithubReader; + +#[async_trait] +impl SourceReader for GithubReader { + fn kind(&self) -> SourceKind { + SourceKind::GithubRepo + } + + async fn list_items( + &self, + source: &MemorySourceEntry, + config: &Config, + ) -> Result, String> { + tinycortex::memory::sources::SourceReader::list_items( + &tinycortex::memory::sources::readers::github::GithubReader, + source, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } + + async fn read_item( + &self, + source: &MemorySourceEntry, + item_id: &str, + config: &Config, + ) -> Result { + tinycortex::memory::sources::SourceReader::read_item( + &tinycortex::memory::sources::readers::github::GithubReader, + source, + item_id, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } +} diff --git a/core/src/sources/readers/mod.rs b/core/src/sources/readers/mod.rs new file mode 100644 index 0000000..0836d76 --- /dev/null +++ b/core/src/sources/readers/mod.rs @@ -0,0 +1,44 @@ +//! Source reader trait and per-kind implementations. + +pub mod composio; +pub mod conversation; +pub mod folder; +pub mod github; +pub mod rss; +pub mod twitter; +pub mod web_page; + +use async_trait::async_trait; + +use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; +use crate::Config; + +/// A reader that can list items and read content from a memory source. +#[async_trait] +pub trait SourceReader: Send + Sync { + fn kind(&self) -> SourceKind; + async fn list_items( + &self, + source: &MemorySourceEntry, + config: &Config, + ) -> Result, String>; + async fn read_item( + &self, + source: &MemorySourceEntry, + item_id: &str, + config: &Config, + ) -> Result; +} + +/// Get the reader for a given source kind. +pub fn reader_for(kind: &SourceKind) -> Box { + match kind { + SourceKind::Composio => Box::new(composio::ComposioReader), + SourceKind::Conversation => Box::new(conversation::ConversationReader), + SourceKind::Folder => Box::new(folder::FolderReader), + SourceKind::GithubRepo => Box::new(github::GithubReader), + SourceKind::TwitterQuery => Box::new(twitter::TwitterReader), + SourceKind::RssFeed => Box::new(rss::RssReader::new()), + SourceKind::WebPage => Box::new(web_page::WebPageReader), + } +} diff --git a/core/src/sources/readers/rss.rs b/core/src/sources/readers/rss.rs new file mode 100644 index 0000000..05a543a --- /dev/null +++ b/core/src/sources/readers/rss.rs @@ -0,0 +1,67 @@ +//! Product `Config` adapter for the tinycortex RSS/Atom feed reader. + +use async_trait::async_trait; + +use crate::sources::readers::SourceReader; +use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; +use crate::Config; + +/// Product adapter retaining the engine reader for a complete sync pass. +/// +/// The engine reader caches a freshly fetched feed between `list_items` and +/// `read_item`, so constructing it per trait call would turn one sync into +/// N+1 downloads. +pub struct RssReader { + inner: tinycortex::memory::sources::readers::rss::RssReader, +} + +impl RssReader { + pub fn new() -> Self { + Self { + inner: tinycortex::memory::sources::readers::rss::RssReader::new(), + } + } +} + +impl Default for RssReader { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl SourceReader for RssReader { + fn kind(&self) -> SourceKind { + SourceKind::RssFeed + } + + async fn list_items( + &self, + source: &MemorySourceEntry, + config: &Config, + ) -> Result, String> { + tinycortex::memory::sources::SourceReader::list_items( + &self.inner, + source, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } + + async fn read_item( + &self, + source: &MemorySourceEntry, + item_id: &str, + config: &Config, + ) -> Result { + tinycortex::memory::sources::SourceReader::read_item( + &self.inner, + source, + item_id, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } +} diff --git a/core/src/sources/readers/twitter.rs b/core/src/sources/readers/twitter.rs new file mode 100644 index 0000000..176f7d2 --- /dev/null +++ b/core/src/sources/readers/twitter.rs @@ -0,0 +1,110 @@ +//! Twitter/X query source reader. +//! +//! Fetches tweets matching a search query. Uses the Twitter API v2 +//! search endpoint. Requires bearer token configuration (not yet +//! wired — this reader validates the source config and returns a +//! clear error when no credentials are available). + +use async_trait::async_trait; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; +use crate::Config; + +use super::SourceReader; + +const DEFAULT_SINCE_DAYS: u32 = 7; + +pub struct TwitterReader; + +#[async_trait] +impl SourceReader for TwitterReader { + fn kind(&self) -> SourceKind { + SourceKind::TwitterQuery + } + + async fn list_items( + &self, + source: &MemorySourceEntry, + _config: &Config, + ) -> Result, String> { + let query = source + .query + .as_deref() + .map(str::trim) + .filter(|q| !q.is_empty()) + .ok_or("twitter source requires a non-empty query")?; + let _since_days = source.since_days.unwrap_or(DEFAULT_SINCE_DAYS); + + tracing::debug!( + query = %query, + "[memory_sources:twitter] list_items" + ); + + // Twitter API v2 requires a bearer token. For now, return an + // informative error until credential wiring lands. + Err(format!( + "Twitter API integration not yet configured. Query '{query}' is saved and will \ + sync once a Twitter bearer token is provided in settings." + )) + } + + async fn read_item( + &self, + _source: &MemorySourceEntry, + item_id: &str, + _config: &Config, + ) -> Result { + tracing::debug!( + item_id = %item_id, + "[memory_sources:twitter] read_item" + ); + + Err("Twitter API integration not yet configured. \ + Individual tweet reading requires a bearer token." + .to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn twitter_source() -> MemorySourceEntry { + MemorySourceEntry { + id: "src_tw".into(), + kind: SourceKind::TwitterQuery, + label: "AI tweets".into(), + enabled: true, + toolkit: None, + connection_id: None, + path: None, + glob: None, + url: None, + branch: None, + paths: Vec::new(), + query: Some("AI safety".into()), + since_days: Some(3), + max_items: None, + max_commits: None, + max_issues: None, + max_prs: None, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days: None, + } + } + + #[tokio::test] + async fn list_items_returns_not_configured_error() { + let reader = TwitterReader; + let result = reader + .list_items(&twitter_source(), &TestHostConfig::default()) + .await; + assert!(result.is_err()); + assert!(result.unwrap_err().contains("not yet configured")); + } +} diff --git a/core/src/sources/readers/web_page.rs b/core/src/sources/readers/web_page.rs new file mode 100644 index 0000000..a2f693c --- /dev/null +++ b/core/src/sources/readers/web_page.rs @@ -0,0 +1,46 @@ +//! Product `Config` adapter for the tinycortex single-page web reader. + +use async_trait::async_trait; + +use crate::sources::readers::SourceReader; +use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; +use crate::Config; + +pub struct WebPageReader; + +#[async_trait] +impl SourceReader for WebPageReader { + fn kind(&self) -> SourceKind { + SourceKind::WebPage + } + + async fn list_items( + &self, + source: &MemorySourceEntry, + config: &Config, + ) -> Result, String> { + tinycortex::memory::sources::SourceReader::list_items( + &tinycortex::memory::sources::readers::web_page::WebPageReader, + source, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } + + async fn read_item( + &self, + source: &MemorySourceEntry, + item_id: &str, + config: &Config, + ) -> Result { + tinycortex::memory::sources::SourceReader::read_item( + &tinycortex::memory::sources::readers::web_page::WebPageReader, + source, + item_id, + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + ) + .await + .map_err(|error| error.to_string()) + } +} diff --git a/core/src/sources/reconcile.rs b/core/src/sources/reconcile.rs new file mode 100644 index 0000000..6f521a9 --- /dev/null +++ b/core/src/sources/reconcile.rs @@ -0,0 +1,397 @@ +//! Startup reconciliation of Composio connections into the memory sources registry. +//! +//! Called once at boot to ensure all active Composio sync targets have +//! a corresponding `MemorySourceEntry` in config. This catches connections +//! created before the memory_sources domain existed. +//! +//! Also owns the retroactive caps migration +//! (`apply_composio_source_caps_migration`) that gives any cap-less Composio +//! source — enabled or disabled — conservative per-toolkit caps. + +use crate::config_loader as config_rpc; +use crate::sources::registry; +use crate::sources::types::{MemorySourceEntry, SourceKind}; +use crate::sync::composio; +use std::collections::HashSet; + +/// Current version of the caps migration. Bump when the migration logic changes +/// so installs that ran an earlier revision re-run it exactly once. +const CURRENT_CAPS_MIGRATION_VERSION: u32 = 1; + +/// Reconcile active Composio connections into the memory sources registry and +/// return the live active-connection set scanned this call. +/// +/// Returns `Some(connection_ids)` — the `connection_id`s of every active sync +/// target — when the live Composio scan **succeeded**, so callers (notably +/// `rpc::list_rpc`) can filter the listing down to connections that are still +/// active and dedupe identical rows. Returns `None` when the scan could not run +/// (config load / network / auth failure); callers must treat `None` as "active +/// set unavailable" and **not** hide any sources — an empty scan from a transient +/// blip must never be read as "everything is inactive". +pub async fn ensure_composio_sources() -> Option> { + tracing::debug!("[memory_sources:reconcile] starting composio reconciliation"); + + let config = match config_rpc::load_config_with_timeout().await { + Ok(c) => c, + Err(e) => { + tracing::warn!( + error = %e, + "[memory_sources:reconcile] failed to load config; skipping" + ); + return None; + } + }; + + // Always hit Composio directly here — using list_sync_targets would + // short-circuit through the registry and miss new connections. + let targets = match composio::scan_active_sync_targets(&*config).await { + Ok(t) => t, + Err(e) => { + tracing::debug!( + error = %e, + "[memory_sources:reconcile] no composio sync targets available; skipping" + ); + return None; + } + }; + + // Build the upsert targets up front, then apply them with a single config + // load + save via the batch path. The per-call upsert does its own + // load-modify-save, so the old loop cost 2N config round-trips for N + // connections; batching collapses that to 2. + let upsert_targets = build_upsert_targets(&targets); + let upserted = match registry::upsert_composio_sources_batch(&upsert_targets).await { + Ok(n) => n, + Err(e) => { + tracing::warn!( + targets = targets.len(), + error = %e, + "[memory_sources:reconcile] batch upsert failed" + ); + 0 + } + }; + + if !targets.is_empty() { + tracing::info!( + targets = targets.len(), + upserted = upserted, + "[memory_sources:reconcile] composio reconciliation complete" + ); + } + + // Run the one-time caps migration after the reconcile loop so any + // sources upserted just above are also considered. + if let Err(e) = apply_composio_source_caps_migration().await { + tracing::warn!( + error = %e, + "[memory_sources:reconcile] caps migration failed (non-fatal, will retry next time)" + ); + } + + // The scan succeeded — surface the live active-connection set so the list + // path can hide rows for connections that are no longer active (re-auth / + // token expiry mints a fresh connection_id, stranding the old row) and + // collapse identical same-id duplicates. + Some(targets.iter().map(|t| t.connection_id.clone()).collect()) +} + +/// Build the `(toolkit, connection_id, label)` upsert targets for a batch +/// reconcile from the scanned Composio sync targets. +/// +/// The label is a title-cased toolkit name plus the truncated connection id so +/// distinct accounts of the same toolkit (e.g. two Gmail logins) don't all show +/// as "Gmail connection". Pure (no I/O) so it can be unit-tested directly. +fn build_upsert_targets(targets: &[composio::SyncTarget]) -> Vec { + targets + .iter() + .map(|target| { + let label = format!( + "{} · {}", + title_case(&target.toolkit), + short_id(&target.connection_id) + ); + (target.toolkit.clone(), target.connection_id.clone(), label) + }) + .collect() +} + +/// Apply conservative default caps in-place to every cap-less source. +/// +/// For a Composio source with no `max_items`/`sync_depth_days`, writes the +/// per-toolkit defaults and enables it (a no-op when already enabled) — an +/// already-enabled, cap-less source would otherwise sync at the provider's large +/// internal ceiling instead of the cheap default. For other kinds, fills any unset +/// kind-specific caps via `apply_kind_defaults`. User-customised caps (non-None) +/// are never overwritten. Returns the number of Composio entries that received +/// defaults. Pure (no I/O) so it can be unit-tested directly. +fn apply_caps_defaults_to_entries(sources: &mut [MemorySourceEntry]) -> u32 { + let mut applied = 0u32; + for source in sources.iter_mut() { + match source.kind { + SourceKind::Composio => { + // Apply to enabled AND disabled cap-less sources; skip entries the + // user has already customised (any non-None cap). + if source.max_items.is_none() && source.sync_depth_days.is_none() { + let toolkit = source.toolkit.as_deref().unwrap_or(""); + let (max_items, sync_depth_days) = + registry::memory_sync_defaults_for_toolkit(toolkit); + tracing::debug!( + id = %source.id, + toolkit = %toolkit, + was_enabled = source.enabled, + max_items = ?max_items, + sync_depth_days = ?sync_depth_days, + "[memory_sources:reconcile] caps migration: applying conservative defaults" + ); + source.enabled = true; + source.max_items = max_items; + source.sync_depth_days = sync_depth_days; + applied += 1; + } + } + // Apply non-composio kind defaults for entries with all-None caps. + _ => { + // Use the rpc::apply_kind_defaults helper so the same + // conservative values are applied consistently. + crate::sources::apply_kind_defaults(source); + } + } + } + applied +} + +/// Retroactive migration: give any cap-less Composio source — enabled or +/// disabled — conservative per-toolkit caps so its first sync stays cheap. +/// +/// Version-gated by `Config.composio_source_caps_migration_version`: runs once per +/// `CURRENT_CAPS_MIGRATION_VERSION` bump (installs that ran an earlier revision +/// re-run it exactly once). Entries the user has already customised (non-None caps) +/// are left untouched. +pub async fn apply_composio_source_caps_migration() -> Result<(), String> { + let _guard = registry::memory_sources_write_guard().await; + let mut config = config_rpc::load_config_with_timeout().await?; + + if config.composio_source_caps_migration_version() >= CURRENT_CAPS_MIGRATION_VERSION { + tracing::debug!( + version = config.composio_source_caps_migration_version(), + "[memory_sources:reconcile] caps migration already at current version; skipping" + ); + return Ok(()); + } + + tracing::info!( + from_version = config.composio_source_caps_migration_version(), + to_version = CURRENT_CAPS_MIGRATION_VERSION, + "[memory_sources:reconcile] applying composio source caps migration" + ); + + // The source registry crosses the host seam as JSON. `MemorySourceEntry` is + // defined by the engine crate, which `tinymemory-api` must not depend on + // (it would drag SQLite into the dependency-light contract crate), so the + // host hands the registry over serialized and takes it back the same way. + let mut entries: Vec = serde_json::from_value( + config + .memory_sources_json() + .map_err(|e| format!("caps migration: failed to read memory sources: {e:#}"))?, + ) + .map_err(|e| format!("caps migration: failed to decode memory sources: {e:#}"))?; + + let migrated_count = apply_caps_defaults_to_entries(&mut entries); + + config + .set_memory_sources_json( + serde_json::to_value(&entries) + .map_err(|e| format!("caps migration: failed to encode memory sources: {e:#}"))?, + ) + .map_err(|e| format!("caps migration: failed to write memory sources: {e:#}"))?; + config.set_composio_source_caps_migration_version(CURRENT_CAPS_MIGRATION_VERSION); + config + .save() + .await + .map_err(|e| format!("caps migration: failed to save config: {e:#}"))?; + + tracing::info!( + migrated = migrated_count, + "[memory_sources:reconcile] caps migration complete" + ); + + Ok(()) +} + +fn title_case(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().chain(chars).collect(), + } +} + +fn short_id(id: &str) -> &str { + // Show only the last 8 Unicode scalar values to keep labels compact. + // Byte-slicing would panic if the cut point isn't a UTF-8 boundary. + let n = id.chars().count(); + if n <= 8 { + return id; + } + let skip = n - 8; + let start = id.char_indices().nth(skip).map(|(idx, _)| idx).unwrap_or(0); + &id[start..] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sources::types::{MemorySourceEntry, SourceKind}; + + fn make_composio_entry( + id: &str, + toolkit: &str, + enabled: bool, + max_items: Option, + sync_depth_days: Option, + ) -> MemorySourceEntry { + MemorySourceEntry { + id: id.to_string(), + kind: SourceKind::Composio, + label: toolkit.to_string(), + enabled, + toolkit: Some(toolkit.to_string()), + connection_id: Some(format!("conn_{id}")), + path: None, + glob: None, + url: None, + branch: None, + paths: Vec::new(), + max_commits: None, + max_issues: None, + max_prs: None, + query: None, + since_days: None, + max_items, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days, + } + } + + /// Exercises the real migration transform (`apply_caps_defaults_to_entries`) + /// so the tests cannot drift from the production predicate. + fn run_migration_on_entries(sources: &mut [MemorySourceEntry]) -> u32 { + apply_caps_defaults_to_entries(sources) + } + + #[test] + fn migration_flips_disabled_capless_entry_to_enabled_with_caps() { + let mut sources = vec![make_composio_entry("s1", "gmail", false, None, None)]; + let count = run_migration_on_entries(&mut sources); + assert_eq!(count, 1); + assert!(sources[0].enabled); + assert_eq!(sources[0].max_items, Some(100)); + assert_eq!(sources[0].sync_depth_days, Some(30)); + } + + #[test] + fn migration_applies_defaults_to_enabled_capless_entry() { + // An already-enabled but cap-less source must also receive defaults — + // otherwise its first sync runs at the provider's large internal ceiling. + let mut sources = vec![make_composio_entry("s2", "slack", true, None, None)]; + let count = run_migration_on_entries(&mut sources); + assert_eq!(count, 1); + assert!(sources[0].enabled); + assert_eq!(sources[0].max_items, Some(50)); + assert_eq!(sources[0].sync_depth_days, Some(14)); + } + + #[test] + fn migration_leaves_user_customised_caps_untouched() { + // User set max_items explicitly → migration should not override. + let mut sources = vec![make_composio_entry("s3", "notion", false, Some(5), None)]; + let count = run_migration_on_entries(&mut sources); + assert_eq!(count, 0, "entry with user-set caps must not be migrated"); + assert!(!sources[0].enabled, "enabled must not be flipped"); + assert_eq!(sources[0].max_items, Some(5), "user cap must be preserved"); + } + + #[test] + fn migration_is_noop_on_empty_list() { + let mut sources: Vec = vec![]; + let count = run_migration_on_entries(&mut sources); + assert_eq!(count, 0); + } + + #[test] + fn migration_applies_correct_defaults_per_toolkit() { + let toolkits = [ + ("gmail", Some(100u32), Some(30u32)), + ("slack", Some(50), Some(14)), + ("notion", Some(30), Some(30)), + ("linear", Some(50), Some(30)), + ("clickup", Some(50), Some(30)), + ("github", Some(50), Some(30)), + ("unknown", Some(30), Some(14)), + ]; + for (toolkit, exp_items, exp_days) in &toolkits { + let mut sources = vec![make_composio_entry("sid", toolkit, false, None, None)]; + run_migration_on_entries(&mut sources); + assert_eq!( + sources[0].max_items, *exp_items, + "max_items mismatch for toolkit={toolkit}" + ); + assert_eq!( + sources[0].sync_depth_days, *exp_days, + "sync_depth_days mismatch for toolkit={toolkit}" + ); + } + } + + fn sync_target(toolkit: &str, connection_id: &str) -> composio::SyncTarget { + composio::SyncTarget { + toolkit: toolkit.to_string(), + connection_id: connection_id.to_string(), + } + } + + #[test] + fn build_upsert_targets_formats_label_and_preserves_order() { + let targets = vec![ + sync_target("gmail", "ca_WaktIDFlZwXO"), + sync_target("slack", "short"), + ]; + let out = build_upsert_targets(&targets); + assert_eq!(out.len(), 2); + // (toolkit, connection_id, label) — toolkit/connection_id carried through verbatim. + assert_eq!(out[0].0, "gmail"); + assert_eq!(out[0].1, "ca_WaktIDFlZwXO"); + assert_eq!(out[0].2, "Gmail · IDFlZwXO"); + assert_eq!(out[1].0, "slack"); + assert_eq!(out[1].1, "short"); + assert_eq!(out[1].2, "Slack · short"); + } + + #[test] + fn build_upsert_targets_empty_is_empty() { + let out = build_upsert_targets(&[]); + assert!(out.is_empty()); + } + + #[test] + fn short_id_truncates_ascii() { + assert_eq!(short_id("ca_WaktIDFlZwXO"), "IDFlZwXO"); + } + + #[test] + fn short_id_short_input_passthrough() { + assert_eq!(short_id("abc"), "abc"); + assert_eq!(short_id("12345678"), "12345678"); + } + + #[test] + fn short_id_utf8_safe() { + // Multi-byte chars would have panicked with byte-slicing. + let s = "🦀🐢🐙🦊🐼🐰🐯🐸🦁"; + let out = short_id(s); + assert_eq!(out.chars().count(), 8); + } +} diff --git a/core/src/sources/registry.rs b/core/src/sources/registry.rs new file mode 100644 index 0000000..a33f532 --- /dev/null +++ b/core/src/sources/registry.rs @@ -0,0 +1,190 @@ +//! Product config discovery and locking around tinycortex source registry CRUD. + +use std::sync::OnceLock; + +use crate::config_loader as config_rpc; +use crate::sources::types::{MemorySourceEntry, SourceKind}; + +pub use tinycortex::memory::sources::{ + memory_sync_defaults_for_toolkit, ComposioUpsertTarget, MemorySourcePatch, +}; + +static MEMORY_SOURCES_WRITE_LOCK: OnceLock> = OnceLock::new(); + +pub(crate) async fn memory_sources_write_guard() -> tokio::sync::MutexGuard<'static, ()> { + MEMORY_SOURCES_WRITE_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await +} + +async fn registry() -> Result { + let config = config_rpc::load_config_with_timeout().await?; + Ok(tinycortex::memory::sources::SourceRegistry::new( + config.config_path(), + )) +} + +pub async fn list_sources() -> Result, String> { + registry().await?.list().map_err(|error| error.to_string()) +} + +pub async fn list_enabled_by_kind(kind: SourceKind) -> Result, String> { + registry() + .await? + .list_enabled_by_kind(kind) + .map_err(|error| error.to_string()) +} + +pub async fn get_source(id: &str) -> Result, String> { + registry().await?.get(id).map_err(|error| error.to_string()) +} + +/// [`get_source`] against an **explicit** config rather than the process-global +/// one. +/// +/// `registry` resolves its config path through +/// `config_rpc::load_config_with_timeout`, i.e. from the process environment. +/// That is right for RPC handlers, which serve the active user, and wrong for +/// the embedded memory driver +/// (`crate::driver::embedded`), which is bound to one +/// workspace and holds a `Config` re-anchored to it. Reading the global path +/// there would let a driver bound to workspace B answer with workspace A's +/// sources — the cross-workspace leak the workspace-keyed binding map exists to +/// prevent. +/// +/// Synchronous because the registry read itself is; only the config lookup in +/// `registry` was ever async. +pub fn get_source_in( + config: &crate::Config, + id: &str, +) -> Result, String> { + tinycortex::memory::sources::SourceRegistry::new(config.config_path().clone()) + .get(id) + .map_err(|error| error.to_string()) +} + +pub async fn add_source(entry: MemorySourceEntry) -> Result { + let _guard = memory_sources_write_guard().await; + log::debug!("[memory_sources] crate add kind={}", entry.kind.as_str()); + registry() + .await? + .add(entry) + .map_err(|error| error.to_string()) +} + +pub async fn update_source( + id: &str, + patch: MemorySourcePatch, +) -> Result { + let _guard = memory_sources_write_guard().await; + log::debug!("[memory_sources] crate update id_len={}", id.len()); + registry() + .await? + .update(id, patch) + .map_err(|error| error.to_string()) +} + +pub async fn remove_source(id: &str) -> Result { + let _guard = memory_sources_write_guard().await; + registry() + .await? + .remove(id) + .map_err(|error| error.to_string()) +} + +pub async fn remove_composio_source_by_connection_id(connection_id: &str) -> Result { + let _guard = memory_sources_write_guard().await; + registry() + .await? + .remove_composio_source_by_connection_id(connection_id) + .map_err(|error| error.to_string()) +} + +pub async fn upsert_composio_source( + toolkit: &str, + connection_id: &str, + label: &str, +) -> Result { + let _guard = memory_sources_write_guard().await; + registry() + .await? + .upsert_composio_source(toolkit, connection_id, label) + .map_err(|error| error.to_string()) +} + +pub async fn upsert_composio_sources_batch( + targets: &[ComposioUpsertTarget], +) -> Result { + let _guard = memory_sources_write_guard().await; + registry() + .await? + .upsert_composio_sources_batch(targets) + .map_err(|error| error.to_string()) +} + +pub async fn apply_all_in() -> Result, String> { + let _guard = memory_sources_write_guard().await; + registry() + .await? + .apply_all_in() + .map_err(|error| error.to_string()) +} + +/// Apply conservative per-kind cap defaults to a new source entry. +/// +/// Only fills fields that are still `None` — never overwrites a +/// caller-supplied value. This mirrors the retroactive migration logic in +/// `reconcile::apply_composio_source_caps_migration` so the same defaults +/// are applied consistently at creation time and during migration. +pub fn apply_kind_defaults(entry: &mut MemorySourceEntry) { + match entry.kind { + SourceKind::GithubRepo => { + if entry.max_prs.is_none() { + entry.max_prs = Some(10); + } + if entry.max_issues.is_none() { + entry.max_issues = Some(10); + } + if entry.max_commits.is_none() { + entry.max_commits = Some(50); + } + } + SourceKind::RssFeed => { + if entry.max_items.is_none() { + entry.max_items = Some(20); + } + } + SourceKind::TwitterQuery if entry.since_days.is_none() => { + entry.since_days = Some(7); + } + // Folder / WebPage / Composio: no defaults to apply here. + // Composio defaults are set at upsert time in registry::upsert_composio_source. + _ => {} + } +} + +/// Decode the source registry a host config carries. +/// +/// The registry crosses the host seam as JSON: [`MemorySourceEntry`] is defined +/// by the engine crate, and `tinymemory-api` must not depend on it — that would +/// drag SQLite into the dependency-light contract crate. So the host hands the +/// registry over serialized and this is where it becomes typed again. +/// +/// A malformed or absent registry yields an empty list rather than an error. +/// Every caller is a background loop deciding what to sync, and "nothing is +/// registered" is the fail-closed answer there; propagating would take the loop +/// down over one bad row. +#[must_use] +pub fn decode_memory_sources(config: &crate::Config) -> Vec { + match config.memory_sources_json() { + Ok(value) => serde_json::from_value(value).unwrap_or_else(|e| { + log::warn!("[memory_sources:registry] could not decode memory sources: {e:#}"); + Vec::new() + }), + Err(e) => { + log::warn!("[memory_sources:registry] could not read memory sources: {e:#}"); + Vec::new() + } + } +} diff --git a/core/src/sources/status.rs b/core/src/sources/status.rs new file mode 100644 index 0000000..f2568f5 --- /dev/null +++ b/core/src/sources/status.rs @@ -0,0 +1,192 @@ +//! Per-source sync status — chunks ingested, freshness, in-flight progress. +//! +//! Queries `mem_tree_chunks` filtered by source-id prefix: +//! - Reader-backed kinds (folder/github/rss/web/twitter) tag chunks +//! with `mem_src:{source.id}:%`, so we count those directly. +//! - Composio sources tag chunks with the toolkit-specific id +//! (e.g. `gmail:user@example.com:msg_xxx`), so we match by toolkit +//! prefix instead. + +use serde::Serialize; + +use crate::sources::types::{MemorySourceEntry, SourceKind}; +use crate::store::chunks::store::with_connection; +use crate::Config; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum FreshnessLabel { + Active, + Recent, + Idle, +} + +impl FreshnessLabel { + pub fn from_age_ms(last_ms: Option, now_ms: i64) -> Self { + match last_ms { + None => Self::Idle, + Some(ts) => { + let age = now_ms.saturating_sub(ts); + if age <= 30_000 { + Self::Active + } else if age <= 5 * 60_000 { + Self::Recent + } else { + Self::Idle + } + } + } + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct SourceStatus { + pub source_id: String, + pub chunks_synced: u64, + pub chunks_pending: u64, + pub last_chunk_at_ms: Option, + pub freshness: FreshnessLabel, +} + +/// Compute status for one source. +pub async fn source_status( + config: &Config, + source: &MemorySourceEntry, +) -> Result { + let cfg = config.to_arc(); + let source_clone = source.clone(); + + tokio::task::spawn_blocking(move || { + with_connection(&*cfg, |conn| { + let prefix = source_id_prefix(&source_clone); + + // Surface real query errors so status telemetry doesn't lie about + // a healthy zero-row state when the DB is actually broken. + let (synced, pending, last_ts): (i64, i64, Option) = conn.query_row( + "SELECT \ + COUNT(*), \ + SUM(CASE WHEN embedding IS NULL THEN 1 ELSE 0 END), \ + MAX(timestamp_ms) \ + FROM mem_tree_chunks \ + WHERE source_id LIKE ?1", + [&prefix], + |r| { + Ok(( + r.get(0)?, + r.get::<_, Option>(1)?.unwrap_or(0), + r.get(2)?, + )) + }, + )?; + + let now_ms = chrono::Utc::now().timestamp_millis(); + Ok(SourceStatus { + source_id: source_clone.id.clone(), + chunks_synced: synced.max(0) as u64, + chunks_pending: pending.max(0) as u64, + last_chunk_at_ms: last_ts, + freshness: FreshnessLabel::from_age_ms(last_ts, now_ms), + }) + }) + .map_err(|e| format!("source_status: {e}")) + }) + .await + .map_err(|e| format!("source_status join: {e}"))? +} + +/// Compute status for all configured sources (one SQL roundtrip per source). +pub async fn status_list(config: &Config) -> Result, String> { + let sources = crate::sources::registry::list_sources().await?; + let mut out = Vec::with_capacity(sources.len()); + for source in sources { + match source_status(config, &source).await { + Ok(s) => out.push(s), + Err(e) => { + tracing::warn!( + source_id = %source.id, + error = %e, + "[memory_sources:status] query failed" + ); + out.push(SourceStatus { + source_id: source.id, + chunks_synced: 0, + chunks_pending: 0, + last_chunk_at_ms: None, + freshness: FreshnessLabel::Idle, + }); + } + } + } + Ok(out) +} + +/// Build the `source_id LIKE` prefix that matches chunks belonging to a source. +fn source_id_prefix(source: &MemorySourceEntry) -> String { + match source.kind { + SourceKind::Composio => { + // Composio providers write chunks with source_id = `{toolkit}:%` + // (e.g. `gmail:user@example.com:msg_xxx`). Match by toolkit only. + source + .toolkit + .as_deref() + .map(|t| format!("{t}:%")) + .unwrap_or_else(|| "__no_toolkit__:%".to_string()) + } + _ => format!("mem_src:{}:%", source.id), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn freshness_thresholds() { + let now = 1_000_000_000_000; + assert_eq!( + FreshnessLabel::from_age_ms(Some(now - 1_000), now), + FreshnessLabel::Active + ); + assert_eq!( + FreshnessLabel::from_age_ms(Some(now - 60_000), now), + FreshnessLabel::Recent + ); + assert_eq!( + FreshnessLabel::from_age_ms(Some(now - 600_000), now), + FreshnessLabel::Idle + ); + assert_eq!(FreshnessLabel::from_age_ms(None, now), FreshnessLabel::Idle); + } + + #[test] + fn source_id_prefix_dispatch() { + let mut entry = MemorySourceEntry { + id: "src_abc".into(), + kind: SourceKind::Folder, + label: "x".into(), + enabled: true, + toolkit: None, + connection_id: None, + path: Some("/tmp".into()), + glob: None, + url: None, + branch: None, + paths: Vec::new(), + query: None, + since_days: None, + max_items: None, + max_commits: None, + max_issues: None, + max_prs: None, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days: None, + }; + assert_eq!(source_id_prefix(&entry), "mem_src:src_abc:%"); + + entry.kind = SourceKind::Composio; + entry.toolkit = Some("gmail".into()); + assert_eq!(source_id_prefix(&entry), "gmail:%"); + } +} diff --git a/core/src/sources/sync.rs b/core/src/sources/sync.rs new file mode 100644 index 0000000..af362a0 --- /dev/null +++ b/core/src/sources/sync.rs @@ -0,0 +1,410 @@ +//! Per-source sync dispatcher. +//! +//! Thin routing layer: dispatches supported sources through tinycortex and +//! retains the product-owned background lock, events, and reconcile shell. +//! - Twitter → placeholder +//! +//! Sync runs in a `tokio::spawn`-ed task so the RPC returns immediately. +//! Progress is published as `MemorySyncStageChanged` events. +//! +//! A per-source mutex prevents duplicate concurrent syncs when the user +//! presses the sync button multiple times. + +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::Mutex; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::sources::types::{MemorySourceEntry, SourceKind}; +use crate::sync::composio::ComposioUsage; +use crate::sync_events::{emit_sync_stage, MemorySyncStage, MemorySyncTrigger}; +use crate::Config; + +static ACTIVE_SYNCS: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| Mutex::new(HashSet::new())); + +/// Trigger a sync for one source. Spawns work in the background and +/// returns immediately. Progress is published as `MemorySyncStageChanged` +/// events with `connection_id = Some(source.id)`. +pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Result<(), String> { + if !source.enabled { + return Err(format!("source '{}' is disabled", source.id)); + } + + // Per-source mutex: reject if this source is already syncing. + { + let mut active = ACTIVE_SYNCS.lock().unwrap_or_else(|e| e.into_inner()); + if !active.insert(source.id.clone()) { + tracing::debug!( + source_id = %source.id, + "[memory_sources:sync] already syncing — skipping duplicate" + ); + return Ok(()); + } + } + + let source_id = source.id.clone(); + let kind_str = source.kind.as_str(); + + tracing::debug!( + source_id = %source_id, + kind = %kind_str, + "[memory_sources:sync] queueing sync" + ); + + emit_sync_stage( + MemorySyncTrigger::Manual, + MemorySyncStage::Requested, + Some(kind_str), + Some(&source_id), + Some(format!("sync requested for {} source", kind_str)), + Some(&source_id), + ); + + tokio::spawn(async move { + let source_id_for_panic = source.id.clone(); + let kind_for_panic = source.kind.as_str(); + let inner = tokio::spawn(async move { + // Retry any previously-failed pipeline jobs so the worker + // resumes processing through all documents. + if let Ok(retried) = crate::queue::store::retry_all_failed(&*config) { + if retried > 0 { + tracing::info!( + retried = retried, + "[memory_sources:sync] retried {retried} failed pipeline job(s)" + ); + } + } + + tracing::debug!( + source_id = %source.id, + kind = %source.kind.as_str(), + "[memory_sources:sync] dispatching by kind" + ); + let sync_start = std::time::Instant::now(); + // Composio billable-action usage for this run, populated by + // `sync_composio` (#3111). Stays zero for non-Composio kinds. + let mut composio_usage = ComposioUsage::default(); + let outcome = match source.kind { + SourceKind::Composio => { + match crate::tinycortex::run_source_pipeline(&source, &*config).await { + Ok(outcome) => { + composio_usage.actions_called = outcome.actions_called; + composio_usage.cost_usd = outcome.provider_cost_usd; + Ok(outcome.records_ingested as usize) + } + Err(error) => { + composio_usage.actions_called = error.actions_called; + composio_usage.cost_usd = error.provider_cost_usd; + Err(format!("composio sync failed: {error}")) + } + } + } + SourceKind::Conversation | SourceKind::Folder => { + crate::tinycortex::run_source_pipeline(&source, &*config) + .await + .map(|outcome| outcome.records_ingested as usize) + .map_err(|error| error.to_string()) + } + SourceKind::GithubRepo => crate::tinycortex::run_source_pipeline(&source, &*config) + .await + .map(|outcome| outcome.records_ingested as usize) + .map_err(|error| error.to_string()), + SourceKind::RssFeed | SourceKind::WebPage => { + crate::tinycortex::run_source_pipeline(&source, &*config) + .await + .map(|outcome| outcome.records_ingested as usize) + .map_err(|error| error.to_string()) + } + SourceKind::TwitterQuery => Err( + "Twitter sync not yet configured. Provide bearer token in settings." + .to_string(), + ), + }; + let duration_ms = sync_start.elapsed().as_millis() as u64; + + match outcome { + Ok(items) => { + tracing::debug!( + source_id = %source.id, + kind = %source.kind.as_str(), + items = items, + "[memory_sources:sync] completed" + ); + emit_sync_stage( + MemorySyncTrigger::Manual, + MemorySyncStage::Completed, + Some(source.kind.as_str()), + Some(&source.id), + Some(format!("ingested {items} item(s)")), + Some(&source.id), + ); + + use crate::tinycortex::{append_audit_entry, SyncAuditEntry}; + append_audit_entry( + &*config, + &SyncAuditEntry { + timestamp: chrono::Utc::now(), + source_id: source.id.clone(), + source_kind: source.kind.as_str().to_string(), + scope: source + .url + .clone() + .or(source.toolkit.clone()) + .unwrap_or_else(|| source.id.clone()), + items_fetched: items as u32, + batches: 0, + input_tokens: 0, + output_tokens: 0, + estimated_cost_usd: 0.0, + composio_actions_called: composio_usage.actions_called, + composio_cost_usd: composio_usage.cost_usd, + actual_charged_usd: None, + duration_ms, + success: true, + error: None, + }, + ); + + // Auto-rebuild: if raw files exist but the tree has + // no summaries, build the tree now. + check_and_rebuild_tree(&source, &*config).await; + + // Auto-snapshot: capture post-sync state for diff tracking. + if let Err(e) = + crate::diff::ops::auto_snapshot_after_sync(&source, &*config).await + { + tracing::warn!( + source_id = %source.id, + error = %e, + "[memory_sources:sync] auto-snapshot failed (non-fatal)" + ); + } + } + Err(error) => { + // Audit failed syncs too. + use crate::tinycortex::{append_audit_entry, SyncAuditEntry}; + append_audit_entry( + &*config, + &SyncAuditEntry { + timestamp: chrono::Utc::now(), + source_id: source.id.clone(), + source_kind: source.kind.as_str().to_string(), + scope: source + .url + .clone() + .or(source.toolkit.clone()) + .unwrap_or_else(|| source.id.clone()), + items_fetched: 0, + batches: 0, + input_tokens: 0, + output_tokens: 0, + estimated_cost_usd: 0.0, + composio_actions_called: composio_usage.actions_called, + composio_cost_usd: composio_usage.cost_usd, + actual_charged_usd: None, + duration_ms, + success: false, + error: Some(error.clone()), + }, + ); + + // Report internal failures to Sentry; known-expected + // conditions (auth/network/rate-limit/missing config) are + // classified by `expected_error_kind` and logged-not-reported + // so we surface real bugs without Sentry-spamming routine + // user/config errors (#3295). The reason is still shown to + // the user via the Failed stage event regardless. + crate::observability::report_error_or_expected( + &error, + "memory_sources", + "sync", + &[ + ("source_id", source.id.as_str()), + ("kind", source.kind.as_str()), + ], + ); + + emit_sync_stage( + MemorySyncTrigger::Manual, + MemorySyncStage::Failed, + Some(source.kind.as_str()), + Some(&source.id), + Some(error.clone()), + Some(&source.id), + ); + tracing::warn!( + source_id = %source.id, + kind = %source.kind.as_str(), + error = %error, + "[memory_sources:sync] failed" + ); + } + } + }); + + if let Err(join_err) = inner.await { + if join_err.is_panic() { + tracing::error!( + source_id = %source_id_for_panic, + kind = %kind_for_panic, + "[memory_sources:sync] sync task panicked" + ); + } + } + + // Release the per-source lock so future syncs can proceed. + if let Ok(mut active) = ACTIVE_SYNCS.lock() { + active.remove(&source_id_for_panic); + } + }); + + Ok(()) +} + +/// Reconcile raw files that are not yet covered by tree summaries. +pub(crate) async fn check_and_rebuild_tree(source: &MemorySourceEntry, config: &Config) { + use crate::tinycortex::{needs_rebuild, rebuild_tree_from_raw}; + + for scope in derive_scopes(source, config) { + if !needs_rebuild(config, &scope.tree_scope, &scope.archive_source_id) { + continue; + } + tracing::info!( + source_id = %source.id, + scope = %scope.tree_scope, + archive = %scope.archive_source_id, + "[memory_sources:sync] reconciling uncovered raw files into tree" + ); + match rebuild_tree_from_raw(config, &scope.tree_scope, &scope.archive_source_id).await { + Ok(outcome) => tracing::info!( + scope = %scope.tree_scope, + files = outcome.files_read, + batches = outcome.batches, + cost = %format!("${:.4}", outcome.actual_charged_usd.unwrap_or(outcome.estimated_cost_usd)), + cost_is_actual = outcome.actual_charged_usd.is_some(), + "[memory_sources:sync] reconcile complete" + ), + Err(error) => tracing::warn!( + scope = %scope.tree_scope, + error = %format!("{error:#}"), + "[memory_sources:sync] reconcile failed" + ), + } + } +} + +/// A source's tree scope paired with its raw-archive source id. The two +/// slugify to DIFFERENT directories for GitHub (`github:owner/repo` vs +/// `github.com/owner/repo`) — conflating them makes reconcile scan an +/// empty directory while the real archive sits uncovered. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SourceScope { + /// Tree registry key, e.g. `"github:owner/repo"`. + pub tree_scope: String, + /// Raw-archive id whose slug names `raw//`, e.g. + /// `"github.com/owner/repo"`. Equal to `tree_scope` for sources that + /// archive under their scope (gmail). + pub archive_source_id: String, +} + +/// Derive the tree scope(s) + raw-archive id(s) that a source maps to. +pub fn derive_scopes(source: &MemorySourceEntry, config: &Config) -> Vec { + use crate::sources::readers::github; + + match source.kind { + SourceKind::GithubRepo => { + let Some(url) = source.url.as_deref() else { + return Vec::new(); + }; + match ( + github::repo_chunk_scope(url), + github::repo_archive_source_id(url), + ) { + (Some(tree_scope), Some(archive_source_id)) => vec![SourceScope { + tree_scope, + archive_source_id, + }], + _ => Vec::new(), + } + } + SourceKind::Composio => { + // Composio sources scope by toolkit + connection email. + // Gmail: "gmail:" — archive dir shares + // the scope. Others: no raw archive to reconcile yet. + let toolkit = source.toolkit.as_deref().unwrap_or("unknown"); + match toolkit { + "gmail" | "GMAIL" => { + // The scope for gmail is "gmail:". + // We scan the raw directory to find it. + let content_root = config.memory_tree_content_root(); + let raw_dir = content_root.join("raw"); + if let Ok(entries) = std::fs::read_dir(&raw_dir) { + entries + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_str() + .map(|n| n.starts_with("gmail-")) + .unwrap_or(false) + }) + .filter_map(|e| { + // Read _source.md to get the scope. + let source_md = e.path().join("_source.md"); + let content = std::fs::read_to_string(&source_md).ok()?; + content.lines().find(|l| l.starts_with("scope:")).map(|l| { + let scope = l + .trim_start_matches("scope:") + .trim() + .trim_matches('"') + .to_string(); + SourceScope { + tree_scope: scope.clone(), + archive_source_id: scope, + } + }) + }) + .collect() + } else { + Vec::new() + } + } + _ => Vec::new(), + } + } + _ => Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The two GitHub coordinate helpers are re-exported from tinycortex and + /// they deliberately differ: `tree_scope` slugifies to + /// `github-tinyhumansai-openhuman` while `archive_source_id` slugifies to + /// `github-com-tinyhumansai-openhuman`. Swapping the two still compiles and + /// still type-checks — it just makes reconcile scan an empty directory at + /// runtime. Pin both spellings. + #[test] + fn derive_scopes_keeps_github_tree_and_archive_ids_distinct() { + let source: MemorySourceEntry = serde_json::from_value(serde_json::json!({ + "id": "gh-scope", + "kind": "github_repo", + "label": "Repo", + "url": "https://github.com/tinyhumansai/openhuman", + })) + .expect("github source entry"); + + let scopes = derive_scopes(&source, &TestHostConfig::default()); + + assert_eq!(scopes.len(), 1); + assert_eq!(scopes[0].tree_scope, "github:tinyhumansai/openhuman"); + assert_eq!( + scopes[0].archive_source_id, + "github.com/tinyhumansai/openhuman" + ); + } +} diff --git a/core/src/sources/types.rs b/core/src/sources/types.rs new file mode 100644 index 0000000..6fbab9f --- /dev/null +++ b/core/src/sources/types.rs @@ -0,0 +1,5 @@ +//! Stable host path for tinycortex-owned memory-source contracts. + +pub use tinycortex::memory::sources::{ + ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, +}; diff --git a/core/src/store/README.md b/core/src/store/README.md new file mode 100644 index 0000000..3eff085 --- /dev/null +++ b/core/src/store/README.md @@ -0,0 +1,60 @@ +# memory_store + +Single home for every persisted memory shape. Owns the storage primitives — +nothing above this module touches SQLite or the on-disk vault directly. + +```text +content/ on-disk .md files — SOURCE OF TRUTH for every body +chunks/ SQLite chunk rows (metadata + tags + md path pointer + + lifecycle status) + the two chunkers that produce them +entities/ mem_tree_entity_index — every entity occurrence per node +trees/ summary tree persistence (one table, kind-parameterized) +vectors [tinycortex::memory::store::vectors] — local vector DB + (cosine, brute-force), moved into the TinyCortex substrate +kv/ global + namespace key-value (kv_global, kv_namespace) +contacts/ [removed] facade over people::store (Person/Handle/Interaction) +namespace_store/ host-retained namespace documents, graph, episodic/event/ + segment/profile tables, and retrieval policy +``` + +## Cross-cutting modules + +| Path | Role | +| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`mod.rs`](mod.rs) | Module root + public re-exports. | +| [`README.md`](README.md) | You are here. | +| [`kinds.rs`](kinds.rs) | `MemoryKind` enum — the authoritative catalog: Raw / Chunk / Entity / Tree / Vector / Kv / Contact — plus per-kind type aliases. | +| [`traits.rs`](traits.rs) | `VectorEmbeddable` + `ObsidianRepresentable` + `ObsidianFile`. Every stored kind implements both — the compiler enforces "everything in memory_store is vector and obsidian compatible". | +| [`types.rs`](types.rs) | Shared serde types used across submodules: `NamespaceDocumentInput`, `NamespaceMemoryHit`, `NamespaceQueryResult`, `NamespaceRetrievalContext`, `RetrievalScoreBreakdown`, `MemoryItemKind`, `MemoryKvRecord`. | +| [`memory_trait.rs`](memory_trait.rs) | `impl Memory for UnifiedMemory` — bridges the generic `Memory` trait surface onto the unified store. | +| [`client.rs`](client.rs) | `MemoryClient` / `MemoryClientRef` / `MemoryState`. Async wrapper over `UnifiedMemory` used by RPC controllers; owns the singleton ingestion-queue handle. | +| [`factories.rs`](factories.rs) | `create_memory*` constructors. Selects the embedding provider per the `MemoryConfig`, probes Ollama health, and builds a `Box` over `UnifiedMemory`. | +| [`retrieval/`](retrieval/) | `RetrievalFacade` — single import surface over the four retrieval modes (tree-walk, vector, keyword, param/tag). | +| [`tools/`](tools/) | Agent tools that read directly from memory_store: `memory_store_raw_search`, `memory_store_raw_chunks`, `memory_store_kinds`. | + +## Storage submodules + +| Path | Owns | +| --- | --- | +| [`content/`](content/) | **Source of truth** for chunk + summary bodies as on-disk `.md` files. Atomic writes, path layout, YAML front-matter compose/parse, tag rewrites, Obsidian vault defaults. See [`content/README.md`](content/README.md). | +| [`chunks/`](chunks/) | Full chunk lifecycle. `types.rs` (`Chunk`, `Metadata`, `SourceKind`, `RawRef`, `ListChunksQuery`) + `store.rs` (SQLite persistence + connection cache) + `produce.rs` (source-kind dispatch chunker used by the ingest pipeline) + `semantic.rs` (heading/paragraph-aware chunker). | +| [`entities.rs`](entities.rs) | Thin re-export of `memory_tree::score::store` — `index_entity`, `index_entities`, `lookup_entity`, `list_entity_ids_for_node`, `clear_entity_index_for_node`, `count_entity_index`, `EntityHit`. Reads/writes the `mem_tree_entity_index` table. | +| [`trees/`](trees/) | `store.rs` (`mem_tree_trees` / `mem_tree_summaries` / `mem_tree_buffers`), `types.rs` (Tree / SummaryNode / TreeKind / TreeStatus / Buffer + topic hotness types), `registry.rs` (kind-parameterized helpers), `hotness.rs` (entity hotness side-table). | +| `vectors/` | Moved into the TinyCortex substrate (`tinycortex::memory::store::vectors`): standalone vector store, `VectorStore` over SQLite, byte-codec for f32 vectors, cosine similarity. | +| [`kv.rs`](kv.rs) | Global + namespace key-value (`kv_global`, `kv_namespace` tables). | +| `contacts/` | Removed. Contact access now lives outside `memory_store` via `people::store`. | +| [`namespace_store/`](namespace_store/) | Host-retained namespace/document tier over the shared SQLite database: documents, persisted product graph relations, episodic/events, segments, profile facets, and host retrieval policy. TinyCortex owns the generic chunk/vector/tree/queue substrate; this tier remains the stable `Memory` implementation. See [`namespace_store/README.md`](namespace_store/README.md). | + +## Layer rules + +- **Content bytes are immutable.** The `.md` file written by `content/` is + the source of truth; SQLite stores a `(content_path, content_sha256)` + pointer. The body never changes after the first write — only YAML + front-matter (`tags:`) is rewritable. +- **SQLite is for indexing and vectors.** Anything keyword/param-searchable + on the body itself should be served by grepping the `.md` files. +- **No upward dependencies.** memory_store does not depend on + `memory_tree`, `memory_tools`, or `memory`. The one documented exception + is `retrieval::RetrievalFacade::tree_walk`, which delegates to + `memory_tree::retrieval::drill_down`; revisit when drill_down's policy bits + can be cleanly separated from its pure traversal. diff --git a/core/src/store/chunks/connection.rs b/core/src/store/chunks/connection.rs new file mode 100644 index 0000000..7fe48ed --- /dev/null +++ b/core/src/store/chunks/connection.rs @@ -0,0 +1,17 @@ +//! `Config` adapters for tinycortex's chunk connection and recovery manager. + +use anyhow::Result; +use rusqlite::Connection; + +use crate::tinycortex::engine_config; +use crate::Config; + +#[doc(hidden)] +pub fn with_connection(config: &Config, f: impl FnOnce(&Connection) -> Result) -> Result { + tinycortex::memory::chunks::with_connection(&engine_config(config), f) +} + +pub(crate) fn recover_corrupt_db(config: &Config) -> Result { + log::warn!("[memory:chunks] checking corrupt database recovery"); + tinycortex::memory::chunks::recover_corrupt_db(&engine_config(config)) +} diff --git a/core/src/store/chunks/embeddings.rs b/core/src/store/chunks/embeddings.rs new file mode 100644 index 0000000..812852a --- /dev/null +++ b/core/src/store/chunks/embeddings.rs @@ -0,0 +1,107 @@ +//! `Config` adapters for tinycortex embedding sidecars and tombstones. + +use std::collections::HashMap; + +use anyhow::Result; +use rusqlite::{Connection, Transaction}; + +use crate::tinycortex::engine_config; +use crate::Config; + +pub(crate) fn tree_active_signature(config: &Config) -> String { + tinycortex::memory::chunks::tree_active_signature(&engine_config(config)) +} + +pub fn set_chunk_embedding(config: &Config, id: &str, embedding: &[f32]) -> Result<()> { + tinycortex::memory::chunks::set_chunk_embedding(&engine_config(config), id, embedding) +} + +pub fn set_chunk_embedding_for_signature( + config: &Config, + id: &str, + signature: &str, + embedding: &[f32], +) -> Result<()> { + tinycortex::memory::chunks::set_chunk_embedding_for_signature( + &engine_config(config), + id, + signature, + embedding, + ) +} + +pub(crate) fn has_uncovered_reembed_work( + conn: &Connection, + signature: &str, +) -> rusqlite::Result { + tinycortex::memory::chunks::has_uncovered_reembed_work(conn, signature) +} + +pub fn mark_chunk_reembed_skipped( + config: &Config, + id: &str, + signature: &str, + reason: &str, +) -> Result<()> { + tinycortex::memory::chunks::mark_chunk_reembed_skipped( + &engine_config(config), + id, + signature, + reason, + ) +} + +pub fn clear_chunk_reembed_skipped(config: &Config, id: &str, signature: &str) -> Result<()> { + tinycortex::memory::chunks::clear_chunk_reembed_skipped(&engine_config(config), id, signature) +} + +pub fn clear_reembed_skipped_for_signature(config: &Config, signature: &str) -> Result { + tinycortex::memory::chunks::clear_reembed_skipped_for_signature( + &engine_config(config), + signature, + ) +} + +pub(crate) fn set_chunk_embedding_for_signature_tx( + tx: &Transaction<'_>, + id: &str, + signature: &str, + embedding: &[f32], +) -> Result<()> { + tinycortex::memory::chunks::set_chunk_embedding_for_signature_tx(tx, id, signature, embedding) +} + +pub fn get_chunk_embedding_for_signature( + config: &Config, + id: &str, + signature: &str, +) -> Result>> { + tinycortex::memory::chunks::get_chunk_embedding_for_signature( + &engine_config(config), + id, + signature, + ) +} + +pub fn get_chunk_embedding(config: &Config, id: &str) -> Result>> { + tinycortex::memory::chunks::get_chunk_embedding(&engine_config(config), id) +} + +pub fn get_chunk_embeddings_for_signature_batch( + config: &Config, + ids: &[String], + signature: &str, +) -> Result>> { + tinycortex::memory::chunks::get_chunk_embeddings_for_signature_batch( + &engine_config(config), + ids, + signature, + ) +} + +pub fn get_chunk_embeddings_batch( + config: &Config, + ids: &[String], +) -> Result>> { + tinycortex::memory::chunks::get_chunk_embeddings_batch(&engine_config(config), ids) +} diff --git a/core/src/store/chunks/mod.rs b/core/src/store/chunks/mod.rs new file mode 100644 index 0000000..9b6fb29 --- /dev/null +++ b/core/src/store/chunks/mod.rs @@ -0,0 +1,26 @@ +//! Chunks — the unit of memory_store persistence. +//! +//! One module for the full chunk lifecycle: +//! +//! - [`types`] — `Chunk`, `Metadata`, `SourceKind`, `RawRef`, +//! `ListChunksQuery`. The persisted shape. +//! - [`store`] — SQLite persistence (`chunks` table + connection cache). +//! - [`semantic`] — heading- and paragraph-aware chunker used by the +//! unified memory writer to split large documents into +//! LLM-context-sized pieces while preserving heading +//! context. +//! +//! The source-kind-dispatch chunker ([`chunk_markdown`], the default — chat / +//! email / document, with stable per-source sequence numbers and bounded +//! segments) is engine-owned and re-exported straight from `tinycortex`. +//! [`chunk_markdown`] and `semantic::chunk_markdown` both yield string-shaped +//! chunks; the store side decides what to do with them. + +pub mod semantic; +pub mod store; +pub mod types; + +pub use semantic::chunk_markdown as chunk_semantic; +pub use store::*; +pub use tinycortex::memory::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions}; +pub use types::*; diff --git a/core/src/store/chunks/raw_refs.rs b/core/src/store/chunks/raw_refs.rs new file mode 100644 index 0000000..bef5c76 --- /dev/null +++ b/core/src/store/chunks/raw_refs.rs @@ -0,0 +1,77 @@ +//! Raw-archive pointers and content-pointer accessors for chunk/summary rows. +//! +//! `RawRef` lets ingest pipelines mirror full message bodies to on-disk +//! archives under `/raw/` while storing only a ≤500-char +//! preview in the SQLite `content` column. Retrieval reads the archive +//! directly instead of going through the SQL preview path. +//! +//! **W3 sub-store flip:** these operations now delegate to +//! [`tinycortex::memory::chunks`] (ported from this exact module — identical SQL +//! against the same `mem_tree_chunks` / `mem_tree_summaries` tables in the shared +//! `chunks.db` the crate now owns). The host signatures are preserved so the ~4 +//! external callers (`content::read`, memory_sync gmail/slack ingest, rebuild) +//! are untouched; only `&Config` is mapped to the crate's `MemoryConfig`. + +use anyhow::Result; +use rusqlite::Transaction; + +use crate::tinycortex::engine_config; +use crate::Config; + +// `RawRef` is re-exported from the crate (identical fields + serde derives), so +// every `chunks::RawRef { path, start, end }` construction site keeps compiling. +pub use tinycortex::memory::chunks::RawRef; + +/// Stash a list of [`RawRef`] entries on a chunk row. Replaces any previous +/// value. +pub fn set_chunk_raw_refs(config: &Config, chunk_id: &str, refs: &[RawRef]) -> Result<()> { + tinycortex::memory::chunks::set_chunk_raw_refs(&engine_config(config), chunk_id, refs) +} + +/// Stash raw archive pointers on a chunk row inside a caller-owned transaction. +pub fn set_chunk_raw_refs_tx(tx: &Transaction<'_>, chunk_id: &str, refs: &[RawRef]) -> Result<()> { + tinycortex::memory::chunks::set_chunk_raw_refs_tx(tx, chunk_id, refs) +} + +/// Return the raw-archive pointers stored in SQLite for `chunk_id`, or `None`. +pub fn get_chunk_raw_refs(config: &Config, chunk_id: &str) -> Result>> { + tinycortex::memory::chunks::get_chunk_raw_refs(&engine_config(config), chunk_id) +} + +/// Collect every raw-archive path referenced by any chunk row, restricted to +/// paths under `rel_prefix`. +pub fn list_chunk_raw_ref_paths_with_prefix( + config: &Config, + rel_prefix: &str, +) -> Result> { + tinycortex::memory::chunks::list_chunk_raw_ref_paths_with_prefix( + &engine_config(config), + rel_prefix, + ) +} + +/// Return both `content_path` and `content_sha256` stored in SQLite for `chunk_id`. +pub fn get_chunk_content_pointers( + config: &Config, + chunk_id: &str, +) -> Result> { + tinycortex::memory::chunks::get_chunk_content_pointers(&engine_config(config), chunk_id) +} + +/// Return the `content_path` stored in SQLite for `chunk_id`, if any. +pub fn get_chunk_content_path(config: &Config, chunk_id: &str) -> Result> { + tinycortex::memory::chunks::get_chunk_content_path(&engine_config(config), chunk_id) +} + +/// Return both `content_path` and `content_sha256` stored in SQLite for `summary_id`. +pub fn get_summary_content_pointers( + config: &Config, + summary_id: &str, +) -> Result> { + tinycortex::memory::chunks::get_summary_content_pointers(&engine_config(config), summary_id) +} + +/// List all summary rows that have a non-NULL `content_path`. +pub fn list_summaries_with_content_path(config: &Config) -> Result> { + tinycortex::memory::chunks::list_summaries_with_content_path(&engine_config(config)) +} diff --git a/core/src/store/chunks/semantic.rs b/core/src/store/chunks/semantic.rs new file mode 100644 index 0000000..a624762 --- /dev/null +++ b/core/src/store/chunks/semantic.rs @@ -0,0 +1,7 @@ +//! Compatibility exports for tinycortex's semantic Markdown chunker. + +pub use tinycortex::memory::chunks::SemanticChunk as Chunk; + +pub fn chunk_markdown(text: &str, max_tokens: usize) -> Vec { + tinycortex::memory::chunks::chunk_semantic(text, max_tokens) +} diff --git a/core/src/store/chunks/store.rs b/core/src/store/chunks/store.rs new file mode 100644 index 0000000..f2f7557 --- /dev/null +++ b/core/src/store/chunks/store.rs @@ -0,0 +1,158 @@ +//! `Config` and transaction adapters for tinycortex chunk persistence. + +use std::collections::HashMap; + +use anyhow::Result; +use rusqlite::Transaction; + +use crate::store::chunks::types::{Chunk, SourceKind}; +use crate::store::content::StagedChunk; +use crate::tinycortex::engine_config; +use crate::Config; + +pub use tinycortex::memory::chunks::{ + ListChunksQuery, RawRef, CHUNK_STATUS_ADMITTED, CHUNK_STATUS_BUFFERED, CHUNK_STATUS_DROPPED, + CHUNK_STATUS_PENDING_EXTRACTION, CHUNK_STATUS_SEALED, RAW_FILE_GATE_KIND, +}; + +pub fn upsert_chunks(config: &Config, chunks: &[Chunk]) -> Result { + tinycortex::memory::chunks::upsert_chunks(&engine_config(config), chunks) +} + +pub fn upsert_chunks_tx(tx: &Transaction<'_>, chunks: &[Chunk]) -> Result { + tinycortex::memory::chunks::upsert_chunks_tx(tx, chunks) +} + +pub fn upsert_staged_chunks_tx(tx: &Transaction<'_>, chunks: &[StagedChunk]) -> Result { + tinycortex::memory::chunks::upsert_staged_chunks_tx(tx, chunks) +} + +pub fn update_chunk_content_sha256(config: &Config, id: &str, sha256: &str) -> Result<()> { + tinycortex::memory::chunks::update_chunk_content_sha256(&engine_config(config), id, sha256) +} + +pub fn update_summary_content_sha256(config: &Config, id: &str, sha256: &str) -> Result<()> { + tinycortex::memory::chunks::update_summary_content_sha256(&engine_config(config), id, sha256) +} + +pub fn list_source_ids_with_prefix( + config: &Config, + kind: SourceKind, + prefix: &str, +) -> Result> { + tinycortex::memory::chunks::list_source_ids_with_prefix(&engine_config(config), kind, prefix) +} + +pub fn get_chunk(config: &Config, id: &str) -> Result> { + tinycortex::memory::chunks::get_chunk(&engine_config(config), id) +} + +pub fn get_chunks_batch(config: &Config, ids: &[String]) -> Result> { + tinycortex::memory::chunks::get_chunks_batch(&engine_config(config), ids) +} + +pub fn list_chunks(config: &Config, query: &ListChunksQuery) -> Result> { + tinycortex::memory::chunks::list_chunks(&engine_config(config), query) +} + +pub fn count_chunks(config: &Config) -> Result { + tinycortex::memory::chunks::count_chunks(&engine_config(config)) +} + +pub fn extraction_coverage(config: &Config) -> Result { + tinycortex::memory::chunks::extraction_coverage(&engine_config(config)) +} + +pub fn set_chunk_lifecycle_status(config: &Config, id: &str, status: &str) -> Result<()> { + tinycortex::memory::chunks::set_chunk_lifecycle_status(&engine_config(config), id, status) +} + +pub(crate) fn set_chunk_lifecycle_status_tx( + tx: &Transaction<'_>, + id: &str, + status: &str, +) -> Result<()> { + tinycortex::memory::chunks::set_chunk_lifecycle_status_tx(tx, id, status) +} + +pub fn get_chunk_lifecycle_status(config: &Config, id: &str) -> Result> { + tinycortex::memory::chunks::get_chunk_lifecycle_status(&engine_config(config), id) +} + +pub fn get_chunk_lifecycle_status_tx(tx: &Transaction<'_>, id: &str) -> Result> { + tinycortex::memory::chunks::get_chunk_lifecycle_status_tx(tx, id) +} + +pub fn count_chunks_by_lifecycle_status(config: &Config, status: &str) -> Result { + tinycortex::memory::chunks::count_chunks_by_lifecycle_status(&engine_config(config), status) +} + +pub fn is_source_ingested(config: &Config, kind: SourceKind, id: &str) -> Result { + tinycortex::memory::chunks::is_source_ingested(&engine_config(config), kind, id) +} + +pub fn claim_source_ingest_tx( + tx: &Transaction<'_>, + kind: SourceKind, + id: &str, + now_ms: i64, +) -> Result { + tinycortex::memory::chunks::claim_source_ingest_tx(tx, kind, id, now_ms) +} + +pub fn mark_raw_paths_ingested(config: &Config, paths: &[String]) -> Result { + tinycortex::memory::chunks::mark_raw_paths_ingested(&engine_config(config), paths) +} + +pub fn filter_raw_paths_not_ingested(config: &Config, paths: &[String]) -> Result> { + tinycortex::memory::chunks::filter_raw_paths_not_ingested(&engine_config(config), paths) +} + +pub fn count_raw_paths_ingested_with_prefix(config: &Config, prefix: &str) -> Result { + tinycortex::memory::chunks::count_raw_paths_ingested_with_prefix(&engine_config(config), prefix) +} + +pub fn delete_chunks_by_source(config: &Config, kind: SourceKind, id: &str) -> Result { + tinycortex::memory::chunks::delete_chunks_by_source(&engine_config(config), kind, id) +} + +pub fn delete_chunks_by_source_prefix( + config: &Config, + kind: SourceKind, + prefix: &str, +) -> Result { + tinycortex::memory::chunks::delete_chunks_by_source_prefix(&engine_config(config), kind, prefix) +} + +pub fn delete_chunks_by_owner(config: &Config, kind: SourceKind, owner: &str) -> Result { + tinycortex::memory::chunks::delete_chunks_by_owner(&engine_config(config), kind, owner) +} + +pub fn delete_orphaned_source_tree(config: &Config, kind: SourceKind, id: &str) -> Result { + tinycortex::memory::chunks::delete_orphaned_source_tree(&engine_config(config), kind, id) +} + +#[path = "connection.rs"] +mod connection; +pub(crate) use connection::recover_corrupt_db; +pub use connection::with_connection; + +#[path = "raw_refs.rs"] +mod raw_refs; +pub use raw_refs::{ + get_chunk_content_path, get_chunk_content_pointers, get_chunk_raw_refs, + get_summary_content_pointers, list_chunk_raw_ref_paths_with_prefix, + list_summaries_with_content_path, set_chunk_raw_refs, set_chunk_raw_refs_tx, +}; + +#[path = "embeddings.rs"] +mod embeddings; +pub use embeddings::{ + clear_chunk_reembed_skipped, clear_reembed_skipped_for_signature, get_chunk_embedding, + get_chunk_embedding_for_signature, get_chunk_embeddings_batch, + get_chunk_embeddings_for_signature_batch, mark_chunk_reembed_skipped, set_chunk_embedding, + set_chunk_embedding_for_signature, +}; +pub(crate) use embeddings::{ + has_uncovered_reembed_work, set_chunk_embedding_for_signature_tx, tree_active_signature, +}; diff --git a/core/src/store/chunks/types.rs b/core/src/store/chunks/types.rs new file mode 100644 index 0000000..bd080fc --- /dev/null +++ b/core/src/store/chunks/types.rs @@ -0,0 +1,24 @@ +//! Core types for the memory tree ingestion layer (Phase 1 / issue #707). +//! +//! This module defines the canonical [`Chunk`] representation produced by the +//! ingestion pipeline along with its provenance [`Metadata`] and back-pointer +//! [`SourceRef`]. These types feed into later phases (#708 scoring, #709 +//! summary trees, #710 retrieval) but are self-contained at Phase 1. +//! +//! All chunk IDs are deterministic: `sha256(source_kind | "\0" | source_id | +//! "\0" | seq | "\0" | content)` truncated to 32 hex chars so re-ingest of the +//! same source material yields stable IDs and idempotent upserts. +//! +//! **W3 type cutover:** these types + chunk-id/token helpers are now +//! **re-exported from the `tinycortex` crate** (ported from this exact module — +//! identical fields, derives, serde wire form, and `chunk_id` derivation, all +//! pinned by `tinycortex::memory::chunks::types_tests`). Re-exporting keeps one source of truth and lets +//! the chunk store operations delegate to the crate without host↔crate type +//! conversions. `DataSource` moved with the ingest cutover and is re-exported +//! here alongside the chunk types. `StagedChunk` remains host-owned in +//! `memory_store::content`. + +pub use tinycortex::memory::chunks::{ + approx_token_count, chunk_id, conservative_token_estimate, truncate_to_conservative_tokens, + Chunk, DataSource, Metadata, SourceKind, SourceRef, +}; diff --git a/core/src/store/client.rs b/core/src/store/client.rs new file mode 100644 index 0000000..569eee1 --- /dev/null +++ b/core/src/store/client.rs @@ -0,0 +1,594 @@ +//! # Memory Client +//! +//! High-level client interface for interacting with the OpenHuman memory system. +//! +//! The `MemoryClient` provides a simplified API for storing and retrieving +//! information from the memory store, handling background tasks like graph +//! extraction and embedding generation. It primarily acts as a wrapper around +//! `UnifiedMemory`. + +use serde_json::json; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::embedding_host::require_embedding_host; +use crate::ingestion::queue as ingestion_queue; +use crate::ingestion::{ + IngestionJob, IngestionQueue, IngestionState, MemoryIngestionConfig, MemoryIngestionRequest, + MemoryIngestionResult, +}; +use crate::store::namespace_store::UnifiedMemory; +use crate::store::types::{ + GraphRelationRecord, MemoryKvRecord, NamespaceDocumentInput, NamespaceMemoryHit, + NamespaceRetrievalContext, StoredMemoryDocument, +}; +use tinymemory_api::host::EmbeddingProvider; + +/// Reference-counted handle to a `MemoryClient`. +pub type MemoryClientRef = Arc; + +/// Thread-safe container for an optional `MemoryClientRef`. +/// +/// Used for global state management where the memory client may or may not +/// be initialized. +pub struct MemoryState(pub std::sync::Mutex>); + +/// SQLite-backed memory client rooted at the user's workspace directory. +/// +/// Storage (documents, vectors, graph) remains on-device via [`UnifiedMemory`]. +/// Embedding generation is delegated to whichever provider the +/// [`MemoryConfig.embedding_provider`](tinymemory_api::host::MemoryConfig) +/// resolves to — cloud (OpenHuman backend, the default returned by +/// [`crate::embedding_host::default_embedding_provider`]) or local Ollama +/// when explicitly opted into. The cloud embedder resolves its session JWT +/// lazily, so an unauthenticated session will surface as a clear error on the +/// first `embed` call rather than at client construction. +/// +/// Callers that need a non-default embedder should construct the underlying +/// store via [`crate::store::create_memory_with_local_ai`] with the +/// appropriate `MemoryConfig.embedding_provider`. +#[derive(Clone)] +pub struct MemoryClient { + /// The underlying memory implementation. + inner: Arc, + /// Queue for background ingestion tasks (e.g., entity extraction). + ingestion_queue: IngestionQueue, +} + +impl MemoryClient { + /// Returns a handle to the underlying SQLite connection backing the + /// profile/facet tables. + /// + /// A raw `Arc>` cannot be wrapped by any decorator, so + /// no caller outside the memory family may hold one. + /// [`Self::profile_store`] is the only door out, and every SQL statement + /// against `user_profile` belongs inside the family. + /// + /// It was `pub(in crate)` before the memory subsystem was extracted, which + /// let the compiler enforce that directly. The family now spans two crates + /// — this one and the host's `openhuman::memory` — so the rule cannot be a + /// visibility any more. It is enforced by + /// `profile_conn_is_confined_to_the_memory_family` in the host, which scans + /// the host tree and names the offending file; a visibility error read as + /// "private method", not as "you are reaching around the guard". + pub fn profile_conn(&self) -> std::sync::Arc> { + std::sync::Arc::clone(&self.inner.conn) + } + + /// Typed access to the profile/facet tables. + /// + /// **Not guarded.** The profile tables have no capability family in the + /// thirteen-family `tinycortex_api` contract, so these reads and writes + /// still run beneath `crate::guard::MemoryGuard`'s + /// seven steps. What this buys is confinement, not policy: the SQL is in + /// the memory family and the compiler keeps it there. + pub fn profile_store(&self) -> crate::store::ProfileStore { + tracing::debug!("[memory::profile_store] handing out typed profile store"); + crate::store::ProfileStore::from_conn(self.profile_conn()) + } + + /// Returns an `Arc` handle backed by the same + /// [`UnifiedMemory`] this client wraps. Used by sub-systems that + /// want to build on top of the `Memory` trait (e.g. the + /// tool-scoped memory layer) without depending on the concrete + /// `MemoryClient` type or holding a reference to it. + /// + /// Intentionally `pub(crate)` — handing a raw `Arc` to an + /// external consumer bypasses any policy decorator wrapped around the + /// `MemoryClient` API, so the escape hatch stays in-crate. Mirrors + /// [`Self::profile_conn`]. + pub fn memory_handle(&self) -> Arc { + Arc::clone(&self.inner) as Arc + } + + /// Create a new local memory client using the default `.openhuman` directory. + /// + /// # Errors + /// + /// Returns an error string if the home directory cannot be resolved or if + /// initialization fails. + pub fn new_local() -> Result { + let workspace_dir = crate::default_openhuman_dir() + .map_err(|e| e.to_string())? + .join("workspace"); + Self::from_workspace_dir(workspace_dir) + } + + /// Create a new memory client from a specific workspace directory. + /// + /// # Arguments + /// + /// * `workspace_dir` - The path where memory databases and assets are stored. + /// + /// # Errors + /// + /// Returns an error string if the directory cannot be created or if the + /// `UnifiedMemory` or `IngestionQueue` fails to start. + pub fn from_workspace_dir(workspace_dir: PathBuf) -> Result { + std::fs::create_dir_all(&workspace_dir) + .map_err(|e| format!("Create workspace dir {}: {e}", workspace_dir.display()))?; + + // Default to cloud embeddings (OpenHuman backend, Voyage-backed). The + // cloud embedder is lazy: JWT + API URL are resolved per call, so an + // unauthenticated session produces a clear error on first embed rather + // than blocking client construction. Callers that need the local + // Ollama path should build their memory store via + // `create_memory_with_local_ai` with the appropriate + // `MemoryConfig.embedding_provider`. + let embedder: Arc = + require_embedding_host()?.default_embedding_provider(); + + // Create the underlying UnifiedMemory instance. + let memory = + UnifiedMemory::new(&workspace_dir, embedder, None).map_err(|e| format!("{e}"))?; + let inner = Arc::new(memory); + + // Start the background worker for document ingestion and graph extraction. + // The worker shares its IngestionState with the synchronous ingest path + // below so all ingestion is singleton-serialised. + let ingestion_queue = + ingestion_queue::start_worker_with_state(Arc::clone(&inner), IngestionState::new()); + + Ok(Self { + inner, + ingestion_queue, + }) + } + + /// Store a document in a specific namespace. + /// + /// This method performs an "upsert" (update or insert). It immediately + /// persists the document and then enqueues a background job for graph + /// extraction (entities and relations). + /// + /// # Arguments + /// + /// * `input` - The document content and metadata. + /// + /// # Returns + /// + /// The unique ID of the stored document. + pub async fn put_doc(&self, input: NamespaceDocumentInput) -> Result { + let document_id = self.inner.upsert_document(input.clone()).await?; + + // Enqueue background graph extraction so entities/relations are + // extracted without blocking the caller. The document is already + // persisted — extract_graph will not upsert again. + self.ingestion_queue.submit(IngestionJob { + document_id: document_id.clone(), + document: input, + config: MemoryIngestionConfig::default(), + }); + + Ok(document_id) + } + + /// Store a document (DB row + markdown file) without vector embedding or + /// graph extraction. Use this for high-frequency, ephemeral writes where + /// the full pipeline would be too expensive (e.g. transient sync + /// checkpoints). The document is still searchable by metadata/FTS but will + /// not appear in semantic vector queries or the knowledge graph. + pub async fn put_doc_light(&self, input: NamespaceDocumentInput) -> Result { + self.inner.upsert_document_metadata_only(input).await + } + + /// Perform a full ingestion (chunking, embedding, extraction) synchronously. + /// + /// Unlike `put_doc`, this waits for the entire process to complete. + /// Serialised against the background worker via the shared + /// [`IngestionState`] singleton lock — only one ingestion runs at a time. + pub async fn ingest_doc( + &self, + request: MemoryIngestionRequest, + ) -> Result { + let state = self.ingestion_queue.state(); + let _guard = state.acquire().await; + + let title = request.document.title.clone(); + let namespace = request.document.namespace.clone(); + // Synthetic id until upsert assigns one — purely for the snapshot. + let placeholder_id = format!("sync:{title}"); + + let queue_depth = state.snapshot().queue_depth; + state.mark_running(&placeholder_id, &title, &namespace); + crate::events::publish(crate::events::MemoryEvent::IngestionStarted { + document_id: placeholder_id.clone(), + title, + namespace: namespace.clone(), + queue_depth, + }); + + let started = std::time::Instant::now(); + let outcome = self.inner.ingest_document(request).await; + let elapsed_ms = started.elapsed().as_millis() as u64; + let success = outcome.is_ok(); + + // Use the same placeholder id as the matching MemoryIngestionStarted + // event so subscribers can correlate start/complete pairs. The real + // upstream-assigned document id is available on `Ok(outcome)` for + // callers that need it. + state.mark_completed( + &placeholder_id, + success, + chrono::Utc::now().timestamp_millis(), + ); + crate::events::publish(crate::events::MemoryEvent::IngestionCompleted { + document_id: placeholder_id, + namespace, + success, + elapsed_ms, + queue_depth: state.snapshot().queue_depth, + }); + + outcome + } + + /// Returns the shared ingestion state — singleton lock + status snapshot. + /// Used by the `openhuman.memory_ingestion_status` RPC handler. + pub fn ingestion_state(&self) -> IngestionState { + self.ingestion_queue.state() + } + + /// Specialized method for syncing skill data into memory. + /// + /// Maps generic skill/integration fields into the `NamespaceDocumentInput` structure. + /// + /// Every write goes in as + /// [`MemoryTaint::ExternalSync`](crate::MemoryTaint::ExternalSync) + /// — this entry point exists specifically for memory_sync providers + /// (Gmail / Slack / Notion / Composio / etc.) that ingest text from + /// third-party services. Routing the call through here is what lets + /// the subconscious gate refuse external_effect tools when these + /// chunks land in a tick's context window. Internal / user-driven + /// writes must use the regular `put_doc` / `Memory::store` paths. + #[allow(clippy::too_many_arguments)] + pub async fn store_skill_sync( + &self, + skill_id: &str, + _integration_id: &str, + title: &str, + content: &str, + source_type: Option, + metadata: Option, + priority: Option, + _created_at: Option, + _updated_at: Option, + document_id: Option, + ) -> Result<(), String> { + let namespace = format!("skill-{}", skill_id.trim()); + // The upsert dedup key must be a stable, opaque identifier — never + // free-form human text. Sync providers pass a `{toolkit}:{id}` + // `document_id` (e.g. `gmail:`); use it as the key so the + // provider's human-readable title (an email subject can legitimately + // contain verification codes, token-rotation notices, or other + // secret-/PII-looking strings) is never run through the + // `upsert_document` secret/PII namespace/key guard. A single such + // subject would otherwise abort the whole non-tolerant provider sync + // with `document namespace/key cannot contain secrets` (see #4947). + // Callers without a stable id (e.g. LinkedIn enrichment) keep the + // title as the key, unchanged. + let stable_id = document_id + .as_deref() + .map(str::trim) + .filter(|id| !id.is_empty()); + let key = match stable_id { + Some(id) => id.to_string(), + None => title.to_string(), + }; + tracing::debug!( + namespace = %namespace, + key_from_document_id = stable_id.is_some(), + "[memory_store] store_skill_sync: upserting synchronized document" + ); + let input = NamespaceDocumentInput { + namespace, + key, + title: title.to_string(), + content: content.to_string(), + source_type: source_type.unwrap_or_else(|| "doc".to_string()), + priority: priority.unwrap_or_else(|| "medium".to_string()), + tags: Vec::new(), + metadata: metadata.unwrap_or_else(|| json!({})), + category: "core".to_string(), + session_id: None, + document_id, + // Every sync entry point is by definition ingesting third- + // party content; mark it so the subconscious gate can see + // the provenance through the persistence layer. + taint: crate::MemoryTaint::ExternalSync, + }; + + let doc_id = self.inner.upsert_document(input.clone()).await?; + + // Enqueue background graph extraction. + self.ingestion_queue.submit(IngestionJob { + document_id: doc_id, + document: input, + config: MemoryIngestionConfig::default(), + }); + + Ok(()) + } + + /// List documents in a namespace (or all namespaces if `None`). + pub async fn list_documents( + &self, + namespace: Option<&str>, + ) -> Result { + self.inner.list_documents(namespace).await + } + + /// Fetch one document by `(namespace, key)`. + /// + /// `pub(crate)` on the same reasoning as [`Self::memory_handle`]: the only + /// in-crate consumer is the embedded memory driver + /// (`crate::driver::embedded`), which needs a read-one + /// path that [`Self::list_documents`] cannot provide — the latter's SELECT + /// carries no `content` column. + pub async fn get_document( + &self, + namespace: &str, + key: &str, + ) -> Result, String> { + self.inner.get_document_by_key(namespace, key).await + } + + /// List all unique namespaces in the memory store. + pub async fn list_namespaces(&self) -> Result, String> { + self.inner.list_namespaces().await + } + + /// Delete a specific document by its ID and namespace. + pub async fn delete_document( + &self, + namespace: &str, + document_id: &str, + ) -> Result { + self.inner.delete_document(namespace, document_id).await + } + + /// Clear all documents and data within a specific namespace. + pub async fn clear_namespace(&self, namespace: &str) -> Result<(), String> { + self.inner.clear_namespace(namespace).await + } + + /// Clear memory associated with a specific skill. + pub async fn clear_skill_memory( + &self, + skill_id: &str, + _integration_id: &str, + ) -> Result<(), String> { + let namespace = format!("skill-{}", skill_id.trim()); + let docs = self.list_documents(Some(&namespace)).await?; + let items = docs + .get("documents") + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default(); + for item in items { + if let Some(document_id) = item.get("documentId").and_then(serde_json::Value::as_str) { + let _ = self.delete_document(&namespace, document_id).await?; + } + } + Ok(()) + } + + /// Query a namespace for context using natural language. + /// + /// Returns a formatted string containing relevant text chunks and context. + pub async fn query_namespace( + &self, + namespace: &str, + query: &str, + max_chunks: u32, + ) -> Result { + self.inner + .query_namespace_context(namespace, query, max_chunks) + .await + } + + /// Query a namespace and return raw context data (hits, relations, etc.). + pub async fn query_namespace_context_data( + &self, + namespace: &str, + query: &str, + max_chunks: u32, + ) -> Result { + self.inner + .query_namespace_context_data(namespace, query, max_chunks) + .await + } + + /// Recall recent context from a namespace without a specific query. + pub async fn recall_namespace( + &self, + namespace: &str, + max_chunks: u32, + ) -> Result, String> { + self.inner + .recall_namespace_context(namespace, max_chunks) + .await + } + + /// Recall raw context data from a namespace without a specific query. + pub async fn recall_namespace_context_data( + &self, + namespace: &str, + max_chunks: u32, + ) -> Result { + self.inner + .recall_namespace_context_data(namespace, max_chunks) + .await + } + + /// Recall a specific number of recent memories (hits) from a namespace. + pub async fn recall_namespace_memories( + &self, + namespace: &str, + limit: u32, + ) -> Result, String> { + self.inner.recall_namespace_memories(namespace, limit).await + } + + /// Store a key-value pair in a namespace (or global if `None`). + pub async fn kv_set( + &self, + namespace: Option<&str>, + key: &str, + value: &serde_json::Value, + ) -> Result<(), String> { + match namespace { + Some(ns) => self.inner.kv_set_namespace(ns, key, value).await, + None => self.inner.kv_set_global(key, value).await, + } + } + + /// Retrieve a key-value pair. + pub async fn kv_get( + &self, + namespace: Option<&str>, + key: &str, + ) -> Result, String> { + match namespace { + Some(ns) => self.inner.kv_get_namespace(ns, key).await, + None => self.inner.kv_get_global(key).await, + } + } + + /// Delete a key-value pair. + pub async fn kv_delete(&self, namespace: Option<&str>, key: &str) -> Result { + match namespace { + Some(ns) => self.inner.kv_delete_namespace(ns, key).await, + None => self.inner.kv_delete_global(key).await, + } + } + + /// Typed key/value records for one namespace, or the global slice when + /// `namespace` is `None`. + /// + /// `pub(crate)` for the embedded driver. Distinct from + /// [`Self::kv_list_namespace`], which returns a camelCase + /// `Vec` with no `updated_at` and no global slice — + /// re-parsing that back into [`MemoryKvRecord`] would be lossy new logic. + pub async fn kv_records(&self, namespace: Option<&str>) -> Result, String> { + match namespace { + Some(ns) => self.inner.kv_records_namespace(ns).await, + None => self.inner.kv_records_global().await, + } + } + + /// Typed relation records, filtered by subject/predicate. + /// + /// `namespace: None` spans every namespace *and* the global graph, matching + /// [`Self::graph_query`]'s `None` behaviour. `pub(crate)` for the embedded + /// driver, for the same reason as [`Self::kv_records`]: `graph_query` + /// returns camelCase JSON, these return the record type directly. + /// + /// Inherits the storage layer's hard `LIMIT 300` per SQL statement. + pub async fn graph_relations( + &self, + namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + match namespace { + Some(ns) => { + self.inner + .graph_relations_namespace(ns, subject, predicate) + .await + } + None => { + let mut rows = self + .inner + .graph_relations_all_namespaces(subject, predicate) + .await?; + rows.extend( + self.inner + .graph_relations_global(subject, predicate) + .await?, + ); + rows.sort_by(|a, b| { + b.updated_at + .partial_cmp(&a.updated_at) + .unwrap_or(std::cmp::Ordering::Equal) + }); + Ok(rows) + } + } + } + + /// List all key-value pairs in a namespace. + pub async fn kv_list_namespace( + &self, + namespace: &str, + ) -> Result, String> { + self.inner.kv_list_namespace(namespace).await + } + + /// Upsert a relationship in the knowledge graph. + pub async fn graph_upsert( + &self, + namespace: Option<&str>, + subject: &str, + predicate: &str, + object: &str, + attrs: &serde_json::Value, + ) -> Result<(), String> { + match namespace { + Some(ns) => { + self.inner + .graph_upsert_namespace(ns, subject, predicate, object, attrs) + .await + } + None => { + self.inner + .graph_upsert_global(subject, predicate, object, attrs) + .await + } + } + } + + /// Query relationships in the knowledge graph using optional filters. + /// + /// When `namespace` is `None`, returns relations from **all** namespaces + /// plus the global graph, so ingested data is always surfaced in the UI. + pub async fn graph_query( + &self, + namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + match namespace { + Some(ns) => { + self.inner + .graph_query_namespace(ns, subject, predicate) + .await + } + None => self.inner.graph_query_all(subject, predicate).await, + } + } +} + +#[cfg(test)] +#[path = "client_tests.rs"] +mod tests; diff --git a/core/src/store/client_tests.rs b/core/src/store/client_tests.rs new file mode 100644 index 0000000..20f6aab --- /dev/null +++ b/core/src/store/client_tests.rs @@ -0,0 +1,381 @@ +//! Tests for `MemoryClient` — exercise the sync storage surface (upsert, list, +//! kv, graph) against a fresh temp workspace. + +use super::*; +use tempfile::TempDir; + +/// Build a MemoryClient pointed at a fresh temp workspace. Ollama is +/// the default embedder — it won't be reachable in tests so anything +/// that exercises the embedding path will surface a retrieval-empty +/// state. That's fine for these tests: we're verifying the sync +/// storage surface (upsert, list, kv, graph) which does not require +/// a working embedder. +fn make_client() -> (TempDir, MemoryClient) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let client = MemoryClient::from_workspace_dir(tmp.path().join("workspace")) + .expect("client should initialise against a fresh workspace"); + (tmp, client) +} + +fn doc(namespace: &str, key: &str, content: &str) -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: namespace.to_string(), + key: key.to_string(), + title: key.to_string(), + content: content.to_string(), + source_type: "doc".to_string(), + priority: "normal".to_string(), + tags: vec![], + metadata: serde_json::Value::Null, + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + } +} + +#[tokio::test] +async fn from_workspace_dir_creates_workspace_and_returns_client() { + let (tmp, client) = make_client(); + assert!(tmp.path().join("workspace").exists()); + // put_doc_light is the cheapest sanity check — it stores a DB row + // without touching the embedder / graph extractor. + let id = client + .put_doc_light(doc("test-ns", "k1", "hello")) + .await + .unwrap(); + assert!(!id.is_empty()); +} + +#[tokio::test] +async fn list_namespaces_returns_what_was_written() { + let (_tmp, client) = make_client(); + client.put_doc_light(doc("alpha", "k1", "a")).await.unwrap(); + client.put_doc_light(doc("beta", "k1", "b")).await.unwrap(); + let mut namespaces = client.list_namespaces().await.unwrap(); + namespaces.sort(); + assert!(namespaces.contains(&"alpha".to_string())); + assert!(namespaces.contains(&"beta".to_string())); +} + +#[tokio::test] +async fn list_documents_and_delete_document_round_trip() { + let (_tmp, client) = make_client(); + let id = client + .put_doc_light(doc("docs", "k1", "some content")) + .await + .unwrap(); + + let docs = client.list_documents(Some("docs")).await.unwrap(); + let docs_arr = docs + .get("documents") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + assert!(docs_arr + .iter() + .any(|d| { d.get("documentId").and_then(|v| v.as_str()) == Some(&id) })); + + let _ = client.delete_document("docs", &id).await.unwrap(); + let docs = client.list_documents(Some("docs")).await.unwrap(); + let docs_arr = docs + .get("documents") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + assert!(docs_arr + .iter() + .all(|d| { d.get("documentId").and_then(|v| v.as_str()) != Some(&id) })); +} + +#[tokio::test] +async fn clear_namespace_removes_all_docs_in_namespace() { + let (_tmp, client) = make_client(); + client + .put_doc_light(doc("throwaway", "k1", "x")) + .await + .unwrap(); + client + .put_doc_light(doc("throwaway", "k2", "y")) + .await + .unwrap(); + client.clear_namespace("throwaway").await.unwrap(); + let docs = client.list_documents(Some("throwaway")).await.unwrap(); + let docs_arr = docs + .get("documents") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + assert!(docs_arr.is_empty()); +} + +#[tokio::test] +async fn store_skill_sync_with_secret_like_title_uses_stable_document_id_as_key() { + // Regression for #4947 (Bug 2): a Composio provider sync passes the + // provider's human-readable title as the document title, but the upsert + // *key* must be the stable opaque `document_id` (`gmail:`), + // NOT the title. Some email subjects legitimately look secret-like + // (verification codes, token-rotation notices), and `upsert_document` + // rejects any secret-like namespace/key. Because gmail's tinycortex + // pipeline does not tolerate scope errors, one such subject would abort + // the entire scheduled sync with `document namespace/key cannot contain + // secrets`, leaving the source stale ("Last synced 17d ago"). + let (_tmp, client) = make_client(); + + // A subject that trips the secret guard. Assert the precondition so the + // test cannot silently pass if the detector's patterns change. + let secret_like_title = "Security alert: token glpat-aaaaaaaaaaaaaaaaaaaa was created"; + assert!( + crate::store::safety::has_likely_secret(secret_like_title), + "test title must trip the secret detector for this regression to be meaningful" + ); + + // With a stable document_id provided, the write must succeed — the key is + // the opaque id, not the secret-like subject. Before the fix the key was + // the title and this returned the secrets error. + client + .store_skill_sync( + "gmail", + "conn-1", + secret_like_title, + "email body", + Some("composio-provider-incremental".into()), + None, + Some("medium".into()), + None, + None, + Some("gmail:19af23bc00112233".into()), + ) + .await + .expect("secret-like subject must not block a stable-id-keyed sync write"); + + // The document is persisted and keyed by the stable id, so a second sync + // of the same message dedups (updates in place) rather than duplicating. + client + .store_skill_sync( + "gmail", + "conn-1", + secret_like_title, + "email body v2", + Some("composio-provider-incremental".into()), + None, + Some("medium".into()), + None, + None, + Some("gmail:19af23bc00112233".into()), + ) + .await + .expect("re-sync of same message id must succeed"); + + let docs = client.list_documents(Some("skill-gmail")).await.unwrap(); + let arr = docs + .get("documents") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + assert_eq!( + arr.len(), + 1, + "stable-id key must dedupe both syncs into a single document" + ); +} + +#[tokio::test] +async fn clear_skill_memory_targets_prefixed_namespace() { + let (_tmp, client) = make_client(); + // `store_skill_sync` prefixes the namespace with "skill-". + client + .store_skill_sync( + "my-skill", "default", "Title", "body", None, None, None, None, None, None, + ) + .await + .unwrap(); + // Verify the doc lives under the prefixed namespace. + let docs = client.list_documents(Some("skill-my-skill")).await.unwrap(); + let arr = docs + .get("documents") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + assert!(!arr.is_empty()); + // Clearing by skill id should remove it. + client + .clear_skill_memory("my-skill", "default") + .await + .unwrap(); + let after = client.list_documents(Some("skill-my-skill")).await.unwrap(); + let after_arr = after + .get("documents") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + assert!(after_arr.is_empty()); +} + +#[tokio::test] +async fn kv_set_get_delete_round_trip() { + let (_tmp, client) = make_client(); + let value = json!("ship-it"); + client.kv_set(Some("team"), "goal", &value).await.unwrap(); + let got = client.kv_get(Some("team"), "goal").await.unwrap(); + assert_eq!(got.as_ref(), Some(&value)); + let removed = client.kv_delete(Some("team"), "goal").await.unwrap(); + assert!(removed); + let after = client.kv_get(Some("team"), "goal").await.unwrap(); + assert!(after.is_none()); +} + +#[tokio::test] +async fn kv_global_set_and_get_uses_none_namespace_branch() { + let (_tmp, client) = make_client(); + let v = json!({"k": 1}); + client.kv_set(None, "global-key", &v).await.unwrap(); + let got = client.kv_get(None, "global-key").await.unwrap(); + assert_eq!(got.as_ref(), Some(&v)); +} + +#[tokio::test] +async fn kv_list_namespace_returns_all_keys() { + let (_tmp, client) = make_client(); + client + .kv_set(Some("cfg"), "env", &json!("dev")) + .await + .unwrap(); + client + .kv_set(Some("cfg"), "region", &json!("us-east")) + .await + .unwrap(); + let entries = client.kv_list_namespace("cfg").await.unwrap(); + // Each entry is a JSON object — we just check that both keys are present. + let s = serde_json::to_string(&entries).unwrap(); + assert!(s.contains("env")); + assert!(s.contains("region")); +} + +#[tokio::test] +async fn graph_upsert_does_not_error_for_namespaced_and_global_writes() { + // We exercise both `Some(ns)` and `None` branches of `graph_upsert` + // — the storage shape returned by `graph_query` is internal and + // varies between unified store versions, so we only assert the + // upsert path completes successfully. + let (_tmp, client) = make_client(); + client + .graph_upsert( + Some("team"), + "Alice", + "OWNS", + "Atlas", + &json!({"evidence": "chat"}), + ) + .await + .unwrap(); + client + .graph_upsert(None, "Bob", "FOLLOWS", "Carol", &json!({})) + .await + .unwrap(); + // graph_query() must not error in either form; we accept any + // returned vec (possibly empty depending on store internals). + let _ = client + .graph_query(Some("team"), Some("Alice"), None) + .await + .unwrap(); + let _ = client.graph_query(None, Some("Bob"), None).await.unwrap(); +} + +#[tokio::test] +async fn profile_conn_returns_arc_shared_connection() { + let (_tmp, client) = make_client(); + let a = client.profile_conn(); + let b = client.profile_conn(); + // Both handles wrap the same Arc. + assert!(Arc::ptr_eq(&a, &b)); +} + +#[tokio::test] +async fn put_doc_full_pipeline_completes() { + // Exercise the full `put_doc` path (vs `put_doc_light`) — the + // ingestion queue submits a background job. The call itself + // returns the document id immediately. + let (_tmp, client) = make_client(); + let id = client + .put_doc(doc( + "ingestion-pipeline", + "k1", + "background-extract content", + )) + .await + .unwrap(); + assert!(!id.is_empty()); +} + +#[tokio::test] +async fn recall_namespace_memories_returns_recent_inputs() { + let (_tmp, client) = make_client(); + for i in 0..3 { + client + .put_doc_light(doc("recall-ns", &format!("k{i}"), &format!("body {i}"))) + .await + .unwrap(); + } + let hits = client + .recall_namespace_memories("recall-ns", 10) + .await + .unwrap(); + // Light docs may not register as queryable hits in every backend, + // but the call must not error. + let _ = hits; +} + +#[tokio::test] +async fn recall_namespace_with_no_data_returns_none_or_empty() { + let (_tmp, client) = make_client(); + let recalled = client + .recall_namespace("never-written-ns", 5) + .await + .unwrap(); + // Either no context (None) or empty string is acceptable. + assert!(recalled.is_none() || recalled.as_deref() == Some("")); +} + +#[tokio::test] +async fn query_namespace_with_no_data_returns_empty_or_short() { + let (_tmp, client) = make_client(); + let result = client + .query_namespace("never-written-ns", "anything", 5) + .await + .unwrap(); + // Empty namespace → either empty result or trivial sentinel. + assert!(result.is_empty() || result.len() < 200); +} + +#[tokio::test] +async fn query_and_recall_namespace_context_data_return_empty_context() { + // Hit the `*_context_data` variants of query / recall so their + // delegation arms in `MemoryClient` get exercised. + let (_tmp, client) = make_client(); + let q = client + .query_namespace_context_data("empty-ns", "q", 5) + .await + .unwrap(); + let r = client + .recall_namespace_context_data("empty-ns", 5) + .await + .unwrap(); + // Ensure the accessor surface is reachable; exact shape varies. + let _ = (q, r); +} + +#[tokio::test] +async fn ingest_doc_completes_and_stores_document() { + let (_tmp, client) = make_client(); + let req = MemoryIngestionRequest { + document: doc("ingest-ns", "direct-k", "inline sync ingest body"), + config: MemoryIngestionConfig::default(), + }; + let result = client.ingest_doc(req).await; + // Depending on whether the embedder is reachable the call may + // error out with a clear message — we only assert that the path + // is exercised (no panic). + let _ = result; +} diff --git a/core/src/store/content/README.md b/core/src/store/content/README.md new file mode 100644 index 0000000..6fee154 --- /dev/null +++ b/core/src/store/content/README.md @@ -0,0 +1,21 @@ +# content_store/ + +On-disk `.md` storage for chunk and summary bodies (Phase MD-content). SQLite holds `content_path` (relative, forward-slash) and `content_sha256` (over body bytes only) as pointers + integrity tokens; the body itself lives at `/`. + +The body is **immutable** once written — only the YAML front-matter `tags:` block may be rewritten post-extraction. + +## Files + +- [`mod.rs`](mod.rs) — public surface: `StagedChunk`, `stage_chunks` (write all chunks atomically before SQLite upsert), `update_summary_tags` re-export. +- `vendor/tinycortex/src/memory/store/content/atomic.rs` — `write_if_new` (tempfile + fsync + rename, parent dir fsync on Unix), `stage_summary` (idempotent re-stage with on-disk SHA check + auto-rewrite on mismatch), `sha256_hex`, `StagedSummary`. +- `vendor/tinycortex/src/memory/store/content/compose/` — YAML front-matter + body composition. `compose_chunk_file` for chunks (with email-only `participants:` / `aliases:` fields parsed from `gmail:{addr1|addr2|…}` source ids), `compose_summary_md` for summary nodes. `rewrite_tags` / `rewrite_summary_tags` swap the `tags:` block in place. `split_front_matter` parses `---\n…\n---\n`. +- `vendor/tinycortex/src/memory/store/content/paths.rs` — path generators. `chunk_rel_path` (`email//.md`, `chat//.md`, `document//.md`); `summary_rel_path` (`summaries/{source,global,topic}/…`). `slugify_source_id` is the canonical filesystem-safe slug. +- [`read.rs`](read.rs) — `read_chunk_file` / `read_summary_file` parse front-matter and return body+SHA. `verify_*` compares against an expected SHA. `read_chunk_body` / `read_summary_body` resolve the path via SQLite and verify the integrity hash; this is the authoritative entry-point for callers that need the **full** body (LLM extractor, summariser, embedder, retrieval API). +- `vendor/tinycortex/src/memory/store/content/raw.rs` — verbatim source-byte mirror under `/raw/`. Writes the unmodified upstream payload (eml, slack json, raw markdown) so downstream callers can re-canonicalise without re-fetching. +- `vendor/tinycortex/src/memory/store/content/obsidian.rs` + `obsidian_defaults/` (`obsidian` feature) — bootstrap an `.obsidian/` config (workspace, graph, app) into the content root on first write so a user opening the vault gets a usable view. +- [`tags.rs`](tags.rs) — post-extraction tag rewrites. `update_chunk_tags` (atomic tempfile rewrite of the `tags:` block) and `update_summary_tags` (fetches entities from `mem_tree_entity_index`, builds Obsidian `kind/Value` tags, rewrites, verifies body SHA is unchanged). `slugify_tag_kind`, `slugify_tag_value`, `entity_tag` build the tag strings. +- `vendor/tinycortex/src/memory/store/content/wiki_git/` (`wiki-git` feature) — initializes `/wiki/.git`, commits only summary-node markdown under `summaries/**` plus the repo `.gitignore`, and stores read high-water marks as lightweight `refs/tags/read/*` pointers. Summary files are staged by `atomic.rs`; seal/ingest callers create descriptive git commits after SQLite persistence succeeds. + +## Integrity contract + +The body bytes never change after the first write. The SHA-256 stored in SQLite is computed over body bytes only — front-matter (including `tags:`) can be rewritten without invalidating the hash. Read paths verify SHA on every fetch and fail loudly on mismatch rather than serve corrupt data into the extractor or summariser. diff --git a/core/src/store/content/mod.rs b/core/src/store/content/mod.rs new file mode 100644 index 0000000..52ec477 --- /dev/null +++ b/core/src/store/content/mod.rs @@ -0,0 +1,37 @@ +//! Content store for memory-tree chunk and summary `.md` files (Phase MD-content). +//! +//! Bodies are stored on disk as `.md` files with YAML front-matter. +//! SQLite holds `content_path` (relative, forward-slash) and `content_sha256` +//! (over body bytes only) as pointers + integrity tokens. +//! +//! ## Module layout +//! +//! - [`paths`] — path generation + `slugify_source_id` + summary path builders +//! - [`compose`] — YAML front-matter + body composition; tag rewriting +//! - [`atomic`] — tempfile+fsync+rename writes; SHA-256; `stage_summary` +//! - [`read`] — read + SHA-256 verification + `split_front_matter`; summary variants +//! - [`tags`] — `update_chunk_tags` + `update_summary_tags` + slugifiers +//! - `obsidian` / `obsidian_registry` / `wiki_git` — on-disk content formats, +//! owned by TinyCortex and re-exported here (see the `pub use` below) + +pub mod read; +pub mod tags; + +pub use tinycortex::memory::chunks::StagedChunk; +/// The git-backed wiki content format. Re-exported only when `memory-git` is +/// on: it lives behind tinycortex's `wiki-git` feature, which the gate carries +/// along with `git-diff` and the libgit2 cohort. +#[cfg(feature = "memory-git")] +pub use tinycortex::memory::store::content::wiki_git; +pub use tinycortex::memory::store::content::{ + atomic, compose, obsidian, obsidian_registry, paths, raw, stage_chunks, StagedSummary, + SummaryComposeInput, SummaryTreeKind, +}; + +/// Update the `tags:` block in a summary's on-disk `.md` file after an +/// extraction job runs. +/// +/// Delegates to [`tags::update_summary_tags`]. +pub fn update_summary_tags(config: &crate::Config, summary_id: &str) -> anyhow::Result<()> { + tags::update_summary_tags(config, summary_id) +} diff --git a/core/src/store/content/read.rs b/core/src/store/content/read.rs new file mode 100644 index 0000000..d15ee13 --- /dev/null +++ b/core/src/store/content/read.rs @@ -0,0 +1,15 @@ +//! Product Config adapters over tinycortex content readers. +use crate::tinycortex::engine_config; + +pub use tinycortex::memory::store::content::{ + read_chunk_file, read_summary_file, verify_chunk_file, verify_summary_file, ChunkFileContents, + VerifyResult, +}; + +pub fn read_chunk_body(config: &crate::Config, chunk_id: &str) -> anyhow::Result { + tinycortex::memory::store::content::read_chunk_body(&engine_config(config), chunk_id) +} + +pub fn read_summary_body(config: &crate::Config, summary_id: &str) -> anyhow::Result { + tinycortex::memory::store::content::read_summary_body(&engine_config(config), summary_id) +} diff --git a/core/src/store/content/tags.rs b/core/src/store/content/tags.rs new file mode 100644 index 0000000..92413b1 --- /dev/null +++ b/core/src/store/content/tags.rs @@ -0,0 +1,134 @@ +//! Host adapter for TinyCortex-owned markdown tag rewriting. +//! +//! Generic chunk rewrites and tag formatting live in TinyCortex. OpenHuman +//! retains only the summary adapter because it resolves product configuration, +//! content pointers, and entity-index rows. + +use std::path::Path; + +use crate::store::chunks::store::get_summary_content_pointers; +use crate::store::content::compose::{ + rewrite_summary_tags, scan_fm_field, source_tag, split_front_matter, +}; +use crate::tree::score::store::list_entity_ids_for_node; +use crate::Config; + +pub use tinycortex::memory::store::content::tags::{ + entity_tag, slugify_tag_kind, slugify_tag_value, update_chunk_tags, +}; + +/// Rewrite a summary's tags from its authoritative entity-index rows. +/// +/// This is host-owned glue: TinyCortex performs the generic markdown rewrite, +/// while OpenHuman supplies configuration and entity-index lookup. +pub fn update_summary_tags(config: &Config, summary_id: &str) -> anyhow::Result<()> { + let Some((rel_path, expected_sha)) = get_summary_content_pointers(config, summary_id)? else { + log::debug!( + "[content_store::tags] update_summary_tags: no content_path for summary {summary_id} — skipping" + ); + return Ok(()); + }; + + let mut abs_path = config.memory_tree_content_root(); + for component in rel_path.split('/') { + abs_path.push(component); + } + if !abs_path.exists() { + log::debug!( + "[content_store::tags] update_summary_tags: file missing for summary {summary_id} at {} — skipping", + abs_path.display() + ); + return Ok(()); + } + + let mut tags = list_entity_ids_for_node(config, summary_id)? + .iter() + .filter_map(|entity_id| { + let (kind, surface) = entity_id.split_once(':')?; + Some(entity_tag(kind, surface)) + }) + .collect::>(); + tags.sort(); + tags.dedup(); + + let old_bytes = std::fs::read(&abs_path) + .map_err(|error| anyhow::anyhow!("read summary {:?}: {error}", abs_path))?; + let tags = augment_with_source_tag(&old_bytes, &tags); + let new_bytes = rewrite_summary_tags(&old_bytes, &tags) + .map_err(|error| anyhow::anyhow!("rewrite_summary_tags {:?}: {error}", abs_path))?; + + write_atomically(&abs_path, &new_bytes)?; + + let verify_bytes = std::fs::read(&abs_path) + .map_err(|error| anyhow::anyhow!("re-read after tag rewrite {:?}: {error}", abs_path))?; + let content = std::str::from_utf8(&verify_bytes) + .map_err(|error| anyhow::anyhow!("UTF-8 after tag rewrite {:?}: {error}", abs_path))?; + let body = split_front_matter(content) + .ok_or_else(|| anyhow::anyhow!("no front-matter after tag rewrite {:?}", abs_path))? + .1; + let actual_sha = super::atomic::sha256_hex(body.as_bytes()); + if actual_sha != expected_sha { + return Err(anyhow::anyhow!( + "[content_store::tags] update_summary_tags body mutated after rewrite summary_id={summary_id} expected_sha={expected_sha} actual_sha={actual_sha}" + )); + } + + log::debug!( + "[content_store::tags] updated summary tags summary_id={summary_id} n_tags={}", + tags.len() + ); + Ok(()) +} + +fn augment_with_source_tag(file_bytes: &[u8], tags: &[String]) -> Vec { + let Some(front_matter) = std::str::from_utf8(file_bytes) + .ok() + .and_then(split_front_matter) + .map(|(front_matter, _)| front_matter) + else { + return tags.to_vec(); + }; + let Some(tree_kind) = scan_fm_field(front_matter, "tree_kind") else { + return tags.to_vec(); + }; + if tree_kind != "source" { + return tags.to_vec(); + } + let Some(tree_scope) = scan_fm_field(front_matter, "tree_scope") else { + return tags.to_vec(); + }; + + let source = source_tag(&tree_scope); + std::iter::once(source.clone()) + .chain(tags.iter().filter(|tag| *tag != &source).cloned()) + .collect() +} + +fn write_atomically(abs_path: &Path, bytes: &[u8]) -> anyhow::Result<()> { + use std::io::Write; + + let parent = abs_path.parent().unwrap_or_else(|| Path::new(".")); + let tmp_path = parent.join(format!( + ".tmp_sum_tags_{}.md", + uuid::Uuid::new_v4().simple() + )); + let result = (|| { + let mut file = std::fs::File::create(&tmp_path) + .map_err(|error| anyhow::anyhow!("create tag tempfile {:?}: {error}", tmp_path))?; + file.write_all(bytes) + .map_err(|error| anyhow::anyhow!("write tag tempfile {:?}: {error}", tmp_path))?; + file.sync_all() + .map_err(|error| anyhow::anyhow!("fsync tag tempfile {:?}: {error}", tmp_path))?; + std::fs::rename(&tmp_path, abs_path).map_err(|error| { + anyhow::anyhow!( + "rename tag tempfile {:?} -> {:?}: {error}", + tmp_path, + abs_path + ) + }) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&tmp_path); + } + result +} diff --git a/core/src/store/entities.rs b/core/src/store/entities.rs new file mode 100644 index 0000000..a571a15 --- /dev/null +++ b/core/src/store/entities.rs @@ -0,0 +1,102 @@ +//! Host adapters for tinycortex's entity occurrence index. + +use std::sync::Arc; + +use anyhow::Result; +use tinycortex::memory::store::entity_index::{ + CanonicalEntity, EntityIndex, EntityKind, SelfIdentity, +}; + +use crate::sync::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind}; +use crate::tinycortex::memory_config_from; +use crate::Config; + +pub use tinycortex::memory::store::entity_index::EntityHit; + +#[derive(Debug)] +struct HostSelfIdentity; + +impl SelfIdentity for HostSelfIdentity { + fn is_self(&self, kind: EntityKind, surface: &str) -> bool { + let identity_kind = match kind { + EntityKind::Email => IdentityKind::Email, + EntityKind::Handle => IdentityKind::Handle, + _ => return false, + }; + is_self_identity_any_toolkit(identity_kind, surface) + } +} + +fn index(config: &Config) -> Result { + let memory = memory_config_from(config, config.workspace_dir().clone()); + let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + EntityIndex::from_shared_connection(connection, Arc::new(HostSelfIdentity)) +} + +pub fn index_entity( + config: &Config, + entity: &CanonicalEntity, + node_id: &str, + node_kind: &str, + timestamp_ms: i64, + tree_id: Option<&str>, +) -> Result<()> { + log::debug!("[memory:entities] index one node_kind={node_kind}"); + index(config)?.index_entity(entity, node_id, node_kind, timestamp_ms, tree_id) +} + +pub fn index_entities( + config: &Config, + entities: &[CanonicalEntity], + node_id: &str, + node_kind: &str, + timestamp_ms: i64, + tree_id: Option<&str>, +) -> Result { + log::debug!( + "[memory:entities] index batch count={} node_kind={node_kind}", + entities.len() + ); + index(config)?.index_entities(entities, node_id, node_kind, timestamp_ms, tree_id) +} + +pub fn clear_entity_index_for_node(config: &Config, node_id: &str) -> Result { + index(config)?.clear_entity_index_for_node(node_id) +} + +pub fn lookup_entity( + config: &Config, + entity_id: &str, + limit: Option, +) -> Result> { + index(config)?.lookup_entity(entity_id, limit) +} + +pub fn list_entity_ids_for_node(config: &Config, node_id: &str) -> Result> { + index(config)?.list_entity_ids_for_node(node_id) +} + +pub fn count_entity_index(config: &Config) -> Result { + index(config)?.count_entity_index() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn crate_entity_hit_is_the_host_facade_type() { + let hit = EntityHit { + entity_id: "person:alice".into(), + node_id: "chunk-1".into(), + node_kind: "leaf".into(), + entity_kind: EntityKind::Person, + surface: "Alice".into(), + score: 1.0, + timestamp_ms: 123, + tree_id: Some("tree-1".into()), + is_user: false, + }; + assert_eq!(hit.entity_id, "person:alice"); + } +} diff --git a/core/src/store/factories.rs b/core/src/store/factories.rs new file mode 100644 index 0000000..4a006f8 --- /dev/null +++ b/core/src/store/factories.rs @@ -0,0 +1,956 @@ +//! # Memory Store Factories +//! +//! Factory functions for creating and initializing various memory store +//! implementations. +//! +//! This module provides a centralized way to instantiate memory stores based on +//! configuration, ensuring that the correct embedding providers and storage +//! backends are used. Currently, it primarily focuses on creating +//! `UnifiedMemory` instances. + +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use parking_lot::Mutex; +use rusqlite::Connection; + +use crate::embedding_host::require_embedding_host; +use crate::store::namespace_store::UnifiedMemory; +use crate::traits::Memory; +use tinyagents::harness::embeddings::{DEFAULT_OLLAMA_DIMENSIONS, DEFAULT_OLLAMA_MODEL}; +use tinymemory_api::host::MemoryConfig; +use tinymemory_api::host::{format_embedding_signature, EmbeddingProvider}; +use tinymemory_api::host::{EmbeddingRouteConfig, StorageProviderConfig}; + +/// One-shot guard so the Ollama health-gate fallback only reports to Sentry +/// once per process lifetime. Memory is constructed many times per session +/// (once per agent in the harness), so an unguarded `report_error` would +/// re-create the per-embed flood the gate exists to suppress — just with a +/// different message. The first failed probe trips this flag; subsequent +/// probes log at debug level and skip the Sentry report. +static OLLAMA_HEALTH_REPORTED: AtomicBool = AtomicBool::new(false); + +/// Reports the Ollama-unreachable fallback to Sentry at most once per +/// process and publishes an [`EmbeddingModelUnhealthy`] domain event. +/// +/// The "once" applies to the Sentry report and the domain event only. The +/// client-facing `user_error` broadcast fires on **every** call, deliberately +/// — see the comment on the first statement. +/// +/// Returns `true` on the firing call, `false` afterwards — callers use the +/// return value only for logging context. +/// +/// [`EmbeddingModelUnhealthy`]: crate::events::MemoryEvent::EmbeddingModelUnhealthy +fn report_ollama_health_gate_once(base_url: &str, model: &str) -> bool { + // Deliberately ABOVE the Sentry latch (#5354). `publish_web_channel_event` + // is a `broadcast::send`: with no socket client attached yet it returns Err + // and the event is dropped, with no buffering and no redelivery. Memory is + // constructed early (once per agent in the harness), so the very first + // failed probe usually lands before the renderer's socket is up — under the + // latch that one dropped send would be the only attempt ever made, and the + // UserErrorCenter would stay empty for the whole outage. + // + // Re-broadcasting per failed probe is safe and intended: the panel store + // dedupes on the descriptor's `kind:scope:provider` identity and bumps + // `count`, so repeats collapse into one entry rather than stacking. Only + // the Sentry report below stays once-per-process, which is what the latch + // was introduced for. + surface_local_model_unavailable_to_clients(); + + if OLLAMA_HEALTH_REPORTED + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + log::debug!( + "[memory::factory] ollama health-gate fallback already reported this process; suppressing duplicate at {base_url} model={model}" + ); + return false; + } + // Tags are indexed and grouped on; keep them low-cardinality and free of + // credentials. Full URL stays in the message body for diagnostics. + let host_tag = redact_ollama_host(base_url); + let sentry_message = format!( + "ollama embeddings opted-in but daemon unreachable at {base_url}; falling back to cloud embeddings for this session" + ); + // Route through `report_error_or_expected` so the GX arm of + // `is_ollama_user_config_rejection` in `expected_error_kind` demotes + // the message to an info breadcrumb (user-state: ollama daemon not + // running). Direct `report_error_message` here bypassed the classifier + // and produced TAURI-RUST-B (~409 events). The `&str` input avoids + // the `format!("{:#}")` round-trip that `report_error` would do on an + // anyhow chain — the wire shape stays bit-identical. + crate::observability::report_error_or_expected( + sentry_message.as_str(), + "memory", + "ollama_health_gate", + &[("ollama_host", host_tag), ("fallback", "cloud")], + ); + + // Publish a user-visible domain event so the UI can surface a notification + // with an actionable fix hint. The event bus is best-effort (no runtime + // present in unit-test contexts without `init_global`), so we fire-and- + // forget and ignore any lagged-receiver errors. + let user_message = format!( + "Local embedding model unreachable — falling back to cloud embeddings. \ + Run `ollama pull {model}` to fix." + ); + log::debug!( + "[memory::factory] publishing EmbeddingModelUnhealthy event: provider=ollama model={model} fallback=cloud" + ); + let event = crate::events::MemoryEvent::EmbeddingModelUnhealthy( + tinymemory_api::host::EmbeddingHealthReason { + provider: "ollama".to_string(), + model: model.to_string(), + fallback_provider: "cloud".to_string(), + message: user_message, + }, + ); + // publish_global is infallible (drops the event when no receivers are + // registered, which is fine for the health-gate use case). + crate::events::publish(event); + + true +} + +/// Surface the Ollama-unreachable fallback in every connected client's +/// UserErrorCenter (#5354). +/// +/// `DomainEvent::EmbeddingModelUnhealthy` is published above, but nothing +/// bridges the domain bus to the product UI — `/events/domain` is consumed only +/// by the developer Event Log panel — so that event alone reaches no user. The +/// payload and publisher live in `memory::tree::health::user_error` so this +/// producer and the embed-failure classifier emit one identical, tested shape. +fn surface_local_model_unavailable_to_clients() { + crate::tree::health::publish_local_model_unavailable_user_error("health_gate"); +} + +/// Resets the once-per-process Sentry latch. Test-only — any test that +/// exercises a fallback path should call this first so it can't be flaked by +/// suite ordering (an earlier test that already tripped the latch). +#[cfg(test)] +fn reset_health_gate_for_test() { + OLLAMA_HEALTH_REPORTED.store(false, Ordering::Release); +} + +/// Effective Ollama base URL. +/// +/// Delegates to the host's [`EmbeddingHost::ollama_base_url`] so the probe +/// always agrees with the rest of the Ollama machinery on the daemon address. +/// If a future change adds another env-var override or shifts precedence, the +/// memory health-gate picks it up automatically. +fn ollama_base_url_for_probe() -> String { + require_embedding_host() + .map(|host| host.ollama_base_url()) + .unwrap_or_default() +} + +/// Canonical `(provider, model, dimensions)` tuple used everywhere the +/// health-gate falls back from Ollama → cloud. Centralised so both the async +/// and sync gate sites agree if the cloud defaults ever change. +fn cloud_embedding_fallback() -> (String, String, usize) { + // The cloud defaults are the host's to state — it owns the managed + // endpoint. Falling back to the ollama defaults when unwired keeps this + // total; a caller with no embedding host has bigger problems than the + // fallback tuple being wrong. + match require_embedding_host() { + Ok(host) => ( + "cloud".to_string(), + host.default_cloud_embedding_model().to_string(), + host.default_cloud_embedding_dimensions(), + ), + Err(_) => ( + "cloud".to_string(), + DEFAULT_OLLAMA_MODEL.to_string(), + DEFAULT_OLLAMA_DIMENSIONS, + ), + } +} + +/// Extracts a low-cardinality `host[:port]` tag from `base_url` for Sentry. +/// +/// Sentry tags are indexed and should not carry secrets or per-instance noise: +/// `http://user:token@host:11434/api/tags?key=v` collapses to `host:11434`. +/// Falls back to `"unknown"` if parsing yields an empty string so we never +/// emit an empty tag value. +fn redact_ollama_host(base_url: &str) -> &str { + let after_scheme = base_url + .split_once("://") + .map(|(_, rest)| rest) + .unwrap_or(base_url); + let after_userinfo = after_scheme + .rsplit_once('@') + .map_or(after_scheme, |(_, h)| h); + let host = after_userinfo + .split(['/', '?', '#']) + .next() + .unwrap_or("") + .trim(); + if host.is_empty() { + "unknown" + } else { + host + } +} + +/// Probe whether an Ollama daemon is reachable at `base_url`. +/// +/// Issues a short-timeout `GET /api/tags` (the standard Ollama +/// "list models" endpoint) and returns `true` only when it responds with a +/// 2xx status. Transport failures, timeouts, and non-2xx responses all +/// return `false`. +/// +/// Kept deliberately small and side-effect-free so it can be called from +/// the memory factory's startup path without pulling in the full +/// `local_ai::service::ollama_admin` machinery. +/// +/// Scoped `pub(crate)` to match `local_ai::ollama_base_url`; the only callers +/// are the factory itself and its sibling tests. Stable external API for the +/// health-gate is [`effective_embedding_settings_probed`]. +pub(crate) async fn probe_ollama_reachable(base_url: &str) -> bool { + let url = format!("{}/api/tags", base_url.trim_end_matches('/')); + let client = match reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(2)) + .build() + { + Ok(c) => c, + Err(e) => { + log::debug!( + "[memory::factory] probe_ollama_reachable: failed to build http client: {e}" + ); + return false; + } + }; + match client.get(&url).send().await { + Ok(resp) => resp.status().is_success(), + Err(e) => { + log::debug!("[memory::factory] probe_ollama_reachable: {url} unreachable: {e}"); + false + } + } +} + +/// Returns the effective `(provider, model, dimensions)` triple for the +/// embedding backend. +/// +/// The user-facing default is `"cloud"` (OpenHuman backend, Voyage-backed) so +/// fresh installs work without a local Ollama daemon. When the user has +/// explicitly opted into local AI for embeddings — +/// `LocalAiConfig::use_local_for_embeddings` — we route through the local +/// Ollama embedder regardless of what `memory.embedding_provider` says, since +/// that toggle is a stronger statement of intent than the per-section default. +/// +/// Note: this is the *intended* setting. It does not check whether the Ollama +/// daemon is actually running. For the live, health-checked variant that +/// falls back to cloud when Ollama is configured but unreachable, see +/// [`effective_embedding_settings_probed`]. +pub fn effective_embedding_settings( + memory: &MemoryConfig, + local_embedding_model: Option<&str>, +) -> (String, String, usize) { + if let Some(raw) = local_embedding_model { + // Trim once and reuse — the emptiness check and the final model + // string must agree, otherwise a value like " bge-m3 " would pass + // through to Ollama with surrounding whitespace and 404. + let trimmed = raw.trim(); + let model = if trimmed.is_empty() { + DEFAULT_OLLAMA_MODEL.to_string() + } else { + trimmed.to_string() + }; + return ("ollama".to_string(), model, DEFAULT_OLLAMA_DIMENSIONS); + } + ( + memory.embedding_provider.clone(), + memory.embedding_model.clone(), + memory.embedding_dimensions, + ) +} + +/// The **active embedding signature** — the canonical key every per-model +/// sidecar read/write is scoped by (#1574). +/// +/// Derived from [`effective_embedding_settings`] (the *intended*, non-probed +/// selection) — deliberately **not** [`effective_embedding_settings_probed`]. +/// A transient Ollama-down fallback to cloud must never silently redefine the +/// signature: that would re-key every read at a different space and trigger a +/// spurious full re-embed on the next cold-Ollama launch (spec §3 oscillation +/// guard). The string is produced by [`format_embedding_signature`], the same +/// formatter [`EmbeddingProvider::signature`] uses, so a config-derived +/// signature is byte-identical to a live provider's. +pub fn active_embedding_signature( + memory: &MemoryConfig, + local_embedding_model: Option<&str>, +) -> String { + let (provider, model, dims) = effective_embedding_settings(memory, local_embedding_model); + format_embedding_signature(&provider, &model, dims) +} + +/// Async, health-checked variant of [`effective_embedding_settings`]. +/// +/// If the intended provider is `"ollama"` but the daemon doesn't respond at +/// `/api/tags` within a short timeout, this falls back to the cloud +/// embedder and logs a single warning. This avoids the failure mode behind +/// OPENHUMAN-TAURI-B7: a user who's flipped `local_ai.usage.embeddings = true` +/// in Settings but doesn't actually have Ollama running ends up firing one +/// `ollama_embed` Sentry event per embed call (226+ events in a day with zero +/// impacted users — pure noise that drowns out real signals). With this +/// gate, embed calls never even reach `OllamaEmbedding` in that state; the +/// cloud embedder serves the session and the user gets a working app. +/// +/// The probe deliberately uses a 2s timeout — long enough to tolerate a +/// briefly-busy daemon, short enough to not block startup if Ollama is +/// genuinely down. +pub async fn effective_embedding_settings_probed( + memory: &MemoryConfig, + local_embedding_model: Option<&str>, +) -> (String, String, usize) { + let intended = effective_embedding_settings(memory, local_embedding_model); + if intended.0 != "ollama" { + return intended; + } + let base_url = ollama_base_url_for_probe(); + if probe_ollama_reachable(&base_url).await { + log::debug!( + "[memory::factory] ollama healthy at {base_url}; using local embeddings (model={}, dims={})", + intended.1, + intended.2, + ); + return intended; + } + // Ollama is configured but not reachable. Report once per process at this + // gate so a genuine misconfiguration still surfaces in Sentry — but no + // more than once, so re-instantiating memory across agents/sessions + // doesn't recreate the per-embed flood we're fixing. Then fall back to + // cloud so the user has a working app. + log::warn!( + "[memory::factory] ollama unreachable at {base_url} (model={}); falling back to cloud embedder for this session", + intended.1 + ); + report_ollama_health_gate_once(&base_url, &intended.1); + cloud_embedding_fallback() +} + +/// Returns the effective name of the memory backend being used. +/// +/// Currently, this always returns "namespace" as the unified memory system +/// is the standard. +pub fn effective_memory_backend_name( + _memory_backend: &str, + _storage_provider: Option<&StorageProviderConfig>, +) -> String { + "namespace".to_string() +} + +/// Create a standard memory instance based on the provided configuration. +pub fn create_memory( + config: &MemoryConfig, + workspace_dir: &Path, +) -> anyhow::Result> { + // No `Config` in scope here (tests + migration), so no credential store to + // read — pass an empty key. Callers that select a keyed BYO provider must + // use `create_memory_with_local_ai`, which resolves the stored credential. + create_memory_full(config, &[], None, None, "", workspace_dir) +} + +/// Create a memory instance honouring the unified per-workload embedding +/// provider. +/// +/// `local_embedding_model` is the parsed Ollama model id when +/// `Config::workload_local_model("embeddings")` returned `Some`, otherwise +/// `None`. Used by top-level entry points (agent harness, channels runtime) +/// that have the full `Config` in scope. The local-AI opt-in flips the +/// embedder to Ollama when `Some`. +/// +/// `embedding_api_key` is the user's stored credential for the selected BYO +/// embedding provider, resolved by the caller via +/// the host's `EmbeddingHost::resolve_api_key` (empty string when none is +/// configured). It is threaded into the keyed providers (cohere/openai/voyage/ +/// custom) so they authenticate instead of sending an empty bearer; cloud / +/// managed / ollama / none ignore it. +pub fn create_memory_with_local_ai( + memory: &MemoryConfig, + local_embedding_model: Option<&str>, + embedding_api_key: &str, + embedding_routes: &[EmbeddingRouteConfig], + storage_provider: Option<&StorageProviderConfig>, + workspace_dir: &Path, +) -> anyhow::Result> { + create_memory_full( + memory, + embedding_routes, + storage_provider, + local_embedding_model, + embedding_api_key, + workspace_dir, + ) +} + +/// Memory resources needed by an agent session. +/// +/// The storage abstraction remains backend-neutral while SQLite-specific +/// consumers receive the concrete shared connection explicitly. +pub struct SessionMemory { + pub memory: Box, + pub sqlite_connection: Arc>, +} + +pub fn create_session_memory_with_local_ai( + memory: &MemoryConfig, + local_embedding_model: Option<&str>, + embedding_api_key: &str, + embedding_routes: &[EmbeddingRouteConfig], + storage_provider: Option<&StorageProviderConfig>, + workspace_dir: &Path, + // Memory subdirectory under `workspace_dir` — `"memory"` for the shared + // tree, `"memory-"` for a profile that opted into dedicated memory + // (derived by the session builder from `effective_memory_suffix`). Routes + // the session's captures + recall (the `UnifiedMemory` SQLite store) into the + // profile's own subtree so `dedicatedMemory` isolation actually takes effect. + memory_subdir: &str, +) -> anyhow::Result { + let memory = create_unified_memory_full( + memory, + embedding_routes, + storage_provider, + local_embedding_model, + embedding_api_key, + workspace_dir, + memory_subdir, + )?; + let sqlite_connection = Arc::clone(&memory.conn); + Ok(SessionMemory { + memory: Box::new(memory), + sqlite_connection, + }) +} + +/// Synchronous health-check shim around [`probe_ollama_reachable`]. +/// +/// Production call sites (`create_memory_with_local_ai` and friends) live in +/// sync code that doesn't want to plumb `async` through the whole agent +/// harness builder chain. They always run inside a multi-thread tokio +/// runtime (the core's main runtime), so we can park the worker via +/// [`tokio::task::block_in_place`] and drive the probe future to completion. +/// +/// When no tokio runtime is available OR the runtime is single-threaded +/// (current-thread flavour), we skip the probe entirely and assume the +/// daemon is reachable. `block_in_place` panics on a current-thread runtime +/// — see — +/// so probing in that context would crash the caller. Skipping preserves +/// the pre-health-gate behaviour (which is what tests rely on) and is safe +/// because the existing `OllamaEmbedding` error path still surfaces a +/// transport failure if the daemon truly is down. +fn probe_ollama_reachable_blocking(base_url: &str) -> bool { + let Ok(handle) = tokio::runtime::Handle::try_current() else { + log::debug!( + "[memory::factory] probe_ollama_reachable_blocking: no tokio runtime in context; skipping probe" + ); + return true; + }; + if !matches!( + handle.runtime_flavor(), + tokio::runtime::RuntimeFlavor::MultiThread + ) { + log::debug!( + "[memory::factory] probe_ollama_reachable_blocking: runtime is current-thread (block_in_place would panic); skipping probe" + ); + return true; + } + tokio::task::block_in_place(move || handle.block_on(probe_ollama_reachable(base_url))) +} + +/// The most comprehensive factory function for creating a memory instance. +/// +/// This function resolves the embedding provider — applying the Ollama +/// health-gate when the user has opted into local embeddings — then +/// initializes the provider and creates a `UnifiedMemory` instance. +fn create_memory_full( + config: &MemoryConfig, + _embedding_routes: &[EmbeddingRouteConfig], + _storage_provider: Option<&StorageProviderConfig>, + local_embedding_model: Option<&str>, + embedding_api_key: &str, + workspace_dir: &Path, +) -> anyhow::Result> { + Ok(Box::new(create_unified_memory_full( + config, + _embedding_routes, + _storage_provider, + local_embedding_model, + embedding_api_key, + workspace_dir, + // Non-session callers (migration, standalone memory) always use the + // shared default subtree. + "memory", + )?)) +} + +fn create_unified_memory_full( + config: &MemoryConfig, + _embedding_routes: &[EmbeddingRouteConfig], + _storage_provider: Option<&StorageProviderConfig>, + local_embedding_model: Option<&str>, + embedding_api_key: &str, + workspace_dir: &Path, + memory_subdir: &str, +) -> anyhow::Result { + // 1. Resolve the intended provider from config. + let intended = effective_embedding_settings(config, local_embedding_model); + let local_ai_opt_in = local_embedding_model + .map(|s| !s.trim().is_empty()) + .unwrap_or(false); + + // 2. Health-gate: if the user has opted into Ollama embeddings but the + // daemon isn't reachable, fall back to cloud for this session. + // Prevents OPENHUMAN-TAURI-B7's 226-event Sentry flood: instead of + // one Sentry event per embed attempt, we report once at the gate + // (low cardinality, high signal) and serve the session from cloud. + let gate_triggered; + let (provider, model, dims) = if intended.0 == "ollama" { + let base_url = ollama_base_url_for_probe(); + if probe_ollama_reachable_blocking(&base_url) { + log::debug!( + "[memory::factory] ollama healthy at {base_url}; using local embeddings (model={}, dims={})", + intended.1, + intended.2, + ); + gate_triggered = false; + intended + } else { + log::warn!( + "[memory::factory] ollama unreachable at {base_url} (model={}); falling back to cloud embedder for this session", + intended.1 + ); + report_ollama_health_gate_once(&base_url, &intended.1); + gate_triggered = true; + cloud_embedding_fallback() + } + } else { + gate_triggered = false; + intended + }; + + log::debug!( + "[memory::factory] effective embedding settings: provider={provider} model={model} dims={dims} \ + (local_ai_opt_in={local_ai_opt_in} gate_triggered={gate_triggered})", + ); + + // 3. Create the embedding provider, threading the user's stored BYO + // credential. The keyless `create_embedding_provider` left the API key + // empty for *every* provider, so a user who selected Cohere — even with + // a valid key configured — sent an empty `Bearer ` and got a guaranteed + // 401 "no api key supplied" on every embed (TAURI-RUST-52S), and the + // same gap silently broke BYO OpenAI / Voyage memory embeddings. + // cloud/managed/ollama/none ignore the key; the keyed providers now + // actually receive it. `embedding_api_key` is "" when no credential is + // stored, which the per-provider guards reject fast. A `custom:` + // provider keeps its inline endpoint (the factory's `custom:` arm strips + // the prefix), so `custom_endpoint` stays `None` here. The key is never + // logged — the warning carries only provider/model/dims. + let embedder: Arc = Arc::from( + require_embedding_host() + .map_err(anyhow::Error::msg)? + .create_embedding_provider_with_credentials( + &provider, + &model, + dims, + embedding_api_key, + None, + ) + .inspect_err(|err| { + log::warn!( + "[memory::factory] create_embedding_provider_with_credentials failed provider={provider} model={model} dims={dims}: {err}", + ); + }) + .map_err(anyhow::Error::msg)?, + ); + + // 4. Instantiate UnifiedMemory which handles SQLite and vector storage, + // rooted at the caller-selected subtree (`memory` shared, `memory-` + // for a dedicated-memory profile). + UnifiedMemory::new_with_memory_dir( + workspace_dir, + memory_subdir, + embedder, + config.sqlite_open_timeout_secs, + ) +} + +/// Create a memory instance specifically for migration purposes. +/// +/// The unified namespace memory core has a single workspace-scoped +/// store, so migration writes into the same `UnifiedMemory` instance the +/// rest of the app reads from — there is no separate "migration +/// backend". This helper delegates to [`create_memory`] so the +/// migration importer (`migrate_openclaw_memory`) gets a real, writable +/// memory handle and the Apply path can actually run end-to-end. +/// +/// Prior to #1440 this function unconditionally bailed with "memory +/// migration is disabled for the unified namespace memory core", which +/// left the OpenClaw importer broken even though the rest of the +/// pipeline (source discovery, dry-run report, backup) worked. +pub fn create_memory_for_migration( + config: &MemoryConfig, + workspace_dir: &Path, +) -> anyhow::Result> { + create_memory(config, workspace_dir) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::{routing::get, Json, Router}; + use std::ffi::OsString; + use std::net::SocketAddr; + + /// RAII helper that swaps `OPENHUMAN_OLLAMA_BASE_URL` to `value` for the + /// duration of the scope while holding the local-AI domain test mutex. + /// The previous value (if any) is restored on drop. + struct EnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + prev: Option, + } + + impl EnvGuard { + fn set(value: &str) -> Self { + let lock = crate::embedding_host::embedding_test_guard(); + let prev = std::env::var_os("OPENHUMAN_OLLAMA_BASE_URL"); + // SAFETY: env mutation is wrapped because Rust 2024 marks it + // unsafe; the call is gated by the local-AI domain mutex so no + // other local-AI test is observing the env concurrently. + unsafe { + std::env::set_var("OPENHUMAN_OLLAMA_BASE_URL", value); + } + Self { _lock: lock, prev } + } + } + + impl Drop for EnvGuard { + fn drop(&mut self) { + // SAFETY: same justification as `set` — still under the same lock. + unsafe { + match self.prev.take() { + Some(v) => std::env::set_var("OPENHUMAN_OLLAMA_BASE_URL", v), + None => std::env::remove_var("OPENHUMAN_OLLAMA_BASE_URL"), + } + } + } + } + + // ── effective_embedding_settings (unprobed selection priority) ──────── + + #[test] + fn embedding_settings_defaults_to_cloud_when_no_local_ai() { + let mem = MemoryConfig::default(); + let (provider, model, dims) = effective_embedding_settings(&mem, None); + assert_eq!( + provider, "cloud", + "no local-AI config must default to cloud" + ); + assert!(!model.is_empty(), "cloud model must be non-empty"); + assert!(dims > 0, "cloud dimensions must be positive"); + } + + #[test] + fn embedding_settings_uses_memory_config_when_local_disabled() { + let mem = MemoryConfig { + embedding_provider: "openai".to_string(), + embedding_model: "text-embedding-3-small".to_string(), + embedding_dimensions: 1536, + ..Default::default() + }; + + // Local embedding model = None means workload routes to cloud. + let (provider, model, dims) = effective_embedding_settings(&mem, None); + assert_eq!( + provider, "openai", + "when local embeddings disabled, memory config must be used" + ); + assert_eq!(model, "text-embedding-3-small"); + assert_eq!(dims, 1536); + } + + #[test] + fn embedding_settings_local_overrides_memory_config() { + // memory.embedding_provider says "cloud" — but a Some(local_model) + // is the stronger signal and must override it. + let mem = MemoryConfig::default(); // cloud by default + let (provider, model, dims) = + effective_embedding_settings(&mem, Some("nomic-embed-text:latest")); + assert_eq!( + provider, "ollama", + "Some(local_model) must override memory.embedding_provider" + ); + assert_eq!(model, "nomic-embed-text:latest"); + assert_eq!( + dims, + tinyagents::harness::embeddings::DEFAULT_OLLAMA_DIMENSIONS, + "dimensions must default to Ollama default" + ); + } + + #[test] + fn embedding_settings_local_with_empty_model_uses_default() { + // When the user has opted in but the model field is empty/whitespace, + // the default Ollama model must be used rather than passing "" to Ollama. + let mem = MemoryConfig::default(); + let (provider, model, dims) = effective_embedding_settings(&mem, Some(" ")); + assert_eq!(provider, "ollama"); + assert_eq!( + model, + tinyagents::harness::embeddings::DEFAULT_OLLAMA_MODEL, + "empty model ID must fall back to default Ollama model" + ); + assert_eq!( + dims, + tinyagents::harness::embeddings::DEFAULT_OLLAMA_DIMENSIONS + ); + } + + #[test] + fn active_signature_ignores_probe_fallback() { + // active_embedding_signature keys off the *intended* selection + // (effective_embedding_settings), NOT the health-checked variant — so + // a transient Ollama-down fallback can't flip it to cloud. The dim is + // base/config-dependent (not what this test pins); the provider+model + // staying the intended ollama/bge-m3 is the probe-stability property. + let mem = MemoryConfig::default(); + let sig = active_embedding_signature(&mem, Some("bge-m3")); + assert!( + sig.starts_with("provider=ollama;model=bge-m3;dims="), + "intended local selection must survive (no cloud fallback); got {sig}" + ); + // And it must equal the non-probed settings, formatted identically. + let (p, m, d) = effective_embedding_settings(&mem, Some("bge-m3")); + assert_eq!(sig, format_embedding_signature(&p, &m, d)); + } + + #[test] + fn effective_memory_backend_name_always_returns_namespace() { + assert_eq!(effective_memory_backend_name("sqlite", None), "namespace"); + assert_eq!(effective_memory_backend_name("anything", None), "namespace"); + assert_eq!(effective_memory_backend_name("", None), "namespace"); + } + + #[test] + fn create_memory_for_migration_returns_writable_memory_on_unified_core() { + // Regression for #1440: prior to that PR this factory unconditionally + // bailed with "memory migration is disabled for the unified namespace + // memory core", which broke the OpenClaw importer's Apply path even + // though the dry-run / preview path worked. Now it delegates to + // `create_memory` so the migration importer gets a real workspace- + // scoped memory handle. Box doesn't impl Debug, so we + // match instead of unwrap. + let tmp = tempfile::tempdir().unwrap(); + let cfg = MemoryConfig::default(); + match create_memory_for_migration(&cfg, tmp.path()) { + Ok(_) => {} + Err(e) => panic!("expected Ok for unified namespace core, got: {e}"), + } + } + + /// Spin up a mock Ollama-shaped server that responds 200 OK on `/api/tags`. + async fn start_mock_ollama() -> String { + let app = Router::new().route( + "/api/tags", + get(|| async { Json(serde_json::json!({ "models": [] })) }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://127.0.0.1:{}", addr.port()) + } + + /// The parsed local-embedding model string that + /// `Config::workload_local_model("embeddings")` would have produced when + /// the legacy `local_ai.usage.embeddings = true` flag was set. Used so + /// the existing test scenarios continue to drive the local code path. + fn local_embedding_for_test() -> &'static str { + tinyagents::harness::embeddings::DEFAULT_OLLAMA_MODEL + } + + #[tokio::test] + async fn probe_returns_true_when_ollama_responds_200() { + let url = start_mock_ollama().await; + assert!(probe_ollama_reachable(&url).await); + } + + #[tokio::test] + async fn probe_returns_false_for_unreachable_host() { + // Port 1 on loopback is reliably refused. + assert!(!probe_ollama_reachable("http://127.0.0.1:1").await); + } + + #[tokio::test] + async fn probe_returns_false_on_non_2xx() { + // Mock that responds 500. + let app = Router::new().route( + "/api/tags", + get(|| async { (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "boom") }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let url = format!("http://127.0.0.1:{}", addr.port()); + assert!(!probe_ollama_reachable(&url).await); + } + + #[tokio::test] + async fn probed_settings_keep_cloud_when_provider_is_cloud() { + // No local-AI opt-in → intended provider is cloud, probe is skipped. + let mem = MemoryConfig::default(); + let (provider, _, _) = effective_embedding_settings_probed(&mem, None).await; + assert_eq!(provider, "cloud"); + } + + /// Sets `OPENHUMAN_OLLAMA_BASE_URL` to a deliberately unreachable address + /// under the local-AI domain mutex, then verifies that the probed settings + /// fall back to cloud when the user has opted into local embeddings. + #[tokio::test] + async fn probed_settings_fall_back_to_cloud_when_ollama_unreachable() { + let _env = EnvGuard::set("http://127.0.0.1:1"); + // Independent of suite ordering: an earlier fallback test must not + // leave the latch tripped and silently turn this assertion green. + reset_health_gate_for_test(); + + // The cloud defaults are the host's to state, so the fallback tuple is + // only meaningful with an embedding host installed. + crate::embedding_host::TestEmbeddingHost::install(); + let mem = MemoryConfig::default(); + + let (provider, model, dims) = + effective_embedding_settings_probed(&mem, Some(local_embedding_for_test())).await; + + assert_eq!( + provider, "cloud", + "opted-in but unreachable Ollama must fall back to cloud" + ); + assert_eq!(model, crate::embedding_host::TestEmbeddingHost::CLOUD_MODEL); + assert_eq!( + dims, + crate::embedding_host::TestEmbeddingHost::CLOUD_DIMENSIONS + ); + } + + #[tokio::test] + async fn probed_settings_keep_ollama_when_daemon_responds() { + let url = start_mock_ollama().await; + let _env = EnvGuard::set(&url); + + let mem = MemoryConfig::default(); + + let (provider, _model, dims) = + effective_embedding_settings_probed(&mem, Some(local_embedding_for_test())).await; + + assert_eq!(provider, "ollama", "healthy Ollama must be honoured"); + assert_eq!(dims, DEFAULT_OLLAMA_DIMENSIONS); + } + + #[test] + fn redact_ollama_host_strips_scheme_userinfo_path_and_query() { + // Strips scheme. + assert_eq!( + redact_ollama_host("http://localhost:11434"), + "localhost:11434" + ); + // Strips userinfo (would be the credential leak vector). + assert_eq!( + redact_ollama_host("http://user:secret@10.0.0.1:11434"), + "10.0.0.1:11434" + ); + // Strips path / query / fragment. + assert_eq!( + redact_ollama_host("https://host:11434/api/tags?key=v#frag"), + "host:11434" + ); + // Scheme-less inputs survive (matches `local_ai::ollama_base_url`'s + // contract: it may or may not prepend `http://`). + assert_eq!(redact_ollama_host("host:1234"), "host:1234"); + // Empty / malformed inputs fall back to a safe constant. + assert_eq!(redact_ollama_host(""), "unknown"); + } + + /// #5354 — the client broadcast must NOT ride the once-per-process Sentry + /// latch. + /// + /// `publish_web_channel_event` is a `broadcast::send` with no buffering: if + /// no socket client is attached the event is dropped outright. Memory is + /// built early (once per agent), so the first failed probe typically fires + /// before the renderer connects. Latched, that single dropped send would be + /// the only attempt ever made and the UserErrorCenter would stay empty for + /// the entire outage. Subscribing here proves a second gate call still + /// broadcasts even though its Sentry half is suppressed. + #[test] + fn user_error_broadcast_is_not_suppressed_by_the_sentry_latch() { + let _lock = crate::embedding_host::embedding_test_guard(); + reset_health_gate_for_test(); + + let sink = crate::events::RecordingSink::install(); + + assert!( + report_ollama_health_gate_once("http://127.0.0.1:1", "bge-m3"), + "first call must fire the Sentry report" + ); + assert!( + !report_ollama_health_gate_once("http://127.0.0.1:1", "bge-m3"), + "second call must suppress the Sentry report" + ); + + // Both calls must still have been announced — the Sentry latch + // suppresses only the *report*, never the user-facing event. + // Count only the user-facing announcement. The health-gate also emits + // `EmbeddingModelUnhealthy` on the first call; that is a different + // event with its own latch and is not what this test pins. + let announcements = sink + .drain() + .into_iter() + .filter(|event| { + matches!( + event, + crate::events::MemoryEvent::LocalModelUnavailable { .. } + ) + }) + .count(); + assert_eq!( + announcements, 2, + "the Sentry latch must suppress the report, never the announcement" + ); + } + + /// First call to `report_ollama_health_gate_once` fires the report; + /// subsequent calls in the same process must be suppressed. We can't + /// observe the Sentry side effect directly here, but the boolean return + /// value is the gate's contract — covers the once-per-process guarantee. + /// Event publication is fire-and-forget via the global event bus and is + /// verified manually/log-side rather than by this unit test. + /// + /// Acquires the local-AI domain mutex to serialize with `probed_settings_*` + /// tests that also touch the latch; without that, parallel test execution + /// can reset the flag between this test's two + /// `report_ollama_health_gate_once` calls and turn the second one into a + /// fresh "first", flaking the suppression assertion. + #[test] + fn ollama_health_gate_reports_at_most_once_per_process() { + let _lock = crate::embedding_host::embedding_test_guard(); + reset_health_gate_for_test(); + + assert!( + report_ollama_health_gate_once("http://127.0.0.1:1", "bge-m3"), + "first call must fire the report" + ); + assert!( + !report_ollama_health_gate_once("http://127.0.0.1:1", "bge-m3"), + "second call must be suppressed" + ); + assert!( + !report_ollama_health_gate_once("http://example.invalid:11434", "nomic-embed-text"), + "different URL also suppressed — gate is process-scoped, not per-URL" + ); + } +} diff --git a/core/src/store/kinds.rs b/core/src/store/kinds.rs new file mode 100644 index 0000000..f50e81f --- /dev/null +++ b/core/src/store/kinds.rs @@ -0,0 +1,112 @@ +//! Catalog of every kind of data the memory_store persists. +//! +//! Every stored object falls into exactly one of these kinds. The enum is the +//! authoritative answer to "what can memory_store store?" and is used by: +//! - The retrieval facade, to fan out a query to the right backends. +//! - The vector/obsidian compatibility traits, to dispatch by kind. +//! - Agent tools, to surface a kind filter to LLM callers. +//! +//! Adding a new storage kind = adding a variant here, an impl of the +//! `VectorEmbeddable` / `ObsidianRepresentable` traits +//! ([`crate::store::traits`]), and a delegation in +//! [`crate::store::retrieval`]. + +use serde::{Deserialize, Serialize}; + +/// Every persisted data shape in memory_store, named once. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MemoryKind { + /// On-disk raw markdown file (the content store). One file per + /// canonicalized source chunk OR per summary node. Source of truth + /// for all content bodies. + Raw, + /// SQLite chunk row — metadata + tags + raw-md pointer + lifecycle. + /// Bodies live in `Raw`; the chunk row is the index entry. + Chunk, + /// Canonical entity row in `mem_tree_entity_index` — every entity + /// occurrence per tree node. The substrate `memory_graph` derives + /// co-occurrence edges from. + Entity, + /// Sealed summary tree node — Source, Global, or Topic flavor. + Tree, + /// Dense vector embedding row in the local vector DB. + Vector, + /// Key-value record (global or namespace-scoped). Lives in the + /// `kv_global` / `kv_namespace` tables. + Kv, + /// Address-book contact (`people::Person`) routed through the contacts + /// facade. + Contact, +} + +impl MemoryKind { + /// Snake-case discriminant used in RPC payloads, logs, and tool args. + pub fn as_str(self) -> &'static str { + match self { + MemoryKind::Raw => "raw", + MemoryKind::Chunk => "chunk", + MemoryKind::Entity => "entity", + MemoryKind::Tree => "tree", + MemoryKind::Vector => "vector", + MemoryKind::Kv => "kv", + MemoryKind::Contact => "contact", + } + } + + /// Every variant, in stable declaration order. Useful for fan-out + /// retrieval and for surfacing the kind catalog to LLM tools. + pub const ALL: &'static [MemoryKind] = &[ + MemoryKind::Raw, + MemoryKind::Chunk, + MemoryKind::Entity, + MemoryKind::Tree, + MemoryKind::Vector, + MemoryKind::Kv, + MemoryKind::Contact, + ]; +} + +/// Per-kind canonical Rust type aliases — one stop to find "what struct +/// represents a Tree row?", "what struct represents a Contact?", etc. +/// Aliases (not re-exports) so the documentation lives here and the +/// source-of-truth types stay in their owning modules. +pub mod types { + pub use crate::people::types::Person as Contact; + pub use crate::store::chunks::types::Chunk; + pub use crate::store::entities::EntityHit as Entity; + pub use crate::store::trees::{SummaryNode as TreeNode, Tree, TreeKind}; + pub use crate::store::types::MemoryKvRecord as Kv; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memory_kind_as_str_matches_all_catalog_entries() { + let kinds = [ + MemoryKind::Raw, + MemoryKind::Chunk, + MemoryKind::Entity, + MemoryKind::Tree, + MemoryKind::Vector, + MemoryKind::Kv, + MemoryKind::Contact, + ]; + let labels: Vec<&str> = kinds.iter().map(|k| k.as_str()).collect(); + let all: Vec<&str> = MemoryKind::ALL.iter().map(|k| k.as_str()).collect(); + assert_eq!(labels, all); + } + + #[test] + fn memory_kind_serde_uses_snake_case() { + let raw = serde_json::to_string(&MemoryKind::Raw).unwrap(); + let tree = serde_json::to_string(&MemoryKind::Tree).unwrap(); + assert_eq!(raw, "\"raw\""); + assert_eq!(tree, "\"tree\""); + + let decoded: MemoryKind = serde_json::from_str("\"contact\"").unwrap(); + assert_eq!(decoded, MemoryKind::Contact); + } +} diff --git a/core/src/store/kv.rs b/core/src/store/kv.rs new file mode 100644 index 0000000..21f9827 --- /dev/null +++ b/core/src/store/kv.rs @@ -0,0 +1,144 @@ +//! Compatibility methods for tinycortex's shared-connection KV store. +//! +//! Every method canonicalizes its namespace/key through +//! [`canonical_identifier`] before delegating. The crate's `set_*` already +//! canonicalizes PII-bearing keys on the way in (#5164) but its `get_*` / +//! `delete_*` / `list_*` address the raw key, so without this shim a write +//! whose key was rewritten reads back as absent — and the caller writes it +//! again. Canonicalizing here is a no-op for the write path (the transform is +//! idempotent and identical) and makes the read path symmetric. + +use tinycortex::memory::store::kv::KvStore; + +use crate::store::namespace_store::UnifiedMemory; +use crate::store::safety::canonical_identifier; +use crate::store::types::MemoryKvRecord; + +impl UnifiedMemory { + fn tinycortex_kv(&self) -> Result { + KvStore::from_shared_connection(self.conn.clone()) + .map_err(|error| format!("initialize tinycortex KV store: {error}")) + } + + pub async fn kv_set_global(&self, key: &str, value: &serde_json::Value) -> Result<(), String> { + self.tinycortex_kv()? + .set_global(&canonical_identifier(key), value) + } + + pub async fn kv_get_global(&self, key: &str) -> Result, String> { + self.tinycortex_kv()?.get_global(&canonical_identifier(key)) + } + + pub async fn kv_set_namespace( + &self, + namespace: &str, + key: &str, + value: &serde_json::Value, + ) -> Result<(), String> { + self.tinycortex_kv()?.set_namespace( + &canonical_identifier(namespace), + &canonical_identifier(key), + value, + ) + } + + pub async fn kv_get_namespace( + &self, + namespace: &str, + key: &str, + ) -> Result, String> { + self.tinycortex_kv()? + .get_namespace(&canonical_identifier(namespace), &canonical_identifier(key)) + } + + pub async fn kv_delete_global(&self, key: &str) -> Result { + self.tinycortex_kv()? + .delete_global(&canonical_identifier(key)) + } + + pub async fn kv_delete_namespace(&self, namespace: &str, key: &str) -> Result { + self.tinycortex_kv()? + .delete_namespace(&canonical_identifier(namespace), &canonical_identifier(key)) + } + + pub async fn kv_list_namespace( + &self, + namespace: &str, + ) -> Result, String> { + self.tinycortex_kv()? + .list_namespace(&canonical_identifier(namespace)) + } + + pub(crate) async fn kv_records_for_scope( + &self, + namespace: &str, + ) -> Result, String> { + self.tinycortex_kv()? + .records_for_scope(&canonical_identifier(namespace)) + .map(convert_records) + } + + pub(crate) async fn kv_records_namespace( + &self, + namespace: &str, + ) -> Result, String> { + self.tinycortex_kv()? + .records_namespace(&canonical_identifier(namespace)) + .map(convert_records) + } + + pub(crate) async fn kv_records_global(&self) -> Result, String> { + self.tinycortex_kv()?.records_global().map(convert_records) + } +} + +fn convert_records(records: Vec) -> Vec { + records + .into_iter() + .map(|record| MemoryKvRecord { + namespace: record.namespace, + key: record.key, + value: record.value, + updated_at: record.updated_at, + }) + .collect() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use tempfile::TempDir; + + use super::*; + use tinymemory_api::host::NoopEmbedding; + + fn test_memory() -> (TempDir, UnifiedMemory) { + let tmp = TempDir::new().unwrap(); + let memory = + UnifiedMemory::new(tmp.path(), std::sync::Arc::new(NoopEmbedding), None).unwrap(); + (tmp, memory) + } + + #[tokio::test] + async fn global_kv_roundtrips_and_deletes_through_tinycortex() { + let (_tmp, memory) = test_memory(); + memory.kv_set_global("theme", &json!("dark")).await.unwrap(); + assert_eq!( + memory.kv_get_global("theme").await.unwrap(), + Some(json!("dark")) + ); + assert!(memory.kv_delete_global("theme").await.unwrap()); + } + + #[tokio::test] + async fn namespace_records_share_the_unified_connection() { + let (_tmp, memory) = test_memory(); + memory + .kv_set_namespace("team alpha/#1", "state", &json!({"open": true})) + .await + .unwrap(); + let records = memory.kv_records_namespace("team alpha/#1").await.unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].namespace.as_deref(), Some("team_alpha/_1")); + } +} diff --git a/core/src/store/memory_trait.rs b/core/src/store/memory_trait.rs new file mode 100644 index 0000000..4f4b3c5 --- /dev/null +++ b/core/src/store/memory_trait.rs @@ -0,0 +1,1135 @@ +//! # Memory Trait Implementation +//! +//! This module implements the core `Memory` trait for the `UnifiedMemory` +//! struct. This allows `UnifiedMemory` to be used as a generic memory backend +//! within the OpenHuman system. +//! +//! Callers pass an explicit `namespace` on `store`/`get`/`forget` and via +//! `RecallOpts` on `recall`. When a `namespace` is omitted on `recall`/`list`, +//! the implementation falls back to `GLOBAL_NAMESPACE` (legacy behavior), which +//! Phase B/C will tighten once the memory tools pass namespace explicitly. + +use async_trait::async_trait; +use chrono::{TimeZone, Utc}; +use rusqlite::{params, OptionalExtension}; +use serde_json::json; + +use crate::store::namespace_store::fts5; +use crate::store::types::{NamespaceDocumentInput, GLOBAL_NAMESPACE}; +use crate::traits::{ + Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, +}; +use anyhow::Context; + +use super::namespace_store::UnifiedMemory; + +/// Convert a UNIX timestamp (f64) to RFC3339 string. +fn timestamp_to_rfc3339(ts: f64) -> String { + let secs = ts.trunc() as i64; + let nanos = ((ts.fract()) * 1_000_000_000.0).round() as u32; + Utc.timestamp_opt(secs, nanos.min(999_999_999)) + .single() + .map(|dt| dt.to_rfc3339()) + .unwrap_or_else(|| format!("{ts}")) +} + +/// Normalize a namespace value: trim whitespace and fall back to +/// `GLOBAL_NAMESPACE` for `None` or blank/whitespace-only inputs. This ensures +/// that `recall`/`list` calls derived from user or RPC input never silently +/// receive an empty string that misses the global namespace. +fn normalize_namespace(namespace: Option<&str>) -> &str { + namespace + .map(str::trim) + .filter(|ns| !ns.is_empty()) + .unwrap_or(GLOBAL_NAMESPACE) +} + +/// Helper to convert a raw string category from the database into a `MemoryCategory`. +/// +/// The store persists a category via its `Display` form, and the current +/// TinyCortex format renders `Custom(name)` as `custom:{name}` (so `Custom("core")` +/// stays distinct from `Core`). Parse back through `FromStr` — the true inverse of +/// `Display` — so the `custom:` prefix is stripped symmetrically. Wrapping the raw +/// string in `Custom(_)` instead (the previous behaviour) double-prefixed on +/// read-back once the wire format gained the prefix. An empty stored value has no +/// `FromStr` mapping, so it falls back to an empty `Custom` (matching the prior +/// catch-all for that degenerate case). +fn memory_category_from_stored(raw: &str) -> MemoryCategory { + raw.parse().unwrap_or_else(|error| { + tracing::debug!( + category_chars = raw.chars().count(), + reason = %error, + "[memory_store] invalid stored category; preserving as custom" + ); + MemoryCategory::Custom(raw.to_string()) + }) +} + +impl UnifiedMemory { + /// Ranked recall with the same-session self-echo exclusion supplied + /// **explicitly** by the caller. + /// + /// This is the engine body: given a query, a limit, [`RecallOpts`], and an + /// optional session id to exclude, it produces the ranked result. It reads + /// no ambient host state, so it is driveable from a test, a CLI, or a + /// future embedding host without an agent harness in the picture. + /// + /// `exclude_session_id` drops documents tagged with that session before + /// ranking (not after), so `limit` is never consumed by rows the caller + /// asked not to see. `None` applies no exclusion at all. + /// + /// The host policy that decides *what* to exclude lives in + /// `crate::store::recall_policy`; the [`Memory::recall`] + /// impl below is the thin adapter that joins the two. + pub async fn recall_excluding_session( + &self, + query: &str, + limit: usize, + opts: RecallOpts<'_>, + exclude_session_id: Option<&str>, + ) -> anyhow::Result> { + let namespace = normalize_namespace(opts.namespace); + + if let Some(excluded) = exclude_session_id { + tracing::debug!( + "[memory-trait] recall applying same-session exclusion namespace={namespace} \ + exclude_session_id={excluded}" + ); + } + let ranked = self + .query_namespace_ranked_excluding_session( + namespace, + query, + limit as u32, + exclude_session_id, + ) + .await + .map_err(anyhow::Error::msg)?; + + let min_score = opts.min_score.unwrap_or(f64::NEG_INFINITY); + let mut out: Vec = ranked + .into_iter() + .enumerate() + .filter(|(_, r)| r.score >= min_score) + .map(|(idx, r)| MemoryEntry { + id: format!("{namespace}:{idx}"), + key: r.key, + content: r.content, + namespace: Some(namespace.to_string()), + category: memory_category_from_stored(&r.category), + timestamp: Utc::now().to_rfc3339(), + session_id: None, + score: Some(r.score), + // Surface the real taint persisted on `memory_docs` so the + // subconscious gate can decide whether to escalate the + // turn origin to `SubconsciousTainted` when this entry + // lands in a tick's context window. + taint: r.taint, + }) + .collect(); + + if let Some(ref cat) = opts.category { + let want = cat.to_string(); + out.retain(|e| e.category.to_string() == want); + } + + if let Some(sid) = opts.session_id { + let episodic_entries = match fts5::episodic_session_entries(&self.conn, sid) { + Ok(entries) => { + tracing::debug!( + "[memory-trait] loaded {} episodic entries for session={sid}", + entries.len() + ); + entries + } + Err(e) => { + tracing::warn!( + "[memory-trait] failed to load episodic entries for session={sid}: {e}" + ); + Vec::new() + } + }; + + let query_lower = query.to_lowercase(); + let query_terms: Vec<&str> = query_lower.split_whitespace().collect(); + for entry in episodic_entries { + let content_lower = entry.content.to_lowercase(); + let matched_count = query_terms + .iter() + .filter(|term| content_lower.contains(*term)) + .count(); + if matched_count == 0 { + continue; + } + let match_score = matched_count as f64 / query_terms.len().max(1) as f64; + if match_score < min_score { + continue; + } + let ts_rfc3339 = timestamp_to_rfc3339(entry.timestamp); + + out.push(MemoryEntry { + id: format!("episodic:{}", entry.id.unwrap_or(0)), + key: format!("{}:{}", entry.session_id, entry.role), + content: entry.content, + namespace: Some(namespace.to_string()), + category: MemoryCategory::Conversation, + timestamp: ts_rfc3339, + session_id: Some(entry.session_id), + score: Some(match_score), + taint: crate::MemoryTaint::Internal, + }); + } + } + + // ── Cross-session episodic recall (#1505) ──────────────────────── + // + // When the caller asks for cross-session memory, pull FTS5-ranked + // hits from every other session in the same workspace. Workspace + // isolation is enforced by the SQLite DB path itself (one DB per + // workspace == one DB per user) so this can never leak across + // users. The current `session_id` (if any) is excluded so the + // caller doesn't double-count its own chat history — those rows + // already came in via the same-session path above. + if opts.cross_session { + let exclude = opts.session_id; + let cross_entries = match fts5::episodic_cross_session_search( + &self.conn, query, limit, exclude, + ) { + Ok(entries) => { + tracing::debug!( + "[memory-trait] cross-session episodic recall returned {} entries (exclude={:?})", + entries.len(), + exclude + ); + entries + } + Err(e) => { + tracing::warn!( + "[memory-trait] cross-session episodic recall failed (non-fatal): {e}" + ); + Vec::new() + } + }; + + // Normalise FTS5 rank into a [0..1] keyword-style score by + // reusing the same matched-terms heuristic as the same-session + // branch. This keeps the score scale consistent across hits so + // the downstream sort doesn't preferentially up-rank one branch + // over the other. + let query_lower = query.to_lowercase(); + let query_terms: Vec<&str> = query_lower.split_whitespace().collect(); + for entry in cross_entries { + let content_lower = entry.content.to_lowercase(); + let matched_count = query_terms + .iter() + .filter(|term| content_lower.contains(*term)) + .count(); + if matched_count == 0 { + // FTS5 surfaced a porter-stemmed match with zero + // literal query-term overlap. Drop it — the previous + // `0.1_f64.max(min_score)` floor defeated the + // downstream `score >= min_relevance_score` gate + // (when min_score==0.4 the floor also became 0.4), + // so those rows always survived. Skip outright. + continue; + } + let match_score = matched_count as f64 / query_terms.len().max(1) as f64; + if match_score < min_score { + continue; + } + let ts_rfc3339 = timestamp_to_rfc3339(entry.timestamp); + out.push(MemoryEntry { + id: format!("episodic-cross:{}", entry.id.unwrap_or(0)), + key: format!("{}:{}", entry.session_id, entry.role), + content: entry.content, + namespace: Some(namespace.to_string()), + category: MemoryCategory::Conversation, + timestamp: ts_rfc3339, + session_id: Some(entry.session_id), + score: Some(match_score), + taint: crate::MemoryTaint::Internal, + }); + } + } + + if opts.session_id.is_some() || opts.cross_session { + out.sort_by(|a, b| { + b.score + .unwrap_or(0.0) + .partial_cmp(&a.score.unwrap_or(0.0)) + .unwrap_or(std::cmp::Ordering::Equal) + }); + out.truncate(limit); + } + + Ok(out) + } +} + +#[async_trait] +impl Memory for UnifiedMemory { + fn name(&self) -> &str { + "namespace" + } + + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + ) -> anyhow::Result<()> { + // The default `store` entry point is user-driven; ingest paths + // come in via `store_with_taint`. + self.store_with_taint( + namespace, + key, + content, + category, + session_id, + MemoryTaint::Internal, + ) + .await + } + + async fn store_with_taint( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> anyhow::Result<()> { + let ns = if namespace.trim().is_empty() { + GLOBAL_NAMESPACE.to_string() + } else { + namespace.to_string() + }; + self.upsert_document(NamespaceDocumentInput { + namespace: ns, + key: key.to_string(), + title: key.to_string(), + content: content.to_string(), + source_type: "chat".to_string(), + priority: "medium".to_string(), + tags: Vec::new(), + metadata: json!({}), + category: category.to_string(), + session_id: session_id.map(str::to_string), + document_id: None, + taint, + }) + .await + .map(|_| ()) + .map_err(anyhow::Error::msg) + } + + async fn recall( + &self, + query: &str, + limit: usize, + opts: RecallOpts<'_>, + ) -> anyhow::Result> { + // Host policy seam: the exclusion the engine applies is resolved + // here, at the trait adapter, and handed down as a parameter. The + // engine itself (`recall_excluding_session`) reads no ambient state. + // See `store::recall_policy` for why the resolution is still ambient + // and what would let it be pushed further up. + let exclude_session_id = super::recall_policy::current_self_echo_exclusion(); + self.recall_excluding_session(query, limit, opts, exclude_session_id.as_deref()) + .await + } + + async fn recall_relevant_by_vector( + &self, + namespace: &str, + query: &str, + limit: usize, + min_vector_similarity: f64, + ) -> anyhow::Result> { + let hits = self + .query_namespace_hits(namespace, query, limit as u32) + .await + .map_err(anyhow::Error::msg)?; + Ok(hits + .into_iter() + .filter(|h| h.score_breakdown.vector_similarity >= min_vector_similarity) + .filter(|h| !h.content.trim().is_empty()) + .map(|h| (h.key, h.content)) + .collect()) + } + + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { + // Address the row the way `store` wrote it: `upsert_document` stores + // `sanitize_namespace(namespace)` and `canonical_document_key(key)`, so + // looking up the raw caller values misses whenever either transform + // changed anything — the caller then reads the row as absent and stores + // it again, which is the retry loop behind #5164. + let ns = UnifiedMemory::sanitize_namespace(namespace); + let key = crate::store::safety::canonical_document_key(key); + let conn = self.conn.lock(); + let row: Option<(String, String, String, f64, String, String)> = conn + .query_row( + "SELECT document_id, key, content, updated_at, category, taint + FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1", + params![ns, key], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, + ) + .optional()?; + Ok(row.map( + |(id, key, content, updated_at, category, taint_str)| MemoryEntry { + id, + key, + content, + namespace: Some(ns.clone()), + category: memory_category_from_stored(&category), + timestamp: timestamp_to_rfc3339(updated_at), + session_id: None, + score: None, + taint: crate::MemoryTaint::from_db_str(&taint_str), + }, + )) + } + + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> anyhow::Result> { + let ns = UnifiedMemory::sanitize_namespace(normalize_namespace(namespace)); + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "SELECT document_id, key, content, category, session_id, updated_at, taint + FROM memory_docs WHERE namespace = ?1 ORDER BY updated_at DESC", + )?; + let rows = stmt.query_map(params![ns], |row| { + let stored_category: String = row.get(3)?; + Ok(MemoryEntry { + id: row.get(0)?, + key: row.get(1)?, + content: row.get(2)?, + namespace: Some(ns.clone()), + category: memory_category_from_stored(&stored_category), + session_id: row.get(4)?, + timestamp: timestamp_to_rfc3339(row.get(5)?), + score: None, + taint: crate::MemoryTaint::from_db_str(&row.get::<_, String>(6)?), + }) + })?; + let mut entries = rows.collect::>>()?; + if let Some(category) = category { + entries.retain(|entry| &entry.category == category); + } + if let Some(session_id) = session_id { + entries.retain(|entry| entry.session_id.as_deref() == Some(session_id)); + } + Ok(entries) + } + + async fn forget(&self, namespace: &str, key: &str) -> anyhow::Result { + // Same write/read symmetry as `get` above (#5164): a `forget` that + // addresses the raw caller identifiers can never delete a row whose + // namespace or key was canonicalized on the way in. + let ns = UnifiedMemory::sanitize_namespace(namespace); + let key = crate::store::safety::canonical_document_key(key); + let row: Option = { + let conn = self.conn.lock(); + conn.query_row( + "SELECT document_id FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1", + params![ns, key], + |row| row.get(0), + ) + .optional()? + }; + let Some(document_id) = row else { + return Ok(false); + }; + self.delete_document(&ns, &document_id) + .await + .map_err(anyhow::Error::msg)?; + Ok(true) + } + + async fn namespace_summaries(&self) -> anyhow::Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "SELECT namespace, COUNT(*) AS n, MAX(updated_at) AS last + FROM memory_docs + GROUP BY namespace + ORDER BY namespace", + )?; + let rows = stmt.query_map([], |row| { + let ns: String = row.get(0)?; + let count: i64 = row.get(1)?; + let last: Option = row.get(2)?; + Ok((ns, count, last)) + })?; + let mut out = Vec::new(); + for r in rows { + let (ns, count, last) = r?; + out.push(NamespaceSummary { + namespace: ns, + count: usize::try_from(count).unwrap_or(0), + last_updated: last.map(timestamp_to_rfc3339), + }); + } + Ok(out) + } + + async fn count(&self) -> anyhow::Result { + let conn = self.conn.lock(); + let count: i64 = + conn.query_row("SELECT COUNT(*) FROM memory_docs", [], |row| row.get(0))?; + usize::try_from(count).context("negative count") + } + + async fn health_check(&self) -> bool { + self.workspace_dir.exists() && self.db_path.exists() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use tempfile::TempDir; + use tinymemory_api::host::NoopEmbedding; + + fn fresh_mem() -> (TempDir, UnifiedMemory) { + let tmp = TempDir::new().unwrap(); + let mem = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + (tmp, mem) + } + + #[tokio::test] + async fn store_and_get_are_namespace_scoped() { + let (_tmp, mem) = fresh_mem(); + mem.store("ns_a", "k1", "value in a", MemoryCategory::Core, None) + .await + .unwrap(); + + let hit = mem.get("ns_a", "k1").await.unwrap(); + assert!(hit.is_some(), "same-namespace get should return entry"); + assert_eq!(hit.unwrap().content, "value in a"); + + let miss = mem.get("ns_b", "k1").await.unwrap(); + assert!(miss.is_none(), "cross-namespace get must not leak"); + } + + #[tokio::test] + async fn list_and_forget_are_namespace_scoped() { + let (_tmp, mem) = fresh_mem(); + mem.store("ns_a", "k1", "a", MemoryCategory::Core, None) + .await + .unwrap(); + mem.store("ns_b", "k1", "b", MemoryCategory::Core, None) + .await + .unwrap(); + + let in_b = mem.list(Some("ns_b"), None, None).await.unwrap(); + assert_eq!(in_b.len(), 1); + assert_eq!(in_b[0].content, "b"); + assert!(in_b.iter().all(|e| e.namespace.as_deref() == Some("ns_b"))); + + // Forget in ns_a must not delete ns_b's row + assert!(mem.forget("ns_a", "k1").await.unwrap()); + assert!(mem.get("ns_b", "k1").await.unwrap().is_some()); + assert!(mem.get("ns_a", "k1").await.unwrap().is_none()); + } + + #[tokio::test] + async fn list_returns_stored_fields_and_applies_category_and_session_filters() { + let (_tmp, mem) = fresh_mem(); + mem.store( + "rules", + "core", + "core body", + MemoryCategory::Core, + Some("session-a"), + ) + .await + .unwrap(); + mem.store( + "rules", + "procedure", + "procedure body", + MemoryCategory::Daily, + Some("session-b"), + ) + .await + .unwrap(); + + let entries = mem + .list( + Some("rules"), + Some(&MemoryCategory::Daily), + Some("session-b"), + ) + .await + .unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].key, "procedure"); + assert_eq!(entries[0].content, "procedure body"); + assert_eq!(entries[0].category, MemoryCategory::Daily); + assert_eq!(entries[0].session_id.as_deref(), Some("session-b")); + assert!(!entries[0].timestamp.starts_with("idx-")); + } + + #[tokio::test] + async fn namespace_summaries_counts_per_namespace() { + let (_tmp, mem) = fresh_mem(); + mem.store("alpha", "k1", "x", MemoryCategory::Core, None) + .await + .unwrap(); + mem.store("alpha", "k2", "y", MemoryCategory::Core, None) + .await + .unwrap(); + mem.store("beta", "k1", "z", MemoryCategory::Core, None) + .await + .unwrap(); + + let summaries = mem.namespace_summaries().await.unwrap(); + let alpha = summaries.iter().find(|s| s.namespace == "alpha").unwrap(); + let beta = summaries.iter().find(|s| s.namespace == "beta").unwrap(); + assert_eq!(alpha.count, 2); + assert_eq!(beta.count, 1); + assert!(alpha.last_updated.is_some()); + } + + #[tokio::test] + async fn legacy_namespace_migration_splits_and_is_idempotent() { + use rusqlite::params; + + let tmp = TempDir::new().unwrap(); + let mem = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // Seed a legacy-shape row: GLOBAL namespace, key="ns_x/real_key". + { + let conn = mem.conn.lock(); + conn.execute( + "INSERT INTO memory_docs ( + document_id, namespace, key, title, content, source_type, + priority, tags_json, metadata_json, category, session_id, + created_at, updated_at, markdown_rel_path + ) VALUES (?1, ?2, ?3, ?4, ?5, 'chat', 'medium', '[]', '{}', 'core', NULL, 0.0, 0.0, '')", + params![ + "legacy-doc-1", + GLOBAL_NAMESPACE, + "ns_x/real_key", + "ns_x/real_key", + "legacy value" + ], + ) + .unwrap(); + } + + drop(mem); + + // Re-open so the startup migration runs again. + let mem = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let hit = mem.get("ns_x", "real_key").await.unwrap(); + assert!(hit.is_some(), "migration should promote ns_x"); + assert_eq!(hit.unwrap().content, "legacy value"); + + // Re-open again — migration must be a no-op (no duplicate / crash). + drop(mem); + let mem = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let still = mem.get("ns_x", "real_key").await.unwrap(); + assert!(still.is_some()); + assert_eq!(mem.count().await.unwrap(), 1); + } + + // ── Cross-session recall (#1505) ───────────────────────────────────── + + fn seed_episodic(mem: &UnifiedMemory, session_id: &str, ts: f64, content: &str) { + fts5::episodic_insert( + &mem.conn, + &fts5::EpisodicEntry { + id: None, + session_id: session_id.into(), + timestamp: ts, + role: "user".into(), + content: content.into(), + lesson: None, + tool_calls_json: None, + cost_microdollars: 0, + }, + ) + .unwrap(); + } + + #[tokio::test] + async fn recall_cross_session_surfaces_other_chat_facts() { + let (_tmp, mem) = fresh_mem(); + // Chat A — durable user fact dropped here + seed_episodic(&mem, "chat-a", 1000.0, "I prefer Postgres for new services"); + // Chat B — current chat (no relevant content yet) + seed_episodic(&mem, "chat-b", 2000.0, "Hello there"); + + // Recall from chat B with cross_session=true should surface chat A's fact + let opts = RecallOpts { + session_id: Some("chat-b"), + cross_session: true, + min_score: Some(0.0), + ..Default::default() + }; + let hits = mem.recall("Postgres", 10, opts).await.unwrap(); + + assert!( + hits.iter() + .any(|h| h.content.to_lowercase().contains("postgres") + && h.session_id.as_deref() == Some("chat-a")), + "cross-session recall must surface chat-a's Postgres fact, got hits={hits:#?}" + ); + assert!( + hits.iter() + .all(|h| h.session_id.as_deref() != Some("chat-b") + || !h.id.starts_with("episodic-cross:")), + "current chat-b session must be excluded from the cross-session sweep" + ); + } + + #[tokio::test] + async fn recall_cross_session_disabled_by_default_no_other_chat_leak() { + let (_tmp, mem) = fresh_mem(); + seed_episodic(&mem, "chat-a", 1000.0, "I prefer Postgres for new services"); + seed_episodic(&mem, "chat-b", 2000.0, "Hello there"); + + // Default RecallOpts (cross_session=false) — no episodic content + // because no session_id is set either, so this exercises the + // pre-#1505 baseline behaviour: documents only. + let hits = mem + .recall("Postgres", 10, RecallOpts::default()) + .await + .unwrap(); + + assert!( + !hits.iter().any(|h| h.id.starts_with("episodic-cross:")), + "cross_session=false must never surface episodic-cross hits, got {hits:#?}" + ); + } + + #[tokio::test] + async fn recall_cross_session_preserves_provenance_via_session_id() { + let (_tmp, mem) = fresh_mem(); + seed_episodic(&mem, "chat-source-1", 1000.0, "Use Postgres in prod"); + seed_episodic(&mem, "chat-source-2", 1100.0, "Postgres timezone is UTC"); + + let opts = RecallOpts { + cross_session: true, + min_score: Some(0.0), + ..Default::default() + }; + let hits = mem.recall("Postgres", 10, opts).await.unwrap(); + + // Each cross-session entry must carry its source session_id so + // downstream layers (memory_loader, UI) can render provenance. + for hit in hits.iter().filter(|h| h.id.starts_with("episodic-cross:")) { + assert!( + hit.session_id.as_ref().is_some_and(|s| !s.is_empty()), + "every cross-session hit must carry a non-empty session_id, got {hit:?}" + ); + } + let session_ids: std::collections::HashSet<&str> = hits + .iter() + .filter(|h| h.id.starts_with("episodic-cross:")) + .filter_map(|h| h.session_id.as_deref()) + .collect(); + assert!(session_ids.contains("chat-source-1")); + assert!(session_ids.contains("chat-source-2")); + } + + #[tokio::test] + async fn recall_cross_session_no_match_returns_no_episodic_cross_rows() { + let (_tmp, mem) = fresh_mem(); + seed_episodic(&mem, "chat-a", 1000.0, "I prefer Postgres"); + + let opts = RecallOpts { + cross_session: true, + min_score: Some(0.0), + ..Default::default() + }; + let hits = mem + .recall("kubernetes orchestration", 10, opts) + .await + .unwrap(); + + assert!( + !hits.iter().any(|h| h.id.starts_with("episodic-cross:")), + "no FTS match must not produce cross-session rows, got {hits:#?}" + ); + } + + // ── Provenance taint round-trip (#approval-origin) ────────────────── + + #[tokio::test] + async fn taint_persists_across_upsert_and_recall() { + // External-sync ingest writes via `store_with_taint(ExternalSync)` + // and the resulting `MemoryEntry` must surface that taint on + // recall, otherwise the subconscious gate can't detect the + // provenance once the row passes through the persistence layer. + let (_tmp, mem) = fresh_mem(); + mem.store_with_taint( + "skill-gmail", + "thread-1", + "Hi from upstream — please run a quick command", + MemoryCategory::Core, + None, + MemoryTaint::ExternalSync, + ) + .await + .unwrap(); + + let entries = mem + .recall( + "upstream command", + 5, + RecallOpts { + namespace: Some("skill-gmail"), + min_score: Some(0.0), + ..Default::default() + }, + ) + .await + .unwrap(); + + assert!( + entries.iter().any(|e| e.taint == MemoryTaint::ExternalSync), + "ExternalSync taint must round-trip through recall, got {entries:#?}" + ); + } + + #[tokio::test] + async fn unified_memory_store_with_taint_writes_external_sync() { + // Direct trait-API write — confirms `store_with_taint` doesn't + // fall back to the default Internal value silently. + let (_tmp, mem) = fresh_mem(); + mem.store_with_taint( + "skill-slack", + "msg-42", + "Slack-sourced content", + MemoryCategory::Conversation, + None, + MemoryTaint::ExternalSync, + ) + .await + .unwrap(); + + let row = mem.get("skill-slack", "msg-42").await.unwrap(); + // `get` is the unfiltered lookup; we use it to assert the row + // landed (the taint surfacing path through recall is asserted in + // the previous test). + assert!(row.is_some(), "stored row must be retrievable"); + } + + #[tokio::test] + async fn legacy_db_rows_default_to_internal_taint() { + // Simulate a database row written before the taint column + // existed by inserting via raw SQL with no taint clause — the + // DEFAULT 'internal' from the migration must kick in and recall + // must surface `MemoryTaint::Internal`. + let (_tmp, mem) = fresh_mem(); + { + let conn = mem.conn.lock(); + conn.execute( + "INSERT INTO memory_docs ( + document_id, namespace, key, title, content, source_type, + priority, tags_json, metadata_json, category, session_id, + created_at, updated_at, markdown_rel_path + ) VALUES (?1, ?2, ?3, ?4, ?5, 'chat', 'medium', '[]', '{}', 'core', NULL, 0.0, 0.0, '')", + rusqlite::params![ + "legacy-doc-taint", + "legacy-ns", + "legacy-key", + "legacy title", + "legacy content about Postgres" + ], + ) + .unwrap(); + } + + let entries = mem + .recall( + "Postgres", + 5, + RecallOpts { + namespace: Some("legacy-ns"), + min_score: Some(0.0), + ..Default::default() + }, + ) + .await + .unwrap(); + + let legacy = entries + .iter() + .find(|e| e.key == "legacy-key") + .expect("legacy row must surface in recall"); + assert_eq!( + legacy.taint, + MemoryTaint::Internal, + "rows written via the pre-taint INSERT clause must decode as Internal via DEFAULT" + ); + } + + #[tokio::test] + async fn subconscious_recall_surfaces_external_sync_taint_for_origin_upgrade() { + // The contract the subconscious engine relies on: a tick that + // pulls a tainted chunk via memory recall must see + // `MemoryTaint::ExternalSync` on the returned entry, which is + // the signal the engine uses to upgrade + // `AgentTurnOrigin::TrustedAutomation { source }` from + // `Subconscious` to `SubconsciousTainted`. + let (_tmp, mem) = fresh_mem(); + mem.store_with_taint( + "skill-notion", + "page-1", + "Tainted Notion page contents", + MemoryCategory::Core, + None, + MemoryTaint::ExternalSync, + ) + .await + .unwrap(); + mem.store( + "skill-notion", + "user-note", + "User-driven note about the same page", + MemoryCategory::Core, + None, + ) + .await + .unwrap(); + + let entries = mem + .recall( + "page", + 10, + RecallOpts { + namespace: Some("skill-notion"), + min_score: Some(0.0), + ..Default::default() + }, + ) + .await + .unwrap(); + + let any_tainted = entries.iter().any(|e| e.taint == MemoryTaint::ExternalSync); + let any_internal = entries.iter().any(|e| e.taint == MemoryTaint::Internal); + assert!( + any_tainted, + "ExternalSync row must surface for the engine's upgrade check" + ); + assert!( + any_internal, + "user-driven row must keep its Internal label so mixed contexts don't over-escalate" + ); + } + + // ── Same-session self-echo exclusion, via the ambient thread scope ──── + // + // `Memory::recall` (backing the agent's `memory_recall` tool) reads the + // ambient chat-thread id set by `tinyagents::thread_context` + // around a live turn, and excludes documents tagged with that same id — + // guarding against the harness's own `user_msg:` autosave being + // recalled as the top "relevant" result for the very request that + // triggered the search. See `agent::harness::session::turn::core` + // (autosave tagging) and `query::query_namespace_hits_excluding_session` + // (the exclusion mechanism). + + #[tokio::test] + async fn recall_excludes_document_from_ambient_current_thread() { + use crate::thread_context::with_thread_id; + + let (_tmp, mem) = fresh_mem(); + mem.store( + "global", + "user_msg:current-turn", + "Please look up Jordan Rivera's chat platform user ID for me.", + MemoryCategory::Conversation, + Some("thread-current"), + ) + .await + .unwrap(); + mem.store( + "global", + "fact:jordan-rivera-platform-id", + "Jordan Rivera's chat platform user ID is U0000042.", + MemoryCategory::Conversation, + Some("thread-other"), + ) + .await + .unwrap(); + + let entries = with_thread_id("thread-current", async { + mem.recall( + "Jordan Rivera chat platform user ID", + 10, + RecallOpts { + namespace: Some("global"), + min_score: Some(0.0), + ..Default::default() + }, + ) + .await + .unwrap() + }) + .await; + + assert!( + !entries.iter().any(|e| e.key == "user_msg:current-turn"), + "recall inside the ambient current-thread scope must exclude that thread's own \ + autosaved request, got {entries:#?}" + ); + assert!( + entries + .iter() + .any(|e| e.key == "fact:jordan-rivera-platform-id"), + "an unrelated document from a different session must still be recalled, got {entries:#?}" + ); + } + + #[tokio::test] + async fn recall_outside_any_thread_scope_is_unaffected() { + let (_tmp, mem) = fresh_mem(); + mem.store( + "global", + "user_msg:current-turn", + "Please look up Jordan Rivera's chat platform user ID for me.", + MemoryCategory::Conversation, + Some("thread-current"), + ) + .await + .unwrap(); + + // No `with_thread_id(...)` scope active — mirrors cron, CLI, + // standalone, and any pre-existing caller. `current_thread_id()` + // returns `None`, so no exclusion applies and behavior is + // byte-for-byte the same as before this fix. + let entries = mem + .recall( + "Jordan Rivera chat platform user ID", + 10, + RecallOpts { + namespace: Some("global"), + min_score: Some(0.0), + ..Default::default() + }, + ) + .await + .unwrap(); + + assert!( + entries.iter().any(|e| e.key == "user_msg:current-turn"), + "with no ambient thread scope, recall must return the document exactly as before \ + this fix, got {entries:#?}" + ); + } + + // ── The engine takes the exclusion as a parameter (H0, piece 1) ────── + // + // `recall_excluding_session` is the policy-free engine body: it must honour + // an exclusion handed to it with **no ambient turn scope active**, and + // apply none when handed `None`. Together these pin that the exclusion + // travels as an argument rather than being re-derived from a task-local + // inside the storage layer — the property that lets the engine move into a + // persistence crate without dragging the chat-turn concept along. + + async fn seed_self_echo_fixture(mem: &UnifiedMemory) { + mem.store( + "global", + "user_msg:current-turn", + "Please look up Jordan Rivera's chat platform user ID for me.", + MemoryCategory::Conversation, + Some("thread-current"), + ) + .await + .unwrap(); + mem.store( + "global", + "fact:jordan-rivera-platform-id", + "Jordan Rivera's chat platform user ID is U0000042.", + MemoryCategory::Conversation, + Some("thread-other"), + ) + .await + .unwrap(); + } + + fn self_echo_opts() -> RecallOpts<'static> { + RecallOpts { + namespace: Some("global"), + min_score: Some(0.0), + ..Default::default() + } + } + + #[tokio::test] + async fn recall_excluding_session_applies_an_explicit_exclusion_with_no_ambient_scope() { + let (_tmp, mem) = fresh_mem(); + seed_self_echo_fixture(&mem).await; + + // Deliberately NOT wrapped in `with_thread_id`: if the engine were + // still reading the ambient task-local rather than the argument, the + // exclusion below would have no effect and the first assert fails. + let entries = mem + .recall_excluding_session( + "Jordan Rivera chat platform user ID", + 10, + self_echo_opts(), + Some("thread-current"), + ) + .await + .unwrap(); + + assert!( + !entries.iter().any(|e| e.key == "user_msg:current-turn"), + "an explicitly passed exclusion must drop that session's own document even with no \ + ambient turn scope, got {entries:#?}" + ); + assert!( + entries + .iter() + .any(|e| e.key == "fact:jordan-rivera-platform-id"), + "a document from a different session must survive the exclusion, got {entries:#?}" + ); + } + + #[tokio::test] + async fn recall_excluding_session_with_none_excludes_nothing() { + let (_tmp, mem) = fresh_mem(); + seed_self_echo_fixture(&mem).await; + + // Inside an ambient turn scope, yet passed `None`: the engine must + // honour the argument, not the task-local. + let entries = crate::thread_context::with_thread_id("thread-current", async { + mem.recall_excluding_session( + "Jordan Rivera chat platform user ID", + 10, + self_echo_opts(), + None, + ) + .await + .unwrap() + }) + .await; + + assert!( + entries.iter().any(|e| e.key == "user_msg:current-turn"), + "`None` must exclude nothing — the engine must not re-derive an exclusion from the \ + ambient turn scope, got {entries:#?}" + ); + } +} diff --git a/core/src/store/mod.rs b/core/src/store/mod.rs new file mode 100644 index 0000000..bf78ba3 --- /dev/null +++ b/core/src/store/mod.rs @@ -0,0 +1,82 @@ +//! # Memory Store +//! +//! This module provides the core storage abstractions and implementations for +//! the OpenHuman memory system. It manages namespaces, documents, text chunks, +//! vector embeddings, and graph relations. +//! +//! The memory system is designed to be pluggable, with the primary implementation +//! being `UnifiedMemory`, which uses SQLite for structured data and Full-Text +//! Search (FTS5), along with vector storage for semantic retrieval. +//! +//! ## Submodules +//! +//! - `types`: Common data structures and types used across the memory store. +//! - `namespace_store`: Host-retained SQLite namespace documents, graph, +//! episodic/event/segment/profile tables, and their query policy. +//! - `client`: High-level client interface for interacting with the memory system. +//! - `factories`: Factory functions for creating and initializing memory instances. +//! - `memory_trait`: Defines the `Memory` trait that all implementations must satisfy. +//! - `recall_policy`: Host *product* policy for recall (the same-session +//! self-echo guard). Deliberately outside `namespace_store` so the storage +//! engine takes its exclusions as parameters instead of reading agent-harness +//! task-locals. +//! - `write_gate`: Host secret/PII policy for document writes. Also outside +//! `namespace_store`, for the same reason: the driver persists +//! already-sanitized input rather than deciding what a secret is. + +pub mod chunks; +pub mod content; +pub mod entities; +pub mod kinds; +pub mod kv; +pub mod namespace_store; +pub mod profile_store; +pub mod retrieval; +pub mod safety; +pub mod traits; +pub mod trees; +pub mod types; + +mod client; +pub mod factories; +/// Golden-workspace fixture seeding / read-back / schema-manifest capture. +/// +/// Public only so `tests/memory_golden_fixture_e2e.rs` can drive it; it needs +/// `pub(crate)` reach (`MemoryClient::profile_conn`, the tree seal helpers) +/// that an integration test does not have. Not part of the product API. +#[doc(hidden)] +mod memory_trait; +mod recall_policy; +mod write_gate; + +pub use kinds::MemoryKind; +pub use traits::{ObsidianFile, ObsidianRepresentable, VectorEmbeddable}; + +pub use client::{MemoryClient, MemoryClientRef, MemoryState}; +pub use factories::{ + active_embedding_signature, create_memory, create_memory_for_migration, + create_memory_with_local_ai, effective_embedding_settings, effective_memory_backend_name, +}; +pub use namespace_store::events; +pub use namespace_store::fts5; +pub use namespace_store::profile; +pub use namespace_store::segments; +pub use namespace_store::UnifiedMemory; +pub use profile_store::ProfileStore; +pub use types::{ + GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput, + NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown, + StoredMemoryDocument, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memory_store_reexports_expected_memory_kind_catalog() { + assert!(MemoryKind::ALL.contains(&MemoryKind::Chunk)); + assert!(MemoryKind::ALL.contains(&MemoryKind::Tree)); + assert!(MemoryKind::ALL.contains(&MemoryKind::Contact)); + } +} diff --git a/core/src/store/namespace_store/README.md b/core/src/store/namespace_store/README.md new file mode 100644 index 0000000..1718dcf --- /dev/null +++ b/core/src/store/namespace_store/README.md @@ -0,0 +1,54 @@ +# Namespace memory store + +Host-retained SQLite namespace/document tier. One `UnifiedMemory` struct owns +the shared connection plus the on-disk markdown sidecar and compatibility +embedding handle; the rest of this directory adds the product-owned document, +graph, episodic, event, segment, profile, and retrieval policy via `impl` +blocks. TinyCortex owns the generic chunk/vector/tree/queue engine beside this +tier; this directory is intentionally not migration staging. + +## Files + +- **`mod.rs`** — declares the `UnifiedMemory` struct (connection + paths + embedder) and wires the submodules. +- **`init.rs`** — constructor, `CREATE TABLE` bootstrap (docs, kv, graph, vector chunks, episodic FTS5, segments, events, profile), idempotent legacy-namespace migrations, plus path / namespace helpers (`sanitize_namespace`, `now_ts`, `namespace_dir`). +- **`documents.rs`** — `memory_docs` CRUD: `upsert_document` (chunks + embeds + writes markdown sidecar), `upsert_document_metadata_only` (light path), `list_documents`, `list_namespaces`, `delete_document`, `clear_namespace`. +- **`kv.rs`** — global and namespace-scoped get/set/delete/list against `kv_global` / `kv_namespace`. +- **`../../safety/`** — secret redaction/validation helpers. Document, KV, and episodic writes sanitize credentials before persistence and emit `[memory:safety]` diagnostics when a payload is rewritten. + +### Identifier canonicalization (namespace / key) + +Content and identifiers are scrubbed by **different** rules, and mixing them up +caused #5164. `safety::canonical_identifier` (namespace, KV key) and +`safety::canonical_document_key` (document key) are the single source of truth: + +- **Strict gating.** Only formatted / keyword-gated national IDs are rewritten + (`has_likely_pii`). The lenient content scrubber (`redact_pii` on its own, + used for titles/bodies/metadata) also rewrites bare digit runs, which the + scanners legitimately use as identifiers — WhatsApp JIDs, iMessage `+1…` chat + ids, timestamps, padded counters. Rewriting those maps two contacts onto one + `(namespace, key)`, and the upsert's `ON CONFLICT … DO UPDATE` then has one + contact's document overwrite the other's. +- **Symmetry.** An identifier is a storage *address*, so every path that + addresses a row canonicalizes the same way: `sanitize_namespace` (`init.rs`) + carries the namespace step for writes, reads, `query.rs`, `graph.rs`, deletes + and the on-disk `namespaces//` directory, and the by-key paths + (`upsert_document*`, `Memory::get`, `Memory::forget`, the `kv.rs` shim) go + through `canonical_document_key` / `canonical_identifier`. A read that skips + the transform silently misses the row the write created, so the caller writes + again — the unthrottled loop #5164 was reported for. +- **Never reject.** Rejecting the write instead returns an `Err` on every retry, + which is what flooded Sentry (3,055 events / 1 user / 1 day). The rejections + that remain deliberate (secret-shaped identifiers, empty keys) are demoted out + of the error stream by `ExpectedErrorKind::MemoryIdentifierRejected`. +- **`graph.rs`** — `graph_namespace` / `graph_global` upserts with attribute merging and evidence accumulation, plus namespace / global / cross-namespace queries and document-scoped relation removal. +- **`query.rs`** — hybrid retrieval. Combines graph relevance, vector similarity, keyword overlap, episodic signal and freshness; exposes `query_namespace_*` (with query) and `recall_namespace_*` (query-less) entry points used by `MemoryClient`. +- **`helpers.rs`** — shared utilities: f32-vector byte codecs, cosine similarity, markdown chunking, text/graph normalisation, JSON attribute merging, recency scoring. +- **`fts5.rs`** — FTS5 episodic memory (`episodic_log` + `episodic_fts`). `EpisodicEntry` plus `episodic_insert` / `episodic_search` / `episodic_session_entries` for the Archivist and `search_memory` tool. +- **`segments.rs`** — conversation segmentation (`conversation_segments`). Boundary detection (time gap, embedding drift, explicit markers, turn count), segment lifecycle (open → closed → summarised), and the `BoundaryConfig` knobs. +- **`events.rs`** — event extraction (`event_log` + `event_fts`). Stores typed atomic events (Fact / Decision / Commitment / Preference / Question / Foresight) extracted from closed segments via heuristic pattern matching. +- **`profile.rs`** — user profile facets (`user_profile`). Evidence-backed `FacetType` rows that accumulate across sessions; on conflict, evidence count is bumped and the value is overwritten only if confidence improves. +- **`*_tests.rs`** — module-local tests for documents, events, profile, query, segments. + +## How it fits + +`MemoryClient` (in `../client.rs`) and the `impl Memory for UnifiedMemory` in `../memory_trait.rs` are the only things that should hold a `UnifiedMemory` directly. The ingestion pipeline (`../../ingestion/`) calls `upsert_document` and `graph_upsert_namespace` after parsing; the agent harness reads via `query_namespace_*` and `recall_namespace_*`; the Archivist writes episodic turns via `fts5::episodic_insert` and segments / events / profile facets via the dedicated submodules. diff --git a/core/src/store/namespace_store/documents.rs b/core/src/store/namespace_store/documents.rs new file mode 100644 index 0000000..9c317f4 --- /dev/null +++ b/core/src/store/namespace_store/documents.rs @@ -0,0 +1,666 @@ +//! Document CRUD against the `memory_docs` table. +//! +//! Owns the upsert pipeline (with chunking + embedding), metadata-only writes +//! for high-frequency callers, list/delete/clear-namespace operations, and the +//! markdown sidecar files in `memory/namespaces//docs/`. + +use rusqlite::{params, OptionalExtension}; +use serde_json::{json, Value}; +use std::collections::BTreeSet; +use uuid::Uuid; + +use crate::store::safety; +use crate::store::types::{NamespaceDocumentInput, StoredMemoryDocument}; + +use super::UnifiedMemory; + +impl UnifiedMemory { + /// Insert or update a document by `(namespace, key)`. Writes the markdown + /// sidecar, replaces vector chunks, and embeds them with the configured + /// provider. + /// + /// **Takes already-sanitized input.** The host secret/PII write gate runs + /// in [`crate::store::write_gate`], which owns this + /// method's only call site; use `UnifiedMemory::upsert_document` instead + /// unless you are that gate. Calling this directly persists caller content + /// verbatim, credentials and all. + pub(crate) async fn upsert_document_presanitized( + &self, + input: NamespaceDocumentInput, + ) -> Result { + let namespace = Self::sanitize_namespace(&input.namespace); + let key = input.key.trim().to_string(); + if key.is_empty() { + return Err("document key cannot be empty".to_string()); + } + let existing_document_id = { + let conn = self.conn.lock(); + conn.query_row( + "SELECT document_id FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1", + params![namespace, key], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(|e| format!("lookup existing document_id: {e}"))? + }; + let document_id = input + .document_id + .or(existing_document_id) + .unwrap_or_else(|| { + let ts = Self::now_ts() as u64; + let short = &Uuid::new_v4().to_string()[..8]; + format!("{ts}_{short}") + }); + let now = Self::now_ts(); + let created_at = { + let conn = self.conn.lock(); + conn.query_row( + "SELECT created_at FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1", + params![namespace, key], + |row| row.get::<_, f64>(0), + ) + .optional() + .map_err(|e| format!("lookup existing created_at: {e}"))? + .unwrap_or(now) + }; + let updated_at = now; + let markdown_rel = self + .write_markdown_doc( + &namespace, + &document_id, + &input.title, + &input.source_type, + &input.priority, + &input.tags, + created_at, + updated_at, + &input.content, + ) + .await + .map_err(|e| e.to_string())?; + + let tags_json = serde_json::to_string(&input.tags).map_err(|e| e.to_string())?; + let metadata_json = input.metadata.to_string(); + + { + let conn = self.conn.lock(); + let tx = conn + .unchecked_transaction() + .map_err(|e| format!("begin tx: {e}"))?; + tx.execute( + "INSERT INTO memory_docs + (document_id, namespace, key, title, content, source_type, priority, tags_json, metadata_json, category, session_id, created_at, updated_at, markdown_rel_path, taint) + VALUES + (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15) + ON CONFLICT(namespace, key) DO UPDATE SET + title = excluded.title, + content = excluded.content, + source_type = excluded.source_type, + priority = excluded.priority, + tags_json = excluded.tags_json, + metadata_json = excluded.metadata_json, + category = excluded.category, + session_id = excluded.session_id, + updated_at = excluded.updated_at, + markdown_rel_path = excluded.markdown_rel_path, + taint = excluded.taint", + params![ + document_id, + namespace, + key, + input.title, + input.content, + input.source_type, + input.priority, + tags_json, + metadata_json, + input.category, + input.session_id, + created_at, + updated_at, + markdown_rel, + input.taint.as_db_str() + ], + ) + .map_err(|e| format!("upsert memory_docs: {e}"))?; + tx.execute( + "DELETE FROM vector_chunks WHERE namespace = ?1 AND document_id = ?2", + params![namespace, document_id], + ) + .map_err(|e| format!("clear vector chunks: {e}"))?; + tx.commit().map_err(|e| format!("commit tx: {e}"))?; + } + + let chunks = Self::chunk_document_content(&input.content, 225); + + // Embed every chunk in a SINGLE provider call rather than one + // round-trip per chunk. All providers implement the batch `embed` + // (`embed_one` is just a convenience wrapper around it), so a document + // that chunks into N pieces previously paid N sequential network + // round-trips on the write path; this collapses them to one. + // + // Result handling preserves the previous per-chunk resilience: + // * a failed batch (provider error) stores all chunks WITHOUT a + // vector — exactly what `embed_one(...).await.ok()` did per chunk; + // * a provider that returns fewer/empty vectors than chunks (e.g. + // `NoopEmbedding` returns an empty Vec, or a blank position from + // NaN recovery) leaves those chunks vector-less by position. + let mut embeddings: Vec>> = if chunks.is_empty() { + Vec::new() + } else { + let chunk_refs: Vec<&str> = chunks.iter().map(String::as_str).collect(); + log::debug!( + "[memory] batch-embedding {} chunk(s) for {namespace}/{document_id}", + chunk_refs.len() + ); + match self.embedder.embed(&chunk_refs).await { + Ok(vectors) => vectors + .into_iter() + .map(|v| (!v.is_empty()).then_some(v)) + .collect(), + Err(e) => { + log::warn!( + "[memory] batch embed failed for {} chunk(s) in {namespace}/{document_id}; storing without vectors: {e}", + chunks.len() + ); + Vec::new() + } + } + }; + + // Computed once; only attached to chunks that actually got a vector. + let signature = self.embedder.signature(); + for (idx, chunk) in chunks.iter().enumerate() { + // Move the vector out by position so recall can exclude vectors + // produced by a different embedding model (cross-model cosine is + // meaningless) and guard against dimension mismatches. Missing + // positions (short/empty provider result) stay vector-less. + let embedded = embeddings.get_mut(idx).and_then(Option::take); + let dim = embedded.as_ref().map(|v| v.len() as i64); + let model_signature = embedded.as_ref().map(|_| signature.clone()); + let embedding = embedded.as_ref().map(|v| Self::vec_to_bytes(v)); + let chunk_id = format!("{document_id}:{idx}"); + let conn = self.conn.lock(); + conn.execute( + "INSERT OR REPLACE INTO vector_chunks + (namespace, document_id, chunk_id, text, embedding, metadata_json, created_at, updated_at, model_signature, dim) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + namespace, + document_id, + chunk_id, + chunk, + embedding, + json!({"lancedb_table": format!("ns_{namespace}"), "chunk_index": idx}).to_string(), + now, + now, + model_signature, + dim + ], + ) + .map_err(|e| format!("insert vector chunk: {e}"))?; + } + + Ok(document_id) + } + + /// Store a document (DB row + markdown file) without chunking, embedding, + /// or graph extraction. Suitable for high-frequency, low-value writes + /// (e.g. transient sync checkpoints) where the full ingestion pipeline + /// would be too expensive. + /// + /// **Takes already-sanitized input** — same contract as + /// [`Self::upsert_document_presanitized`]; go through + /// `UnifiedMemory::upsert_document_metadata_only` instead. + pub(crate) async fn upsert_document_metadata_only_presanitized( + &self, + input: NamespaceDocumentInput, + ) -> Result { + let namespace = Self::sanitize_namespace(&input.namespace); + let key = input.key.trim().to_string(); + if key.is_empty() { + return Err("document key cannot be empty".to_string()); + } + let existing_document_id = { + let conn = self.conn.lock(); + conn.query_row( + "SELECT document_id FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1", + params![namespace, key], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(|e| format!("lookup existing document_id: {e}"))? + }; + let document_id = input + .document_id + .or(existing_document_id) + .unwrap_or_else(|| { + let ts = Self::now_ts() as u64; + let short = &Uuid::new_v4().to_string()[..8]; + format!("{ts}_{short}") + }); + let now = Self::now_ts(); + let created_at = { + let conn = self.conn.lock(); + conn.query_row( + "SELECT created_at FROM memory_docs WHERE namespace = ?1 AND key = ?2 LIMIT 1", + params![namespace, key], + |row| row.get::<_, f64>(0), + ) + .optional() + .map_err(|e| format!("lookup existing created_at: {e}"))? + .unwrap_or(now) + }; + let updated_at = now; + let markdown_rel = self + .write_markdown_doc( + &namespace, + &document_id, + &input.title, + &input.source_type, + &input.priority, + &input.tags, + created_at, + updated_at, + &input.content, + ) + .await + .map_err(|e| e.to_string())?; + + let tags_json = serde_json::to_string(&input.tags).map_err(|e| e.to_string())?; + let metadata_json = input.metadata.to_string(); + + { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO memory_docs + (document_id, namespace, key, title, content, source_type, priority, tags_json, metadata_json, category, session_id, created_at, updated_at, markdown_rel_path, taint) + VALUES + (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15) + ON CONFLICT(namespace, key) DO UPDATE SET + title = excluded.title, + content = excluded.content, + source_type = excluded.source_type, + priority = excluded.priority, + tags_json = excluded.tags_json, + metadata_json = excluded.metadata_json, + category = excluded.category, + session_id = excluded.session_id, + updated_at = excluded.updated_at, + markdown_rel_path = excluded.markdown_rel_path, + taint = excluded.taint", + params![ + document_id, + namespace, + key, + input.title, + input.content, + input.source_type, + input.priority, + tags_json, + metadata_json, + input.category, + input.session_id, + created_at, + updated_at, + markdown_rel, + input.taint.as_db_str() + ], + ) + .map_err(|e| format!("upsert memory_docs: {e}"))?; + } + + Ok(document_id) + } + + /// Fetch a single document by `(namespace, key)`. + /// + /// The same SELECT as [`Self::load_documents_for_scope`] with a `key` + /// predicate bolted on — deliberately *not* implemented as + /// `load_documents_for_scope(ns).find(…)`, which would load every document + /// body in the namespace to return one. + /// + /// `key` goes through [`safety::canonical_document_key`], the exact + /// transform `upsert_document` applies before writing the column. Reading + /// the raw key here would reproduce #5164: the lookup misses, the caller + /// treats the row as absent, and writes it again. + pub(crate) async fn get_document_by_key( + &self, + namespace: &str, + key: &str, + ) -> Result, String> { + let conn = self.conn.lock(); + let ns = Self::sanitize_namespace(namespace); + let key = safety::canonical_document_key(key); + let mut stmt = conn + .prepare( + "SELECT + document_id, + namespace, + key, + title, + content, + source_type, + priority, + tags_json, + metadata_json, + category, + session_id, + created_at, + updated_at, + markdown_rel_path, + taint + FROM memory_docs + WHERE namespace = ?1 AND key = ?2 + LIMIT 1", + ) + .map_err(|e| format!("prepare get_document_by_key: {e}"))?; + let mut rows = stmt + .query(params![ns, key]) + .map_err(|e| format!("query get_document_by_key: {e}"))?; + let Some(row) = rows + .next() + .map_err(|e| format!("row get_document_by_key: {e}"))? + else { + return Ok(None); + }; + let tags_json: String = row.get(7).map_err(|e| e.to_string())?; + let metadata_json: String = row.get(8).map_err(|e| e.to_string())?; + let taint_str: String = row.get(14).map_err(|e| e.to_string())?; + Ok(Some(StoredMemoryDocument { + document_id: row.get(0).map_err(|e| e.to_string())?, + namespace: row.get(1).map_err(|e| e.to_string())?, + key: row.get(2).map_err(|e| e.to_string())?, + title: row.get(3).map_err(|e| e.to_string())?, + content: row.get(4).map_err(|e| e.to_string())?, + source_type: row.get(5).map_err(|e| e.to_string())?, + priority: row.get(6).map_err(|e| e.to_string())?, + tags: serde_json::from_str(&tags_json).unwrap_or_default(), + metadata: serde_json::from_str(&metadata_json).unwrap_or_else(|_| json!({})), + category: row.get(9).map_err(|e| e.to_string())?, + session_id: row.get(10).map_err(|e| e.to_string())?, + created_at: row.get(11).map_err(|e| e.to_string())?, + updated_at: row.get(12).map_err(|e| e.to_string())?, + markdown_rel_path: row.get(13).map_err(|e| e.to_string())?, + taint: crate::MemoryTaint::from_db_str(&taint_str), + })) + } + + pub(crate) async fn load_documents_for_scope( + &self, + namespace: &str, + ) -> Result, String> { + let conn = self.conn.lock(); + let ns = Self::sanitize_namespace(namespace); + let mut stmt = conn + .prepare( + "SELECT + document_id, + namespace, + key, + title, + content, + source_type, + priority, + tags_json, + metadata_json, + category, + session_id, + created_at, + updated_at, + markdown_rel_path, + taint + FROM memory_docs + WHERE namespace = ?1 + ORDER BY updated_at DESC", + ) + .map_err(|e| format!("prepare load_documents_for_scope: {e}"))?; + let mut rows = stmt + .query(params![ns]) + .map_err(|e| format!("query load_documents_for_scope: {e}"))?; + let mut docs = Vec::new(); + while let Some(row) = rows + .next() + .map_err(|e| format!("row load_documents_for_scope: {e}"))? + { + let tags_json: String = row.get(7).map_err(|e| e.to_string())?; + let metadata_json: String = row.get(8).map_err(|e| e.to_string())?; + // The `taint` column has a NOT NULL DEFAULT 'internal' clause + // from the migration, so legacy rows that pre-date the column + // surface as "internal" string and round-trip back to + // `MemoryTaint::Internal`. Unknown / corrupted values fail + // closed to `MemoryTaint::ExternalSync` inside `from_db_str`, + // so a forward-rolled schema variant or a bad UPDATE can't + // silently downgrade a row to user-authored content. + let taint_str: String = row.get(14).map_err(|e| e.to_string())?; + let taint = crate::MemoryTaint::from_db_str(&taint_str); + docs.push(StoredMemoryDocument { + document_id: row.get(0).map_err(|e| e.to_string())?, + namespace: row.get(1).map_err(|e| e.to_string())?, + key: row.get(2).map_err(|e| e.to_string())?, + title: row.get(3).map_err(|e| e.to_string())?, + content: row.get(4).map_err(|e| e.to_string())?, + source_type: row.get(5).map_err(|e| e.to_string())?, + priority: row.get(6).map_err(|e| e.to_string())?, + tags: serde_json::from_str(&tags_json).unwrap_or_default(), + metadata: serde_json::from_str(&metadata_json).unwrap_or_else(|_| json!({})), + category: row.get(9).map_err(|e| e.to_string())?, + session_id: row.get(10).map_err(|e| e.to_string())?, + created_at: row.get(11).map_err(|e| e.to_string())?, + updated_at: row.get(12).map_err(|e| e.to_string())?, + markdown_rel_path: row.get(13).map_err(|e| e.to_string())?, + taint, + }); + } + Ok(docs) + } + + /// List documents in a namespace, or across all namespaces when `None`. + /// Returns `{ "documents": [...], "count": N }` JSON. + pub async fn list_documents(&self, namespace: Option<&str>) -> Result { + let conn = self.conn.lock(); + let mut docs = Vec::new(); + if let Some(ns) = namespace { + let mut stmt = conn + .prepare( + "SELECT document_id, namespace, key, title, source_type, priority, created_at, updated_at, taint + FROM memory_docs WHERE namespace = ?1 ORDER BY updated_at DESC", + ) + .map_err(|e| format!("prepare list_documents: {e}"))?; + let mut rows = stmt + .query(params![Self::sanitize_namespace(ns)]) + .map_err(|e| format!("query list_documents: {e}"))?; + while let Some(row) = rows + .next() + .map_err(|e| format!("row list_documents: {e}"))? + { + docs.push(json!({ + "documentId": row.get::<_, String>(0).map_err(|e| e.to_string())?, + "namespace": row.get::<_, String>(1).map_err(|e| e.to_string())?, + "key": row.get::<_, String>(2).map_err(|e| e.to_string())?, + "title": row.get::<_, String>(3).map_err(|e| e.to_string())?, + "sourceType": row.get::<_, String>(4).map_err(|e| e.to_string())?, + "priority": row.get::<_, String>(5).map_err(|e| e.to_string())?, + "createdAt": row.get::<_, f64>(6).map_err(|e| e.to_string())?, + "updatedAt": row.get::<_, f64>(7).map_err(|e| e.to_string())?, + "taint": row.get::<_, String>(8).map_err(|e| e.to_string())?, + })); + } + } else { + let mut stmt = conn + .prepare( + "SELECT document_id, namespace, key, title, source_type, priority, created_at, updated_at, taint + FROM memory_docs ORDER BY updated_at DESC", + ) + .map_err(|e| format!("prepare list_documents: {e}"))?; + let mut rows = stmt + .query([]) + .map_err(|e| format!("query list_documents: {e}"))?; + while let Some(row) = rows + .next() + .map_err(|e| format!("row list_documents: {e}"))? + { + docs.push(json!({ + "documentId": row.get::<_, String>(0).map_err(|e| e.to_string())?, + "namespace": row.get::<_, String>(1).map_err(|e| e.to_string())?, + "key": row.get::<_, String>(2).map_err(|e| e.to_string())?, + "title": row.get::<_, String>(3).map_err(|e| e.to_string())?, + "sourceType": row.get::<_, String>(4).map_err(|e| e.to_string())?, + "priority": row.get::<_, String>(5).map_err(|e| e.to_string())?, + "createdAt": row.get::<_, f64>(6).map_err(|e| e.to_string())?, + "updatedAt": row.get::<_, f64>(7).map_err(|e| e.to_string())?, + "taint": row.get::<_, String>(8).map_err(|e| e.to_string())?, + })); + } + } + Ok(json!({ "documents": docs, "count": docs.len() })) + } + + /// Return every distinct namespace that has at least one document. + pub async fn list_namespaces(&self) -> Result, String> { + let conn = self.conn.lock(); + let mut stmt = conn + .prepare("SELECT DISTINCT namespace FROM memory_docs ORDER BY namespace") + .map_err(|e| format!("prepare list_namespaces: {e}"))?; + let mut rows = stmt + .query([]) + .map_err(|e| format!("query list_namespaces: {e}"))?; + let mut out = BTreeSet::new(); + while let Some(row) = rows + .next() + .map_err(|e| format!("row list_namespaces: {e}"))? + { + let ns: String = row.get(0).map_err(|e| e.to_string())?; + if !ns.trim().is_empty() { + out.insert(ns); + } + } + Ok(out.into_iter().collect()) + } + + /// Delete all documents, vector chunks, KV entries, and graph relations + /// for the given namespace in a single transaction. Also removes the + /// on-disk markdown directory (`namespaces/{ns}/docs/`). + pub async fn clear_namespace(&self, namespace: &str) -> Result<(), String> { + let ns = Self::sanitize_namespace(namespace); + log::debug!("[memory] clear_namespace: starting for namespace={ns}"); + + { + let conn = self.conn.lock(); + let tx = conn + .unchecked_transaction() + .map_err(|e| format!("clear_namespace begin tx: {e}"))?; + + let doc_count = tx + .execute( + "DELETE FROM memory_docs WHERE namespace = ?1", + rusqlite::params![ns], + ) + .map_err(|e| format!("clear_namespace delete memory_docs: {e}"))?; + log::debug!("[memory] clear_namespace: deleted {doc_count} rows from memory_docs"); + + let chunk_count = tx + .execute( + "DELETE FROM vector_chunks WHERE namespace = ?1", + rusqlite::params![ns], + ) + .map_err(|e| format!("clear_namespace delete vector_chunks: {e}"))?; + log::debug!("[memory] clear_namespace: deleted {chunk_count} rows from vector_chunks"); + + let kv_count = tx + .execute( + "DELETE FROM kv_namespace WHERE namespace = ?1", + rusqlite::params![ns], + ) + .map_err(|e| format!("clear_namespace delete kv_namespace: {e}"))?; + log::debug!("[memory] clear_namespace: deleted {kv_count} rows from kv_namespace"); + + let graph_count = tx + .execute( + "DELETE FROM graph_namespace WHERE namespace = ?1", + rusqlite::params![ns], + ) + .map_err(|e| format!("clear_namespace delete graph_namespace: {e}"))?; + log::debug!( + "[memory] clear_namespace: deleted {graph_count} rows from graph_namespace" + ); + + tx.commit() + .map_err(|e| format!("clear_namespace commit tx: {e}"))?; + } + + // Remove on-disk markdown files for this namespace. + let docs_dir = self.namespace_dir(&ns).join("docs"); + if docs_dir.exists() { + tokio::fs::remove_dir_all(&docs_dir).await.map_err(|e| { + format!( + "clear_namespace remove docs dir {}: {e}", + docs_dir.display() + ) + })?; + log::debug!( + "[memory] clear_namespace: removed docs directory {}", + docs_dir.display() + ); + } + + log::debug!("[memory] clear_namespace: completed for namespace={ns}"); + Ok(()) + } + + /// Delete a single document plus its vector chunks, graph relations, and + /// markdown sidecar. Returns `{ "deleted": bool, "namespace", "documentId" }`. + pub async fn delete_document( + &self, + namespace: &str, + document_id: &str, + ) -> Result { + let ns = Self::sanitize_namespace(namespace); + let rel_path: Option = { + let conn = self.conn.lock(); + conn.query_row( + "SELECT markdown_rel_path FROM memory_docs WHERE namespace = ?1 AND document_id = ?2", + params![ns, document_id], + |row| row.get(0), + ) + .optional() + .map_err(|e| format!("query delete_document path: {e}"))? + }; + + self.graph_remove_document_namespace(&ns, document_id) + .await?; + + let deleted = { + let conn = self.conn.lock(); + let deleted = conn + .execute( + "DELETE FROM memory_docs WHERE namespace = ?1 AND document_id = ?2", + params![ns, document_id], + ) + .map_err(|e| format!("delete memory_doc: {e}"))? + > 0; + conn.execute( + "DELETE FROM vector_chunks WHERE namespace = ?1 AND document_id = ?2", + params![ns, document_id], + ) + .map_err(|e| format!("delete vector_chunks: {e}"))?; + deleted + }; + + if let Some(rel) = rel_path { + let abs = self.workspace_dir.join(rel); + // Surface non-NotFound failures so storage drift between the DB + // row and the markdown sidecar is diagnosable. + if let Err(e) = tokio::fs::remove_file(&abs).await { + if e.kind() != std::io::ErrorKind::NotFound { + log::warn!("[memory] failed to remove sidecar {}: {e}", abs.display()); + } + } + } + Ok(json!({"deleted": deleted, "namespace": ns, "documentId": document_id })) + } +} + +#[cfg(test)] +#[path = "documents_tests.rs"] +mod tests; diff --git a/core/src/store/namespace_store/documents_tests.rs b/core/src/store/namespace_store/documents_tests.rs new file mode 100644 index 0000000..7b2f7ea --- /dev/null +++ b/core/src/store/namespace_store/documents_tests.rs @@ -0,0 +1,1508 @@ +//! Tests for the `documents` module — upsert / list / delete / clear-namespace. + +use std::sync::Arc; + +use serde_json::json; +use tempfile::TempDir; + +use crate::store::{NamespaceDocumentInput, UnifiedMemory}; +use tinymemory_api::host::NoopEmbedding; + +fn make_doc_input( + namespace: &str, + key: &str, + title: &str, + content: &str, +) -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: namespace.to_string(), + key: key.to_string(), + title: title.to_string(), + content: content.to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + } +} + +fn count_vector_chunks(memory: &UnifiedMemory, namespace: &str, document_id: &str) -> i64 { + let conn = memory.conn.lock(); + conn.query_row( + "SELECT COUNT(*) FROM vector_chunks WHERE namespace = ?1 AND document_id = ?2", + rusqlite::params![UnifiedMemory::sanitize_namespace(namespace), document_id], + |row| row.get(0), + ) + .unwrap() +} + +#[tokio::test] +async fn list_documents_without_namespace_returns_all_docs_across_namespaces() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input("test:one", "doc-a", "Doc A", "A body")) + .await + .unwrap(); + memory + .upsert_document(make_doc_input("test:two", "doc-b", "Doc B", "B body")) + .await + .unwrap(); + + let docs = memory.list_documents(None).await.unwrap(); + assert_eq!(docs["count"].as_u64().unwrap(), 2); + let namespaces: std::collections::BTreeSet<_> = docs["documents"] + .as_array() + .unwrap() + .iter() + .filter_map(|doc| doc["namespace"].as_str()) + .collect(); + assert!(namespaces.contains("test_one")); + assert!(namespaces.contains("test_two")); +} + +#[tokio::test] +async fn list_namespaces_returns_distinct_sorted_sanitized_namespaces() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input("team alpha/#1", "doc-a", "Doc A", "A body")) + .await + .unwrap(); + memory + .upsert_document(make_doc_input("team alpha/#1", "doc-b", "Doc B", "B body")) + .await + .unwrap(); + memory + .upsert_document(make_doc_input("zeta", "doc-c", "Doc C", "C body")) + .await + .unwrap(); + + let namespaces = memory.list_namespaces().await.unwrap(); + assert_eq!( + namespaces, + vec!["team_alpha/_1".to_string(), "zeta".to_string()] + ); +} + +#[tokio::test] +async fn list_documents_with_namespace_filters_by_sanitized_namespace_and_orders_newest_first() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input( + "team alpha/#1", + "doc-a", + "Older Doc", + "A body", + )) + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + memory + .upsert_document(make_doc_input( + "team alpha/#1", + "doc-b", + "Newer Doc", + "B body", + )) + .await + .unwrap(); + memory + .upsert_document(make_doc_input("other", "doc-c", "Other Doc", "C body")) + .await + .unwrap(); + + let docs = memory.list_documents(Some("team alpha/#1")).await.unwrap(); + let documents = docs["documents"].as_array().unwrap(); + + assert_eq!(docs["count"].as_u64().unwrap(), 2); + assert_eq!(documents[0]["namespace"], json!("team_alpha/_1")); + assert_eq!(documents[0]["key"], json!("doc-b")); + assert_eq!(documents[1]["key"], json!("doc-a")); +} + +#[tokio::test] +async fn load_documents_for_scope_defaults_invalid_json_fields_from_persisted_rows() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let namespace = UnifiedMemory::sanitize_namespace("broken/json"); + + { + let conn = memory.conn.lock(); + conn.execute( + "INSERT INTO memory_docs + (document_id, namespace, key, title, content, source_type, priority, tags_json, metadata_json, category, session_id, created_at, updated_at, markdown_rel_path) + VALUES + (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + rusqlite::params![ + "doc-invalid-json", + namespace, + "doc-a", + "Doc A", + "Body", + "doc", + "medium", + "{not json", + "also not json", + "core", + Option::::None, + 10.0_f64, + 20.0_f64, + "memory/namespaces/broken_json/docs/doc-invalid-json.md" + ], + ) + .unwrap(); + } + + let docs = memory + .load_documents_for_scope("broken/json") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + assert!( + docs[0].tags.is_empty(), + "invalid tags_json should fall back to []" + ); + assert_eq!( + docs[0].metadata, + json!({}), + "invalid metadata_json should fall back to an empty object" + ); +} + +#[tokio::test] +async fn upsert_document_metadata_only_reuses_document_id_for_same_namespace_and_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let first_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta", + "doc-a", + "Doc A", + "Initial body", + )) + .await + .unwrap(); + let second_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta", + "doc-a", + "Doc A v2", + "Updated body", + )) + .await + .unwrap(); + + assert_eq!( + first_id, second_id, + "metadata-only upsert should reuse the document id" + ); + let docs = memory.load_documents_for_scope("test:meta").await.unwrap(); + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].document_id, first_id); + assert_eq!(docs[0].title, "Doc A v2"); + assert_eq!(docs[0].content, "Updated body"); + assert_eq!( + count_vector_chunks(&memory, "test:meta", &first_id), + 0, + "metadata-only writes must not enqueue vector chunks" + ); +} + +#[tokio::test] +async fn upsert_document_metadata_only_preserves_created_at_and_rewrites_sidecar() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let first_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta-sidecar", + "doc-a", + "Doc A", + "Initial body", + )) + .await + .unwrap(); + let first_doc = memory + .load_documents_for_scope("test:meta-sidecar") + .await + .unwrap()[0] + .clone(); + let sidecar = tmp.path().join(&first_doc.markdown_rel_path); + let first_markdown = std::fs::read_to_string(&sidecar).unwrap(); + assert!(first_markdown.contains("Initial body")); + + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + + let second_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta-sidecar", + "doc-a", + "Doc A v2", + "Updated body", + )) + .await + .unwrap(); + + assert_eq!(first_id, second_id); + let updated_doc = memory + .load_documents_for_scope("test:meta-sidecar") + .await + .unwrap()[0] + .clone(); + assert_eq!(updated_doc.created_at, first_doc.created_at); + assert!(updated_doc.updated_at >= first_doc.updated_at); + let updated_markdown = std::fs::read_to_string(sidecar).unwrap(); + assert!(updated_markdown.contains("Updated body")); + assert!(updated_markdown.contains("Doc A v2")); +} + +#[tokio::test] +async fn upsert_document_metadata_only_over_existing_document_preserves_vector_chunks() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(make_doc_input( + "test:meta-preserve-chunks", + "doc-a", + "Doc A", + &"alpha ".repeat(400), + )) + .await + .unwrap(); + let original_chunk_count = + count_vector_chunks(&memory, "test:meta-preserve-chunks", &document_id); + assert!(original_chunk_count > 0); + + let updated_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta-preserve-chunks", + "doc-a", + "Doc A v2", + "Updated body without re-embedding", + )) + .await + .unwrap(); + + assert_eq!(updated_id, document_id); + let docs = memory + .load_documents_for_scope("test:meta-preserve-chunks") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].content, "Updated body without re-embedding"); + assert_eq!( + count_vector_chunks(&memory, "test:meta-preserve-chunks", &document_id), + original_chunk_count, + "metadata-only writes should not delete existing semantic chunks" + ); +} + +#[tokio::test] +async fn upsert_document_after_metadata_only_reuses_document_id_and_adds_vector_chunks() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let metadata_only_id = memory + .upsert_document_metadata_only(make_doc_input( + "test:meta-then-full", + "doc-a", + "Doc A", + "Short body", + )) + .await + .unwrap(); + assert_eq!( + count_vector_chunks(&memory, "test:meta-then-full", &metadata_only_id), + 0 + ); + + let full_id = memory + .upsert_document(make_doc_input( + "test:meta-then-full", + "doc-a", + "Doc A Embedded", + &"beta ".repeat(400), + )) + .await + .unwrap(); + + assert_eq!(full_id, metadata_only_id); + let docs = memory + .load_documents_for_scope("test:meta-then-full") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].title, "Doc A Embedded"); + assert!( + count_vector_chunks(&memory, "test:meta-then-full", &full_id) > 0, + "full upsert should backfill chunks for a metadata-only document" + ); +} + +#[tokio::test] +async fn upsert_document_writes_vector_chunks_for_chunked_content() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let long_body = "alpha ".repeat(400); + let document_id = memory + .upsert_document(make_doc_input("test:vector", "doc-a", "Doc A", &long_body)) + .await + .unwrap(); + + assert!( + count_vector_chunks(&memory, "test:vector", &document_id) > 0, + "full document upsert should replace vector chunks for semantic retrieval" + ); +} + +/// Embedder that records how many times `embed` is invoked and returns one +/// fixed-dimension vector per input text. Used to prove `upsert_document` +/// embeds all chunks in a SINGLE batch call rather than one call per chunk. +struct CountingEmbedder { + calls: std::sync::atomic::AtomicUsize, +} + +#[async_trait::async_trait] +impl tinymemory_api::host::EmbeddingProvider for CountingEmbedder { + fn name(&self) -> &str { + "counting" + } + + fn model_id(&self) -> &str { + "counting-test" + } + + fn dimensions(&self) -> usize { + 3 + } + + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(texts.iter().map(|_| vec![0.1, 0.2, 0.3]).collect()) + } +} + +#[tokio::test] +async fn upsert_document_batch_embeds_all_chunks_in_one_call() { + let tmp = TempDir::new().unwrap(); + let embedder = Arc::new(CountingEmbedder { + calls: std::sync::atomic::AtomicUsize::new(0), + }); + let memory = UnifiedMemory::new(tmp.path(), embedder.clone(), None).unwrap(); + + // Long enough to chunk into several pieces (chunk size is 225 chars). + let long_body = "alpha ".repeat(400); + let document_id = memory + .upsert_document(make_doc_input("test:batch", "doc-a", "Doc A", &long_body)) + .await + .unwrap(); + + let chunk_count = count_vector_chunks(&memory, "test:batch", &document_id); + assert!( + chunk_count >= 3, + "test body should chunk into >=3 pieces, got {chunk_count}" + ); + assert_eq!( + embedder.calls.load(std::sync::atomic::Ordering::SeqCst), + 1, + "all chunks must be embedded in a single batch call, not one call per chunk" + ); +} + +#[tokio::test] +async fn upsert_document_reuses_document_id_preserves_created_at_and_replaces_vector_chunks() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let first_id = memory + .upsert_document(make_doc_input( + "test:update", + "doc-a", + "Doc A", + &"alpha ".repeat(400), + )) + .await + .unwrap(); + let first_doc = memory + .load_documents_for_scope("test:update") + .await + .unwrap()[0] + .clone(); + let first_chunk_count = count_vector_chunks(&memory, "test:update", &first_id); + assert!(first_chunk_count > 0); + + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + + let second_id = memory + .upsert_document(make_doc_input( + "test:update", + "doc-a", + "Doc A v2", + &"beta ".repeat(40), + )) + .await + .unwrap(); + + assert_eq!( + first_id, second_id, + "upsert should reuse the existing document id" + ); + let updated_doc = memory + .load_documents_for_scope("test:update") + .await + .unwrap()[0] + .clone(); + assert_eq!(updated_doc.document_id, first_id); + assert_eq!(updated_doc.created_at, first_doc.created_at); + assert!(updated_doc.updated_at >= first_doc.updated_at); + assert_eq!(updated_doc.title, "Doc A v2"); + assert_eq!(updated_doc.content, "beta ".repeat(40)); + let second_chunk_count = count_vector_chunks(&memory, "test:update", &second_id); + assert!(second_chunk_count > 0); + assert!( + second_chunk_count <= first_chunk_count, + "replacing with shorter content should not leave stale vector chunks behind" + ); +} + +#[tokio::test] +async fn delete_document_removes_doc_sidecar_and_is_idempotent() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(make_doc_input("test:delete", "doc-a", "Doc A", "Delete me")) + .await + .unwrap(); + + let docs = memory + .load_documents_for_scope("test:delete") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + let sidecar = tmp.path().join(&docs[0].markdown_rel_path); + assert!(sidecar.exists(), "sidecar should exist before delete"); + + memory + .graph_upsert_namespace( + "test:delete", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_id": document_id.clone(), + "chunk_id": format!("{document_id}:0") + }), + ) + .await + .unwrap(); + + let deleted = memory + .delete_document("test:delete", &document_id) + .await + .unwrap(); + assert_eq!(deleted["deleted"], json!(true)); + assert_eq!(deleted["documentId"], json!(document_id.clone())); + assert!(!sidecar.exists(), "sidecar should be removed on delete"); + assert!(memory + .load_documents_for_scope("test:delete") + .await + .unwrap() + .is_empty()); + assert!( + memory + .graph_relations_namespace("test:delete", None, None) + .await + .unwrap() + .is_empty(), + "document-linked graph relations should be pruned" + ); + + let second = memory + .delete_document("test:delete", &document_id) + .await + .unwrap(); + assert_eq!(second["deleted"], json!(false)); + assert_eq!(second["documentId"], json!(document_id)); +} + +#[tokio::test] +async fn delete_document_succeeds_when_sidecar_is_already_missing() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(make_doc_input( + "test:delete-missing-sidecar", + "doc-a", + "Doc A", + "Delete me", + )) + .await + .unwrap(); + + let docs = memory + .load_documents_for_scope("test:delete-missing-sidecar") + .await + .unwrap(); + assert_eq!(docs.len(), 1); + let sidecar = tmp.path().join(&docs[0].markdown_rel_path); + assert!(sidecar.exists()); + std::fs::remove_file(&sidecar).unwrap(); + assert!(!sidecar.exists()); + + let deleted = memory + .delete_document("test:delete-missing-sidecar", &document_id) + .await + .unwrap(); + assert_eq!(deleted["deleted"], json!(true)); + assert!(memory + .load_documents_for_scope("test:delete-missing-sidecar") + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn delete_document_accepts_unsanitized_namespace_and_removes_chunks() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(make_doc_input( + "Team Alpha/#1", + "doc-a", + "Doc A", + &"delete ".repeat(300), + )) + .await + .unwrap(); + assert!(count_vector_chunks(&memory, "Team Alpha/#1", &document_id) > 0); + + let deleted = memory + .delete_document("Team Alpha/#1", &document_id) + .await + .unwrap(); + assert_eq!(deleted["deleted"], json!(true)); + assert_eq!(deleted["namespace"], json!("Team_Alpha/_1")); + assert_eq!( + count_vector_chunks(&memory, "Team Alpha/#1", &document_id), + 0 + ); + assert!(memory + .load_documents_for_scope("Team Alpha/#1") + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn clear_namespace_removes_all_data_and_preserves_other_namespaces() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // --- Populate "test:cleanup" namespace --- + + // 3 documents + memory + .upsert_document(make_doc_input( + "test:cleanup", + "doc-a", + "Document A", + "Content of document A for cleanup.", + )) + .await + .unwrap(); + memory + .upsert_document(make_doc_input( + "test:cleanup", + "doc-b", + "Document B", + "Content of document B for cleanup.", + )) + .await + .unwrap(); + memory + .upsert_document(make_doc_input( + "test:cleanup", + "doc-c", + "Document C", + "Content of document C for cleanup.", + )) + .await + .unwrap(); + + // 2 KV entries + memory + .kv_set_namespace("test:cleanup", "pref-1", &json!({"theme": "dark"})) + .await + .unwrap(); + memory + .kv_set_namespace("test:cleanup", "pref-2", &json!({"lang": "en"})) + .await + .unwrap(); + + // 2 graph relations + memory + .graph_upsert_namespace( + "test:cleanup", + "Alice", + "knows", + "Bob", + &json!({"source": "test"}), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "test:cleanup", + "Bob", + "works_at", + "Acme", + &json!({"source": "test"}), + ) + .await + .unwrap(); + + // --- Populate "test:other" namespace (control) --- + + memory + .upsert_document(make_doc_input( + "test:other", + "other-doc", + "Other Document", + "Content of document in the other namespace.", + )) + .await + .unwrap(); + memory + .kv_set_namespace("test:other", "other-key", &json!({"value": true})) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "test:other", + "X", + "relates_to", + "Y", + &json!({"source": "other"}), + ) + .await + .unwrap(); + + // --- Verify pre-conditions --- + + let cleanup_docs = memory.list_documents(Some("test:cleanup")).await.unwrap(); + assert_eq!( + cleanup_docs["count"].as_u64().unwrap(), + 3, + "test:cleanup should have 3 documents before clear" + ); + + let cleanup_kv = memory.kv_list_namespace("test:cleanup").await.unwrap(); + assert_eq!( + cleanup_kv.len(), + 2, + "test:cleanup should have 2 KV entries before clear" + ); + + let cleanup_graph = memory + .graph_relations_namespace("test:cleanup", None, None) + .await + .unwrap(); + assert_eq!( + cleanup_graph.len(), + 2, + "test:cleanup should have 2 graph relations before clear" + ); + + let other_docs = memory.list_documents(Some("test:other")).await.unwrap(); + assert_eq!( + other_docs["count"].as_u64().unwrap(), + 1, + "test:other should have 1 document before clear" + ); + + // --- Execute clear_namespace --- + + memory.clear_namespace("test:cleanup").await.unwrap(); + + // --- Assert: "test:cleanup" is empty --- + + let cleanup_docs_after = memory.list_documents(Some("test:cleanup")).await.unwrap(); + assert_eq!( + cleanup_docs_after["count"].as_u64().unwrap(), + 0, + "test:cleanup documents should be empty after clear" + ); + + let cleanup_kv_after = memory.kv_list_namespace("test:cleanup").await.unwrap(); + assert!( + cleanup_kv_after.is_empty(), + "test:cleanup KV entries should be empty after clear" + ); + + let cleanup_graph_after = memory + .graph_relations_namespace("test:cleanup", None, None) + .await + .unwrap(); + assert!( + cleanup_graph_after.is_empty(), + "test:cleanup graph relations should be empty after clear" + ); + + // --- Assert: "test:other" is untouched (critical) --- + + let other_docs_after = memory.list_documents(Some("test:other")).await.unwrap(); + assert_eq!( + other_docs_after["count"].as_u64().unwrap(), + 1, + "test:other document must still exist after clearing test:cleanup" + ); + + let other_kv_after = memory.kv_list_namespace("test:other").await.unwrap(); + assert_eq!( + other_kv_after.len(), + 1, + "test:other KV entry must still exist after clearing test:cleanup" + ); + + let other_graph_after = memory + .graph_relations_namespace("test:other", None, None) + .await + .unwrap(); + assert_eq!( + other_graph_after.len(), + 1, + "test:other graph relation must still exist after clearing test:cleanup" + ); +} + +#[tokio::test] +async fn clear_namespace_on_empty_namespace_is_noop() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // Clearing a namespace that has never been used should succeed without error. + memory.clear_namespace("nonexistent").await.unwrap(); + + let docs = memory.list_documents(Some("nonexistent")).await.unwrap(); + assert_eq!(docs["count"].as_u64().unwrap(), 0); +} + +#[tokio::test] +async fn clear_namespace_removes_on_disk_markdown_files() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input( + "test:diskcheck", + "disk-doc", + "Disk Doc", + "This doc has a markdown file on disk.", + )) + .await + .unwrap(); + + let docs_dir = tmp + .path() + .join("memory") + .join("namespaces") + .join("test_diskcheck") + .join("docs"); + assert!( + docs_dir.exists(), + "docs directory should exist after upsert" + ); + + memory.clear_namespace("test:diskcheck").await.unwrap(); + + assert!( + !docs_dir.exists(), + "docs directory should be removed after clear_namespace" + ); +} + +#[tokio::test] +async fn clear_namespace_accepts_unsanitized_namespace_and_removes_sanitized_docs_dir() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input( + "Team Alpha/#1", + "doc-a", + "Doc A", + "Namespace cleanup body", + )) + .await + .unwrap(); + memory + .kv_set_namespace("Team Alpha/#1", "pref-1", &json!({"theme": "dark"})) + .await + .unwrap(); + + let docs_dir = tmp + .path() + .join("memory") + .join("namespaces") + .join("Team_Alpha/_1") + .join("docs"); + assert!(docs_dir.exists()); + + memory.clear_namespace("Team Alpha/#1").await.unwrap(); + + assert!(memory + .load_documents_for_scope("Team Alpha/#1") + .await + .unwrap() + .is_empty()); + assert!(memory + .kv_list_namespace("Team Alpha/#1") + .await + .unwrap() + .is_empty()); + assert!(!docs_dir.exists()); +} + +#[tokio::test] +async fn list_namespaces_skips_blank_rows_inserted_outside_normal_writes() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + { + let conn = memory.conn.lock(); + conn.execute( + "INSERT INTO memory_docs + (document_id, namespace, key, title, content, source_type, priority, tags_json, metadata_json, category, session_id, created_at, updated_at, markdown_rel_path) + VALUES + (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + rusqlite::params![ + "doc-blank-ns", + " ", + "doc-a", + "Doc A", + "Body", + "doc", + "medium", + "[]", + "{}", + "core", + Option::::None, + 10.0_f64, + 20.0_f64, + "memory/namespaces/blank/docs/doc-blank-ns.md" + ], + ) + .unwrap(); + } + memory + .upsert_document(make_doc_input("valid/ns", "doc-b", "Doc B", "Body")) + .await + .unwrap(); + + let namespaces = memory.list_namespaces().await.unwrap(); + assert_eq!(namespaces, vec!["valid/ns".to_string()]); +} + +#[tokio::test] +async fn upsert_document_redacts_secret_like_content_before_persisting() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(NamespaceDocumentInput { + namespace: "safe".to_string(), + key: "secret-note".to_string(), + title: "Bearer abcdefghijklmnop".to_string(), + content: "token=abc123\n-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----" + .to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec!["sk-1234567890123456789012345".to_string()], + metadata: json!({ + "token": "raw", + "notes": "api_key=really-secret" + }), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .unwrap(); + + let docs = memory.load_documents_for_scope("safe").await.unwrap(); + assert_eq!(docs.len(), 1); + let doc = &docs[0]; + assert!(!doc.title.contains("abcdefghijklmnop")); + assert!(doc.title.contains("[REDACTED]")); + assert!(!doc.content.contains("BEGIN PRIVATE KEY")); + assert!(doc.content.contains("[REDACTED_PRIVATE_KEY]")); + assert_eq!(doc.metadata["token"], json!("[REDACTED_SECRET]")); + assert_eq!(doc.metadata["notes"], json!("api_key=[REDACTED]")); + assert_eq!(doc.tags[0], "[REDACTED]"); + + let markdown = std::fs::read_to_string(tmp.path().join(&doc.markdown_rel_path)).unwrap(); + assert!(!markdown.contains("BEGIN PRIVATE KEY")); + assert!(markdown.contains("[REDACTED_PRIVATE_KEY]")); +} + +#[tokio::test] +async fn kv_set_namespace_redacts_secret_like_payloads() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .kv_set_namespace( + "safe", + "key-1", + &json!({ + "token": "super-secret", + "note": "Bearer abcdefghijklmnop" + }), + ) + .await + .unwrap(); + + let rows = memory.kv_list_namespace("safe").await.unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["key"], json!("key-1")); + assert_eq!(rows[0]["value"]["token"], json!("[REDACTED_SECRET]")); + assert_eq!(rows[0]["value"]["note"], json!("Bearer [REDACTED]")); +} + +#[tokio::test] +async fn kv_set_namespace_rejects_secret_like_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let err = memory + .kv_set_namespace( + "safe", + "api_key=sk-1234567890123456789012345", + &json!({"value": "ok"}), + ) + .await + .expect_err("secret-like key should be rejected"); + assert!(err.contains("cannot contain secrets")); +} + +#[tokio::test] +async fn kv_set_namespace_rejects_secret_like_namespace() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let err = memory + .kv_set_namespace( + "Bearer abcdefghijklmnop", + "safe-key", + &json!({"value": "ok"}), + ) + .await + .expect_err("secret-like namespace should be rejected"); + assert!(err.contains("cannot contain secrets")); +} + +#[tokio::test] +async fn kv_set_global_rejects_secret_like_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let err = memory + .kv_set_global( + "authorization=Bearer abcdefghijklmnop", + &json!({"value": "ok"}), + ) + .await + .expect_err("secret-like global key should be rejected"); + assert!(err.contains("cannot contain secrets")); +} + +#[tokio::test] +async fn upsert_document_rejects_secret_like_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let err = memory + .upsert_document(NamespaceDocumentInput { + namespace: "safe".to_string(), + key: "api_key=sk-1234567890123456789012345".to_string(), + title: "Title".to_string(), + content: "Body".to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .expect_err("secret-like key should be rejected"); + assert!(err.contains("cannot contain secrets")); +} + +#[tokio::test] +async fn upsert_document_rejects_secret_like_namespace() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let err = memory + .upsert_document(NamespaceDocumentInput { + namespace: "Bearer abcdefghijklmnop".to_string(), + key: "k1".to_string(), + title: "Title".to_string(), + content: "Body".to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .expect_err("secret-like namespace should be rejected"); + assert!(err.contains("cannot contain secrets")); +} + +#[tokio::test] +async fn upsert_document_metadata_only_rejects_secret_like_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let err = memory + .upsert_document_metadata_only(NamespaceDocumentInput { + namespace: "safe".to_string(), + key: "refresh_token=abcdef".to_string(), + title: "Title".to_string(), + content: "Body".to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .expect_err("secret-like key should be rejected"); + assert!(err.contains("cannot contain secrets")); +} + +// --------------------------------------------------------------------------- +// Personal-identifier (PII) at the namespace/key boundary — auto-sanitize. +// +// Rather than rejecting writes with PII-like keys/namespaces (which caused +// unthrottled retry loops, see #5164), the store now auto-sanitizes the +// namespace and key using `redact_pii` before persisting. These tests verify +// the upsert succeeds and the stored key/namespace contains redacted tokens. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn kv_set_global_auto_sanitizes_pii_like_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // SSN-like key should be auto-sanitized, not rejected. + memory + .kv_set_global("ssn-123-45-6789", &json!({"value": "ok"})) + .await + .expect("PII-like global key should be auto-sanitized, not rejected"); + + // ... and the caller must be able to read it back with the identifier it + // wrote. The canonicalization is a storage-address transform, so the read + // path applies the same one (#5164). A miss here is what made the caller + // write again, which is the loop the issue was reported for. + let stored = memory.kv_get_global("ssn-123-45-6789").await.unwrap(); + assert_eq!( + stored, + Some(json!({"value": "ok"})), + "a canonicalized KV key must stay readable by its original identifier" + ); + assert!( + memory.kv_delete_global("ssn-123-45-6789").await.unwrap(), + "delete must address the same canonicalized row the write created" + ); +} + +#[tokio::test] +async fn kv_set_namespace_auto_sanitizes_pii_like_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .kv_set_namespace("safe", "ssn-123-45-6789", &json!({"value": "ok"})) + .await + .expect("PII-like key should be auto-sanitized, not rejected"); + + let records = memory.kv_records_namespace("safe").await.unwrap(); + // The record should still exist; the key gets redacted internally. + assert_eq!(records.len(), 1); + assert_eq!( + memory + .kv_get_namespace("safe", "ssn-123-45-6789") + .await + .unwrap(), + Some(json!({"value": "ok"})), + "a canonicalized KV key must stay readable by its original identifier" + ); +} + +#[tokio::test] +async fn kv_set_namespace_auto_sanitizes_pii_like_namespace() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .kv_set_namespace("user/111.444.777-35", "safe-key", &json!({"value": "ok"})) + .await + .expect("PII-like namespace should be auto-sanitized, not rejected"); + + assert_eq!( + memory + .kv_get_namespace("user/111.444.777-35", "safe-key") + .await + .unwrap(), + Some(json!({"value": "ok"})), + "a canonicalized KV namespace must stay readable by its original value" + ); +} + +#[tokio::test] +async fn upsert_document_auto_sanitizes_pii_like_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let doc_id = memory + .upsert_document(NamespaceDocumentInput { + namespace: "safe".to_string(), + key: "cuit-20-11111111-2".to_string(), + title: "Title".to_string(), + content: "Body".to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .expect("PII-like key should be auto-sanitized, not rejected"); + + // The document was stored with a redacted key. + let docs = memory.load_documents_for_scope("safe").await.unwrap(); + let doc = docs.iter().find(|d| d.document_id == doc_id).unwrap(); + assert!(!doc.key.contains("20-11111111-2"), "key should be redacted"); + assert!( + doc.key.contains("REDACTED"), // matches [REDACTED_PII_CUIT] + "key should contain a redaction token, got: {}", + doc.key + ); +} + +#[tokio::test] +async fn upsert_document_auto_sanitizes_pii_like_namespace() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let doc_id = memory + .upsert_document(NamespaceDocumentInput { + namespace: "cliente-RFC-VECJ880326XK4".to_string(), + key: "k1".to_string(), + title: "Title".to_string(), + content: "Body".to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .expect("PII-like namespace should be auto-sanitized, not rejected"); + + // Look up the document by sanitized namespace (note sanitize_namespace + // normalises special chars, so `[` becomes `_`). + let docs = memory + .load_documents_for_scope("cliente-RFC-VECJ880326XK4") + .await + .unwrap(); + let doc = docs.iter().find(|d| d.document_id == doc_id).unwrap(); + assert!( + doc.namespace.contains("REDACTED"), // [REDACTED_PII_RFC] after sanitize_namespace + "namespace should contain a redaction token, got: {}", + doc.namespace + ); +} + +#[tokio::test] +async fn upsert_document_metadata_only_auto_sanitizes_pii_like_key() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let doc_id = memory + .upsert_document_metadata_only(NamespaceDocumentInput { + namespace: "safe".to_string(), + key: "ssn-123-45-6789".to_string(), + title: "Title".to_string(), + content: "Body".to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .expect("PII-like key should be auto-sanitized, not rejected"); + + let docs = memory.load_documents_for_scope("safe").await.unwrap(); + let doc = docs.iter().find(|d| d.document_id == doc_id).unwrap(); + assert!( + doc.key.contains("REDACTED"), // [REDACTED_PII_SSN] + "key should contain a redaction token, got: {}", + doc.key + ); +} + +#[tokio::test] +async fn upsert_document_metadata_only_auto_sanitizes_pii_like_namespace() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let doc_id = memory + .upsert_document_metadata_only(NamespaceDocumentInput { + namespace: "user/111.444.777-35".to_string(), + key: "safe-key".to_string(), + title: "Title".to_string(), + content: "Body".to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .expect("PII-like namespace should be auto-sanitized, not rejected"); + + let docs = memory + .load_documents_for_scope("user/111.444.777-35") + .await + .unwrap(); + let doc = docs.iter().find(|d| d.document_id == doc_id).unwrap(); + assert!( + doc.namespace.contains("REDACTED"), // [REDACTED_PII_CPF] after sanitize_namespace + "namespace should contain a redaction token, got: {}", + doc.namespace + ); +} + +// --------------------------------------------------------------------------- +// #5164 — the identifier canonicalization has to be symmetric, and it has to +// leave scanner-built identifiers alone. +// +// Canonicalizing a namespace/key rewrites the row's *address*. Two failure +// modes follow, and both re-create the unthrottled write loop the issue was +// filed for (silently, this time): +// +// 1. a read path that addresses the raw caller identifier never finds the +// canonicalized row, so the caller writes it again; +// 2. canonicalizing with the lenient *content* scrubber maps every +// phone-shaped identifier onto one placeholder, so distinct chats collapse +// onto one `(namespace, key)` and the upsert's `ON CONFLICT … DO UPDATE` +// has one contact's document overwrite another's. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn pii_like_document_key_round_trips_through_get_and_forget() { + use crate::traits::Memory; + + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input( + "clients", + "ssn-123-45-6789", + "Title", + "Body", + )) + .await + .expect("PII-like key should be canonicalized, not rejected"); + + let entry = memory + .get("clients", "ssn-123-45-6789") + .await + .unwrap() + .expect("a canonicalized key must stay readable by its original identifier"); + assert_eq!(entry.content, "Body"); + assert!( + !entry.key.contains("123-45-6789"), + "the SSN must not be persisted as the storage address, got: {}", + entry.key + ); + + assert!( + memory.forget("clients", "ssn-123-45-6789").await.unwrap(), + "forget must address the same canonicalized row the write created" + ); + assert!(memory + .get("clients", "ssn-123-45-6789") + .await + .unwrap() + .is_none()); +} + +#[tokio::test] +async fn pii_like_namespace_round_trips_through_get_and_list() { + use crate::traits::Memory; + + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(make_doc_input( + "cliente-RFC-VECJ880326XK4", + "notes", + "Title", + "Body", + )) + .await + .expect("PII-like namespace should be canonicalized, not rejected"); + + assert!( + memory + .get("cliente-RFC-VECJ880326XK4", "notes") + .await + .unwrap() + .is_some(), + "a canonicalized namespace must stay readable by its original value" + ); + let listed = memory + .list(Some("cliente-RFC-VECJ880326XK4"), None, None) + .await + .unwrap(); + assert_eq!( + listed.len(), + 1, + "list() must canonicalize its namespace the same way the write did" + ); +} + +#[tokio::test] +async fn scanner_built_phone_shaped_keys_stay_distinct_documents() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // Two different WhatsApp contacts, same day. The digit runs differ only in + // the phone number — the lenient content scrubber replaces both with one + // `[REDACTED_PII_PHONE]` token, which would collapse them onto a single row. + for (key, content) in [ + ("12025551234@c.us:2026-05-30", "alice thread"), + ("12025559999@c.us:2026-05-30", "bob thread"), + ] { + memory + .upsert_document(make_doc_input("whatsapp-web", key, "Chat", content)) + .await + .unwrap(); + } + + let docs = memory + .load_documents_for_scope("whatsapp-web") + .await + .unwrap(); + assert_eq!( + docs.len(), + 2, + "scanner-built phone-shaped keys must stay distinct documents, got: {:?}", + docs.iter().map(|d| d.key.clone()).collect::>() + ); + for key in ["12025551234@c.us:2026-05-30", "12025559999@c.us:2026-05-30"] { + assert!( + docs.iter().any(|d| d.key == key), + "key {key} must be stored verbatim, got: {:?}", + docs.iter().map(|d| d.key.clone()).collect::>() + ); + } +} + +#[tokio::test] +async fn scanner_built_identifiers_are_preserved_verbatim() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // The strict boundary predicate deliberately tolerates these shapes + // (WhatsApp group JID, iMessage E.164 chat id, padded ms timestamp); the + // content scrubber does not. Canonicalization must follow the strict set, + // or every scanner rewrites its own storage addresses. + for key in [ + "12025551234-1543890267@g.us:2026-05-30", + "imessage:+12025551234:2026-05-30", + "accepted:000001747729035001", + ] { + let doc_id = memory + .upsert_document(make_doc_input("scanner", key, "Title", "Body")) + .await + .unwrap(); + let docs = memory.load_documents_for_scope("scanner").await.unwrap(); + let doc = docs.iter().find(|d| d.document_id == doc_id).unwrap(); + assert_eq!( + doc.key, key, + "scanner-built identifier must not be rewritten" + ); + } +} + +#[tokio::test] +async fn metadata_only_write_round_trips_through_pii_like_key() { + use crate::traits::Memory; + + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document_metadata_only(make_doc_input( + "clients", + "cuit-20-11111111-2", + "Title", + "Body", + )) + .await + .expect("PII-like key should be canonicalized, not rejected"); + + assert!( + memory + .get("clients", "cuit-20-11111111-2") + .await + .unwrap() + .is_some(), + "the metadata-only path must canonicalize keys the same way the full upsert does" + ); +} diff --git a/core/src/store/namespace_store/events.rs b/core/src/store/namespace_store/events.rs new file mode 100644 index 0000000..bb3aca4 --- /dev/null +++ b/core/src/store/namespace_store/events.rs @@ -0,0 +1,496 @@ +//! Event extraction and storage — atomic facts, decisions, commitments, and +//! preferences extracted from closed conversation segments. +//! +//! Two-tier extraction: +//! - Tier A (heuristic/regex): always runs, free — pattern matching for +//! decisions, commitments, preferences, and facts. +//! - Tier B (local LLM): runs on segment close if local AI is enabled. + +use parking_lot::Mutex; +use rusqlite::{params, Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +/// SQL to create the event tables. Called during UnifiedMemory init. +pub const EVENTS_INIT_SQL: &str = r#" +CREATE TABLE IF NOT EXISTS event_log ( + event_id TEXT PRIMARY KEY, + segment_id TEXT NOT NULL, + session_id TEXT NOT NULL, + namespace TEXT NOT NULL DEFAULT 'global', + event_type TEXT NOT NULL, + content TEXT NOT NULL, + subject TEXT, + timestamp_ref TEXT, + confidence REAL NOT NULL, + embedding BLOB, + source_turn_ids TEXT, + created_at REAL NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_events_segment + ON event_log(segment_id); + +CREATE INDEX IF NOT EXISTS idx_events_namespace + ON event_log(namespace, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_events_type + ON event_log(event_type, namespace); + +CREATE VIRTUAL TABLE IF NOT EXISTS event_fts USING fts5( + content, + subject, + event_type, + content=event_log, + content_rowid=rowid, + tokenize='porter unicode61' +); + +CREATE TRIGGER IF NOT EXISTS event_ai AFTER INSERT ON event_log BEGIN + INSERT INTO event_fts(rowid, content, subject, event_type) + VALUES (new.rowid, new.content, new.subject, new.event_type); +END; + +CREATE TRIGGER IF NOT EXISTS event_ad AFTER DELETE ON event_log BEGIN + INSERT INTO event_fts(event_fts, rowid, content, subject, event_type) + VALUES ('delete', old.rowid, old.content, old.subject, old.event_type); +END; + +CREATE TRIGGER IF NOT EXISTS event_au AFTER UPDATE ON event_log BEGIN + INSERT INTO event_fts(event_fts, rowid, content, subject, event_type) + VALUES ('delete', old.rowid, old.content, old.subject, old.event_type); + INSERT INTO event_fts(rowid, content, subject, event_type) + VALUES (new.rowid, new.content, new.subject, new.event_type); +END; + +-- Per-(event, embedding model) vectors (#1574). The legacy event_log.embedding +-- column stays available during the dual-write migration; this table records +-- vector-space provenance for safe provider/model switches. +CREATE TABLE IF NOT EXISTS event_embeddings ( + event_id TEXT NOT NULL REFERENCES event_log(event_id) ON DELETE CASCADE, + model_signature TEXT NOT NULL, + vector BLOB NOT NULL, + dim INTEGER NOT NULL, + created_at REAL NOT NULL, + PRIMARY KEY (event_id, model_signature) +); + +CREATE INDEX IF NOT EXISTS idx_event_embeddings_model + ON event_embeddings(model_signature); +"#; + +/// Event types extracted from conversations. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EventType { + Fact, + Decision, + Commitment, + Preference, + Question, + Foresight, +} + +impl EventType { + /// Stable lowercase identifier persisted in the `event_log` table. + pub fn as_str(&self) -> &'static str { + match self { + Self::Fact => "fact", + Self::Decision => "decision", + Self::Commitment => "commitment", + Self::Preference => "preference", + Self::Question => "question", + Self::Foresight => "foresight", + } + } + + /// Parse a stored string back to an `EventType`; unknown values fall back + /// to `Fact`. + pub fn parse_or_default(s: &str) -> Self { + match s { + "decision" => Self::Decision, + "commitment" => Self::Commitment, + "preference" => Self::Preference, + "question" => Self::Question, + "foresight" => Self::Foresight, + _ => Self::Fact, + } + } +} + +/// An extracted event record. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EventRecord { + pub event_id: String, + pub segment_id: String, + pub session_id: String, + pub namespace: String, + pub event_type: EventType, + pub content: String, + pub subject: Option, + pub timestamp_ref: Option, + pub confidence: f64, + pub embedding: Option>, + pub source_turn_ids: Option, + pub created_at: f64, +} + +/// Insert an event record. +pub fn event_insert(conn: &Arc>, event: &EventRecord) -> anyhow::Result<()> { + let embedding_bytes: Option> = event.embedding.as_ref().map(|v| vec_to_bytes(v)); + let conn = conn.lock(); + conn.execute( + "INSERT OR REPLACE INTO event_log + (event_id, segment_id, session_id, namespace, event_type, content, + subject, timestamp_ref, confidence, embedding, source_turn_ids, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + params![ + event.event_id, + event.segment_id, + event.session_id, + event.namespace, + event.event_type.as_str(), + event.content, + event.subject, + event.timestamp_ref, + event.confidence, + embedding_bytes, + event.source_turn_ids, + event.created_at, + ], + )?; + tracing::debug!( + "[events] inserted event {} type={} for segment={}", + event.event_id, + event.event_type.as_str(), + event.segment_id + ); + Ok(()) +} + +/// Store an event embedding for a specific provider/model/dimension signature. +/// +/// This writes only the per-model table introduced for #1574. The legacy +/// `event_log.embedding` column remains available for dual-read fallback. +pub fn event_embedding_upsert( + conn: &Arc>, + event_id: &str, + model_signature: &str, + embedding: &[f32], + created_at: f64, +) -> anyhow::Result<()> { + let bytes = vec_to_bytes(embedding); + let dim = i64::try_from(embedding.len())?; + let conn = conn.lock(); + conn.execute( + "INSERT INTO event_embeddings (event_id, model_signature, vector, dim, created_at) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(event_id, model_signature) DO UPDATE SET + vector = excluded.vector, + dim = excluded.dim, + created_at = excluded.created_at", + params![event_id, model_signature, bytes, dim, created_at], + )?; + Ok(()) +} + +/// Fetch an event embedding for exactly one provider/model/dimension signature. +pub fn event_embedding_get( + conn: &Arc>, + event_id: &str, + model_signature: &str, +) -> anyhow::Result>> { + let conn = conn.lock(); + let row: Option<(Vec, i64)> = conn + .query_row( + "SELECT vector, dim + FROM event_embeddings + WHERE event_id = ?1 AND model_signature = ?2", + params![event_id, model_signature], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()?; + match row { + None => Ok(None), + Some((bytes, dim)) => decode_embedding_row(&bytes, dim), + } +} + +/// Search events via FTS5, scoped to a namespace. +pub fn event_search_fts( + conn: &Arc>, + namespace: &str, + query: &str, + limit: usize, +) -> anyhow::Result> { + let conn = conn.lock(); + let trimmed = query.trim(); + if trimmed.is_empty() { + tracing::debug!("[events] FTS search skipped — empty query"); + return Ok(Vec::new()); + } + let phrase_query = super::fts5::sanitize_fts_query(trimmed); + if phrase_query.is_empty() { + tracing::debug!("[events] FTS search skipped — sanitised query is empty"); + return Ok(Vec::new()); + } + + let mut stmt = conn.prepare( + "SELECT el.event_id, el.segment_id, el.session_id, el.namespace, + el.event_type, el.content, el.subject, el.timestamp_ref, + el.confidence, el.embedding, el.source_turn_ids, el.created_at + FROM event_fts AS ef + JOIN event_log AS el ON ef.rowid = el.rowid + WHERE event_fts MATCH ?1 AND el.namespace = ?2 + ORDER BY rank + LIMIT ?3", + )?; + let rows = stmt + .query_map(params![phrase_query, namespace, limit as i64], |row| { + row_to_event(row) + })? + .collect::, _>>()?; + tracing::debug!( + "[events] FTS search ns={} returned {} results", + namespace, + rows.len() + ); + Ok(rows) +} + +/// Get all events for a segment. +pub fn events_for_segment( + conn: &Arc>, + segment_id: &str, +) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT event_id, segment_id, session_id, namespace, + event_type, content, subject, timestamp_ref, + confidence, embedding, source_turn_ids, created_at + FROM event_log + WHERE segment_id = ?1 + ORDER BY created_at ASC", + )?; + let rows = stmt + .query_map(params![segment_id], row_to_event)? + .collect::, _>>()?; + Ok(rows) +} + +/// Get events by type within a namespace. +pub fn events_by_type( + conn: &Arc>, + namespace: &str, + event_type: &str, + limit: usize, +) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT event_id, segment_id, session_id, namespace, + event_type, content, subject, timestamp_ref, + confidence, embedding, source_turn_ids, created_at + FROM event_log + WHERE namespace = ?1 AND event_type = ?2 + ORDER BY created_at DESC + LIMIT ?3", + )?; + let rows = stmt + .query_map(params![namespace, event_type, limit as i64], |row| { + row_to_event(row) + })? + .collect::, _>>()?; + Ok(rows) +} + +// ── Heuristic extraction patterns ── + +/// Patterns that indicate a decision. +const DECISION_PATTERNS: &[&str] = &[ + "let's go with", + "lets go with", + "i've decided", + "ive decided", + "i decided", + "we decided", + "we agreed", + "the decision is", + "going with", + "we'll use", + "well use", + "i'll use", + "chosen to", + "i choose", + "we choose", +]; + +/// Patterns that indicate a commitment or deadline. +const COMMITMENT_PATTERNS: &[&str] = &[ + "by friday", + "by monday", + "by tuesday", + "by wednesday", + "by thursday", + "by saturday", + "by sunday", + "by tomorrow", + "by next week", + "by end of", + "deadline is", + "due date", + "i will", + "i'll do", + "ill do", + "i promise", + "i commit", + "we need to finish", + "scheduled for", + "plan to", + "planning to", +]; + +/// Patterns that indicate a preference. +const PREFERENCE_PATTERNS: &[&str] = &[ + "i prefer", + "i like", + "i love", + "i hate", + "i dislike", + "i always", + "i never", + "my favorite", + "my favourite", + "i usually", + "i tend to", + "i'm used to", + "im used to", +]; + +/// Patterns that indicate a personal fact. +const FACT_PATTERNS: &[&str] = &[ + "i'm based in", + "im based in", + "i live in", + "i work at", + "i work for", + "my name is", + "i'm a", + "im a", + "i am a", + "my role is", + "i've been", + "ive been", + "i have been", + "i'm from", + "im from", + "my timezone", + "my time zone", +]; + +/// Extract events from text using heuristic pattern matching. +/// Returns a list of (event_type, matched_sentence) pairs. +pub fn extract_events_heuristic(text: &str) -> Vec<(EventType, String)> { + let mut events = Vec::new(); + + // Split into sentences (rough heuristic). + let sentences: Vec<&str> = text + .split(['.', '!', '?', '\n']) + .map(str::trim) + .filter(|s| s.len() > 5) + .collect(); + + for sentence in sentences { + let lower = sentence.to_lowercase(); + + // Check each pattern category. + for pattern in DECISION_PATTERNS { + if lower.contains(pattern) { + events.push((EventType::Decision, sentence.to_string())); + break; + } + } + for pattern in COMMITMENT_PATTERNS { + if lower.contains(pattern) { + // Avoid duplicate if already matched as decision. + if !events.iter().any(|(_, s)| s == sentence) { + events.push((EventType::Commitment, sentence.to_string())); + } + break; + } + } + for pattern in PREFERENCE_PATTERNS { + if lower.contains(pattern) { + if !events.iter().any(|(_, s)| s == sentence) { + events.push((EventType::Preference, sentence.to_string())); + } + break; + } + } + for pattern in FACT_PATTERNS { + if lower.contains(pattern) { + if !events.iter().any(|(_, s)| s == sentence) { + events.push((EventType::Fact, sentence.to_string())); + } + break; + } + } + } + + events +} + +// ── helpers ── + +fn row_to_event(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let embedding_blob: Option> = row.get(9)?; + let event_type_str: String = row.get(4)?; + Ok(EventRecord { + event_id: row.get(0)?, + segment_id: row.get(1)?, + session_id: row.get(2)?, + namespace: row.get(3)?, + event_type: EventType::parse_or_default(&event_type_str), + content: row.get(5)?, + subject: row.get(6)?, + timestamp_ref: row.get(7)?, + confidence: row.get(8)?, + embedding: embedding_blob.as_deref().map(bytes_to_vec), + source_turn_ids: row.get(10)?, + created_at: row.get(11)?, + }) +} + +fn vec_to_bytes(v: &[f32]) -> Vec { + v.iter().flat_map(|f| f.to_le_bytes()).collect() +} + +fn bytes_to_vec(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(4) + .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) + .collect() +} + +fn decode_embedding_row(bytes: &[u8], dim: i64) -> anyhow::Result>> { + if dim < 0 { + anyhow::bail!("event embedding has negative dimension {dim}"); + } + if bytes.len() % 4 != 0 { + anyhow::bail!( + "event embedding blob length {} not a multiple of 4", + bytes.len() + ); + } + let vector = bytes_to_vec(bytes); + if vector.len() != dim as usize { + anyhow::bail!( + "event embedding dimension mismatch: dim column says {dim}, blob contains {} floats", + vector.len() + ); + } + Ok(Some(vector)) +} + +#[cfg(test)] +#[path = "events_tests.rs"] +mod tests; diff --git a/core/src/store/namespace_store/events_tests.rs b/core/src/store/namespace_store/events_tests.rs new file mode 100644 index 0000000..cc96354 --- /dev/null +++ b/core/src/store/namespace_store/events_tests.rs @@ -0,0 +1,287 @@ +//! Tests for the `events` module — heuristic extraction and FTS5 storage. + +use super::*; + +fn setup_db() -> Arc> { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(EVENTS_INIT_SQL).unwrap(); + Arc::new(Mutex::new(conn)) +} + +#[test] +fn insert_and_search_event() { + let conn = setup_db(); + let event = EventRecord { + event_id: "evt-1".into(), + segment_id: "seg-1".into(), + session_id: "s1".into(), + namespace: "global".into(), + event_type: EventType::Decision, + content: "We decided to use Rust for the backend".into(), + subject: Some("backend language".into()), + timestamp_ref: None, + confidence: 0.8, + embedding: None, + source_turn_ids: None, + created_at: 1000.0, + }; + event_insert(&conn, &event).unwrap(); + + let results = event_search_fts(&conn, "global", "Rust backend", 10).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].event_type, EventType::Decision); +} + +#[test] +fn heuristic_extraction_finds_patterns() { + let text = "I prefer dark mode for coding. We decided to use PostgreSQL. \ + The deadline is by Friday. I live in Berlin. \ + This is a regular sentence with no pattern."; + let events = extract_events_heuristic(text); + + let types: Vec<&EventType> = events.iter().map(|(t, _)| t).collect(); + assert!(types.contains(&&EventType::Preference)); + assert!(types.contains(&&EventType::Decision)); + assert!(types.contains(&&EventType::Commitment)); + assert!(types.contains(&&EventType::Fact)); + // Regular sentence should NOT be extracted. + assert!(!events.iter().any(|(_, s)| s.contains("regular sentence"))); +} + +#[test] +fn events_for_segment_returns_ordered() { + let conn = setup_db(); + for i in 0..3 { + event_insert( + &conn, + &EventRecord { + event_id: format!("evt-{i}"), + segment_id: "seg-1".into(), + session_id: "s1".into(), + namespace: "global".into(), + event_type: EventType::Fact, + content: format!("Fact number {i}"), + subject: None, + timestamp_ref: None, + confidence: 0.7, + embedding: None, + source_turn_ids: None, + created_at: 1000.0 + i as f64, + }, + ) + .unwrap(); + } + + let events = events_for_segment(&conn, "seg-1").unwrap(); + assert_eq!(events.len(), 3); + assert!(events[0].created_at < events[2].created_at); +} + +#[test] +fn event_insert_idempotent() { + let conn = setup_db(); + let event = EventRecord { + event_id: "evt-idem".into(), + segment_id: "seg-1".into(), + session_id: "s1".into(), + namespace: "global".into(), + event_type: EventType::Fact, + content: "Rust is a systems language".into(), + subject: None, + timestamp_ref: None, + confidence: 0.9, + embedding: None, + source_turn_ids: None, + created_at: 1000.0, + }; + // Insert same event_id twice — OR REPLACE semantics; no duplicate row. + event_insert(&conn, &event).unwrap(); + event_insert(&conn, &event).unwrap(); + + let events = events_for_segment(&conn, "seg-1").unwrap(); + assert_eq!( + events.len(), + 1, + "Duplicate insert should not create a second row" + ); +} + +#[test] +fn events_by_type_filters_correctly() { + let conn = setup_db(); + + let make_event = |id: &str, event_type: EventType, ns: &str| EventRecord { + event_id: id.to_string(), + segment_id: "seg-x".into(), + session_id: "s1".into(), + namespace: ns.to_string(), + event_type, + content: format!("Content for {id}"), + subject: None, + timestamp_ref: None, + confidence: 0.7, + embedding: None, + source_turn_ids: None, + created_at: 1000.0, + }; + + event_insert(&conn, &make_event("e-dec", EventType::Decision, "ns1")).unwrap(); + event_insert(&conn, &make_event("e-pref", EventType::Preference, "ns1")).unwrap(); + event_insert(&conn, &make_event("e-fact", EventType::Fact, "ns1")).unwrap(); + + let decisions = events_by_type(&conn, "ns1", "decision", 10).unwrap(); + assert_eq!(decisions.len(), 1); + assert_eq!(decisions[0].event_id, "e-dec"); + assert_eq!(decisions[0].event_type, EventType::Decision); + + let prefs = events_by_type(&conn, "ns1", "preference", 10).unwrap(); + assert_eq!(prefs.len(), 1); + assert_eq!(prefs[0].event_id, "e-pref"); + + // Different namespace should return nothing. + let other = events_by_type(&conn, "ns2", "decision", 10).unwrap(); + assert!( + other.is_empty(), + "No events expected for unrelated namespace" + ); +} + +#[test] +fn heuristic_extracts_multiple_from_same_sentence() { + // A sentence that simultaneously satisfies a preference pattern AND a fact + // pattern will only produce one event (dedup guard). Use two separate + // sentences to confirm both types are emitted. + let text = "I prefer Python for scripting. I live in Berlin."; + let events = extract_events_heuristic(text); + + let types: Vec<&EventType> = events.iter().map(|(t, _)| t).collect(); + assert!( + types.contains(&&EventType::Preference), + "Expected a Preference event from 'I prefer Python'" + ); + assert!( + types.contains(&&EventType::Fact), + "Expected a Fact event from 'I live in Berlin'" + ); + assert!( + events.len() >= 2, + "Expected at least 2 events, got {}", + events.len() + ); +} + +#[test] +fn heuristic_handles_empty_and_whitespace() { + assert!( + extract_events_heuristic("").is_empty(), + "Empty string should yield no events" + ); + assert!( + extract_events_heuristic(" \n\t ").is_empty(), + "Whitespace-only string should yield no events" + ); +} + +#[test] +fn event_fts_matches_subject_field() { + let conn = setup_db(); + let event = EventRecord { + event_id: "evt-subj".into(), + segment_id: "seg-1".into(), + session_id: "s1".into(), + namespace: "global".into(), + event_type: EventType::Decision, + content: "We agreed on the final design".into(), + subject: Some("microservice architecture".into()), + timestamp_ref: None, + confidence: 0.85, + embedding: None, + source_turn_ids: None, + created_at: 1000.0, + }; + event_insert(&conn, &event).unwrap(); + + // Search by content (should match). + let by_content = event_search_fts(&conn, "global", "design", 5).unwrap(); + assert_eq!(by_content.len(), 1, "FTS should match on content field"); + + // Search by subject text (should also match via event_fts). + let by_subject = event_search_fts(&conn, "global", "microservice", 5).unwrap(); + assert_eq!(by_subject.len(), 1, "FTS should match on subject field"); + assert_eq!(by_subject[0].event_id, "evt-subj"); +} + +#[test] +fn event_fts_sanitises_punctuation_safely() { + let conn = setup_db(); + let event = EventRecord { + event_id: "evt-punct".into(), + segment_id: "seg-1".into(), + session_id: "s1".into(), + namespace: "global".into(), + event_type: EventType::Decision, + content: "We decided to use Rust for backend deployment".into(), + subject: Some("backend deployment".into()), + timestamp_ref: None, + confidence: 0.85, + embedding: None, + source_turn_ids: None, + created_at: 1000.0, + }; + event_insert(&conn, &event).unwrap(); + + let results = event_search_fts(&conn, "global", "\"Rust\"，(backend)?", 5) + .expect("punctuated user query should not trip FTS5 syntax errors"); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].event_id, "evt-punct"); +} + +#[test] +fn event_embeddings_are_scoped_by_model_signature() { + let conn = setup_db(); + let event = EventRecord { + event_id: "evt-embed".into(), + segment_id: "seg-1".into(), + session_id: "s1".into(), + namespace: "global".into(), + event_type: EventType::Fact, + content: "The user prefers Korean summaries".into(), + subject: Some("language preference".into()), + timestamp_ref: None, + confidence: 0.9, + embedding: None, + source_turn_ids: None, + created_at: 1000.0, + }; + event_insert(&conn, &event).unwrap(); + + event_embedding_upsert( + &conn, + "evt-embed", + "openai/text-embedding-3-small@1536", + &[0.1, 0.2], + 1001.0, + ) + .unwrap(); + event_embedding_upsert( + &conn, + "evt-embed", + "local/bge-small@384", + &[0.3, 0.4, 0.5], + 1002.0, + ) + .unwrap(); + + assert_eq!( + event_embedding_get(&conn, "evt-embed", "openai/text-embedding-3-small@1536").unwrap(), + Some(vec![0.1, 0.2]) + ); + assert_eq!( + event_embedding_get(&conn, "evt-embed", "local/bge-small@384").unwrap(), + Some(vec![0.3, 0.4, 0.5]) + ); + assert!(event_embedding_get(&conn, "evt-embed", "missing/model@1") + .unwrap() + .is_none()); +} diff --git a/core/src/store/namespace_store/fts5.rs b/core/src/store/namespace_store/fts5.rs new file mode 100644 index 0000000..90b1f2c --- /dev/null +++ b/core/src/store/namespace_store/fts5.rs @@ -0,0 +1,595 @@ +//! FTS5 episodic memory — full-text search over past sessions. +//! +//! Adds an FTS5 virtual table backed by an `episodic_log` table for storing +//! turn-level records with optional extracted lessons. The Archivist uses +//! this for post-session knowledge extraction and the `search_memory` tool +//! uses it for episodic recall. + +use parking_lot::Mutex; +use rusqlite::Connection; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +use crate::store::safety; + +/// A single episodic record (one turn or event). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EpisodicEntry { + pub id: Option, + pub session_id: String, + pub timestamp: f64, + pub role: String, + pub content: String, + pub lesson: Option, + pub tool_calls_json: Option, + pub cost_microdollars: u64, +} + +/// SQL to create the episodic tables. Called during `UnifiedMemory` init. +pub const EPISODIC_INIT_SQL: &str = r#" +CREATE TABLE IF NOT EXISTS episodic_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + timestamp REAL NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + lesson TEXT, + tool_calls_json TEXT, + cost_microdollars INTEGER DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_episodic_session + ON episodic_log(session_id, timestamp); + +CREATE VIRTUAL TABLE IF NOT EXISTS episodic_fts USING fts5( + session_id, + role, + content, + lesson, + content=episodic_log, + content_rowid=id, + tokenize='porter unicode61' +); + +-- Triggers to keep FTS5 in sync with the backing table. +CREATE TRIGGER IF NOT EXISTS episodic_ai AFTER INSERT ON episodic_log BEGIN + INSERT INTO episodic_fts(rowid, session_id, role, content, lesson) + VALUES (new.id, new.session_id, new.role, new.content, new.lesson); +END; + +CREATE TRIGGER IF NOT EXISTS episodic_ad AFTER DELETE ON episodic_log BEGIN + INSERT INTO episodic_fts(episodic_fts, rowid, session_id, role, content, lesson) + VALUES ('delete', old.id, old.session_id, old.role, old.content, old.lesson); +END; + +CREATE TRIGGER IF NOT EXISTS episodic_au AFTER UPDATE ON episodic_log BEGIN + INSERT INTO episodic_fts(episodic_fts, rowid, session_id, role, content, lesson) + VALUES ('delete', old.id, old.session_id, old.role, old.content, old.lesson); + INSERT INTO episodic_fts(rowid, session_id, role, content, lesson) + VALUES (new.id, new.session_id, new.role, new.content, new.lesson); +END; +"#; + +/// Insert an episodic entry. +pub fn episodic_insert(conn: &Arc>, entry: &EpisodicEntry) -> anyhow::Result<()> { + if safety::has_likely_secret(&entry.session_id) || safety::has_likely_secret(&entry.role) { + tracing::warn!( + "[memory:safety] episodic insert rejected secret-like session/role session_chars={} role_chars={}", + entry.session_id.chars().count(), + entry.role.chars().count() + ); + anyhow::bail!("episodic session_id/role cannot contain secrets"); + } + + let content = safety::sanitize_text(&entry.content); + let lesson = entry + .lesson + .as_ref() + .map(|value| safety::sanitize_text(value)); + let tool_calls_json = entry.tool_calls_json.as_ref().map(|value| { + if let Ok(parsed) = serde_json::from_str::(value) { + let sanitized = safety::sanitize_json(&parsed); + safety::Sanitized { + value: sanitized.value.to_string(), + report: sanitized.report, + } + } else { + safety::sanitize_text(value) + } + }); + + let report = content + .report + .merge( + lesson + .as_ref() + .map(|value| value.report) + .unwrap_or_default(), + ) + .merge( + tool_calls_json + .as_ref() + .map(|value| value.report) + .unwrap_or_default(), + ); + if report.changed() { + tracing::warn!( + "[memory:safety] episodic insert sanitized session_chars={} role_chars={} text_redactions={} key_redactions={} blocked_secret_hits={} depth_redactions={} pii_redactions={}", + entry.session_id.chars().count(), + entry.role.chars().count(), + report.text_redactions, + report.key_redactions, + report.blocked_secret_hits, + report.depth_redactions, + report.pii_redactions + ); + } + + let conn = conn.lock(); + conn.execute( + "INSERT INTO episodic_log (session_id, timestamp, role, content, lesson, tool_calls_json, cost_microdollars) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + rusqlite::params![ + &entry.session_id, + entry.timestamp, + &entry.role, + content.value, + lesson.map(|value| value.value), + tool_calls_json.map(|value| value.value), + entry.cost_microdollars as i64, + ], + )?; + tracing::debug!( + "[fts5] inserted episodic entry: session={}, role={}", + entry.session_id, + entry.role + ); + Ok(()) +} + +/// Full-text search over episodic entries. +pub fn episodic_search( + conn: &Arc>, + query: &str, + limit: usize, +) -> anyhow::Result> { + let conn = conn.lock(); + let trimmed = query.trim(); + if trimmed.is_empty() { + tracing::debug!("[fts5] search skipped — empty query"); + return Ok(Vec::new()); + } + let phrase_query = sanitize_fts_query(trimmed); + if phrase_query.is_empty() { + tracing::debug!("[fts5] search skipped — sanitised query is empty"); + return Ok(Vec::new()); + } + + let mut stmt = conn.prepare( + "SELECT el.id, el.session_id, el.timestamp, el.role, el.content, el.lesson, + el.tool_calls_json, el.cost_microdollars + FROM episodic_fts AS ef + JOIN episodic_log AS el ON ef.rowid = el.id + WHERE episodic_fts MATCH ?1 + ORDER BY rank + LIMIT ?2", + )?; + + let rows = stmt + .query_map(rusqlite::params![phrase_query, limit as i64], |row| { + Ok(EpisodicEntry { + id: row.get(0)?, + session_id: row.get(1)?, + timestamp: row.get(2)?, + role: row.get(3)?, + content: row.get(4)?, + lesson: row.get(5)?, + tool_calls_json: row.get(6)?, + cost_microdollars: row.get::<_, i64>(7)? as u64, + }) + })? + .collect::, _>>()?; + + tracing::debug!("[fts5] search returned {} results", rows.len()); + Ok(rows) +} + +/// FTS5 search across **all** sessions, optionally excluding one session +/// from the result set. Used by [`crate`] to surface +/// cross-chat conversational context for the same user/workspace (issue +/// #1505) without leaking the current chat's own history into the +/// "other chats" block. +/// +/// `exclude_session` should be the active session_id when the caller is +/// also pulling same-session entries via [`episodic_session_entries`] — +/// passing `None` returns hits from every session indexed in this DB. +/// +/// Workspace/user scope is enforced at the connection level: the SQLite +/// database lives at `/memory/...` so one DB == one workspace. +/// This helper cannot cross that boundary. +pub fn episodic_cross_session_search( + conn: &Arc>, + query: &str, + limit: usize, + exclude_session: Option<&str>, +) -> anyhow::Result> { + let conn = conn.lock(); + let trimmed = query.trim(); + if trimmed.is_empty() { + tracing::debug!("[fts5] cross-session search skipped — empty query"); + return Ok(Vec::new()); + } + + // FTS5 MATCH expressions are picky about syntax — bare phrases with + // punctuation can fail to parse. Wrap the query in double quotes so + // it's treated as a phrase (FTS5 will still tokenize it). This mirrors + // how the unified store sanitises queries before MATCH. + let phrase_query = sanitize_fts_query(trimmed); + if phrase_query.is_empty() { + tracing::debug!("[fts5] cross-session search skipped — sanitised query is empty"); + return Ok(Vec::new()); + } + + let mut stmt = match exclude_session { + Some(_) => conn.prepare( + "SELECT el.id, el.session_id, el.timestamp, el.role, el.content, el.lesson, + el.tool_calls_json, el.cost_microdollars + FROM episodic_fts AS ef + JOIN episodic_log AS el ON ef.rowid = el.id + WHERE episodic_fts MATCH ?1 AND el.session_id != ?2 + ORDER BY rank + LIMIT ?3", + )?, + None => conn.prepare( + "SELECT el.id, el.session_id, el.timestamp, el.role, el.content, el.lesson, + el.tool_calls_json, el.cost_microdollars + FROM episodic_fts AS ef + JOIN episodic_log AS el ON ef.rowid = el.id + WHERE episodic_fts MATCH ?1 + ORDER BY rank + LIMIT ?2", + )?, + }; + + let map_row = |row: &rusqlite::Row<'_>| -> rusqlite::Result { + Ok(EpisodicEntry { + id: row.get(0)?, + session_id: row.get(1)?, + timestamp: row.get(2)?, + role: row.get(3)?, + content: row.get(4)?, + lesson: row.get(5)?, + tool_calls_json: row.get(6)?, + cost_microdollars: row.get::<_, i64>(7)? as u64, + }) + }; + + let rows: Vec = match exclude_session { + Some(sid) => stmt + .query_map(rusqlite::params![phrase_query, sid, limit as i64], map_row)? + .collect::, _>>()?, + None => stmt + .query_map(rusqlite::params![phrase_query, limit as i64], map_row)? + .collect::, _>>()?, + }; + + // Never log the raw query string — may contain secrets / PII. Emit a + // stable non-reversible hash + length instead so cross-session + // diagnostics stay grep-friendly without leaking user content. + let query_hash = { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + trimmed.hash(&mut hasher); + hasher.finish() + }; + tracing::debug!( + "[fts5] cross-session search query_hash={:016x} query_len={} (exclude={:?}) returned {} results", + query_hash, + trimmed.chars().count(), + exclude_session, + rows.len() + ); + Ok(rows) +} + +/// Best-effort FTS5 query sanitiser: split user text on punctuation and +/// symbols that break the MATCH grammar, then quote each surviving token +/// so FTS5 treats it as literal text. Returns an empty string when +/// nothing usable survives — callers short-circuit to "no hits". +pub(super) fn sanitize_fts_query(query: &str) -> String { + let cleaned: String = query + .chars() + .map(|c| { + if c.is_alphanumeric() || c == '_' { + c + } else { + ' ' + } + }) + .collect(); + let tokens: Vec = cleaned + .split_whitespace() + .filter(|tok| !tok.is_empty()) + .take(8) + .map(|tok| format!("\"{tok}\"")) + .collect(); + tokens.join(" ") +} + +/// Get all entries for a session (for post-session summary). +pub fn episodic_session_entries( + conn: &Arc>, + session_id: &str, +) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT id, session_id, timestamp, role, content, lesson, tool_calls_json, cost_microdollars + FROM episodic_log + WHERE session_id = ?1 + ORDER BY timestamp ASC", + )?; + + let rows = stmt + .query_map(rusqlite::params![session_id], |row| { + Ok(EpisodicEntry { + id: row.get(0)?, + session_id: row.get(1)?, + timestamp: row.get(2)?, + role: row.get(3)?, + content: row.get(4)?, + lesson: row.get(5)?, + tool_calls_json: row.get(6)?, + cost_microdollars: row.get::<_, i64>(7)? as u64, + }) + })? + .collect::, _>>()?; + + Ok(rows) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn setup_db() -> Arc> { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(EPISODIC_INIT_SQL).unwrap(); + Arc::new(Mutex::new(conn)) + } + + #[test] + fn insert_and_search() { + let conn = setup_db(); + let entry = EpisodicEntry { + id: None, + session_id: "s1".into(), + timestamp: 1000.0, + role: "user".into(), + content: "How do I deploy to production?".into(), + lesson: Some("User frequently asks about deployment".into()), + tool_calls_json: None, + cost_microdollars: 100, + }; + episodic_insert(&conn, &entry).unwrap(); + + let results = episodic_search(&conn, "deploy production", 10).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].session_id, "s1"); + assert!(results[0].content.contains("deploy")); + } + + #[test] + fn session_entries() { + let conn = setup_db(); + for i in 0..3 { + episodic_insert( + &conn, + &EpisodicEntry { + id: None, + session_id: "s2".into(), + timestamp: 1000.0 + i as f64, + role: if i % 2 == 0 { "user" } else { "assistant" }.into(), + content: format!("Turn {i} content"), + lesson: None, + tool_calls_json: None, + cost_microdollars: 0, + }, + ) + .unwrap(); + } + + let entries = episodic_session_entries(&conn, "s2").unwrap(); + assert_eq!(entries.len(), 3); + assert!(entries[0].timestamp < entries[2].timestamp); + } + + #[test] + fn empty_search_returns_empty() { + let conn = setup_db(); + let results = episodic_search(&conn, "nonexistent query", 10).unwrap(); + assert!(results.is_empty()); + } + + #[test] + fn insert_redacts_secret_like_content() { + let conn = setup_db(); + episodic_insert( + &conn, + &EpisodicEntry { + id: None, + session_id: "s1".into(), + timestamp: 1000.0, + role: "user".into(), + content: "Bearer abcdefghijklmnop".into(), + lesson: Some("token=abc123".into()), + tool_calls_json: Some("{\"api_key\":\"sk-1234567890123456789012345\"}".into()), + cost_microdollars: 0, + }, + ) + .unwrap(); + + let rows = episodic_session_entries(&conn, "s1").unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].content, "Bearer [REDACTED]"); + assert_eq!(rows[0].lesson.as_deref(), Some("[REDACTED]")); + assert_eq!( + rows[0].tool_calls_json.as_deref(), + Some("{\"api_key\":\"[REDACTED_SECRET]\"}") + ); + } + + #[test] + fn insert_rejects_secret_like_session_id() { + let conn = setup_db(); + let err = episodic_insert( + &conn, + &EpisodicEntry { + id: None, + session_id: "Bearer abcdefghijklmnop".into(), + timestamp: 1000.0, + role: "user".into(), + content: "hello".into(), + lesson: None, + tool_calls_json: None, + cost_microdollars: 0, + }, + ) + .expect_err("secret-like session_id should be rejected"); + assert!(err.to_string().contains("cannot contain secrets")); + } + + // ── Cross-session search (#1505) ───────────────────────────────────── + + fn insert_turn(conn: &Arc>, session_id: &str, ts: f64, content: &str) { + episodic_insert( + conn, + &EpisodicEntry { + id: None, + session_id: session_id.into(), + timestamp: ts, + role: "user".into(), + content: content.into(), + lesson: None, + tool_calls_json: None, + cost_microdollars: 0, + }, + ) + .unwrap(); + } + + #[test] + fn cross_session_search_surfaces_other_sessions_excluding_current() { + let conn = setup_db(); + // Chat A — user shared the durable fact + insert_turn( + &conn, + "session-a", + 1000.0, + "I prefer Postgres for new services", + ); + // Chat B — current chat, where the question is being asked + insert_turn( + &conn, + "session-b", + 2000.0, + "What database should I use today?", + ); + // Chat C — yet another chat with a related fact + insert_turn(&conn, "session-c", 1500.0, "Postgres timezone is UTC"); + + // Asking from chat B: should see session-a + session-c (not session-b) + let hits = episodic_cross_session_search(&conn, "Postgres", 10, Some("session-b")).unwrap(); + assert!( + !hits.is_empty(), + "cross-session search must surface hits from other sessions" + ); + for hit in &hits { + assert_ne!( + hit.session_id, "session-b", + "current session must be excluded from cross-session sweep, got {}", + hit.session_id + ); + } + let session_ids: std::collections::HashSet<&str> = + hits.iter().map(|h| h.session_id.as_str()).collect(); + assert!(session_ids.contains("session-a")); + assert!(session_ids.contains("session-c")); + } + + #[test] + fn cross_session_search_returns_empty_for_unknown_query() { + let conn = setup_db(); + insert_turn(&conn, "session-a", 1000.0, "I prefer Postgres"); + let hits = episodic_cross_session_search(&conn, "kubernetes", 10, None).unwrap(); + assert!( + hits.is_empty(), + "no FTS match should produce zero hits, not all rows" + ); + } + + #[test] + fn cross_session_search_handles_empty_query() { + let conn = setup_db(); + insert_turn(&conn, "session-a", 1000.0, "anything"); + let hits = episodic_cross_session_search(&conn, " ", 10, None).unwrap(); + assert!(hits.is_empty(), "empty query short-circuits to zero hits"); + } + + #[test] + fn cross_session_search_sanitises_punctuation_safely() { + let conn = setup_db(); + insert_turn(&conn, "session-a", 1000.0, "Postgres deployment notes"); + // Query with FTS5-hostile punctuation — should not panic. Tokens + // shared with the indexed row should still match (FTS5 phrase + // ANDs every quoted token, so we use words that all appear in + // the row to avoid AND-mismatch false negatives). + let hits = + episodic_cross_session_search(&conn, "\"Postgres\" (deployment)?", 10, None).unwrap(); + assert!( + !hits.is_empty(), + "punctuated query whose surviving tokens match the indexed row must still surface it" + ); + assert!(hits[0].content.contains("Postgres")); + } + + #[test] + fn episodic_search_sanitises_punctuation_safely() { + let conn = setup_db(); + insert_turn(&conn, "session-a", 1000.0, "Postgres deployment notes"); + + let hits = episodic_search(&conn, "\"Postgres\"，(deployment)?", 10) + .expect("punctuated user query should not trip FTS5 syntax errors"); + + assert!( + !hits.is_empty(), + "punctuated query whose surviving tokens match the indexed row must still surface it" + ); + assert!(hits[0].content.contains("Postgres")); + } + + #[test] + fn cross_session_search_does_not_panic_on_pure_punctuation() { + let conn = setup_db(); + insert_turn(&conn, "session-a", 1000.0, "Postgres deployment notes"); + // All-punctuation query should normalise to empty and produce + // zero hits without panicking. + let hits = episodic_cross_session_search(&conn, "()*\":", 10, None).unwrap(); + assert!( + hits.is_empty(), + "punctuation-only query must produce zero hits" + ); + } + + #[test] + fn cross_session_search_no_exclusion_includes_all_matches() { + let conn = setup_db(); + insert_turn(&conn, "session-a", 1000.0, "Postgres preference"); + insert_turn(&conn, "session-b", 2000.0, "Postgres setup"); + + let hits = episodic_cross_session_search(&conn, "Postgres", 10, None).unwrap(); + let session_ids: std::collections::HashSet<&str> = + hits.iter().map(|h| h.session_id.as_str()).collect(); + assert!(session_ids.contains("session-a")); + assert!(session_ids.contains("session-b")); + } +} diff --git a/core/src/store/namespace_store/graph.rs b/core/src/store/namespace_store/graph.rs new file mode 100644 index 0000000..3a8a308 --- /dev/null +++ b/core/src/store/namespace_store/graph.rs @@ -0,0 +1,842 @@ +//! Knowledge-graph relations stored in `graph_namespace` and `graph_global`. +//! +//! Provides upsert (with attribute merging + evidence accumulation), namespace +//! / global / cross-namespace queries, and the document-scoped removal used +//! when a source document is deleted or re-ingested. + +use rusqlite::{params, OptionalExtension}; +use serde_json::{json, Map, Value}; + +use crate::store::types::GraphRelationRecord; + +use super::UnifiedMemory; + +impl UnifiedMemory { + pub(crate) async fn graph_remove_document_namespace( + &self, + namespace: &str, + document_id: &str, + ) -> Result<(), String> { + let relations = self + .graph_relations_namespace(namespace, None, None) + .await?; + if relations.is_empty() { + return Ok(()); + } + + let doc_prefix = format!("{document_id}:"); + let updated_at = Self::now_ts(); + let conn = self.conn.lock(); + let tx = conn + .unchecked_transaction() + .map_err(|e| format!("graph_remove_document_namespace begin tx: {e}"))?; + + for relation in relations { + let touches_document = relation.document_ids.iter().any(|id| id == document_id) + || relation + .chunk_ids + .iter() + .any(|chunk_id| chunk_id.starts_with(&doc_prefix)); + if !touches_document { + continue; + } + + let mut attrs = relation.attrs.as_object().cloned().unwrap_or_default(); + let document_ids = relation + .document_ids + .iter() + .filter(|id| id.as_str() != document_id) + .cloned() + .collect::>(); + let chunk_ids = relation + .chunk_ids + .iter() + .filter(|chunk_id| !chunk_id.starts_with(&doc_prefix)) + .cloned() + .collect::>(); + + if document_ids.is_empty() && chunk_ids.is_empty() { + tx.execute( + "DELETE FROM graph_namespace + WHERE namespace = ?1 AND subject = ?2 AND predicate = ?3 AND object = ?4", + params![ + Self::sanitize_namespace(namespace), + relation.subject, + relation.predicate, + relation.object + ], + ) + .map_err(|e| format!("graph_remove_document_namespace delete: {e}"))?; + continue; + } + + attrs.insert("document_ids".to_string(), json!(document_ids)); + if chunk_ids.is_empty() { + attrs.remove("chunk_ids"); + } else { + attrs.insert("chunk_ids".to_string(), json!(chunk_ids.clone())); + } + attrs.insert("evidence_count".to_string(), json!(chunk_ids.len().max(1))); + attrs.insert("updated_at".to_string(), json!(updated_at)); + + tx.execute( + "UPDATE graph_namespace + SET attrs_json = ?1, updated_at = ?2 + WHERE namespace = ?3 AND subject = ?4 AND predicate = ?5 AND object = ?6", + params![ + Value::Object(attrs).to_string(), + updated_at, + Self::sanitize_namespace(namespace), + relation.subject, + relation.predicate, + relation.object + ], + ) + .map_err(|e| format!("graph_remove_document_namespace update: {e}"))?; + } + + tx.commit() + .map_err(|e| format!("graph_remove_document_namespace commit: {e}"))?; + Ok(()) + } + + /// Upsert a relation into the cross-namespace `graph_global` table. + pub async fn graph_upsert_global( + &self, + subject: &str, + predicate: &str, + object: &str, + attrs: &serde_json::Value, + ) -> Result<(), String> { + self.graph_upsert_internal(None, subject, predicate, object, attrs) + .await + } + + /// Upsert a relation into the namespace-scoped `graph_namespace` table, + /// merging attributes (evidence count, document/chunk ids) with any + /// existing edge. + pub async fn graph_upsert_namespace( + &self, + namespace: &str, + subject: &str, + predicate: &str, + object: &str, + attrs: &serde_json::Value, + ) -> Result<(), String> { + self.graph_upsert_internal(Some(namespace), subject, predicate, object, attrs) + .await + } + + /// Query relations in the global graph with optional subject/predicate filters. + pub async fn graph_query_global( + &self, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + let rows = self.graph_relations_global(subject, predicate).await?; + Ok(rows + .into_iter() + .map(Self::graph_relation_to_json) + .collect::>()) + } + + /// Query all graph relations across every namespace AND global, with + /// optional subject/predicate filters. Used when the caller passes no + /// namespace so that ingested (namespace-scoped) data is still surfaced. + pub async fn graph_query_all( + &self, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + let mut rows = self + .graph_relations_all_namespaces(subject, predicate) + .await?; + rows.extend(self.graph_relations_global(subject, predicate).await?); + rows.sort_by(|a, b| { + b.updated_at + .partial_cmp(&a.updated_at) + .unwrap_or(std::cmp::Ordering::Equal) + }); + rows.truncate(300); + Ok(rows + .into_iter() + .map(Self::graph_relation_to_json) + .collect::>()) + } + + /// Query relations within a single namespace with optional subject/predicate filters. + pub async fn graph_query_namespace( + &self, + namespace: &str, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + let rows = self + .graph_relations_namespace(namespace, subject, predicate) + .await?; + Ok(rows + .into_iter() + .map(Self::graph_relation_to_json) + .collect::>()) + } + + pub(crate) async fn graph_relations_for_scope( + &self, + namespace: &str, + ) -> Result, String> { + let mut rows = self + .graph_relations_namespace(namespace, None, None) + .await?; + rows.extend(self.graph_relations_global(None, None).await?); + rows.sort_by(|a, b| { + b.updated_at + .partial_cmp(&a.updated_at) + .unwrap_or(std::cmp::Ordering::Equal) + }); + Ok(rows) + } + + pub(crate) async fn graph_relations_namespace( + &self, + namespace: &str, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + let conn = self.conn.lock(); + let ns = Self::sanitize_namespace(namespace); + let subject = subject.map(Self::normalize_graph_entity); + let predicate = predicate.map(Self::normalize_graph_predicate); + let mut stmt = conn + .prepare( + "SELECT subject, predicate, object, attrs_json, updated_at + FROM graph_namespace + WHERE namespace = ?1 + AND (?2 IS NULL OR subject = ?2) + AND (?3 IS NULL OR predicate = ?3) + ORDER BY updated_at DESC + LIMIT 300", + ) + .map_err(|e| format!("graph_relations_namespace prepare: {e}"))?; + let mut rows = stmt + .query(params![ns, subject, predicate]) + .map_err(|e| format!("graph_relations_namespace query: {e}"))?; + let mut out = Vec::new(); + while let Some(row) = rows + .next() + .map_err(|e| format!("graph_relations_namespace row: {e}"))? + { + let attrs_raw: String = row.get(3).map_err(|e| e.to_string())?; + out.push(Self::graph_relation_from_parts( + Some(Self::sanitize_namespace(namespace)), + row.get(0).map_err(|e| e.to_string())?, + row.get(1).map_err(|e| e.to_string())?, + row.get(2).map_err(|e| e.to_string())?, + &attrs_raw, + row.get(4).map_err(|e| e.to_string())?, + )); + } + Ok(out) + } + + pub(crate) async fn graph_relations_global( + &self, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + let conn = self.conn.lock(); + let subject = subject.map(Self::normalize_graph_entity); + let predicate = predicate.map(Self::normalize_graph_predicate); + let mut stmt = conn + .prepare( + "SELECT subject, predicate, object, attrs_json, updated_at + FROM graph_global + WHERE (?1 IS NULL OR subject = ?1) + AND (?2 IS NULL OR predicate = ?2) + ORDER BY updated_at DESC + LIMIT 300", + ) + .map_err(|e| format!("graph_relations_global prepare: {e}"))?; + let mut rows = stmt + .query(params![subject, predicate]) + .map_err(|e| format!("graph_relations_global query: {e}"))?; + let mut out = Vec::new(); + while let Some(row) = rows + .next() + .map_err(|e| format!("graph_relations_global row: {e}"))? + { + let attrs_raw: String = row.get(3).map_err(|e| e.to_string())?; + out.push(Self::graph_relation_from_parts( + None, + row.get(0).map_err(|e| e.to_string())?, + row.get(1).map_err(|e| e.to_string())?, + row.get(2).map_err(|e| e.to_string())?, + &attrs_raw, + row.get(4).map_err(|e| e.to_string())?, + )); + } + Ok(out) + } + + /// Query relations from `graph_namespace` across ALL namespaces, with + /// optional subject/predicate filters. + pub(crate) async fn graph_relations_all_namespaces( + &self, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + let conn = self.conn.lock(); + let subject = subject.map(Self::normalize_graph_entity); + let predicate = predicate.map(Self::normalize_graph_predicate); + let mut stmt = conn + .prepare( + "SELECT namespace, subject, predicate, object, attrs_json, updated_at + FROM graph_namespace + WHERE (?1 IS NULL OR subject = ?1) + AND (?2 IS NULL OR predicate = ?2) + ORDER BY updated_at DESC + LIMIT 300", + ) + .map_err(|e| format!("graph_relations_all_namespaces prepare: {e}"))?; + let mut rows = stmt + .query(params![subject, predicate]) + .map_err(|e| format!("graph_relations_all_namespaces query: {e}"))?; + let mut out = Vec::new(); + while let Some(row) = rows + .next() + .map_err(|e| format!("graph_relations_all_namespaces row: {e}"))? + { + let namespace: String = row.get(0).map_err(|e| e.to_string())?; + let attrs_raw: String = row.get(4).map_err(|e| e.to_string())?; + out.push(Self::graph_relation_from_parts( + Some(namespace), + row.get(1).map_err(|e| e.to_string())?, + row.get(2).map_err(|e| e.to_string())?, + row.get(3).map_err(|e| e.to_string())?, + &attrs_raw, + row.get(5).map_err(|e| e.to_string())?, + )); + } + Ok(out) + } + + async fn graph_upsert_internal( + &self, + namespace: Option<&str>, + subject: &str, + predicate: &str, + object: &str, + attrs: &serde_json::Value, + ) -> Result<(), String> { + let subject = Self::normalize_graph_entity(subject); + let predicate = Self::normalize_graph_predicate(predicate); + let object = Self::normalize_graph_entity(object); + let updated_at = Self::now_ts(); + let conn = self.conn.lock(); + + let existing_attrs: Option = match namespace { + Some(ns) => conn + .query_row( + "SELECT attrs_json + FROM graph_namespace + WHERE namespace = ?1 AND subject = ?2 AND predicate = ?3 AND object = ?4", + params![Self::sanitize_namespace(ns), subject, predicate, object], + |row| row.get(0), + ) + .optional() + .map_err(|e| format!("graph_upsert_namespace lookup: {e}"))?, + None => conn + .query_row( + "SELECT attrs_json + FROM graph_global + WHERE subject = ?1 AND predicate = ?2 AND object = ?3", + params![subject, predicate, object], + |row| row.get(0), + ) + .optional() + .map_err(|e| format!("graph_upsert_global lookup: {e}"))?, + }; + + let merged_attrs = Self::merge_graph_attrs(existing_attrs.as_deref(), attrs, updated_at); + let merged_attrs_json = merged_attrs.to_string(); + + match namespace { + Some(ns) => { + conn.execute( + "INSERT INTO graph_namespace (namespace, subject, predicate, object, attrs_json, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(namespace, subject, predicate, object) + DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at", + params![ + Self::sanitize_namespace(ns), + subject, + predicate, + object, + merged_attrs_json, + updated_at + ], + ) + .map_err(|e| format!("graph_upsert_namespace: {e}"))?; + } + None => { + conn.execute( + "INSERT INTO graph_global (subject, predicate, object, attrs_json, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(subject, predicate, object) + DO UPDATE SET attrs_json = excluded.attrs_json, updated_at = excluded.updated_at", + params![subject, predicate, object, merged_attrs_json, updated_at], + ) + .map_err(|e| format!("graph_upsert_global: {e}"))?; + } + } + + Ok(()) + } + + fn merge_graph_attrs( + existing_attrs_raw: Option<&str>, + incoming_attrs: &Value, + updated_at: f64, + ) -> Value { + let existing = existing_attrs_raw + .and_then(|raw| serde_json::from_str::(raw).ok()) + .unwrap_or_else(|| json!({})); + let existing_evidence = Self::json_i64(&existing, "evidence_count") + .unwrap_or(0) + .max(0) as u64; + let existing_document_ids = + Self::json_string_array(&existing, "document_ids", "document_id"); + let existing_chunk_ids = Self::json_string_array(&existing, "chunk_ids", "chunk_id"); + + let mut merged = match existing { + Value::Object(map) => map, + _ => Map::new(), + }; + let incoming_map = incoming_attrs.as_object().cloned().unwrap_or_default(); + let existing_order_index = Self::json_i64(&Value::Object(merged.clone()), "order_index"); + let incoming_order_index = Self::json_i64(incoming_attrs, "order_index"); + let merged_order_index = match (existing_order_index, incoming_order_index) { + (Some(left), Some(right)) => Some(left.min(right)), + (Some(left), None) => Some(left), + (None, Some(right)) => Some(right), + (None, None) => None, + }; + + for (key, value) in incoming_map { + merged.insert(key, value); + } + + let incoming_evidence = Self::json_i64(incoming_attrs, "evidence_count") + .unwrap_or(1) + .max(0) as u64; + let evidence_count = existing_evidence.saturating_add(incoming_evidence).max(1); + + merged.insert("evidence_count".to_string(), json!(evidence_count)); + merged.insert("updated_at".to_string(), json!(updated_at)); + + let mut document_ids = existing_document_ids; + document_ids.extend(Self::json_string_array( + incoming_attrs, + "document_ids", + "document_id", + )); + document_ids.sort(); + document_ids.dedup(); + if !document_ids.is_empty() { + merged.insert("document_ids".to_string(), json!(document_ids)); + } + + let mut chunk_ids = existing_chunk_ids; + chunk_ids.extend(Self::json_string_array( + incoming_attrs, + "chunk_ids", + "chunk_id", + )); + chunk_ids.sort(); + chunk_ids.dedup(); + if !chunk_ids.is_empty() { + merged.insert("chunk_ids".to_string(), json!(chunk_ids)); + } + + if !merged.contains_key("created_at") { + merged.insert("created_at".to_string(), json!(updated_at)); + } + if let Some(order_index) = merged_order_index { + merged.insert("order_index".to_string(), json!(order_index)); + } + + Value::Object(merged) + } + + fn graph_relation_from_parts( + namespace: Option, + subject: String, + predicate: String, + object: String, + attrs_raw: &str, + updated_at: f64, + ) -> GraphRelationRecord { + let attrs = serde_json::from_str::(attrs_raw).unwrap_or_else(|_| json!({})); + let evidence_count = Self::json_i64(&attrs, "evidence_count").unwrap_or(1).max(1) as u32; + let order_index = Self::json_i64(&attrs, "order_index"); + let document_ids = Self::json_string_array(&attrs, "document_ids", "document_id"); + let chunk_ids = Self::json_string_array(&attrs, "chunk_ids", "chunk_id"); + + GraphRelationRecord { + namespace, + subject, + predicate, + object, + attrs, + updated_at, + evidence_count, + order_index, + document_ids, + chunk_ids, + } + } + + fn graph_relation_to_json(record: GraphRelationRecord) -> serde_json::Value { + json!({ + "namespace": record.namespace, + "subject": record.subject, + "predicate": record.predicate, + "object": record.object, + "attrs": record.attrs, + "updatedAt": record.updated_at, + "evidenceCount": record.evidence_count, + "orderIndex": record.order_index, + "documentIds": record.document_ids, + "chunkIds": record.chunk_ids, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use tempfile::TempDir; + use tinymemory_api::host::NoopEmbedding; + + #[test] + fn merge_graph_attrs_accumulates_evidence_and_dedupes_ids() { + let existing = json!({ + "evidence_count": 2, + "document_ids": ["doc-1"], + "chunk_ids": ["doc-1:chunk-1"], + "order_index": 7, + "created_at": 1.0 + }); + let incoming = json!({ + "evidence_count": 3, + "document_ids": ["doc-1", "doc-2"], + "chunk_ids": ["doc-2:chunk-9"], + "order_index": 3, + "attrs_only": true + }); + + let merged = UnifiedMemory::merge_graph_attrs(Some(&existing.to_string()), &incoming, 9.0); + assert_eq!(merged["evidence_count"], json!(5)); + assert_eq!(merged["document_ids"], json!(["doc-1", "doc-2"])); + assert_eq!( + merged["chunk_ids"], + json!(["doc-1:chunk-1", "doc-2:chunk-9"]) + ); + assert_eq!(merged["order_index"], json!(3)); + assert_eq!(merged["created_at"], json!(1.0)); + assert_eq!(merged["updated_at"], json!(9.0)); + assert_eq!(merged["attrs_only"], json!(true)); + } + + #[test] + fn graph_relation_from_parts_extracts_counts_and_ids() { + let record = UnifiedMemory::graph_relation_from_parts( + Some("global".into()), + "Alice".into(), + "OWNS".into(), + "OpenHuman".into(), + r#"{"evidence_count":2,"order_index":4,"document_ids":["doc-1"],"chunk_ids":["doc-1:chunk-1"]}"#, + 5.0, + ); + assert_eq!(record.namespace.as_deref(), Some("global")); + assert_eq!(record.evidence_count, 2); + assert_eq!(record.order_index, Some(4)); + assert_eq!(record.document_ids, vec!["doc-1".to_string()]); + assert_eq!(record.chunk_ids, vec!["doc-1:chunk-1".to_string()]); + } + + #[test] + fn merge_graph_attrs_recovers_from_invalid_existing_json_and_negative_evidence() { + let incoming = json!({ + "evidence_count": -4, + "document_id": "doc-2", + "chunk_id": "doc-2:chunk-9", + "order_index": 8 + }); + + let merged = UnifiedMemory::merge_graph_attrs(Some("not-json"), &incoming, 11.0); + assert_eq!( + merged["evidence_count"], + json!(1), + "negative evidence should clamp to the minimum count" + ); + assert_eq!(merged["document_ids"], json!(["doc-2"])); + assert_eq!(merged["chunk_ids"], json!(["doc-2:chunk-9"])); + assert_eq!(merged["order_index"], json!(8)); + assert_eq!(merged["created_at"], json!(11.0)); + assert_eq!(merged["updated_at"], json!(11.0)); + } + + #[test] + fn graph_relation_from_parts_defaults_invalid_attrs_payload() { + let record = UnifiedMemory::graph_relation_from_parts( + None, + "Alice".into(), + "OWNS".into(), + "Phoenix".into(), + "not-json", + 7.5, + ); + assert_eq!(record.evidence_count, 1); + assert_eq!(record.order_index, None); + assert!(record.document_ids.is_empty()); + assert!(record.chunk_ids.is_empty()); + assert_eq!(record.attrs, json!({})); + } + + #[test] + fn graph_relation_to_json_uses_expected_public_keys() { + let value = UnifiedMemory::graph_relation_to_json(GraphRelationRecord { + namespace: None, + subject: "Alice".into(), + predicate: "OWNS".into(), + object: "OpenHuman".into(), + attrs: json!({"extra": true}), + updated_at: 1.5, + evidence_count: 1, + order_index: Some(2), + document_ids: vec!["doc-1".into()], + chunk_ids: vec!["doc-1:chunk-1".into()], + }); + assert_eq!(value["subject"], "Alice"); + assert_eq!(value["predicate"], "OWNS"); + assert_eq!(value["evidenceCount"], 1); + assert_eq!(value["orderIndex"], 2); + assert_eq!(value["documentIds"], json!(["doc-1"])); + assert_eq!(value["chunkIds"], json!(["doc-1:chunk-1"])); + } + + fn test_memory() -> (TempDir, UnifiedMemory) { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + (tmp, memory) + } + + #[tokio::test] + async fn graph_upsert_namespace_merges_attrs_and_query_returns_json() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_namespace( + "team alpha/#1", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_id": "doc-1", + "chunk_id": "doc-1:chunk-1", + "evidence_count": 1 + }), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "team alpha/#1", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_ids": ["doc-2"], + "chunk_ids": ["doc-2:chunk-9"], + "order_index": 2 + }), + ) + .await + .unwrap(); + + let rows = memory + .graph_query_namespace("team alpha/#1", Some("Alice"), Some("OWNS")) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["subject"], "ALICE"); + assert_eq!(rows[0]["predicate"], "OWNS"); + assert_eq!(rows[0]["object"], "PHOENIX"); + assert_eq!(rows[0]["evidenceCount"], 2); + assert_eq!(rows[0]["orderIndex"], 2); + assert_eq!(rows[0]["documentIds"], json!(["doc-1", "doc-2"])); + assert_eq!( + rows[0]["chunkIds"], + json!(["doc-1:chunk-1", "doc-2:chunk-9"]) + ); + + let scoped = memory + .graph_relations_for_scope("team alpha/#1") + .await + .unwrap(); + assert_eq!(scoped.len(), 1); + assert_eq!(scoped[0].namespace.as_deref(), Some("team_alpha/_1")); + } + + #[tokio::test] + async fn graph_global_and_all_queries_include_expected_rows() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_global( + "Bob", + "MENTIONED", + "Launch", + &json!({"document_id": "doc-global"}), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "project", + "Alice", + "OWNS", + "Phoenix", + &json!({"document_id": "doc-local"}), + ) + .await + .unwrap(); + + let global = memory + .graph_query_global(Some("Bob"), Some("MENTIONED")) + .await + .unwrap(); + assert_eq!(global.len(), 1); + assert_eq!(global[0]["namespace"], Value::Null); + assert_eq!(global[0]["subject"], "BOB"); + + let all = memory.graph_query_all(None, None).await.unwrap(); + assert_eq!(all.len(), 2); + assert!(all.iter().any(|row| row["subject"] == "ALICE")); + assert!(all.iter().any(|row| row["subject"] == "BOB")); + } + + #[tokio::test] + async fn graph_relations_for_scope_includes_global_rows_and_sorts_newest_first() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_namespace( + "scope-a", + "Alice", + "OWNS", + "Phoenix", + &json!({"document_id": "doc-local"}), + ) + .await + .unwrap(); + memory + .graph_upsert_global( + "Bob", + "MENTIONED", + "Launch", + &json!({"document_id": "doc-global"}), + ) + .await + .unwrap(); + + let scoped = memory.graph_relations_for_scope("scope-a").await.unwrap(); + assert_eq!(scoped.len(), 2); + assert!(scoped + .iter() + .any(|row| row.namespace.as_deref() == Some("scope-a"))); + assert!(scoped.iter().any(|row| row.namespace.is_none())); + assert!( + scoped[0].updated_at >= scoped[1].updated_at, + "scope queries should stay sorted newest-first across namespace+global rows" + ); + } + + #[tokio::test] + async fn graph_remove_document_namespace_prunes_or_deletes_relations() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_namespace( + "cleanup", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_ids": ["doc-1", "doc-2"], + "chunk_ids": ["doc-1:chunk-1", "doc-2:chunk-2"] + }), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "cleanup", + "Alice", + "BLOCKED", + "Atlas", + &json!({ + "document_id": "doc-1", + "chunk_id": "doc-1:chunk-9" + }), + ) + .await + .unwrap(); + + memory + .graph_remove_document_namespace("cleanup", "doc-1") + .await + .unwrap(); + + let rows = memory + .graph_query_namespace("cleanup", None, None) + .await + .unwrap(); + assert_eq!( + rows.len(), + 1, + "single-doc relation should be deleted entirely" + ); + assert_eq!(rows[0]["predicate"], "OWNS"); + assert_eq!(rows[0]["documentIds"], json!(["doc-2"])); + assert_eq!(rows[0]["chunkIds"], json!(["doc-2:chunk-2"])); + } + + #[tokio::test] + async fn graph_remove_document_namespace_is_noop_for_unrelated_document() { + let (_tmp, memory) = test_memory(); + memory + .graph_upsert_namespace( + "cleanup", + "Alice", + "OWNS", + "Phoenix", + &json!({ + "document_ids": ["doc-2"], + "chunk_ids": ["doc-2:chunk-2"] + }), + ) + .await + .unwrap(); + + memory + .graph_remove_document_namespace("cleanup", "doc-missing") + .await + .unwrap(); + + let rows = memory + .graph_query_namespace("cleanup", None, None) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["documentIds"], json!(["doc-2"])); + assert_eq!(rows[0]["chunkIds"], json!(["doc-2:chunk-2"])); + } +} diff --git a/core/src/store/namespace_store/helpers.rs b/core/src/store/namespace_store/helpers.rs new file mode 100644 index 0000000..11d184b --- /dev/null +++ b/core/src/store/namespace_store/helpers.rs @@ -0,0 +1,411 @@ +//! Shared helpers used across the unified store: byte/float vector codecs, +//! cosine similarity, markdown chunking, text/predicate normalization, JSON +//! attribute merging, and recency scoring. + +use crate::store::chunks::chunk_semantic as chunk_markdown; + +use super::UnifiedMemory; + +impl UnifiedMemory { + #[allow(clippy::too_many_arguments)] + pub(crate) async fn write_markdown_doc( + &self, + namespace: &str, + doc_id: &str, + title: &str, + source_type: &str, + priority: &str, + tags: &[String], + created_at: f64, + updated_at: f64, + content: &str, + ) -> anyhow::Result { + let docs_dir = self.namespace_dir(namespace).join("docs"); + tokio::fs::create_dir_all(&docs_dir).await?; + let memory_subdir = self + .memory_dir + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("memory"); + let rel_path = format!( + "{memory_subdir}/namespaces/{}/docs/{doc_id}.md", + Self::sanitize_namespace(namespace) + ); + let abs_path = self.workspace_dir.join(&rel_path); + + let header = format!( + "---\ndoc_id: {doc_id}\nnamespace: {}\ntitle: {}\nsource_type: {}\npriority: {}\ntags: {}\ncreated_at: {}\nupdated_at: {}\n---\n\n", + namespace.replace('\n', " "), + title.replace('\n', " "), + source_type.replace('\n', " "), + priority.replace('\n', " "), + serde_json::to_string(tags).unwrap_or_else(|_| "[]".to_string()), + created_at, + updated_at + ); + tokio::fs::write(abs_path, format!("{header}{content}\n")).await?; + Ok(rel_path) + } + + pub(crate) fn vec_to_bytes(v: &[f32]) -> Vec { + let mut bytes = Vec::with_capacity(v.len() * 4); + for &f in v { + bytes.extend_from_slice(&f.to_le_bytes()); + } + bytes + } + + pub(crate) fn bytes_to_vec(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(4) + .map(|chunk| { + let arr: [u8; 4] = chunk.try_into().unwrap_or([0; 4]); + f32::from_le_bytes(arr) + }) + .collect() + } + + pub(crate) fn cosine_similarity(a: &[f32], b: &[f32]) -> f64 { + if a.len() != b.len() || a.is_empty() { + return 0.0; + } + let mut dot = 0.0_f64; + let mut norm_a = 0.0_f64; + let mut norm_b = 0.0_f64; + for (x, y) in a.iter().zip(b.iter()) { + let x = f64::from(*x); + let y = f64::from(*y); + dot += x * y; + norm_a += x * x; + norm_b += y * y; + } + let denom = norm_a.sqrt() * norm_b.sqrt(); + if denom <= f64::EPSILON { + return 0.0; + } + (dot / denom).clamp(0.0, 1.0) + } + + pub(crate) fn chunk_document_content(content: &str, max_tokens: usize) -> Vec { + let mut chunks: Vec = chunk_markdown(content, max_tokens.max(1)) + .into_iter() + .map(|chunk| chunk.content.trim().to_string()) + .filter(|chunk: &String| !chunk.is_empty()) + .collect(); + if chunks.is_empty() && !content.trim().is_empty() { + chunks.push(content.trim().to_string()); + } + chunks + } + + pub(crate) fn collapse_whitespace(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") + } + + pub(crate) fn normalize_search_text(text: &str) -> String { + let collapsed = Self::collapse_whitespace(text); + let mut normalized = String::with_capacity(collapsed.len()); + for ch in collapsed.chars() { + if ch.is_alphanumeric() { + normalized.extend(ch.to_lowercase()); + } else if ch.is_whitespace() || matches!(ch, '_' | '-' | '/' | '.') { + normalized.push(' '); + } + } + normalized.split_whitespace().collect::>().join(" ") + } + + pub(crate) fn tokenize_search_terms(text: &str) -> Vec { + Self::normalize_search_text(text) + .split_whitespace() + .map(ToOwned::to_owned) + .collect() + } + + pub(crate) fn normalize_graph_entity(text: &str) -> String { + Self::collapse_whitespace(text.trim()).to_uppercase() + } + + pub(crate) fn normalize_graph_predicate(text: &str) -> String { + let mut out = String::new(); + let mut last_was_sep = false; + for ch in Self::collapse_whitespace(text.trim()).chars() { + if ch.is_alphanumeric() { + out.extend(ch.to_uppercase()); + last_was_sep = false; + } else if !last_was_sep { + out.push('_'); + last_was_sep = true; + } + } + out.trim_matches('_').to_string() + } + + pub(crate) fn json_string_array( + value: &serde_json::Value, + primary_key: &str, + singular_key: &str, + ) -> Vec { + let mut items = Vec::new(); + if let Some(array) = value.get(primary_key).and_then(serde_json::Value::as_array) { + for item in array { + if let Some(text) = item.as_str() { + let trimmed = text.trim(); + if !trimmed.is_empty() { + items.push(trimmed.to_string()); + } + } + } + } + if let Some(text) = value.get(singular_key).and_then(serde_json::Value::as_str) { + let trimmed = text.trim(); + if !trimmed.is_empty() { + items.push(trimmed.to_string()); + } + } + items.sort(); + items.dedup(); + items + } + + pub(crate) fn json_i64(value: &serde_json::Value, key: &str) -> Option { + value.get(key).and_then(|raw| { + raw.as_i64().or_else(|| { + raw.as_u64() + .and_then(|v| i64::try_from(v).ok()) + .or_else(|| raw.as_f64().map(|v| v as i64)) + }) + }) + } + + pub(crate) fn recency_score(updated_at: f64, now: f64) -> f64 { + let age_secs = (now - updated_at).max(0.0); + let age_hours = age_secs / 3600.0; + (1.0 / (1.0 + age_hours / 24.0)).clamp(0.0, 1.0) + } +} + +#[cfg(test)] +mod tests { + use super::UnifiedMemory; + use serde_json::json; + + // ── vec_to_bytes / bytes_to_vec ────────────────────────────────── + + #[test] + fn vec_bytes_roundtrip() { + let original = vec![1.0_f32, 2.5, -3.0, 0.0]; + let bytes = UnifiedMemory::vec_to_bytes(&original); + assert_eq!(bytes.len(), 16); // 4 floats * 4 bytes + let back = UnifiedMemory::bytes_to_vec(&bytes); + assert_eq!(back, original); + } + + #[test] + fn vec_to_bytes_empty() { + let bytes = UnifiedMemory::vec_to_bytes(&[]); + assert!(bytes.is_empty()); + let back = UnifiedMemory::bytes_to_vec(&bytes); + assert!(back.is_empty()); + } + + // ── cosine_similarity ──────────────────────────────────────────── + + #[test] + fn cosine_similarity_identical_vectors() { + let v = vec![1.0_f32, 0.0, 0.0]; + let sim = UnifiedMemory::cosine_similarity(&v, &v); + assert!((sim - 1.0).abs() < 1e-6); + } + + #[test] + fn cosine_similarity_orthogonal_vectors() { + let a = vec![1.0_f32, 0.0]; + let b = vec![0.0_f32, 1.0]; + let sim = UnifiedMemory::cosine_similarity(&a, &b); + assert!(sim.abs() < 1e-6); + } + + #[test] + fn cosine_similarity_different_lengths_returns_zero() { + let a = vec![1.0_f32, 0.0]; + let b = vec![1.0_f32, 0.0, 0.0]; + assert_eq!(UnifiedMemory::cosine_similarity(&a, &b), 0.0); + } + + #[test] + fn cosine_similarity_empty_vectors_returns_zero() { + assert_eq!(UnifiedMemory::cosine_similarity(&[], &[]), 0.0); + } + + #[test] + fn cosine_similarity_zero_vector_returns_zero() { + let a = vec![0.0_f32, 0.0]; + let b = vec![1.0_f32, 0.0]; + assert_eq!(UnifiedMemory::cosine_similarity(&a, &b), 0.0); + } + + // ── collapse_whitespace ────────────────────────────────────────── + + #[test] + fn collapse_whitespace_normalizes() { + assert_eq!( + UnifiedMemory::collapse_whitespace(" hello world "), + "hello world" + ); + } + + #[test] + fn collapse_whitespace_empty() { + assert_eq!(UnifiedMemory::collapse_whitespace(""), ""); + } + + // ── normalize_search_text ──────────────────────────────────────── + + #[test] + fn normalize_search_text_lowercases_and_strips_special() { + let result = UnifiedMemory::normalize_search_text("Hello, World! @#$ test"); + assert_eq!(result, "hello world test"); + } + + #[test] + fn normalize_search_text_preserves_separators() { + let result = UnifiedMemory::normalize_search_text("path/to_file-name.txt"); + assert_eq!(result, "path to file name txt"); + } + + // ── tokenize_search_terms ──────────────────────────────────────── + + #[test] + fn tokenize_search_terms_splits_correctly() { + let terms = UnifiedMemory::tokenize_search_terms("Hello World"); + assert_eq!(terms, vec!["hello", "world"]); + } + + #[test] + fn tokenize_search_terms_empty() { + assert!(UnifiedMemory::tokenize_search_terms("").is_empty()); + assert!(UnifiedMemory::tokenize_search_terms(" @#$ ").is_empty()); + } + + // ── normalize_graph_entity / predicate ─────────────────────────── + + #[test] + fn normalize_graph_entity_uppercases() { + assert_eq!( + UnifiedMemory::normalize_graph_entity(" rust language "), + "RUST LANGUAGE" + ); + } + + #[test] + fn normalize_graph_predicate_underscores_separators() { + assert_eq!( + UnifiedMemory::normalize_graph_predicate("is written in"), + "IS_WRITTEN_IN" + ); + } + + #[test] + fn normalize_graph_predicate_strips_trailing_underscores() { + assert_eq!(UnifiedMemory::normalize_graph_predicate(" has -- "), "HAS"); + } + + // ── json_string_array ──────────────────────────────────────────── + + #[test] + fn json_string_array_from_array_and_singular() { + let val = json!({"tags": ["a", "b"], "tag": "c"}); + let result = UnifiedMemory::json_string_array(&val, "tags", "tag"); + assert_eq!(result, vec!["a", "b", "c"]); + } + + #[test] + fn json_string_array_deduplicates() { + let val = json!({"tags": ["a", "a"], "tag": "a"}); + let result = UnifiedMemory::json_string_array(&val, "tags", "tag"); + assert_eq!(result, vec!["a"]); + } + + #[test] + fn json_string_array_empty_when_missing() { + let val = json!({}); + let result = UnifiedMemory::json_string_array(&val, "tags", "tag"); + assert!(result.is_empty()); + } + + #[test] + fn json_string_array_filters_empty_strings() { + let val = json!({"tags": ["", " ", "valid"]}); + let result = UnifiedMemory::json_string_array(&val, "tags", "tag"); + assert_eq!(result, vec!["valid"]); + } + + // ── json_i64 ───────────────────────────────────────────────────── + + #[test] + fn json_i64_from_integer() { + assert_eq!(UnifiedMemory::json_i64(&json!({"n": 42}), "n"), Some(42)); + } + + #[test] + fn json_i64_from_float() { + assert_eq!(UnifiedMemory::json_i64(&json!({"n": 3.9}), "n"), Some(3)); + } + + #[test] + fn json_i64_missing_key() { + assert_eq!(UnifiedMemory::json_i64(&json!({}), "n"), None); + } + + #[test] + fn json_i64_from_string_returns_none() { + assert_eq!(UnifiedMemory::json_i64(&json!({"n": "42"}), "n"), None); + } + + // ── recency_score ──────────────────────────────────────────────── + + #[test] + fn recency_score_current_time_is_one() { + let now = 1_700_000_000.0; + let score = UnifiedMemory::recency_score(now, now); + assert!((score - 1.0).abs() < 1e-6); + } + + #[test] + fn recency_score_old_document_is_lower() { + let now = 1_700_000_000.0; + let one_day_ago = now - 86400.0; + let score = UnifiedMemory::recency_score(one_day_ago, now); + assert!(score < 1.0); + assert!(score > 0.0); + } + + #[test] + fn recency_score_future_clamped_to_one() { + let now = 1_700_000_000.0; + let future = now + 86400.0; + let score = UnifiedMemory::recency_score(future, now); + assert!((score - 1.0).abs() < 1e-6); + } + + // ── chunk_document_content ─────────────────────────────────────── + + #[test] + fn chunk_document_content_returns_nonempty_for_content() { + let chunks = UnifiedMemory::chunk_document_content("Hello world. This is a test.", 100); + assert!(!chunks.is_empty()); + } + + #[test] + fn chunk_document_content_empty_input_returns_empty() { + let chunks = UnifiedMemory::chunk_document_content("", 100); + assert!(chunks.is_empty()); + } + + #[test] + fn chunk_document_content_whitespace_only_returns_empty() { + let chunks = UnifiedMemory::chunk_document_content(" \n \t ", 100); + assert!(chunks.is_empty()); + } +} diff --git a/core/src/store/namespace_store/init.rs b/core/src/store/namespace_store/init.rs new file mode 100644 index 0000000..a8d0093 --- /dev/null +++ b/core/src/store/namespace_store/init.rs @@ -0,0 +1,605 @@ +//! `UnifiedMemory` constructor + schema bootstrap. +//! +//! Creates the workspace directories, opens the SQLite connection in WAL mode, +//! materialises every table the unified store owns (docs, kv, graph, vector +//! chunks, episodic FTS5, segments, events, profile), and runs idempotent +//! legacy-namespace migrations. Also exposes path / namespace helpers shared +//! by the rest of the unified module. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::Context as _; +use parking_lot::Mutex; +use rusqlite::Connection; + +use crate::store::safety::canonical_identifier; +use crate::store::types::GLOBAL_NAMESPACE; +use tinymemory_api::host::EmbeddingProvider; + +use super::UnifiedMemory; + +/// What an idempotent additive `ALTER TABLE … ADD COLUMN` actually did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AdditiveMigration { + /// The column did not exist and was added. + Applied, + /// SQLite reported `duplicate column name` — an older DB already has it. + AlreadyPresent, + /// SQLite reported `no such table` — the table has not been created yet + /// (the fresh-install case for the profile Phase-3 columns, which run + /// *before* `PROFILE_INIT_SQL`). + TableAbsent, +} + +/// Classify a rusqlite error as one of the two benign, expected outcomes of an +/// idempotent additive migration, or `None` when it is a genuine failure. +/// +/// SQLite reports both conditions as the generic `SQLITE_ERROR` (code 1) with +/// no distinguishing extended code, so the message is the only signal. Every +/// other error — a malformed statement, a read-only or corrupt database, disk +/// I/O — must surface rather than be swallowed as "column already exists". +fn classify_additive_migration_error(err: &rusqlite::Error) -> Option { + let rusqlite::Error::SqliteFailure(_, Some(message)) = err else { + return None; + }; + let lowered = message.to_ascii_lowercase(); + if lowered.contains("duplicate column name") { + Some(AdditiveMigration::AlreadyPresent) + } else if lowered.contains("no such table") { + Some(AdditiveMigration::TableAbsent) + } else { + None + } +} + +/// Run one idempotent additive migration, swallowing **only** the +/// duplicate-column and missing-table cases. +/// +/// Before this existed, every `ALTER TABLE` on the boot path matched +/// `Err(_)` and logged at `trace`, so a genuinely broken statement or an +/// unwritable database was indistinguishable from a no-op re-run and the store +/// came up silently missing a column. +pub(crate) fn apply_additive_migration( + conn: &Connection, + sql: &str, + scope: &str, +) -> anyhow::Result { + match conn.execute(sql, []) { + Ok(_) => { + tracing::debug!("[{scope}:init] additive migration applied: {sql}"); + Ok(AdditiveMigration::Applied) + } + Err(e) => match classify_additive_migration_error(&e) { + Some(outcome @ AdditiveMigration::AlreadyPresent) => { + tracing::trace!("[{scope}:init] column already present, skipping: {sql}"); + Ok(outcome) + } + Some(outcome @ AdditiveMigration::TableAbsent) => { + tracing::trace!("[{scope}:init] table not created yet, skipping: {sql}"); + Ok(outcome) + } + Some(AdditiveMigration::Applied) => unreachable!("Applied is not an error outcome"), + None => { + tracing::error!("[{scope}:init] additive migration FAILED: {sql}: {e}"); + Err(anyhow::Error::new(e) + .context(format!("[{scope}:init] additive migration failed: {sql}"))) + } + }, + } +} + +impl UnifiedMemory { + /// Open (or create) the unified store rooted at `workspace_dir`. + /// + /// Delegates to [`Self::new_with_memory_dir`] using the default + /// `"memory"` subdirectory name. Safe to call on every boot. + pub fn new( + workspace_dir: &Path, + embedder: Arc, + _open_timeout_secs: Option, + ) -> anyhow::Result { + Self::new_with_memory_dir(workspace_dir, "memory", embedder, _open_timeout_secs) + } + + /// Open (or create) a unified store using an explicit memory subdirectory + /// name under `workspace_dir`. + /// + /// This enables multiple independent stores inside the same workspace (e.g. + /// per-personality databases) without path collisions. `memory_subdir` is + /// joined directly to `workspace_dir`, so `"memory-1"` yields + /// `workspace_dir/memory-1/memory.db`. + /// + /// Creates the on-disk layout, runs all `CREATE TABLE` statements, and + /// applies idempotent legacy-namespace migrations. Safe to call on every + /// boot. + pub fn new_with_memory_dir( + workspace_dir: &Path, + memory_subdir: &str, + embedder: Arc, + _open_timeout_secs: Option, + ) -> anyhow::Result { + use std::path::Component; + anyhow::ensure!(!memory_subdir.is_empty(), "memory_subdir must not be empty"); + let subdir_path = Path::new(memory_subdir); + anyhow::ensure!( + subdir_path.components().count() == 1 + && subdir_path + .components() + .all(|c| matches!(c, Component::Normal(_))), + "memory_subdir must be a single relative path component without traversal" + ); + let memory_dir = workspace_dir.join(subdir_path); + let namespaces_dir = memory_dir.join("namespaces"); + let vectors_dir = memory_dir.join("vectors"); + std::fs::create_dir_all(&namespaces_dir)?; + std::fs::create_dir_all(&vectors_dir)?; + + let db_path = memory_dir.join("memory.db"); + let conn = Connection::open(&db_path)?; + // Active storage layout for the core memory domain: + // - memory_docs: namespace-scoped source documents and markdown metadata. + // - vector_chunks: chunked document text plus optional local embedding bytes. + // - graph_namespace: namespace graph edges used for relation-first retrieval. + // - graph_global: cross-namespace graph edges used as fallback/shared memory. + // - kv_namespace: namespace-scoped durable preferences, decisions, and state. + // - kv_global: global durable key-value memories outside a namespace scope. + // Absorb concurrent write contention under cargo-llvm-cov and any other + // scenario where background workers hold the write lock while a second + // connection attempts a write. Without a timeout the driver returns + // SQLITE_BUSY immediately, which causes test flakes and runtime errors. + conn.busy_timeout(std::time::Duration::from_secs(15)) + .context("configure unified memory busy_timeout")?; + + conn.execute_batch( + "PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + + CREATE TABLE IF NOT EXISTS memory_docs ( + document_id TEXT PRIMARY KEY, + namespace TEXT NOT NULL, + key TEXT NOT NULL, + title TEXT NOT NULL, + content TEXT NOT NULL, + source_type TEXT NOT NULL, + priority TEXT NOT NULL, + tags_json TEXT NOT NULL, + metadata_json TEXT NOT NULL, + category TEXT NOT NULL, + session_id TEXT, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + markdown_rel_path TEXT NOT NULL, + taint TEXT NOT NULL DEFAULT 'internal', + UNIQUE(namespace, key) + ); + CREATE INDEX IF NOT EXISTS idx_memory_docs_ns_updated ON memory_docs(namespace, updated_at DESC); + + CREATE TABLE IF NOT EXISTS kv_global ( + key TEXT PRIMARY KEY, + value_json TEXT NOT NULL, + updated_at REAL NOT NULL + ); + + CREATE TABLE IF NOT EXISTS kv_namespace ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + value_json TEXT NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY(namespace, key) + ); + CREATE INDEX IF NOT EXISTS idx_kv_namespace_ns ON kv_namespace(namespace); + + CREATE TABLE IF NOT EXISTS graph_global ( + subject TEXT NOT NULL, + predicate TEXT NOT NULL, + object TEXT NOT NULL, + attrs_json TEXT NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY(subject, predicate, object) + ); + CREATE INDEX IF NOT EXISTS idx_graph_global_subject ON graph_global(subject, predicate); + + CREATE TABLE IF NOT EXISTS graph_namespace ( + namespace TEXT NOT NULL, + subject TEXT NOT NULL, + predicate TEXT NOT NULL, + object TEXT NOT NULL, + attrs_json TEXT NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY(namespace, subject, predicate, object) + ); + CREATE INDEX IF NOT EXISTS idx_graph_namespace_ns ON graph_namespace(namespace); + CREATE INDEX IF NOT EXISTS idx_graph_namespace_subject ON graph_namespace(namespace, subject, predicate); + + CREATE TABLE IF NOT EXISTS vector_chunks ( + namespace TEXT NOT NULL, + document_id TEXT NOT NULL, + chunk_id TEXT NOT NULL, + text TEXT NOT NULL, + embedding BLOB, + metadata_json TEXT NOT NULL, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + model_signature TEXT, + dim INTEGER, + PRIMARY KEY(namespace, chunk_id) + ); + CREATE INDEX IF NOT EXISTS idx_vector_chunks_ns_doc ON vector_chunks(namespace, document_id);", + )?; + + // Tag vector_chunks with the embedding model that produced each vector + // on existing databases (idempotent). Fresh installs get these from the + // CREATE TABLE above; older DBs need the ALTERs so recall can exclude + // vectors generated by a different embedding model (cross-model cosine is + // garbage) and skip dimension mismatches instead of silently scoring 0. + for sql in [ + "ALTER TABLE vector_chunks ADD COLUMN model_signature TEXT", + "ALTER TABLE vector_chunks ADD COLUMN dim INTEGER", + ] { + apply_additive_migration(&conn, sql, "vector_chunks")?; + } + + // Backfill the `taint` column on existing `memory_docs` databases. + // Fresh installs get this via the CREATE TABLE above; older DBs need + // the ALTER so retrieval can carry the provenance signal up to the + // subconscious gate. Idempotent: a duplicate-column error on + // re-application is expected (logged at trace). + apply_additive_migration( + &conn, + "ALTER TABLE memory_docs ADD COLUMN taint TEXT NOT NULL DEFAULT 'internal'", + "memory_docs", + )?; + + // Create FTS5 episodic tables (episodic_log, episodic_fts, and their + // triggers) so the Archivist can call episodic_insert immediately after + // the store is initialised. + conn.execute_batch(super::fts5::EPISODIC_INIT_SQL)?; + + // Conversation segmentation tables. + conn.execute_batch(super::segments::SEGMENTS_INIT_SQL)?; + + // Backfill the (start_seq, end_seq) columns on existing databases + // — fresh installs get them from SEGMENTS_INIT_SQL above; older DBs + // need the ALTER TABLEs. Idempotent: a duplicate-column error is + // expected and logged at trace level. + for sql in super::segments::SEGMENTS_MIGRATIONS_SQL { + apply_additive_migration(&conn, sql, "segments")?; + } + + // Event extraction tables. + conn.execute_batch(super::events::EVENTS_INIT_SQL)?; + + // Phase 3 (#566): add new columns to existing databases BEFORE running + // PROFILE_INIT_SQL. PROFILE_INIT_SQL includes indexes that reference + // state/stability/user_state — those CREATE INDEX statements fail on + // pre-Phase-3 databases unless the columns already exist. + // On fresh installs the table doesn't exist yet so ALTER TABLE fails + // silently here; PROFILE_INIT_SQL then creates it with all columns. + { + use super::profile::PHASE3_COLUMNS_SQL; + for sql in PHASE3_COLUMNS_SQL.iter() { + // `TableAbsent` is the expected fresh-install outcome here: + // these run *before* `PROFILE_INIT_SQL` creates `user_profile`. + apply_additive_migration(&conn, sql, "profile")?; + } + } + + // User profile accumulation table (CREATE TABLE IF NOT EXISTS + all indexes). + // On existing databases the table creation is a no-op; the index creation + // succeeds because PHASE3_COLUMNS_SQL above has already added the columns. + conn.execute_batch(super::profile::PROFILE_INIT_SQL)?; + + // Phase 3 indexes: idempotently restore performance indexes removed in #1616. + // New installs get these from PROFILE_INIT_SQL; existing DBs get them here, + // after PHASE3_COLUMNS_SQL has ensured the columns exist. + { + use super::profile::PHASE3_INDEXES_SQL; + for sql in PHASE3_INDEXES_SQL { + match conn.execute_batch(sql) { + Ok(_) => tracing::debug!("[profile:init] index applied: {sql}"), + Err(e) => { + tracing::warn!( + "[profile:init] index creation failed (non-fatal): {sql}: {e}" + ); + } + } + } + } + + // Idempotent legacy-namespace migration. + // + // Older writes via MemoryStoreTool packed the intended namespace into + // the key as `"{namespace}/{actual_key}"` and stored the row under the + // GLOBAL_NAMESPACE. Split those rows now so the new trait surface can + // rely on the `namespace` column. + // + // The anti-join guard prevents duplicate-split collisions if a + // post-split row already exists (UNIQUE(namespace, key) would otherwise + // fail). Safe to run on every boot. + let migrated = conn.execute( + "UPDATE memory_docs + SET namespace = substr(key, 1, instr(key, '/') - 1), + key = substr(key, instr(key, '/') + 1) + WHERE namespace = ?1 + AND instr(key, '/') > 0 + AND NOT EXISTS ( + SELECT 1 FROM memory_docs m2 + WHERE m2.namespace = substr(memory_docs.key, 1, instr(memory_docs.key, '/') - 1) + AND m2.key = substr(memory_docs.key, instr(memory_docs.key, '/') + 1) + )", + rusqlite::params![GLOBAL_NAMESPACE], + )?; + if migrated > 0 { + log::info!( + "[memory] migrated {migrated} legacy `ns/key` rows out of the `{GLOBAL_NAMESPACE}` namespace" + ); + } + + // Companion migration: `vector_chunks` rows keyed by `document_id` still + // point at `GLOBAL_NAMESPACE` after the `memory_docs` split above, so + // namespace-scoped recall would miss them. Re-home each chunk to its + // document's new namespace. Idempotent: after both migrations run, no + // chunk under GLOBAL_NAMESPACE maps to a document in another namespace. + let chunks_migrated = conn.execute( + "UPDATE vector_chunks + SET namespace = ( + SELECT namespace FROM memory_docs + WHERE memory_docs.document_id = vector_chunks.document_id + ) + WHERE namespace = ?1 + AND document_id IN ( + SELECT document_id FROM memory_docs WHERE namespace != ?1 + )", + rusqlite::params![GLOBAL_NAMESPACE], + )?; + if chunks_migrated > 0 { + log::info!( + "[memory] migrated {chunks_migrated} vector_chunks rows out of the `{GLOBAL_NAMESPACE}` namespace" + ); + } + + Ok(Self { + workspace_dir: workspace_dir.to_path_buf(), + memory_dir, + db_path, + vectors_dir, + conn: Arc::new(Mutex::new(conn)), + embedder, + }) + } + + /// Root workspace directory holding `memory/` and its subtrees. + pub fn workspace_dir(&self) -> &Path { + &self.workspace_dir + } + + /// Filesystem path of the SQLite database file. + pub fn db_path(&self) -> &Path { + &self.db_path + } + + /// Directory used for vector-related sidecar files. + pub fn vectors_dir(&self) -> &Path { + &self.vectors_dir + } + + pub(crate) fn now_ts() -> f64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0) + } + + /// Canonical storage form of a namespace: PII-bearing namespaces are + /// canonicalized (#5164), then path-hostile characters collapse to `_`. + /// + /// The PII step lives here, in the one funnel every namespace path already + /// goes through — writes, reads, recall/search (`query.rs`), graph relations + /// (`graph.rs`), deletes, and the on-disk `namespaces//` directory — so + /// a canonicalized write stays addressable by its original namespace + /// instead of looking like a missing row and driving the caller to retry. + pub(crate) fn sanitize_namespace(namespace: &str) -> String { + let trimmed = canonical_identifier(namespace.trim()); + if trimmed.is_empty() { + return GLOBAL_NAMESPACE.to_string(); + } + trimmed + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' || ch == '/' { + ch + } else { + '_' + } + }) + .collect() + } + + /// Resolved memory subdirectory for this store instance (e.g. + /// `workspace_dir/memory` for the default store, or a custom subdir for + /// personality-specific stores). + pub fn memory_dir(&self) -> &Path { + &self.memory_dir + } + + pub(crate) fn namespace_dir(&self, namespace: &str) -> PathBuf { + self.memory_dir + .join("namespaces") + .join(Self::sanitize_namespace(namespace)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + use tinymemory_api::host::NoopEmbedding; + + #[test] + fn sanitize_namespace_defaults_and_scrubs() { + assert_eq!(UnifiedMemory::sanitize_namespace(""), GLOBAL_NAMESPACE); + assert_eq!(UnifiedMemory::sanitize_namespace(" "), GLOBAL_NAMESPACE); + assert_eq!( + UnifiedMemory::sanitize_namespace("team alpha/#1"), + "team_alpha/_1" + ); + assert_eq!(UnifiedMemory::sanitize_namespace("a-b_c/ok"), "a-b_c/ok"); + } + + /// #5164: the PII step lives in this one funnel so every namespace path + /// (write, read, recall/search, graph, delete, on-disk dir) derives the same + /// address. Strict-gated — scanner-built namespaces keep their identity. + #[test] + fn sanitize_namespace_canonicalizes_pii_and_preserves_scanner_namespaces() { + let canonical = UnifiedMemory::sanitize_namespace("cliente-RFC-VECJ880326XK4"); + assert!( + !canonical.contains("VECJ880326XK4"), + "the national ID must not become the storage address, got: {canonical}" + ); + assert!( + canonical.contains("REDACTED_PII"), + "expected a redaction placeholder, got: {canonical}" + ); + // Idempotent, so read paths can canonicalize unconditionally. + assert_eq!(UnifiedMemory::sanitize_namespace(&canonical), canonical); + + for namespace in ["whatsapp-web:12025551234@c.us", "skill-gmail", "global"] { + assert_eq!( + UnifiedMemory::sanitize_namespace(namespace), + namespace.replace(['@', ':', '.'], "_"), + "scanner-built namespace must only get the character scrub: {namespace}" + ); + } + } + + #[test] + fn namespace_dir_uses_sanitized_namespace() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let dir = memory.namespace_dir("team alpha/#1"); + assert_eq!( + dir, + tmp.path() + .join("memory") + .join("namespaces") + .join("team_alpha/_1") + ); + } + + #[test] + fn new_with_memory_dir_creates_separate_db() { + let tmp = TempDir::new().unwrap(); + let mem1 = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let mem2 = UnifiedMemory::new_with_memory_dir( + tmp.path(), + "memory-1", + Arc::new(NoopEmbedding), + None, + ) + .unwrap(); + assert_ne!(mem1.db_path(), mem2.db_path()); + assert!( + mem1.db_path().ends_with("memory/memory.db"), + "expected mem1 db under memory/memory.db, got {:?}", + mem1.db_path() + ); + assert!( + mem2.db_path().ends_with("memory-1/memory.db"), + "expected mem2 db under memory-1/memory.db, got {:?}", + mem2.db_path() + ); + assert!(mem1.db_path().exists(), "mem1 db file must exist on disk"); + assert!(mem2.db_path().exists(), "mem2 db file must exist on disk"); + } + + // ── Additive-migration error narrowing ────────────────────────────── + // + // Before `apply_additive_migration` existed these four boot-path + // `ALTER TABLE`s matched `Err(_)` and logged at `trace`, so a genuinely + // failing statement was indistinguishable from "column already exists". + + fn scratch_conn() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch("CREATE TABLE t (a TEXT);").unwrap(); + conn + } + + #[test] + fn additive_migration_applies_a_new_column() { + let conn = scratch_conn(); + assert_eq!( + apply_additive_migration(&conn, "ALTER TABLE t ADD COLUMN b TEXT", "test").unwrap(), + AdditiveMigration::Applied + ); + } + + #[test] + fn additive_migration_swallows_duplicate_column() { + let conn = scratch_conn(); + apply_additive_migration(&conn, "ALTER TABLE t ADD COLUMN b TEXT", "test").unwrap(); + assert_eq!( + apply_additive_migration(&conn, "ALTER TABLE t ADD COLUMN b TEXT", "test").unwrap(), + AdditiveMigration::AlreadyPresent + ); + } + + #[test] + fn additive_migration_swallows_missing_table() { + let conn = scratch_conn(); + assert_eq!( + apply_additive_migration(&conn, "ALTER TABLE nope ADD COLUMN b TEXT", "test").unwrap(), + AdditiveMigration::TableAbsent + ); + } + + #[test] + fn additive_migration_surfaces_a_genuine_failure() { + let conn = scratch_conn(); + // Not a duplicate column and not a missing table: a malformed + // statement. Swallowing this would leave the store silently missing a + // column that recall depends on, with only a trace-level breadcrumb. + let err = apply_additive_migration(&conn, "ALTER TABLE t ADD COLUMN", "test") + .expect_err("a real ALTER TABLE failure must surface, not be swallowed as idempotent"); + let rendered = format!("{err:#}"); + assert!( + rendered.contains("additive migration failed"), + "error must name the failing migration, got: {rendered}" + ); + } + + #[test] + fn additive_migration_surfaces_a_readonly_database() { + // A read-only DB is the real-world shape of this defect: every ALTER + // fails, the old code logged each at trace, and the store came up + // missing columns that recall depends on. + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("ro.db"); + { + let conn = Connection::open(&path).unwrap(); + conn.execute_batch("CREATE TABLE t (a TEXT);").unwrap(); + } + let conn = + Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY).unwrap(); + assert!( + apply_additive_migration(&conn, "ALTER TABLE t ADD COLUMN b TEXT", "test").is_err(), + "a read-only database must fail the migration, not look idempotent" + ); + } + + #[test] + fn connection_has_busy_timeout_set() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + let conn = memory.conn.lock(); + // SQLite reports busy_timeout as a PRAGMA; 0 means no timeout. + let timeout: i64 = conn + .query_row("PRAGMA busy_timeout", [], |row| row.get(0)) + .unwrap(); + assert!( + timeout > 0, + "busy_timeout must be non-zero to absorb write contention, got {timeout}" + ); + } +} diff --git a/core/src/store/namespace_store/mod.rs b/core/src/store/namespace_store/mod.rs new file mode 100644 index 0000000..e3ae140 --- /dev/null +++ b/core/src/store/namespace_store/mod.rs @@ -0,0 +1,36 @@ +//! SQLite-backed unified namespace memory store. + +use parking_lot::Mutex; +use rusqlite::Connection; +use std::path::PathBuf; +use std::sync::Arc; + +use tinymemory_api::host::EmbeddingProvider; + +/// SQLite-backed unified memory store. +/// +/// Owns a single connection (WAL-mode) plus the on-disk markdown sidecar +/// directory and vector storage path. Methods are added across the sibling +/// modules (`documents`, `kv`, `graph`, `query`, …) via `impl` blocks. +pub struct UnifiedMemory { + pub(crate) workspace_dir: PathBuf, + /// Resolved memory subdirectory (e.g. `workspace_dir/memory` or a custom + /// per-personality path). Drives `db_path`, `vectors_dir`, and + /// `namespace_dir()` so that multiple stores rooted in the same workspace + /// don't collide. + pub(crate) memory_dir: PathBuf, + pub(crate) db_path: PathBuf, + pub(crate) vectors_dir: PathBuf, + pub(crate) conn: Arc>, + pub(crate) embedder: Arc, +} + +mod documents; +pub mod events; +pub mod fts5; +mod graph; +mod helpers; +mod init; +pub mod profile; +mod query; +pub mod segments; diff --git a/core/src/store/namespace_store/profile.rs b/core/src/store/namespace_store/profile.rs new file mode 100644 index 0000000..1e8fe47 --- /dev/null +++ b/core/src/store/namespace_store/profile.rs @@ -0,0 +1,721 @@ +//! User profile accumulation — structured, evidence-backed profile facets +//! that accumulate across sessions. +//! +//! Profile facets are extracted from conversation events (preferences, +//! facts about the user, skills, roles) and stored with confidence scores +//! and evidence counts. On conflict (same facet_type + key), evidence_count +//! is incremented; the value is only overwritten if the new confidence is +//! higher. +//! +//! ## Phase 3 schema additions (#566) +//! +//! Added `state`, `stability`, `user_state`, and `evidence_refs_json` columns. +//! Existing databases are migrated idempotently via `ALTER TABLE … ADD COLUMN` +//! wrapped in `migrate_profile_schema()`. + +use parking_lot::Mutex; +use rusqlite::{params, Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; + +use tinymemory_api::host::EvidenceRef; + +/// SQL to create the user_profile table. Called during UnifiedMemory init. +pub const PROFILE_INIT_SQL: &str = r#" +CREATE TABLE IF NOT EXISTS user_profile ( + facet_id TEXT PRIMARY KEY, + facet_type TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.5, + evidence_count INTEGER NOT NULL DEFAULT 1, + source_segment_ids TEXT, + first_seen_at REAL NOT NULL, + last_seen_at REAL NOT NULL, + state TEXT NOT NULL DEFAULT 'active', + stability REAL NOT NULL DEFAULT 0.0, + user_state TEXT NOT NULL DEFAULT 'auto', + evidence_refs_json TEXT, + class TEXT, + cue_families_json TEXT, + UNIQUE(facet_type, key) +); + +CREATE INDEX IF NOT EXISTS idx_profile_type + ON user_profile(facet_type); + +CREATE INDEX IF NOT EXISTS idx_profile_state_stability + ON user_profile(state, stability DESC); + +CREATE INDEX IF NOT EXISTS idx_profile_key + ON user_profile(key); + +CREATE INDEX IF NOT EXISTS idx_profile_state_user_stability + ON user_profile(state, user_state, stability); +"#; + +/// Phase 3 ALTER TABLE statements for adding new columns to existing databases. +/// +/// Used by both `migrate_profile_schema` (post-Arc-wrap path) and +/// `init.rs` (pre-Arc-wrap path) to avoid duplicating the SQL. +pub const PHASE3_COLUMNS_SQL: &[&str] = &[ + "ALTER TABLE user_profile ADD COLUMN state TEXT NOT NULL DEFAULT 'active'", + "ALTER TABLE user_profile ADD COLUMN stability REAL NOT NULL DEFAULT 0.0", + "ALTER TABLE user_profile ADD COLUMN user_state TEXT NOT NULL DEFAULT 'auto'", + "ALTER TABLE user_profile ADD COLUMN evidence_refs_json TEXT", + "ALTER TABLE user_profile ADD COLUMN class TEXT", + "ALTER TABLE user_profile ADD COLUMN cue_families_json TEXT", +]; + +/// Phase 3 index definitions for idempotent restoration on existing databases. +/// +/// New installs get these via `PROFILE_INIT_SQL`. Existing databases (where the +/// indexes were removed in #1616) need them applied after `PHASE3_COLUMNS_SQL` +/// has ensured the columns exist. +pub const PHASE3_INDEXES_SQL: &[&str] = &[ + "CREATE INDEX IF NOT EXISTS idx_profile_state_stability ON user_profile(state, stability DESC)", + "CREATE INDEX IF NOT EXISTS idx_profile_key ON user_profile(key)", + "CREATE INDEX IF NOT EXISTS idx_profile_state_user_stability ON user_profile(state, user_state, stability)", +]; + +/// Idempotent schema migration for existing databases. +/// +/// New installs get the full schema from `PROFILE_INIT_SQL`. Existing databases +/// may be missing the Phase 3 columns. This function adds each new column if it +/// doesn't exist, ignoring the "duplicate column name" error that SQLite returns +/// when the column is already present. +pub fn migrate_profile_schema(conn: &Arc>) { + let conn = conn.lock(); + for sql in PHASE3_COLUMNS_SQL { + match conn.execute(sql, []) { + Ok(_) => { + tracing::debug!("[profile] schema migration applied: {sql}"); + } + Err(rusqlite::Error::SqliteFailure(err, _)) + if err.extended_code == rusqlite::ffi::SQLITE_ERROR => + { + // "duplicate column name" is not a named SQLite error code; it comes + // back as a generic SQLITE_ERROR with the text "duplicate column name". + // We tolerate any SQLITE_ERROR here because that's the only class of + // error this ALTER TABLE can produce when the column already exists. + tracing::trace!("[profile] column already present (ok): {sql}"); + } + Err(e) => { + tracing::warn!("[profile] schema migration failed (non-fatal): {sql}: {e}"); + } + } + } +} + +// ── FacetState ─────────────────────────────────────────────────────────────── + +/// Lifecycle state of a profile facet in the stability detector. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum FacetState { + /// Facet has cleared τ_promote and is included in the ambient cache. + #[default] + Active, + /// Facet is between τ_provisional and τ_promote — included at lower weight. + Provisional, + /// Facet is between τ_evict and τ_provisional — held as a candidate. + Candidate, + /// Facet fell below τ_evict — will be removed on next rebuild. + Dropped, +} + +impl FacetState { + pub fn as_str(self) -> &'static str { + match self { + Self::Active => "active", + Self::Provisional => "provisional", + Self::Candidate => "candidate", + Self::Dropped => "dropped", + } + } + + pub fn parse_or_default(s: &str) -> Self { + match s { + "provisional" => Self::Provisional, + "candidate" => Self::Candidate, + "dropped" => Self::Dropped, + _ => Self::Active, + } + } +} + +// ── UserState ──────────────────────────────────────────────────────────────── + +/// User-controlled override for a profile facet. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum UserState { + /// No user override — stability detector manages the lifecycle. + #[default] + Auto, + /// User has explicitly pinned this facet; it stays Active regardless of score. + Pinned, + /// User has explicitly forgotten this facet; it stays Dropped and cannot be + /// re-promoted by new evidence. + Forgotten, +} + +impl UserState { + pub fn as_str(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::Pinned => "pinned", + Self::Forgotten => "forgotten", + } + } + + pub fn parse_or_default(s: &str) -> Self { + match s { + "pinned" => Self::Pinned, + "forgotten" => Self::Forgotten, + _ => Self::Auto, + } + } +} + +// ── FacetType ──────────────────────────────────────────────────────────────── + +/// Profile facet types. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FacetType { + Preference, + Workflow, + Role, + Personality, + Context, +} + +impl FacetType { + /// Stable lowercase identifier persisted in the `user_profile` table. + pub fn as_str(&self) -> &'static str { + match self { + Self::Preference => "preference", + Self::Workflow => "skill", + Self::Role => "role", + Self::Personality => "personality", + Self::Context => "context", + } + } + + /// Parse a stored string back to a `FacetType`; unknown values fall back + /// to `Preference`. + pub fn parse_or_default(s: &str) -> Self { + match s { + "skill" => Self::Workflow, + "role" => Self::Role, + "personality" => Self::Personality, + "context" => Self::Context, + _ => Self::Preference, + } + } +} + +// ── ProfileFacet ───────────────────────────────────────────────────────────── + +/// A single profile facet — extended with Phase 3 state + stability fields. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProfileFacet { + pub facet_id: String, + pub facet_type: FacetType, + pub key: String, + pub value: String, + pub confidence: f64, + pub evidence_count: i32, + pub source_segment_ids: Option, + pub first_seen_at: f64, + pub last_seen_at: f64, + // ── Phase 3 additions ── + /// Lifecycle state assigned by the stability detector. + pub state: FacetState, + /// Computed stability score from the last rebuild cycle. + pub stability: f64, + /// User-controlled override. + pub user_state: UserState, + /// Provenance references deserialized from `evidence_refs_json`. + pub evidence_refs: Vec, + /// Facet class (style / identity / tooling / veto / goal / channel). + /// + /// Derived from the key prefix (e.g. `"style/verbosity"` → `"style"`) for + /// learning-path rows. `None` for legacy provider rows whose key prefix + /// doesn't match a known class. + pub class: Option, + /// Per-cue-family evidence counts serialized as JSON. + /// + /// Shape: `{"explicit": N, "structural": N, "behavioral": N, "recurrence": N}`. + /// `None` until the stability detector writes the first rebuild. + pub cue_families: Option>, +} + +// ── Write helpers ───────────────────────────────────────────────────────────── + +/// Upsert a profile facet (legacy / provider path). On conflict (same facet_type + key): +/// - Increments evidence_count +/// - Updates last_seen_at +/// - Appends segment_id to source_segment_ids +/// - Only overwrites value if new confidence > existing confidence +/// +/// The new Phase 3 columns (`state`, `stability`, `user_state`, +/// `evidence_refs_json`) default to `active`, `0.0`, `auto`, and `NULL` +/// respectively, so existing callers need no changes. +#[allow(clippy::too_many_arguments)] +pub fn profile_upsert( + conn: &Arc>, + facet_id: &str, + facet_type: &FacetType, + key: &str, + value: &str, + confidence: f64, + segment_id: Option<&str>, + now: f64, +) -> anyhow::Result<()> { + let conn = conn.lock(); + + // Check if this facet already exists. + let existing: Option<(String, f64, i32, Option)> = conn + .query_row( + "SELECT facet_id, confidence, evidence_count, source_segment_ids + FROM user_profile WHERE facet_type = ?1 AND key = ?2", + params![facet_type.as_str(), key], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .ok(); + + if let Some((existing_id, existing_confidence, existing_count, existing_segments)) = existing { + let new_segments = merge_segments(existing_segments, segment_id); + + if confidence >= existing_confidence { + // Higher or equal confidence: overwrite value + update metadata. + conn.execute( + "UPDATE user_profile + SET value = ?2, confidence = ?3, evidence_count = ?4, + source_segment_ids = ?5, last_seen_at = ?6 + WHERE facet_id = ?1", + params![ + existing_id, + value, + confidence, + existing_count + 1, + new_segments, + now, + ], + )?; + } else { + // Lower confidence: keep existing value, only bump evidence. + conn.execute( + "UPDATE user_profile + SET evidence_count = ?2, source_segment_ids = ?3, last_seen_at = ?4 + WHERE facet_id = ?1", + params![existing_id, existing_count + 1, new_segments, now], + )?; + } + tracing::debug!( + "[profile] updated facet {}:{} (evidence_count={})", + facet_type.as_str(), + key, + existing_count + 1 + ); + } else { + // Insert new facet. Derive class from the key prefix for learning rows. + let segments = segment_id.unwrap_or("").to_string(); + let class = infer_class_from_key(key, facet_type); + conn.execute( + "INSERT INTO user_profile + (facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json) + VALUES (?1, ?2, ?3, ?4, ?5, 1, ?6, ?7, ?7, 'active', 0.0, 'auto', NULL, + ?8, NULL)", + params![ + facet_id, + facet_type.as_str(), + key, + value, + confidence, + segments, + now, + class, + ], + )?; + tracing::debug!( + "[profile] inserted new facet {}:{} = {}", + facet_type.as_str(), + key, + value + ); + } + + Ok(()) +} + +/// Full upsert used by the stability detector rebuild path. +/// +/// Writes all Phase 3 columns explicitly. On conflict (same facet_type + key) +/// the row is replaced in full — the rebuild owns these rows. +pub fn profile_upsert_full( + conn: &Arc>, + facet: &ProfileFacet, +) -> anyhow::Result<()> { + let evidence_refs_json = if facet.evidence_refs.is_empty() { + None + } else { + Some(serde_json::to_string(&facet.evidence_refs)?) + }; + + let cue_families_json = facet + .cue_families + .as_ref() + .filter(|m| !m.is_empty()) + .map(serde_json::to_string) + .transpose()?; + + // Derive class from the facet's own class field or fall back to key prefix. + let class = facet + .class + .clone() + .or_else(|| infer_class_from_key(&facet.key, &facet.facet_type)); + + let conn = conn.lock(); + + // Use INSERT OR REPLACE to atomically update all columns including + // state/stability without reading the row first. Note: on a UNIQUE(facet_type, + // key) conflict, SQLite performs DELETE + INSERT rather than an in-place + // update, which means facet_id will change for conflicting rows. This is + // intentional: the stability detector owns these rows during rebuild and + // provides consistent facet_id values; external references by facet_id are + // not expected. + conn.execute( + "INSERT OR REPLACE INTO user_profile + (facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, + ?14, ?15)", + params![ + facet.facet_id, + facet.facet_type.as_str(), + facet.key, + facet.value, + facet.confidence, + facet.evidence_count, + facet.source_segment_ids, + facet.first_seen_at, + facet.last_seen_at, + facet.state.as_str(), + facet.stability, + facet.user_state.as_str(), + evidence_refs_json, + class, + cue_families_json, + ], + )?; + + tracing::debug!( + "[profile] full-upsert facet {}:{} = {} (state={}, stability={:.3}, class={:?})", + facet.facet_type.as_str(), + facet.key, + facet.value, + facet.state.as_str(), + facet.stability, + class, + ); + Ok(()) +} + +/// Update the `user_state` column for a facet by key. +/// +/// Returns `Ok(true)` if a row was updated, `Ok(false)` if not found. +pub fn profile_set_user_state( + conn: &Arc>, + key: &str, + user_state: UserState, +) -> anyhow::Result { + let conn = conn.lock(); + let rows = conn.execute( + "UPDATE user_profile SET user_state = ?1 WHERE key = ?2", + params![user_state.as_str(), key], + )?; + Ok(rows > 0) +} + +/// Delete a facet by key. Returns `true` if a row was deleted. +pub fn profile_delete_by_key(conn: &Arc>, key: &str) -> anyhow::Result { + let conn = conn.lock(); + let rows = conn.execute("DELETE FROM user_profile WHERE key = ?1", params![key])?; + Ok(rows > 0) +} + +/// Delete all facets whose stability is below the given threshold. +/// +/// Facets with `user_state = 'pinned'` are never deleted regardless of score. +/// Returns the number of rows deleted. +pub fn profile_delete_below_threshold( + conn: &Arc>, + threshold: f64, +) -> anyhow::Result { + let conn = conn.lock(); + let rows = conn.execute( + "DELETE FROM user_profile + WHERE stability < ?1 + AND user_state != 'pinned' + AND state = 'dropped'", + params![threshold], + )?; + Ok(rows) +} + +// ── Read helpers ────────────────────────────────────────────────────────────── + +/// Load all profile facets. +pub fn profile_load_all(conn: &Arc>) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json + FROM user_profile + ORDER BY facet_type, evidence_count DESC", + )?; + let rows = stmt + .query_map([], row_to_facet)? + .collect::, _>>()?; + Ok(rows) +} + +/// Load all facets with `state = 'active'` ordered by stability descending. +pub fn profile_select_active(conn: &Arc>) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json + FROM user_profile + WHERE state = 'active' + ORDER BY stability DESC", + )?; + let rows = stmt + .query_map([], row_to_facet)? + .collect::, _>>()?; + Ok(rows) +} + +/// Load all facets regardless of state (used by the rebuild cycle for a full view). +pub fn profile_select_all(conn: &Arc>) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json + FROM user_profile + ORDER BY stability DESC", + )?; + let rows = stmt + .query_map([], row_to_facet)? + .collect::, _>>()?; + Ok(rows) +} + +/// Load profile facets by type (legacy path). +pub fn profile_facets_by_type( + conn: &Arc>, + facet_type: &FacetType, +) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json + FROM user_profile + WHERE facet_type = ?1 + ORDER BY evidence_count DESC", + )?; + let rows = stmt + .query_map(params![facet_type.as_str()], row_to_facet)? + .collect::, _>>()?; + Ok(rows) +} + +/// Load a single facet by key. Returns `None` if not found. +pub fn profile_get_by_key( + conn: &Arc>, + key: &str, +) -> anyhow::Result> { + let conn = conn.lock(); + conn.query_row( + "SELECT facet_id, facet_type, key, value, confidence, evidence_count, + source_segment_ids, first_seen_at, last_seen_at, + state, stability, user_state, evidence_refs_json, + class, cue_families_json + FROM user_profile WHERE key = ?1", + params![key], + row_to_facet, + ) + .optional() + .map_err(Into::into) +} + +/// Count facets grouped by class prefix (the portion of `key` before the first `/`). +/// +/// For example, `style/verbosity` → class `"style"`. +/// Facets whose key contains no `/` are grouped under `"_other"`. +pub fn profile_count_by_class( + conn: &Arc>, +) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare("SELECT key FROM user_profile WHERE state = 'active'")?; + let keys: Vec = stmt + .query_map([], |row| row.get(0))? + .collect::, _>>()?; + + let mut counts: HashMap = HashMap::new(); + for key in keys { + let class = key + .split_once('/') + .map(|(prefix, _)| prefix.to_string()) + .unwrap_or_else(|| "_other".to_string()); + *counts.entry(class).or_insert(0) += 1; + } + Ok(counts) +} + +// ── Rendering ───────────────────────────────────────────────────────────────── + +/// Render profile facets as a markdown section for context assembly. +pub fn render_profile_context(facets: &[ProfileFacet]) -> String { + if facets.is_empty() { + return String::new(); + } + + let mut sections: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + + for facet in facets { + let section = facet.facet_type.as_str().to_string(); + let evidence = if facet.evidence_count > 1 { + format!(" (confirmed {}x)", facet.evidence_count) + } else { + String::new() + }; + sections + .entry(section) + .or_default() + .push(format!("- {}: {}{}", facet.key, facet.value, evidence)); + } + + let mut parts = Vec::new(); + for (section, items) in §ions { + parts.push(format!("### {}\n{}", capitalize(section), items.join("\n"))); + } + + parts.join("\n\n") +} + +// ── Internal helpers ────────────────────────────────────────────────────────── + +/// Infer the class label for a facet row from its key prefix and facet_type. +/// +/// Learning-path rows use a key like `"style/verbosity"` where the prefix +/// directly encodes the class. Legacy provider rows use `"skill:..."` keys +/// and are mapped via `facet_type`. +fn infer_class_from_key(key: &str, facet_type: &FacetType) -> Option { + // Try key prefix first (learning path: "style/verbosity" → "style"). + if let Some((prefix, _)) = key.split_once('/') { + let known = matches!( + prefix, + "style" | "identity" | "tooling" | "veto" | "goal" | "channel" + ); + if known { + return Some(prefix.to_string()); + } + } + // Legacy provider rows: skill:* keys → "tooling". + if key.starts_with("skill:") { + return Some("tooling".to_string()); + } + // Fall back on facet_type. + Some( + match facet_type { + FacetType::Role | FacetType::Personality => "identity", + FacetType::Workflow => "tooling", + FacetType::Preference => "style", + FacetType::Context => "identity", + } + .to_string(), + ) +} + +fn merge_segments(existing: Option, new_sid: Option<&str>) -> String { + match (existing, new_sid) { + (Some(existing), Some(sid)) => { + if existing.contains(sid) { + existing + } else { + format!("{existing},{sid}") + } + } + (Some(existing), None) => existing, + (None, Some(sid)) => sid.to_string(), + (None, None) => String::new(), + } +} + +fn capitalize(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + None => String::new(), + Some(first) => first.to_uppercase().to_string() + chars.as_str(), + } +} + +fn row_to_facet(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let facet_type_str: String = row.get(1)?; + let state_str: String = row.get(9)?; + let stability: f64 = row.get(10)?; + let user_state_str: String = row.get(11)?; + let evidence_refs_json: Option = row.get(12)?; + let class: Option = row.get(13)?; + let cue_families_json: Option = row.get(14)?; + + let evidence_refs = evidence_refs_json + .as_deref() + .and_then(|json| serde_json::from_str(json).ok()) + .unwrap_or_default(); + + let cue_families = cue_families_json + .as_deref() + .and_then(|json| serde_json::from_str(json).ok()); + + Ok(ProfileFacet { + facet_id: row.get(0)?, + facet_type: FacetType::parse_or_default(&facet_type_str), + key: row.get(2)?, + value: row.get(3)?, + confidence: row.get(4)?, + evidence_count: row.get(5)?, + source_segment_ids: row.get(6)?, + first_seen_at: row.get(7)?, + last_seen_at: row.get(8)?, + state: FacetState::parse_or_default(&state_str), + stability, + user_state: UserState::parse_or_default(&user_state_str), + evidence_refs, + class, + cue_families, + }) +} + +#[cfg(test)] +#[path = "profile_tests.rs"] +mod tests; diff --git a/core/src/store/namespace_store/profile_tests.rs b/core/src/store/namespace_store/profile_tests.rs new file mode 100644 index 0000000..332c343 --- /dev/null +++ b/core/src/store/namespace_store/profile_tests.rs @@ -0,0 +1,763 @@ +//! Tests for the `profile` module — facet upsert with confidence merging. + +use super::*; + +// ── Migration test ──────────────────────────────────────────────────────────── + +/// Verify that `migrate_profile_schema` adds Phase 3 columns to a database +/// that was created with the pre-Phase-3 schema (missing state/stability/…). +#[test] +fn migrate_adds_new_columns_to_existing_db() { + // Create the pre-Phase-3 schema manually (only original columns). + let pre_phase3_sql = r#" + CREATE TABLE IF NOT EXISTS user_profile ( + facet_id TEXT PRIMARY KEY, + facet_type TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.5, + evidence_count INTEGER NOT NULL DEFAULT 1, + source_segment_ids TEXT, + first_seen_at REAL NOT NULL, + last_seen_at REAL NOT NULL, + UNIQUE(facet_type, key) + ); + "#; + let raw_conn = Connection::open_in_memory().unwrap(); + raw_conn.execute_batch(pre_phase3_sql).unwrap(); + let conn = Arc::new(Mutex::new(raw_conn)); + + // Insert a row using the old schema. + { + let c = conn.lock(); + c.execute( + "INSERT INTO user_profile + (facet_id, facet_type, key, value, confidence, evidence_count, + first_seen_at, last_seen_at) + VALUES ('f-old', 'preference', 'theme', 'dark', 0.8, 1, 1000.0, 1000.0)", + [], + ) + .unwrap(); + } + + // Run the migration — should succeed without panicking. + migrate_profile_schema(&conn); + + // The new columns must be present and readable. + let facets = profile_load_all(&conn).unwrap(); + assert_eq!(facets.len(), 1); + let f = &facets[0]; + assert_eq!(f.key, "theme"); + // Defaults applied by ALTER TABLE … DEFAULT. + assert_eq!(f.state, FacetState::Active); + assert!((f.stability - 0.0).abs() < f64::EPSILON); + assert_eq!(f.user_state, UserState::Auto); + assert!(f.evidence_refs.is_empty()); +} + +/// Running migrate twice is idempotent (no panic on duplicate column). +#[test] +fn migrate_is_idempotent() { + let conn = setup_db(); + // First call — columns already exist in PROFILE_INIT_SQL. + migrate_profile_schema(&conn); + // Second call — must not panic. + migrate_profile_schema(&conn); +} + +// ── New column round-trip ───────────────────────────────────────────────────── + +#[test] +fn profile_upsert_full_persists_phase3_fields() { + use tinymemory_api::host::EvidenceRef; + let conn = setup_db(); + let facet = ProfileFacet { + facet_id: "f-full".into(), + facet_type: FacetType::Preference, + key: "style/verbosity".into(), + value: "terse".into(), + confidence: 0.9, + evidence_count: 3, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1200.0, + state: FacetState::Active, + stability: 1.8, + user_state: UserState::Auto, + evidence_refs: vec![EvidenceRef::Episodic { episodic_id: 42 }], + class: Some("style".into()), + cue_families: None, + }; + profile_upsert_full(&conn, &facet).unwrap(); + + let loaded = profile_load_all(&conn).unwrap(); + assert_eq!(loaded.len(), 1); + let f = &loaded[0]; + assert_eq!(f.key, "style/verbosity"); + assert_eq!(f.state, FacetState::Active); + assert!((f.stability - 1.8).abs() < 1e-9); + assert_eq!(f.user_state, UserState::Auto); + assert_eq!(f.evidence_refs.len(), 1); + assert_eq!( + f.evidence_refs[0], + EvidenceRef::Episodic { episodic_id: 42 } + ); +} + +#[test] +fn profile_select_active_filters_by_state() { + let conn = setup_db(); + + let active = ProfileFacet { + facet_id: "f-active".into(), + facet_type: FacetType::Preference, + key: "style/tone".into(), + value: "formal".into(), + confidence: 0.85, + evidence_count: 2, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1100.0, + state: FacetState::Active, + stability: 1.6, + user_state: UserState::Auto, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + let provisional = ProfileFacet { + facet_id: "f-prov".into(), + facet_type: FacetType::Preference, + key: "style/length".into(), + value: "short".into(), + confidence: 0.6, + evidence_count: 1, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Provisional, + stability: 0.8, + user_state: UserState::Auto, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + profile_upsert_full(&conn, &active).unwrap(); + profile_upsert_full(&conn, &provisional).unwrap(); + + let actives = profile_select_active(&conn).unwrap(); + assert_eq!(actives.len(), 1); + assert_eq!(actives[0].key, "style/tone"); +} + +#[test] +fn profile_count_by_class_groups_keys() { + let conn = setup_db(); + for (id, key) in [ + ("f1", "style/verbosity"), + ("f2", "style/tone"), + ("f3", "identity/name"), + ("f4", "no_slash"), + ] { + let f = ProfileFacet { + facet_id: id.into(), + facet_type: FacetType::Preference, + key: key.into(), + value: "v".into(), + confidence: 0.8, + evidence_count: 1, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Active, + stability: 1.6, + user_state: UserState::Auto, + evidence_refs: vec![], + class: None, + cue_families: None, + }; + profile_upsert_full(&conn, &f).unwrap(); + } + + let counts = profile_count_by_class(&conn).unwrap(); + assert_eq!(counts.get("style"), Some(&2)); + assert_eq!(counts.get("identity"), Some(&1)); + assert_eq!(counts.get("_other"), Some(&1)); +} + +#[test] +fn profile_set_user_state_persists() { + let conn = setup_db(); + profile_upsert( + &conn, + "f-us", + &FacetType::Preference, + "tool/editor", + "neovim", + 0.8, + None, + 1000.0, + ) + .unwrap(); + let updated = profile_set_user_state(&conn, "tool/editor", UserState::Pinned).unwrap(); + assert!(updated); + let f = profile_get_by_key(&conn, "tool/editor").unwrap().unwrap(); + assert_eq!(f.user_state, UserState::Pinned); +} + +#[test] +fn profile_delete_below_threshold_removes_dropped_only() { + let conn = setup_db(); + + let dropped_low = ProfileFacet { + facet_id: "f-drop".into(), + facet_type: FacetType::Preference, + key: "style/dropped".into(), + value: "x".into(), + confidence: 0.3, + evidence_count: 1, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Dropped, + stability: 0.1, + user_state: UserState::Auto, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + let active_low = ProfileFacet { + facet_id: "f-act".into(), + facet_type: FacetType::Preference, + key: "style/active".into(), + value: "y".into(), + confidence: 0.9, + evidence_count: 5, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Active, + stability: 0.1, + user_state: UserState::Auto, + evidence_refs: vec![], + class: Some("style".into()), + cue_families: None, + }; + profile_upsert_full(&conn, &dropped_low).unwrap(); + profile_upsert_full(&conn, &active_low).unwrap(); + + let deleted = profile_delete_below_threshold(&conn, 0.3).unwrap(); + assert_eq!(deleted, 1); // Only the Dropped one. + let all = profile_load_all(&conn).unwrap(); + assert_eq!(all.len(), 1); + assert_eq!(all[0].key, "style/active"); +} + +fn setup_db() -> Arc> { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + Arc::new(Mutex::new(conn)) +} + +#[test] +fn insert_and_load_facet() { + let conn = setup_db(); + profile_upsert( + &conn, + "f-1", + &FacetType::Preference, + "theme", + "dark mode", + 0.8, + Some("seg-1"), + 1000.0, + ) + .unwrap(); + + let facets = profile_load_all(&conn).unwrap(); + assert_eq!(facets.len(), 1); + assert_eq!(facets[0].key, "theme"); + assert_eq!(facets[0].value, "dark mode"); + assert_eq!(facets[0].evidence_count, 1); +} + +#[test] +fn upsert_increments_evidence() { + let conn = setup_db(); + profile_upsert( + &conn, + "f-1", + &FacetType::Preference, + "language", + "Rust", + 0.7, + Some("seg-1"), + 1000.0, + ) + .unwrap(); + + // Same facet_type + key, lower confidence — value should NOT change. + profile_upsert( + &conn, + "f-2", + &FacetType::Preference, + "language", + "Python", + 0.5, + Some("seg-2"), + 1001.0, + ) + .unwrap(); + + let facets = profile_facets_by_type(&conn, &FacetType::Preference).unwrap(); + assert_eq!(facets.len(), 1); + assert_eq!(facets[0].value, "Rust"); // Not overwritten. + assert_eq!(facets[0].evidence_count, 2); + + // Higher confidence — value SHOULD change. + profile_upsert( + &conn, + "f-3", + &FacetType::Preference, + "language", + "Go", + 0.9, + Some("seg-3"), + 1002.0, + ) + .unwrap(); + + let facets = profile_facets_by_type(&conn, &FacetType::Preference).unwrap(); + assert_eq!(facets[0].value, "Go"); + assert_eq!(facets[0].evidence_count, 3); +} + +#[test] +fn render_profile_context_formats_correctly() { + let facets = vec![ + ProfileFacet { + facet_id: "f-1".into(), + facet_type: FacetType::Preference, + key: "theme".into(), + value: "dark mode".into(), + confidence: 0.8, + evidence_count: 3, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1002.0, + state: FacetState::Active, + stability: 0.0, + user_state: UserState::Auto, + evidence_refs: vec![], + class: None, + cue_families: None, + }, + ProfileFacet { + facet_id: "f-2".into(), + facet_type: FacetType::Role, + key: "title".into(), + value: "backend engineer".into(), + confidence: 0.9, + evidence_count: 1, + source_segment_ids: None, + first_seen_at: 1000.0, + last_seen_at: 1000.0, + state: FacetState::Active, + stability: 0.0, + user_state: UserState::Auto, + evidence_refs: vec![], + class: None, + cue_families: None, + }, + ]; + + let rendered = render_profile_context(&facets); + assert!(rendered.contains("### Preference")); + assert!(rendered.contains("theme: dark mode (confirmed 3x)")); + assert!(rendered.contains("### Role")); + assert!(rendered.contains("title: backend engineer")); + // Single evidence should not show "(confirmed 1x)". + assert!(!rendered.contains("(confirmed 1x)")); +} + +#[test] +fn empty_profile_renders_empty() { + let rendered = render_profile_context(&[]); + assert!(rendered.is_empty()); +} + +#[test] +fn profile_upsert_appends_segment_ids() { + let conn = setup_db(); + + // First upsert — creates the facet with seg-1. + profile_upsert( + &conn, + "f-seg-1", + &FacetType::Preference, + "editor", + "neovim", + 0.7, + Some("seg-1"), + 1000.0, + ) + .unwrap(); + + // Second upsert — same facet_type + key, different segment_id. + profile_upsert( + &conn, + "f-seg-2", + &FacetType::Preference, + "editor", + "neovim", + 0.5, + Some("seg-2"), + 1001.0, + ) + .unwrap(); + + // Third upsert — again different segment_id. + profile_upsert( + &conn, + "f-seg-3", + &FacetType::Preference, + "editor", + "neovim", + 0.5, + Some("seg-3"), + 1002.0, + ) + .unwrap(); + + let facets = profile_facets_by_type(&conn, &FacetType::Preference).unwrap(); + assert_eq!( + facets.len(), + 1, + "All upserts should resolve to a single row" + ); + assert_eq!(facets[0].evidence_count, 3); + + let seg_ids = facets[0] + .source_segment_ids + .as_deref() + .expect("source_segment_ids should be present"); + assert!( + seg_ids.contains("seg-1"), + "seg-1 should be in source_segment_ids" + ); + assert!( + seg_ids.contains("seg-2"), + "seg-2 should be in source_segment_ids" + ); + assert!( + seg_ids.contains("seg-3"), + "seg-3 should be in source_segment_ids" + ); +} + +#[test] +fn profile_facets_by_type_returns_empty_for_no_matches() { + let conn = setup_db(); + // Insert a Preference facet; querying for Workflow should yield nothing. + profile_upsert( + &conn, + "f-pref", + &FacetType::Preference, + "theme", + "dark", + 0.8, + None, + 1000.0, + ) + .unwrap(); + + let skills = profile_facets_by_type(&conn, &FacetType::Workflow).unwrap(); + assert!( + skills.is_empty(), + "Querying Workflow type should return empty when only Preference exists" + ); +} + +#[test] +fn profile_multiple_types_coexist() { + let conn = setup_db(); + + profile_upsert( + &conn, + "f-pref", + &FacetType::Preference, + "theme", + "dark mode", + 0.8, + None, + 1000.0, + ) + .unwrap(); + profile_upsert( + &conn, + "f-skill", + &FacetType::Workflow, + "language", + "Rust", + 0.9, + None, + 1001.0, + ) + .unwrap(); + profile_upsert( + &conn, + "f-role", + &FacetType::Role, + "title", + "backend engineer", + 0.85, + None, + 1002.0, + ) + .unwrap(); + + let all = profile_load_all(&conn).unwrap(); + assert_eq!( + all.len(), + 3, + "All three distinct facet types should be stored" + ); + + let types_present: Vec = all + .iter() + .map(|f| f.facet_type.as_str().to_string()) + .collect(); + assert!(types_present.contains(&"preference".to_string())); + assert!(types_present.contains(&"skill".to_string())); + assert!(types_present.contains(&"role".to_string())); +} + +#[test] +fn render_profile_context_groups_by_type() { + let conn = setup_db(); + + profile_upsert( + &conn, + "f-1", + &FacetType::Preference, + "theme", + "dark", + 0.8, + None, + 1000.0, + ) + .unwrap(); + profile_upsert( + &conn, + "f-2", + &FacetType::Preference, + "font", + "mono", + 0.7, + None, + 1001.0, + ) + .unwrap(); + profile_upsert( + &conn, + "f-3", + &FacetType::Role, + "title", + "engineer", + 0.9, + None, + 1002.0, + ) + .unwrap(); + + let all = profile_load_all(&conn).unwrap(); + let rendered = render_profile_context(&all); + + // Each type should appear as a distinct section header. + assert!( + rendered.contains("### Preference"), + "Should have a Preference section" + ); + assert!(rendered.contains("### Role"), "Should have a Role section"); + + // Both preference facets should appear under the Preference section. + assert!( + rendered.contains("theme: dark"), + "theme preference should appear" + ); + assert!( + rendered.contains("font: mono"), + "font preference should appear" + ); + + // Role facet should appear under the Role section. + assert!( + rendered.contains("title: engineer"), + "role facet should appear" + ); + + // The two sections should be separated (not merged into one block). + let pref_pos = rendered.find("### Preference").unwrap(); + let role_pos = rendered.find("### Role").unwrap(); + assert_ne!( + pref_pos, role_pos, + "Preference and Role sections should be at different positions" + ); +} + +#[test] +fn fresh_db_has_phase3_indexes() { + let conn = setup_db(); + let c = conn.lock(); + let indexes: Vec = c + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'user_profile'", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + + assert!( + indexes.contains(&"idx_profile_state_stability".to_string()), + "Missing idx_profile_state_stability; found: {indexes:?}" + ); + assert!( + indexes.contains(&"idx_profile_key".to_string()), + "Missing idx_profile_key; found: {indexes:?}" + ); + assert!( + indexes.contains(&"idx_profile_state_user_stability".to_string()), + "Missing idx_profile_state_user_stability; found: {indexes:?}" + ); + assert!( + indexes.contains(&"idx_profile_type".to_string()), + "Missing idx_profile_type; found: {indexes:?}" + ); +} + +#[test] +fn phase3_indexes_applied_to_existing_db() { + use super::super::profile::{PHASE3_COLUMNS_SQL, PHASE3_INDEXES_SQL}; + use rusqlite::Connection; + + let pre_phase3_sql = " + CREATE TABLE IF NOT EXISTS user_profile ( + facet_id TEXT PRIMARY KEY, + facet_type TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.5, + evidence_count INTEGER NOT NULL DEFAULT 1, + source_segment_ids TEXT, + first_seen_at REAL NOT NULL, + last_seen_at REAL NOT NULL, + UNIQUE(facet_type, key) + ); + CREATE INDEX IF NOT EXISTS idx_profile_type ON user_profile(facet_type); + "; + let raw_conn = Connection::open_in_memory().unwrap(); + raw_conn.execute_batch(pre_phase3_sql).unwrap(); + + for sql in PHASE3_COLUMNS_SQL { + let _ = raw_conn.execute(sql, []); + } + for sql in PHASE3_INDEXES_SQL { + raw_conn.execute_batch(sql).unwrap(); + } + + let indexes: Vec = raw_conn + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'user_profile'", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + + assert!(indexes.contains(&"idx_profile_state_stability".to_string())); + assert!(indexes.contains(&"idx_profile_key".to_string())); + assert!(indexes.contains(&"idx_profile_state_user_stability".to_string())); +} + +#[test] +fn phase3_indexes_idempotent() { + use super::super::profile::PHASE3_INDEXES_SQL; + let conn = setup_db(); + let c = conn.lock(); + for sql in PHASE3_INDEXES_SQL { + c.execute_batch(sql).unwrap(); + } + for sql in PHASE3_INDEXES_SQL { + c.execute_batch(sql).unwrap(); + } +} + +/// Verify that the real `UnifiedMemory::new` bootstrap path applies Phase 3 +/// indexes when opened over a pre-Phase-3 database file (the exact scenario +/// that caused the original crash in initialization ordering). +#[test] +fn unified_memory_new_applies_phase3_indexes_to_existing_db() { + use super::super::UnifiedMemory; + use rusqlite::Connection; + use std::sync::Arc; + use tinymemory_api::host::NoopEmbedding; + + let dir = tempfile::tempdir().unwrap(); + let workspace = dir.path(); + + // Seed a pre-Phase-3 database at the path UnifiedMemory::new will open. + let memory_dir = workspace.join("memory"); + std::fs::create_dir_all(&memory_dir).unwrap(); + let db_path = memory_dir.join("memory.db"); + { + let conn = Connection::open(&db_path).unwrap(); + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS user_profile ( + facet_id TEXT PRIMARY KEY, + facet_type TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.5, + evidence_count INTEGER NOT NULL DEFAULT 1, + source_segment_ids TEXT, + first_seen_at REAL NOT NULL, + last_seen_at REAL NOT NULL, + UNIQUE(facet_type, key) + ); + CREATE INDEX IF NOT EXISTS idx_profile_type ON user_profile(facet_type);", + ) + .unwrap(); + } + + // Call the real bootstrap path — must not fail on a pre-Phase-3 DB. + let mem = UnifiedMemory::new(workspace, Arc::new(NoopEmbedding), None) + .expect("UnifiedMemory::new must succeed on a pre-Phase-3 database"); + + // The Phase 3 indexes must exist after initialization. + let conn = mem.conn.lock(); + let indexes: Vec = conn + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'user_profile'", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + + assert!( + indexes.contains(&"idx_profile_state_stability".to_string()), + "Missing idx_profile_state_stability; found: {indexes:?}" + ); + assert!( + indexes.contains(&"idx_profile_key".to_string()), + "Missing idx_profile_key; found: {indexes:?}" + ); + assert!( + indexes.contains(&"idx_profile_state_user_stability".to_string()), + "Missing idx_profile_state_user_stability; found: {indexes:?}" + ); +} diff --git a/core/src/store/namespace_store/query.rs b/core/src/store/namespace_store/query.rs new file mode 100644 index 0000000..8dee248 --- /dev/null +++ b/core/src/store/namespace_store/query.rs @@ -0,0 +1,1395 @@ +//! Hybrid retrieval over the unified store. +//! +//! Combines graph relevance, vector similarity, keyword overlap, episodic +//! signal, and freshness into a single score per hit. Owns the query planner +//! (`build_retrieval_plan`), per-document score composition, and the +//! `query_namespace_hits` / `query_namespace_ranked` / `recall_namespace_*` +//! entry points used by `MemoryClient`. + +use rusqlite::params; +use std::collections::{HashMap, HashSet}; + +use crate::store::types::{ + GraphRelationRecord, MemoryItemKind, NamespaceMemoryHit, NamespaceQueryResult, + NamespaceRetrievalContext, RetrievalScoreBreakdown, +}; + +use super::events; +use super::fts5; +use super::UnifiedMemory; + +const GRAPH_WEIGHT: f64 = 0.55; +const VECTOR_WEIGHT: f64 = 0.30; +const KEYWORD_WEIGHT: f64 = 0.15; +const EPISODIC_WEIGHT: f64 = 0.20; + +// Adjusted weights when episodic signal is present +const GRAPH_WEIGHT_WITH_EPISODIC: f64 = 0.45; +const VECTOR_WEIGHT_WITH_EPISODIC: f64 = 0.25; +const KEYWORD_WEIGHT_WITH_EPISODIC: f64 = 0.10; + +const RECALL_PRIORITY_WEIGHT: f64 = 0.45; +const RECALL_GRAPH_WEIGHT: f64 = 0.30; +const RECALL_FRESHNESS_WEIGHT: f64 = 0.25; + +#[derive(Debug, Clone)] +struct StoredChunk { + document_id: String, + chunk_id: String, + embedding: Option>, + /// Signature of the embedding model that produced `embedding`. `None` for + /// rows written before model tagging was introduced. Used to exclude + /// cross-model vectors from cosine scoring. + model_signature: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TemporalOperator { + Latest, + Earliest, + Before, + After, + All, +} + +#[derive(Debug, Clone)] +struct RetrievalPlan { + query_terms: Vec, + seed_entities: Vec, + relation_types: Vec, + chains: Vec>, + temporal: TemporalOperator, + anchor_entity: Option, +} + +#[derive(Debug, Clone)] +struct RelationMatch { + relation: GraphRelationRecord, + hop: usize, +} + +impl UnifiedMemory { + /// Relation-first retrieval: + /// - graph relevance is the primary signal + /// - vector similarity is the secondary verification signal + /// - keyword overlap remains as a lexical backstop + pub async fn query_namespace_ranked( + &self, + namespace: &str, + query: &str, + limit: u32, + ) -> Result, String> { + self.query_namespace_ranked_excluding_session(namespace, query, limit, None) + .await + } + + /// Same as [`Self::query_namespace_ranked`], but excludes same-session + /// documents — see [`Self::query_namespace_hits_excluding_session`] for + /// the exact semantics and backward-compatibility guarantee. + pub async fn query_namespace_ranked_excluding_session( + &self, + namespace: &str, + query: &str, + limit: u32, + exclude_session_id: Option<&str>, + ) -> Result, String> { + let hits = self + .query_namespace_hits_excluding_session(namespace, query, limit, exclude_session_id) + .await?; + let mut out = Vec::new(); + for hit in hits { + if hit.kind != MemoryItemKind::Document { + continue; + } + out.push(NamespaceQueryResult { + key: hit.key, + content: hit.content, + score: hit.score, + category: hit.category, + taint: hit.taint, + }); + } + Ok(out) + } + + /// Hybrid retrieval: returns ranked hits across documents and KV records, + /// scored by graph relevance + vector similarity + keyword overlap + + /// freshness. + pub async fn query_namespace_hits( + &self, + namespace: &str, + query: &str, + limit: u32, + ) -> Result, String> { + self.query_namespace_hits_excluding_session(namespace, query, limit, None) + .await + } + + /// Same as [`Self::query_namespace_hits`], but drops any document-kind + /// hit whose stored `session_id` matches `exclude_session_id`. + /// + /// This is the self-echo guard for agent-invoked search (`memory_recall`, + /// `memory_hybrid_search`): the harness auto-saves the user's own turn as + /// a `[conversation]` document tagged with the ambient chat thread id + /// (see `agent::harness::session::turn::core`), so without this filter a + /// search issued *during that same turn* can retrieve its own triggering + /// request as the top "relevant" result. Only documents are + /// session-filtered — KV rows carry no session concept, and + /// episodic/event hits already have their own dedicated session-scoping + /// (`RecallOpts::session_id` / `cross_session`). + /// + /// `exclude_session_id = None` (or an empty/whitespace string) is + /// identical to [`Self::query_namespace_hits`] — no filtering is + /// applied, so every existing caller (and every caller with no ambient + /// session context) keeps its exact prior behavior. + pub async fn query_namespace_hits_excluding_session( + &self, + namespace: &str, + query: &str, + limit: u32, + exclude_session_id: Option<&str>, + ) -> Result, String> { + let ns = Self::sanitize_namespace(namespace); + let exclude_session_id = exclude_session_id + .map(str::trim) + .filter(|id| !id.is_empty()); + let mut docs = self.load_documents_for_scope(&ns).await?; + if let Some(exclude) = exclude_session_id { + let before = docs.len(); + docs.retain(|doc| doc.session_id.as_deref() != Some(exclude)); + let dropped = before - docs.len(); + tracing::debug!( + "[query] session-exclusion filter namespace={ns} exclude_session_id={exclude} \ + dropped={dropped} remaining={}", + docs.len() + ); + } + let kvs = self.kv_records_for_scope(&ns).await?; + + let graph_relations = self + .graph_relations_for_scope(&ns) + .await + .unwrap_or_default(); + let chunks = self.load_chunks_for_scope(&ns).await?; + let plan = self.build_retrieval_plan(query, &docs, &graph_relations); + let matched_relations = self.collect_relation_matches(&plan, &graph_relations); + let graph_scores = self.compute_graph_document_scores(&docs, &chunks, &matched_relations); + let vector_scores = self + .query_vector_scores_from_chunks(&chunks, query) + .await + .unwrap_or_default(); + let query_terms = plan.query_terms.clone(); + let now = Self::now_ts(); + + let has_graph_signal = graph_scores.values().any(|score| *score > 0.0); + let mut hits = Vec::new(); + + for doc in docs { + let keyword = self.keyword_score_for_text( + &query_terms, + &[doc.key.as_str(), doc.title.as_str(), doc.content.as_str()], + ); + let vector = vector_scores + .get(&doc.document_id) + .map(|(score, _)| *score) + .unwrap_or(0.0); + let graph = graph_scores.get(&doc.document_id).copied().unwrap_or(0.0); + let breakdown = if has_graph_signal { + Self::compose_query_score(keyword, vector, graph) + } else { + Self::compose_fallback_query_score(keyword, vector) + }; + if breakdown.final_score <= 0.0 { + continue; + } + + let best_chunk_id = vector_scores + .get(&doc.document_id) + .and_then(|(_, chunk_id)| chunk_id.clone()); + let supporting_relations = self.supporting_relations_for_document( + &doc.document_id, + &doc.content, + &matched_relations, + ); + + hits.push(NamespaceMemoryHit { + id: doc.document_id.clone(), + kind: MemoryItemKind::Document, + namespace: doc.namespace.clone(), + key: doc.key.clone(), + title: Some(doc.title.clone()), + content: doc.content.clone(), + category: doc.category.clone(), + source_type: Some(doc.source_type.clone()), + updated_at: doc.updated_at, + score: breakdown.final_score, + score_breakdown: breakdown, + document_id: Some(doc.document_id.clone()), + chunk_id: best_chunk_id, + supporting_relations, + taint: doc.taint, + }); + } + + for kv in kvs { + let rendered = Self::render_kv_value(&kv.value); + let keyword = + self.keyword_score_for_text(&query_terms, &[kv.key.as_str(), rendered.as_str()]); + if keyword <= 0.0 { + continue; + } + let freshness = Self::recency_score(kv.updated_at, now); + let final_score = (keyword * 0.8) + (freshness * 0.2); + hits.push(NamespaceMemoryHit { + id: format!( + "kv:{}:{}", + kv.namespace.as_deref().unwrap_or("global"), + kv.key + ), + kind: MemoryItemKind::Kv, + namespace: kv.namespace.unwrap_or_else(|| "global".to_string()), + key: kv.key, + title: None, + content: rendered, + category: "kv".to_string(), + source_type: None, + updated_at: kv.updated_at, + score: final_score, + score_breakdown: RetrievalScoreBreakdown { + keyword_relevance: keyword, + vector_similarity: 0.0, + graph_relevance: 0.0, + episodic_relevance: 0.0, + freshness, + final_score, + }, + document_id: None, + chunk_id: None, + supporting_relations: Vec::new(), + // KV rows have no provenance column; conservatively + // surface as Internal so the subconscious gate doesn't + // mis-escalate user-state writes. + taint: crate::MemoryTaint::Internal, + }); + } + + // Episodic FTS5 search — search past conversation turns. + // Only merge episodic results when querying the global namespace, + // since episodic entries are session-scoped, not namespace-scoped. + let episodic_hits = if ns == "global" { + fts5::episodic_search(&self.conn, query, limit as usize).unwrap_or_else(|e| { + tracing::warn!("[query] episodic search failed: {e}"); + Vec::new() + }) + } else { + Vec::new() + }; + + if !episodic_hits.is_empty() { + tracing::debug!( + "[query] merging {} episodic hits for '{}'", + episodic_hits.len(), + query + ); + + // Reweight existing document/KV hits when episodic signal is present. + let has_episodic = true; + if has_episodic { + for hit in &mut hits { + if hit.kind == MemoryItemKind::Document { + let bd = &hit.score_breakdown; + let new_score = (bd.graph_relevance * GRAPH_WEIGHT_WITH_EPISODIC) + + (bd.vector_similarity * VECTOR_WEIGHT_WITH_EPISODIC) + + (bd.keyword_relevance * KEYWORD_WEIGHT_WITH_EPISODIC); + hit.score = new_score; + hit.score_breakdown.final_score = new_score; + } + } + } + + for (position_idx, entry) in episodic_hits.iter().enumerate() { + let freshness = Self::recency_score(entry.timestamp, now); + // Episodic FTS5 returns results ordered by rank (best first). + // Normalize position to a 0-1 relevance score. + let fts_relevance = 1.0 - (position_idx as f64 / episodic_hits.len().max(1) as f64); + + let episodic_score = (fts_relevance * 0.7) + (freshness * 0.3); + let final_score = episodic_score * EPISODIC_WEIGHT; + + // Truncate long episodic content for context display (UTF-8 safe). + let content = match entry.content.char_indices().nth(500) { + Some((byte_idx, _)) => format!("{}...", &entry.content[..byte_idx]), + None => entry.content.clone(), + }; + + hits.push(NamespaceMemoryHit { + id: format!("episodic:{}", entry.id.unwrap_or(0)), + kind: MemoryItemKind::Episodic, + namespace: ns.clone(), + key: format!("{}:{}", entry.session_id, entry.role), + title: entry.lesson.clone(), + content, + category: "episodic".to_string(), + source_type: Some(entry.role.clone()), + updated_at: entry.timestamp, + score: final_score, + score_breakdown: RetrievalScoreBreakdown { + keyword_relevance: 0.0, + vector_similarity: 0.0, + graph_relevance: 0.0, + episodic_relevance: fts_relevance, + freshness, + final_score, + }, + document_id: None, + chunk_id: None, + supporting_relations: Vec::new(), + // Episodic rows are derived from user chat turns and + // never carry sync-ingest content; surface as + // Internal so the subconscious gate trusts them. + taint: crate::MemoryTaint::Internal, + }); + } + } + + // Event FTS5 search — search extracted facts, decisions, preferences. + let event_hits = events::event_search_fts(&self.conn, &ns, query, limit as usize) + .unwrap_or_else(|e| { + tracing::warn!("[query] event search failed: {e}"); + Vec::new() + }); + + for (idx, event) in event_hits.iter().enumerate() { + let freshness = Self::recency_score(event.created_at, now); + let fts_relevance = 1.0 - (idx as f64 / event_hits.len().max(1) as f64); + let final_score = (fts_relevance * 0.6) + (freshness * 0.4); + + hits.push(NamespaceMemoryHit { + id: format!("event:{}", event.event_id), + kind: MemoryItemKind::Event, + namespace: event.namespace.clone(), + key: format!("{}:{}", event.event_type.as_str(), event.segment_id), + title: event.subject.clone(), + content: event.content.clone(), + category: event.event_type.as_str().to_string(), + source_type: Some("event".to_string()), + updated_at: event.created_at, + score: final_score, + score_breakdown: RetrievalScoreBreakdown { + keyword_relevance: fts_relevance, + vector_similarity: 0.0, + graph_relevance: 0.0, + episodic_relevance: 0.0, + freshness, + final_score, + }, + document_id: None, + chunk_id: None, + supporting_relations: Vec::new(), + // Event extractions are derived from chat segments; + // treat them as Internal until a future migration + // surfaces per-event provenance. + taint: crate::MemoryTaint::Internal, + }); + } + + hits.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + hits.truncate(limit as usize); + Ok(hits) + } + + /// Run a hybrid query and return only the rendered context text. + pub async fn query_namespace_context( + &self, + namespace: &str, + query: &str, + limit: u32, + ) -> Result { + let context = self + .query_namespace_context_data(namespace, query, limit) + .await?; + Ok(context.context_text) + } + + /// Run a hybrid query and return both the rendered context text and the + /// underlying ranked hits. + pub async fn query_namespace_context_data( + &self, + namespace: &str, + query: &str, + limit: u32, + ) -> Result { + let ns = Self::sanitize_namespace(namespace); + let hits = self.query_namespace_hits(&ns, query, limit).await?; + Ok(NamespaceRetrievalContext { + namespace: ns, + query: Some(query.to_string()), + context_text: Self::format_context_text(&hits, Some(query)), + hits, + }) + } + + /// Query-less recall: rank documents and KV records by priority + graph + /// relevance + freshness without a search query. + pub async fn recall_namespace_memories( + &self, + namespace: &str, + limit: u32, + ) -> Result, String> { + let ns = Self::sanitize_namespace(namespace); + let docs = self.load_documents_for_scope(&ns).await?; + let kvs = self.kv_records_for_scope(&ns).await?; + let graph_relations = self + .graph_relations_for_scope(&ns) + .await + .unwrap_or_default(); + let now = Self::now_ts(); + let mut hits = Vec::new(); + + // Loop-invariant: every document sees the same graph relations, so build + // the RelationMatch view once instead of cloning all rows per document. + let relation_matches = graph_relations + .iter() + .cloned() + .map(|relation| RelationMatch { relation, hop: 1 }) + .collect::>(); + + for doc in docs { + let freshness = Self::recency_score(doc.updated_at, now); + let priority = Self::document_priority_signal( + &doc.category, + &doc.priority, + &doc.tags, + &doc.metadata, + ); + let graph = + self.document_recall_graph_signal(&doc.document_id, &doc.content, &graph_relations); + let final_score = (priority * RECALL_PRIORITY_WEIGHT) + + (graph * RECALL_GRAPH_WEIGHT) + + (freshness * RECALL_FRESHNESS_WEIGHT); + hits.push(NamespaceMemoryHit { + id: doc.document_id.clone(), + kind: MemoryItemKind::Document, + namespace: doc.namespace.clone(), + key: doc.key.clone(), + title: Some(doc.title.clone()), + content: doc.content.clone(), + category: doc.category.clone(), + source_type: Some(doc.source_type.clone()), + updated_at: doc.updated_at, + score: final_score, + score_breakdown: RetrievalScoreBreakdown { + keyword_relevance: priority, + vector_similarity: 0.0, + graph_relevance: graph, + episodic_relevance: 0.0, + freshness, + final_score, + }, + document_id: Some(doc.document_id.clone()), + chunk_id: None, + supporting_relations: self.supporting_relations_for_document( + &doc.document_id, + &doc.content, + &relation_matches, + ), + taint: doc.taint, + }); + } + + for kv in kvs { + let freshness = Self::recency_score(kv.updated_at, now); + let priority = Self::kv_priority_signal(&kv.key, &kv.value); + let final_score = + (priority * RECALL_PRIORITY_WEIGHT) + (freshness * (1.0 - RECALL_PRIORITY_WEIGHT)); + hits.push(NamespaceMemoryHit { + id: format!( + "kv:{}:{}", + kv.namespace.as_deref().unwrap_or("global"), + kv.key + ), + kind: MemoryItemKind::Kv, + namespace: kv.namespace.unwrap_or_else(|| "global".to_string()), + key: kv.key, + title: None, + content: Self::render_kv_value(&kv.value), + category: "kv".to_string(), + source_type: None, + updated_at: kv.updated_at, + score: final_score, + score_breakdown: RetrievalScoreBreakdown { + keyword_relevance: priority, + vector_similarity: 0.0, + graph_relevance: 0.0, + episodic_relevance: 0.0, + freshness, + final_score, + }, + document_id: None, + chunk_id: None, + supporting_relations: Vec::new(), + taint: crate::MemoryTaint::Internal, + }); + } + + hits.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + hits.truncate(limit as usize); + Ok(hits) + } + + /// Query-less recall returning only rendered context text. `None` when + /// the namespace is empty. + pub async fn recall_namespace_context( + &self, + namespace: &str, + max_chunks: u32, + ) -> Result, String> { + let hits = self + .recall_namespace_memories(namespace, max_chunks) + .await?; + if hits.is_empty() { + return Ok(None); + } + Ok(Some(Self::format_context_text(&hits, None))) + } + + /// Query-less recall returning both rendered text and ranked hits. + pub async fn recall_namespace_context_data( + &self, + namespace: &str, + limit: u32, + ) -> Result { + let ns = Self::sanitize_namespace(namespace); + let hits = self.recall_namespace_memories(&ns, limit).await?; + Ok(NamespaceRetrievalContext { + namespace: ns, + query: None, + context_text: Self::format_context_text(&hits, None), + hits, + }) + } + + async fn load_chunks_for_scope(&self, namespace: &str) -> Result, String> { + let conn = self.conn.lock(); + let mut stmt = conn + .prepare( + "SELECT document_id, chunk_id, embedding, model_signature + FROM vector_chunks + WHERE namespace = ?1", + ) + .map_err(|e| format!("prepare load_chunks_for_scope: {e}"))?; + let mut rows = stmt + .query(params![Self::sanitize_namespace(namespace)]) + .map_err(|e| format!("query load_chunks_for_scope: {e}"))?; + let mut chunks = Vec::new(); + while let Some(row) = rows + .next() + .map_err(|e| format!("row load_chunks_for_scope: {e}"))? + { + let embedding_blob: Option> = row.get(2).map_err(|e| e.to_string())?; + chunks.push(StoredChunk { + document_id: row.get(0).map_err(|e| e.to_string())?, + chunk_id: row.get(1).map_err(|e| e.to_string())?, + embedding: embedding_blob.as_deref().map(Self::bytes_to_vec), + model_signature: row.get(3).map_err(|e| e.to_string())?, + }); + } + Ok(chunks) + } + + async fn query_vector_scores_from_chunks( + &self, + chunks: &[StoredChunk], + query: &str, + ) -> Result)>, String> { + if chunks.is_empty() { + return Ok(HashMap::new()); + } + let query_embedding = self + .embedder + .embed_one(query) + .await + .map_err(|e| format!("embedding query: {e}"))?; + let active_signature = self.embedder.signature(); + let mut scores = HashMap::new(); + for chunk in chunks { + let Some(embedding) = chunk.embedding.as_ref() else { + continue; + }; + // Skip vectors produced by a different embedding model — cosine across + // two embedding spaces is meaningless. Rows with no signature (written + // before model tagging) fall through to the dimension guard below. + if let Some(sig) = chunk.model_signature.as_deref() { + if sig != active_signature { + continue; + } + } + // Dimension guard: a model swap that changed dimensionality leaves + // legacy/untagged vectors at the old length; skip them rather than + // letting cosine_similarity silently return 0. + if embedding.len() != query_embedding.len() { + continue; + } + let similarity = Self::cosine_similarity(&query_embedding, embedding); + let entry = scores + .entry(chunk.document_id.clone()) + .or_insert((0.0, None::)); + if similarity > entry.0 { + *entry = (similarity, Some(chunk.chunk_id.clone())); + } + } + Ok(scores) + } + + fn build_retrieval_plan( + &self, + query: &str, + docs: &[crate::store::types::StoredMemoryDocument], + graph_relations: &[GraphRelationRecord], + ) -> RetrievalPlan { + let query_terms = Self::tokenize_search_terms(query); + let temporal = Self::infer_temporal_operator(&query_terms); + let relation_types = Self::infer_relation_types(&query_terms); + let entity_candidates = self.match_query_entities(query, docs, graph_relations); + let anchor_entity = match temporal { + TemporalOperator::Before | TemporalOperator::After => { + self.resolve_anchor_entity(query, &entity_candidates) + } + _ => None, + }; + let seed_entities = entity_candidates + .into_iter() + .filter(|entity| anchor_entity.as_ref() != Some(entity)) + .collect::>(); + let chains = Self::infer_relation_chains(&query_terms, &relation_types); + + RetrievalPlan { + query_terms, + seed_entities, + relation_types, + chains, + temporal, + anchor_entity, + } + } + + fn match_query_entities( + &self, + query: &str, + docs: &[crate::store::types::StoredMemoryDocument], + graph_relations: &[GraphRelationRecord], + ) -> Vec { + let normalized_query = Self::normalize_search_text(query); + let mut entities = HashSet::new(); + + for relation in graph_relations { + for candidate in [&relation.subject, &relation.object] { + let normalized = Self::normalize_search_text(candidate); + if !normalized.is_empty() && normalized_query.contains(&normalized) { + entities.insert(candidate.clone()); + } + } + } + + for doc in docs { + for candidate in [&doc.key, &doc.title] { + let normalized = Self::normalize_search_text(candidate); + if !normalized.is_empty() && normalized_query.contains(&normalized) { + entities.insert(Self::normalize_graph_entity(candidate)); + } + } + } + + let mut out = entities.into_iter().collect::>(); + out.sort(); + out + } + + fn resolve_anchor_entity(&self, query: &str, entities: &[String]) -> Option { + let normalized_query = Self::normalize_search_text(query); + let mut best: Option<(usize, String)> = None; + for entity in entities { + let normalized_entity = Self::normalize_search_text(entity); + if normalized_entity.is_empty() { + continue; + } + if let Some(pos) = normalized_query.rfind(&normalized_entity) { + if best + .as_ref() + .map(|(best_pos, _)| pos > *best_pos) + .unwrap_or(true) + { + best = Some((pos, entity.clone())); + } + } + } + best.map(|(_, entity)| entity) + } + + fn collect_relation_matches( + &self, + plan: &RetrievalPlan, + graph_relations: &[GraphRelationRecord], + ) -> Vec { + let matches = self.direct_relation_matches(plan, graph_relations); + let chain_matches = self.multi_hop_relation_matches(plan, graph_relations); + let mut merged = matches; + for item in chain_matches { + let identity = Self::relation_identity(&item.relation); + if merged + .iter() + .any(|existing| Self::relation_identity(&existing.relation) == identity) + { + continue; + } + merged.push(item); + } + + let anchor_order = self.resolve_anchor_order(plan, graph_relations); + Self::apply_temporal_filter(plan, anchor_order, merged) + } + + fn direct_relation_matches( + &self, + plan: &RetrievalPlan, + graph_relations: &[GraphRelationRecord], + ) -> Vec { + let seed_entities = plan.seed_entities.iter().collect::>(); + graph_relations + .iter() + .filter(|relation| { + let touches_seed = seed_entities.is_empty() + || seed_entities.contains(&relation.subject) + || seed_entities.contains(&relation.object); + let predicate_match = plan.relation_types.is_empty() + || plan.relation_types.contains(&relation.predicate) + || Self::predicate_matches_query(&relation.predicate, &plan.query_terms); + let entity_overlap = seed_entities.is_empty() + || Self::relation_matches_terms(relation, &plan.query_terms); + touches_seed && predicate_match && entity_overlap + }) + .cloned() + .map(|relation| RelationMatch { relation, hop: 1 }) + .collect() + } + + fn multi_hop_relation_matches( + &self, + plan: &RetrievalPlan, + graph_relations: &[GraphRelationRecord], + ) -> Vec { + if plan.chains.is_empty() || plan.seed_entities.is_empty() { + return Vec::new(); + } + + let mut chain_results: Vec> = Vec::new(); + for chain in &plan.chains { + let mut frontier = plan.seed_entities.clone(); + let mut path = Vec::new(); + let mut used = HashSet::new(); + + for (hop_idx, step) in chain.iter().enumerate() { + let mut candidates = graph_relations + .iter() + .filter(|relation| { + relation.predicate == *step + && (frontier.contains(&relation.subject) + || frontier.contains(&relation.object)) + }) + .cloned() + .collect::>(); + + if candidates.is_empty() { + path.clear(); + break; + } + + candidates.sort_by(|a, b| { + Self::relation_order_value(b) + .cmp(&Self::relation_order_value(a)) + .then_with(|| { + b.updated_at + .partial_cmp(&a.updated_at) + .unwrap_or(std::cmp::Ordering::Equal) + }) + }); + + let mut next_frontier = Vec::new(); + for relation in candidates { + let identity = Self::relation_identity(&relation); + if !used.insert(identity) { + continue; + } + if frontier.contains(&relation.subject) { + next_frontier.push(relation.object.clone()); + } + if frontier.contains(&relation.object) { + next_frontier.push(relation.subject.clone()); + } + path.push(RelationMatch { + relation, + hop: hop_idx + 2, + }); + } + + next_frontier.sort(); + next_frontier.dedup(); + frontier = next_frontier; + } + + if !path.is_empty() { + chain_results.push(path); + } + } + + if chain_results.is_empty() { + return Vec::new(); + } + + if plan.temporal == TemporalOperator::All { + return chain_results.into_iter().flatten().collect(); + } + + let choose_max = matches!( + plan.temporal, + TemporalOperator::Latest | TemporalOperator::Before + ); + chain_results + .into_iter() + .max_by(|a, b| { + let a_order = a + .iter() + .map(|item| Self::relation_order_value(&item.relation)) + .max() + .unwrap_or_default(); + let b_order = b + .iter() + .map(|item| Self::relation_order_value(&item.relation)) + .max() + .unwrap_or_default(); + if choose_max { + a_order.cmp(&b_order) + } else { + b_order.cmp(&a_order) + } + }) + .unwrap_or_default() + } + + fn apply_temporal_filter( + plan: &RetrievalPlan, + anchor_order: Option, + relations: Vec, + ) -> Vec { + if plan.temporal == TemporalOperator::All { + return relations; + } + + let filtered = match plan.temporal { + TemporalOperator::Before => relations + .into_iter() + .filter(|item| { + anchor_order + .map(|anchor| Self::relation_order_value(&item.relation) < anchor) + .unwrap_or(true) + }) + .collect::>(), + TemporalOperator::After => relations + .into_iter() + .filter(|item| { + anchor_order + .map(|anchor| Self::relation_order_value(&item.relation) > anchor) + .unwrap_or(true) + }) + .collect::>(), + _ => relations, + }; + + let mut groups: HashMap<(String, String), Vec> = HashMap::new(); + for item in filtered { + let pivot = if plan.seed_entities.contains(&item.relation.subject) { + item.relation.subject.clone() + } else if plan.seed_entities.contains(&item.relation.object) { + item.relation.object.clone() + } else { + item.relation.subject.clone() + }; + groups + .entry((pivot, item.relation.predicate.clone())) + .or_default() + .push(item); + } + + let mut out = Vec::new(); + for mut items in groups.into_values() { + items.sort_by(|a, b| { + Self::relation_order_value(&a.relation) + .cmp(&Self::relation_order_value(&b.relation)) + }); + match plan.temporal { + TemporalOperator::Earliest | TemporalOperator::After => { + if let Some(item) = items.into_iter().next() { + out.push(item); + } + } + TemporalOperator::Latest | TemporalOperator::Before => { + if let Some(item) = items.into_iter().last() { + out.push(item); + } + } + TemporalOperator::All => out.extend(items), + } + } + + out + } + + fn resolve_anchor_order( + &self, + plan: &RetrievalPlan, + graph_relations: &[GraphRelationRecord], + ) -> Option { + let anchor = plan.anchor_entity.as_ref()?; + let mut orders = graph_relations + .iter() + .filter(|relation| relation.subject == *anchor || relation.object == *anchor) + .map(Self::relation_order_value) + .collect::>(); + if orders.is_empty() { + return None; + } + orders.sort(); + match plan.temporal { + TemporalOperator::Before => orders.into_iter().max(), + TemporalOperator::After => orders.into_iter().min(), + _ => orders.into_iter().max(), + } + } + + fn compute_graph_document_scores( + &self, + docs: &[crate::store::types::StoredMemoryDocument], + chunks: &[StoredChunk], + relations: &[RelationMatch], + ) -> HashMap { + if relations.is_empty() { + return HashMap::new(); + } + + let mut doc_scores: HashMap = + HashMap::with_capacity(docs.len().max(relations.len())); + let chunk_to_doc = chunks + .iter() + .map(|chunk| (chunk.chunk_id.as_str(), chunk.document_id.as_str())) + .collect::>(); + let normalized_docs = docs + .iter() + .map(|doc| { + ( + doc.document_id.as_str(), + Self::normalize_search_text(&doc.content), + ) + }) + .collect::>(); + + for relation in relations { + let base = f64::from(relation.relation.evidence_count) / relation.hop.max(1) as f64; + for document_id in &relation.relation.document_ids { + *doc_scores.entry(document_id.clone()).or_insert(0.0) += base; + } + for chunk_id in &relation.relation.chunk_ids { + if let Some(document_id) = chunk_to_doc.get(chunk_id.as_str()) { + *doc_scores.entry((*document_id).to_string()).or_insert(0.0) += base * 0.9; + } + } + + let subject = Self::normalize_search_text(&relation.relation.subject); + let object = Self::normalize_search_text(&relation.relation.object); + if subject.is_empty() && object.is_empty() { + continue; + } + + for (document_id, normalized) in &normalized_docs { + if (!subject.is_empty() && normalized.contains(&subject)) + || (!object.is_empty() && normalized.contains(&object)) + { + *doc_scores.entry((*document_id).to_string()).or_insert(0.0) += base * 0.35; + } + } + } + + Self::normalize_scores(doc_scores) + } + + fn supporting_relations_for_document( + &self, + document_id: &str, + content: &str, + relations: &[RelationMatch], + ) -> Vec { + let normalized_content = Self::normalize_search_text(content); + let mut out = relations + .iter() + .filter(|relation| { + relation + .relation + .document_ids + .iter() + .any(|id| id == document_id) + || relation + .relation + .chunk_ids + .iter() + .any(|chunk_id| chunk_id.starts_with(document_id)) + || normalized_content + .contains(&Self::normalize_search_text(&relation.relation.subject)) + || normalized_content + .contains(&Self::normalize_search_text(&relation.relation.object)) + }) + .map(|relation| relation.relation.clone()) + .collect::>(); + out.sort_by(|a, b| { + b.evidence_count.cmp(&a.evidence_count).then_with(|| { + b.updated_at + .partial_cmp(&a.updated_at) + .unwrap_or(std::cmp::Ordering::Equal) + }) + }); + out.truncate(3); + out + } + + fn document_recall_graph_signal( + &self, + document_id: &str, + content: &str, + relations: &[GraphRelationRecord], + ) -> f64 { + let normalized_content = Self::normalize_search_text(content); + let mut score = 0.0; + for relation in relations { + if relation.document_ids.iter().any(|id| id == document_id) { + score += f64::from(relation.evidence_count); + continue; + } + let subject = Self::normalize_search_text(&relation.subject); + let object = Self::normalize_search_text(&relation.object); + if (!subject.is_empty() && normalized_content.contains(&subject)) + || (!object.is_empty() && normalized_content.contains(&object)) + { + score += f64::from(relation.evidence_count) * 0.35; + } + } + score.clamp(0.0, 10.0) / 10.0 + } + + fn keyword_score_for_text(&self, query_terms: &[String], text_parts: &[&str]) -> f64 { + if query_terms.is_empty() { + return 0.0; + } + let haystack = text_parts + .iter() + .map(|part| Self::normalize_search_text(part)) + .collect::>() + .join(" "); + if haystack.is_empty() { + return 0.0; + } + let matched = query_terms + .iter() + .filter(|term| haystack.contains(term.as_str())) + .count(); + matched as f64 / query_terms.len().max(1) as f64 + } + + fn compose_query_score( + keyword_relevance: f64, + vector_similarity: f64, + graph_relevance: f64, + ) -> RetrievalScoreBreakdown { + let final_score = (graph_relevance * GRAPH_WEIGHT) + + (vector_similarity * VECTOR_WEIGHT) + + (keyword_relevance * KEYWORD_WEIGHT); + RetrievalScoreBreakdown { + keyword_relevance, + vector_similarity, + graph_relevance, + episodic_relevance: 0.0, + freshness: 0.0, + final_score, + } + } + + fn compose_fallback_query_score( + keyword_relevance: f64, + vector_similarity: f64, + ) -> RetrievalScoreBreakdown { + let final_score = (vector_similarity * 0.65) + (keyword_relevance * 0.35); + RetrievalScoreBreakdown { + keyword_relevance, + vector_similarity, + graph_relevance: 0.0, + episodic_relevance: 0.0, + freshness: 0.0, + final_score, + } + } + + fn normalize_scores(scores: HashMap) -> HashMap { + let max_score = scores.values().copied().fold(0.0_f64, f64::max); + if max_score <= f64::EPSILON { + return HashMap::new(); + } + scores + .into_iter() + .map(|(key, score)| (key, (score / max_score).clamp(0.0, 1.0))) + .collect() + } + + fn infer_temporal_operator(query_terms: &[String]) -> TemporalOperator { + if query_terms.iter().any(|term| term == "before") { + TemporalOperator::Before + } else if query_terms.iter().any(|term| term == "after") { + TemporalOperator::After + } else if query_terms + .iter() + .any(|term| matches!(term.as_str(), "history" | "timeline" | "all")) + { + TemporalOperator::All + } else if query_terms + .iter() + .any(|term| matches!(term.as_str(), "first" | "earliest" | "initial")) + { + TemporalOperator::Earliest + } else { + TemporalOperator::Latest + } + } + + fn infer_relation_types(query_terms: &[String]) -> Vec { + let mut relation_types = HashSet::new(); + for term in query_terms { + match term.as_str() { + "where" | "location" | "located" | "place" => { + relation_types.insert("LOCATED_IN".to_string()); + relation_types.insert("RESIDES_AT".to_string()); + relation_types.insert("TRAVELS_TO".to_string()); + } + "owner" | "owns" | "owned" | "has" | "holding" => { + relation_types.insert("OWNS".to_string()); + relation_types.insert("USES".to_string()); + } + "works" | "employer" | "company" | "organization" => { + relation_types.insert("WORKS_FOR".to_string()); + } + "north" => { + relation_types.insert("NORTH_OF".to_string()); + } + "south" => { + relation_types.insert("SOUTH_OF".to_string()); + } + "east" => { + relation_types.insert("EAST_OF".to_string()); + } + "west" => { + relation_types.insert("WEST_OF".to_string()); + } + "give" | "gave" | "sent" | "handed" | "passed" | "received" | "receive" => { + relation_types.insert("USES".to_string()); + } + _ => {} + } + } + let mut out = relation_types.into_iter().collect::>(); + out.sort(); + out + } + + fn infer_relation_chains( + query_terms: &[String], + relation_types: &[String], + ) -> Vec> { + let mut chains = Vec::new(); + let asks_where = query_terms.iter().any(|term| term == "where"); + let transfer_like = query_terms.iter().any(|term| { + matches!( + term.as_str(), + "give" | "gave" | "sent" | "handed" | "passed" + ) + }); + + if asks_where { + chains.push(vec!["OWNS".to_string(), "TRAVELS_TO".to_string()]); + chains.push(vec!["USES".to_string(), "TRAVELS_TO".to_string()]); + chains.push(vec!["OWNS".to_string(), "LOCATED_IN".to_string()]); + chains.push(vec!["USES".to_string(), "LOCATED_IN".to_string()]); + } else if transfer_like { + chains.push(vec!["USES".to_string()]); + } else if !relation_types.is_empty() { + chains.push(relation_types.to_vec()); + } + + chains.truncate(4); + chains + } + + fn predicate_matches_query(predicate: &str, query_terms: &[String]) -> bool { + let normalized = Self::normalize_search_text(predicate); + query_terms.iter().any(|term| normalized.contains(term)) + } + + fn relation_matches_terms(relation: &GraphRelationRecord, query_terms: &[String]) -> bool { + let subject = Self::normalize_search_text(&relation.subject); + let object = Self::normalize_search_text(&relation.object); + let predicate = Self::normalize_search_text(&relation.predicate); + query_terms.iter().any(|term| { + subject.contains(term.as_str()) + || object.contains(term.as_str()) + || predicate.contains(term.as_str()) + }) + } + + fn relation_identity(relation: &GraphRelationRecord) -> String { + format!( + "{}|{}|{}|{}", + relation.namespace.as_deref().unwrap_or("global"), + relation.subject, + relation.predicate, + relation.object + ) + } + + fn relation_order_value(relation: &GraphRelationRecord) -> i64 { + relation + .order_index + .unwrap_or_else(|| relation.updated_at.round() as i64) + } + + fn document_priority_signal( + category: &str, + priority: &str, + tags: &[String], + metadata: &serde_json::Value, + ) -> f64 { + let mut score: f64 = 0.25; + if matches!(category, "core" | "conversation") { + score += 0.25; + } + if matches!(priority, "high" | "critical") { + score += 0.20; + } + if tags.iter().any(|tag| { + matches!( + tag.as_str(), + "decision" | "preference" | "owner" | "durable" | "profile" + ) + }) { + score += 0.20; + } + if metadata + .get("kind") + .and_then(serde_json::Value::as_str) + .map(|kind| matches!(kind, "decision" | "preference" | "profile")) + .unwrap_or(false) + { + score += 0.10; + } + score.clamp(0.0, 1.0) + } + + fn kv_priority_signal(key: &str, value: &serde_json::Value) -> f64 { + let key_norm = Self::normalize_search_text(key); + let value_norm = Self::normalize_search_text(&Self::render_kv_value(value)); + let mut score: f64 = 0.30; + if ["preference", "decision", "profile", "setting", "owner"] + .iter() + .any(|needle| key_norm.contains(needle) || value_norm.contains(needle)) + { + score += 0.35; + } + if value.is_object() || value.is_array() { + score += 0.15; + } + score.clamp(0.0, 1.0) + } + + fn render_kv_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(text) => text.clone(), + _ => serde_json::to_string(value).unwrap_or_else(|_| value.to_string()), + } + } + + fn entity_label_with_type(name: &str, attrs: &serde_json::Value, role: &str) -> String { + let entity_type = attrs + .get("entity_types") + .and_then(|et| et.get(role)) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()); + match entity_type { + Some(t) => format!("{name} ({t})"), + None => name.to_string(), + } + } + + fn format_context_text(hits: &[NamespaceMemoryHit], query: Option<&str>) -> String { + let mut parts = Vec::new(); + if let Some(query) = query { + parts.push(format!("Query: {query}")); + } + for hit in hits { + let summary = match hit.kind { + MemoryItemKind::Document => { + let title = hit.title.clone().unwrap_or_else(|| hit.key.clone()); + format!("{title}: {}", hit.content.trim()) + } + MemoryItemKind::Kv => format!("[kv:{}] {}", hit.key, hit.content.trim()), + MemoryItemKind::Episodic => { + format!("[episodic:{}] {}", hit.key, hit.content.trim()) + } + MemoryItemKind::Event => { + format!("[event:{}] {}", hit.key, hit.content.trim()) + } + }; + parts.push(summary); + + if !hit.supporting_relations.is_empty() { + let relations = hit + .supporting_relations + .iter() + .map(|relation| { + let subject_label = Self::entity_label_with_type( + &relation.subject, + &relation.attrs, + "subject", + ); + let object_label = Self::entity_label_with_type( + &relation.object, + &relation.attrs, + "object", + ); + format!( + "{} -[{}]-> {}", + subject_label, relation.predicate, object_label + ) + }) + .collect::>() + .join("; "); + parts.push(format!("Relations: {relations}")); + } + } + parts.join("\n\n") + } +} + +#[cfg(test)] +#[path = "query_tests.rs"] +mod tests; diff --git a/core/src/store/namespace_store/query_tests.rs b/core/src/store/namespace_store/query_tests.rs new file mode 100644 index 0000000..0b7d2cf --- /dev/null +++ b/core/src/store/namespace_store/query_tests.rs @@ -0,0 +1,1000 @@ +//! Tests for the `query` module — hybrid retrieval scoring. + +use std::sync::Arc; + +use serde_json::json; +use tempfile::TempDir; + +use crate::store::{NamespaceDocumentInput, UnifiedMemory}; +use crate::Memory; +use tinymemory_api::host::NoopEmbedding; + +#[tokio::test] +async fn graph_duplicate_upsert_aggregates_evidence_count() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .graph_upsert_namespace( + "team", + "alice", + "owns", + "atlas", + &json!({"document_id": "doc-1"}), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "team", + "ALICE", + "OWNS", + "ATLAS", + &json!({"document_ids": ["doc-2"], "evidence_count": 2}), + ) + .await + .unwrap(); + + let rows = memory.graph_relations_for_scope("team").await.unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].subject, "ALICE"); + assert_eq!(rows[0].predicate, "OWNS"); + assert_eq!(rows[0].object, "ATLAS"); + assert_eq!(rows[0].evidence_count, 3); + assert_eq!(rows[0].document_ids, vec!["doc-1", "doc-2"]); +} + +#[tokio::test] +async fn query_namespace_uses_graph_signal_for_document_ranking() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(NamespaceDocumentInput { + namespace: "team".to_string(), + key: "atlas-status".to_string(), + title: "Atlas status".to_string(), + content: "Project Atlas is currently owned by Alice.".to_string(), + source_type: "doc".to_string(), + priority: "high".to_string(), + tags: vec!["decision".to_string()], + metadata: json!({"kind": "decision"}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .unwrap(); + + memory + .graph_upsert_namespace( + "team", + "Alice", + "owns", + "Atlas", + &json!({"document_id": document_id}), + ) + .await + .unwrap(); + + let results = memory + .query_namespace_ranked("team", "who owns atlas", 5) + .await + .unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].key, "atlas-status"); + assert!(results[0].score > 0.5); +} + +#[tokio::test] +async fn query_scores_relation_entities_found_in_document_content() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(NamespaceDocumentInput { + namespace: "team".to_string(), + key: "atlas-background".to_string(), + title: "Atlas background".to_string(), + content: "Alice coordinates the Atlas rollout notes.".to_string(), + source_type: "doc".to_string(), + priority: "high".to_string(), + tags: vec!["project".to_string()], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .unwrap(); + + memory + .graph_upsert_namespace("team", "Alice", "owns", "Atlas", &json!({})) + .await + .unwrap(); + + let hits = memory + .query_namespace_hits("team", "who owns atlas", 5) + .await + .unwrap(); + let hit = hits + .iter() + .find(|hit| hit.key == "atlas-background") + .expect("document content should receive graph relevance"); + + assert!(hit.score_breakdown.graph_relevance > 0.0); + assert!(!hit.supporting_relations.is_empty()); +} + +#[tokio::test] +async fn recall_namespace_memories_includes_namespace_kv() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .kv_set_namespace( + "team", + "user.preference.theme", + &json!({"value": "sunrise", "kind": "preference"}), + ) + .await + .unwrap(); + + let hits = memory.recall_namespace_memories("team", 5).await.unwrap(); + assert!(hits + .iter() + .any(|hit| matches!(hit.kind, crate::store::MemoryItemKind::Kv))); +} + +#[tokio::test] +async fn query_returns_episodic_hits_when_available() { + use crate::store::fts5::{self, EpisodicEntry}; + + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // Insert an episodic entry that matches the query. + fts5::episodic_insert( + &memory.conn, + &EpisodicEntry { + id: None, + session_id: "sess-1".into(), + timestamp: 1000.0, + role: "user".into(), + content: "I have been using Tokio for async Rust development".into(), + lesson: None, + tool_calls_json: None, + cost_microdollars: 0, + }, + ) + .unwrap(); + + let hits = memory + .query_namespace_hits("global", "Tokio async Rust", 10) + .await + .unwrap(); + + let episodic_hits: Vec<_> = hits + .iter() + .filter(|h| h.kind == crate::store::MemoryItemKind::Episodic) + .collect(); + assert!( + !episodic_hits.is_empty(), + "Expected at least one Episodic hit for 'Tokio async Rust'" + ); +} + +#[tokio::test] +async fn query_returns_event_hits_when_available() { + use crate::store::events::{self, EventRecord, EventType}; + + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // Insert an event that matches the query. + events::event_insert( + &memory.conn, + &EventRecord { + event_id: "evt-q-1".into(), + segment_id: "seg-q-1".into(), + session_id: "s1".into(), + namespace: "global".into(), + event_type: EventType::Decision, + content: "We decided to use PostgreSQL as the primary database".into(), + subject: Some("database choice".into()), + timestamp_ref: None, + confidence: 0.85, + embedding: None, + source_turn_ids: None, + created_at: 1000.0, + }, + ) + .unwrap(); + + let hits = memory + .query_namespace_hits("global", "PostgreSQL database", 10) + .await + .unwrap(); + + let event_hits: Vec<_> = hits + .iter() + .filter(|h| h.kind == crate::store::MemoryItemKind::Event) + .collect(); + assert!( + !event_hits.is_empty(), + "Expected at least one Event hit for 'PostgreSQL database'" + ); +} + +#[tokio::test] +async fn query_episodic_hits_have_correct_kind() { + use crate::store::fts5::{self, EpisodicEntry}; + + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + fts5::episodic_insert( + &memory.conn, + &EpisodicEntry { + id: None, + session_id: "sess-kind".into(), + timestamp: 2000.0, + role: "assistant".into(), + content: "The deployment pipeline uses GitHub Actions for CI".into(), + lesson: Some("CI runs on push to main".into()), + tool_calls_json: None, + cost_microdollars: 0, + }, + ) + .unwrap(); + + let hits = memory + .query_namespace_hits("global", "GitHub Actions deployment", 10) + .await + .unwrap(); + + for hit in hits.iter().filter(|h| h.id.starts_with("episodic:")) { + assert_eq!( + hit.kind, + crate::store::MemoryItemKind::Episodic, + "Hits with 'episodic:' id prefix must have kind Episodic" + ); + } +} + +/// Episodic FTS relevance is derived from each hit's rank position +/// (`1.0 - idx / len`). With two equally-fresh matches the only +/// differentiator is rank, so the relevance scores must be exactly the +/// per-position values {1.0, 0.5}. This pins the position-indexing math +/// for n > 1 — the single-entry tests above cannot, since idx is always 0. +#[tokio::test] +async fn query_episodic_relevance_tracks_rank_position() { + use crate::store::fts5::{self, EpisodicEntry}; + + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // Two distinct entries, identical timestamp (equal freshness), both + // matching the query so episodic_hits has len == 2. + for content in [ + "I have been using Tokio for async Rust development", + "Tokio async runtime powers our backend services", + ] { + fts5::episodic_insert( + &memory.conn, + &EpisodicEntry { + id: None, + session_id: "sess-rank".into(), + timestamp: 1000.0, + role: "user".into(), + content: content.into(), + lesson: None, + tool_calls_json: None, + cost_microdollars: 0, + }, + ) + .unwrap(); + } + + let hits = memory + .query_namespace_hits("global", "Tokio async", 10) + .await + .unwrap(); + + let mut relevances: Vec = hits + .iter() + .filter(|h| h.kind == crate::store::MemoryItemKind::Episodic) + .map(|h| h.score_breakdown.episodic_relevance) + .collect(); + relevances.sort_by(|a, b| a.partial_cmp(b).unwrap()); + + assert_eq!( + relevances.len(), + 2, + "expected exactly two episodic hits, got {relevances:?}" + ); + assert!( + (relevances[0] - 0.5).abs() < 1e-9 && (relevances[1] - 1.0).abs() < 1e-9, + "episodic relevance must be {{0.5, 1.0}} for two-element rank order, got {relevances:?}" + ); +} + +#[tokio::test] +async fn query_supporting_relations_contain_entity_types() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(NamespaceDocumentInput { + namespace: "team".to_string(), + key: "alice-google".to_string(), + title: "Alice at Google".to_string(), + content: "Alice works on Project Alpha at Google.".to_string(), + source_type: "doc".to_string(), + priority: "high".to_string(), + tags: vec!["decision".to_string()], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .unwrap(); + + // Upsert graph relations with entity types in attrs (mimics ingestion pipeline). + memory + .graph_upsert_namespace( + "team", + "Alice", + "WORKS_FOR", + "Google", + &json!({ + "document_id": document_id, + "entity_types": { + "subject": "PERSON", + "object": "ORGANIZATION" + } + }), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "team", + "Alice", + "OWNS", + "Project Alpha", + &json!({ + "document_id": document_id, + "entity_types": { + "subject": "PERSON", + "object": "PROJECT" + } + }), + ) + .await + .unwrap(); + + // Query path: entity types should appear in supporting_relations attrs. + let hits = memory + .query_namespace_hits("team", "Alice", 5) + .await + .unwrap(); + assert!(!hits.is_empty(), "should return at least one hit"); + + let hit = &hits[0]; + assert!( + !hit.supporting_relations.is_empty(), + "hit should have supporting relations" + ); + + // Verify entity types are present in the attrs of supporting relations. + for relation in &hit.supporting_relations { + let entity_types = relation.attrs.get("entity_types"); + assert!( + entity_types.is_some(), + "relation {} -[{}]-> {} should have entity_types in attrs", + relation.subject, + relation.predicate, + relation.object + ); + let et = entity_types.unwrap(); + let subject_type = et.get("subject").and_then(|v| v.as_str()); + assert_eq!( + subject_type, + Some("PERSON"), + "subject_type should be PERSON for Alice" + ); + } + + // Recall path: entity types should also appear. + let recall_hits = memory.recall_namespace_memories("team", 5).await.unwrap(); + assert!(!recall_hits.is_empty(), "recall should return hits"); + + let recall_hit = &recall_hits[0]; + assert!( + !recall_hit.supporting_relations.is_empty(), + "recall hit should have supporting relations" + ); + for relation in &recall_hit.supporting_relations { + let entity_types = relation.attrs.get("entity_types"); + assert!( + entity_types.is_some(), + "recall relation should have entity_types in attrs" + ); + } +} + +/// `recall_namespace_memories` builds one shared `RelationMatch` view for all +/// documents (hoisted out of the per-document loop). This pins that the shared +/// input is still filtered per-document: with two documents each carrying their +/// own graph relation, neither hit may surface the other's relation. A naive +/// hoist that leaked the wrong relations across documents would fail here, where +/// the single-document recall tests above cannot. +#[tokio::test] +async fn recall_supporting_relations_stay_scoped_per_document() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let alpha_id = memory + .upsert_document(NamespaceDocumentInput { + namespace: "team".to_string(), + key: "alpha-doc".to_string(), + title: "Alpha".to_string(), + content: "Alice leads the Atlas project.".to_string(), + source_type: "doc".to_string(), + priority: "high".to_string(), + tags: vec!["project".to_string()], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .unwrap(); + let beta_id = memory + .upsert_document(NamespaceDocumentInput { + namespace: "team".to_string(), + key: "beta-doc".to_string(), + title: "Beta".to_string(), + content: "Bob manages the Borealis launch.".to_string(), + source_type: "doc".to_string(), + priority: "high".to_string(), + tags: vec!["project".to_string()], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .unwrap(); + + memory + .graph_upsert_namespace( + "team", + "Alice", + "OWNS", + "Atlas", + &json!({ "document_id": alpha_id }), + ) + .await + .unwrap(); + memory + .graph_upsert_namespace( + "team", + "Bob", + "OWNS", + "Borealis", + &json!({ "document_id": beta_id }), + ) + .await + .unwrap(); + + let hits = memory.recall_namespace_memories("team", 10).await.unwrap(); + let alpha = hits + .iter() + .find(|hit| hit.key == "alpha-doc") + .expect("recall should return alpha-doc"); + let beta = hits + .iter() + .find(|hit| hit.key == "beta-doc") + .expect("recall should return beta-doc"); + + let objects = |hit: &crate::store::NamespaceMemoryHit| { + hit.supporting_relations + .iter() + .map(|relation| relation.object.to_uppercase()) + .collect::>() + }; + let alpha_objects = objects(alpha); + let beta_objects = objects(beta); + + assert!( + alpha_objects.iter().any(|object| object.contains("ATLAS")), + "alpha-doc should keep its own relation, got {alpha_objects:?}" + ); + assert!( + !alpha_objects + .iter() + .any(|object| object.contains("BOREALIS")), + "alpha-doc must not surface beta-doc's relation, got {alpha_objects:?}" + ); + assert!( + beta_objects + .iter() + .any(|object| object.contains("BOREALIS")), + "beta-doc should keep its own relation, got {beta_objects:?}" + ); + assert!( + !beta_objects.iter().any(|object| object.contains("ATLAS")), + "beta-doc must not surface alpha-doc's relation, got {beta_objects:?}" + ); +} + +#[tokio::test] +async fn format_context_text_includes_entity_types() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + let document_id = memory + .upsert_document(NamespaceDocumentInput { + namespace: "team".to_string(), + key: "atlas-status".to_string(), + title: "Atlas status".to_string(), + content: "Project Atlas is owned by Alice at Google.".to_string(), + source_type: "doc".to_string(), + priority: "high".to_string(), + tags: vec!["decision".to_string()], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + }) + .await + .unwrap(); + + memory + .graph_upsert_namespace( + "team", + "Alice", + "OWNS", + "Atlas", + &json!({ + "document_id": document_id, + "entity_types": { + "subject": "PERSON", + "object": "PROJECT" + } + }), + ) + .await + .unwrap(); + + let context = memory + .query_namespace_context_data("team", "who owns atlas", 5) + .await + .unwrap(); + // Entity names are normalized to uppercase during graph upsert. + assert!( + context.context_text.contains("ALICE (PERSON)"), + "context_text should include entity type for Alice, got: {}", + context.context_text + ); + assert!( + context.context_text.contains("ATLAS (PROJECT)"), + "context_text should include entity type for Atlas, got: {}", + context.context_text + ); +} + +// ── vector_chunks model-signature guard (embedding model-swap safety) ───────── + +use async_trait::async_trait; + +use tinymemory_api::host::EmbeddingProvider; + +/// Embedder stub that returns a fixed vector for any text, with a controllable +/// name + dimension so tests can produce distinct embedding signatures and +/// dimensionalities. +struct StubEmbedder { + name: &'static str, + vector: Vec, +} + +#[async_trait] +impl EmbeddingProvider for StubEmbedder { + fn name(&self) -> &str { + self.name + } + fn model_id(&self) -> &str { + self.name + } + fn dimensions(&self) -> usize { + self.vector.len() + } + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(texts.iter().map(|_| self.vector.clone()).collect()) + } +} + +fn pref_doc(key: &str, content: &str) -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: "user_pref".to_string(), + key: key.to_string(), + title: key.to_string(), + content: content.to_string(), + source_type: "pref".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + } +} + +#[tokio::test] +async fn upsert_tags_vector_chunks_with_signature_and_dim() { + let tmp = TempDir::new().unwrap(); + let embedder = Arc::new(StubEmbedder { + name: "stub-a", + vector: vec![1.0, 0.0, 0.0], + }); + let memory = UnifiedMemory::new(tmp.path(), embedder.clone(), None).unwrap(); + + memory + .upsert_document(pref_doc("reply_language", "Reply in British English.")) + .await + .unwrap(); + + // The stored chunk carries the active model's signature. + let chunks = memory.load_chunks_for_scope("user_pref").await.unwrap(); + assert_eq!(chunks.len(), 1, "expected exactly one chunk for the doc"); + assert_eq!( + chunks[0].model_signature.as_deref(), + Some(embedder.signature().as_str()), + "chunk should be tagged with the embedder signature" + ); + + // The `dim` column reflects the embedding dimensionality. + let dim: Option = memory + .conn + .lock() + .query_row( + "SELECT dim FROM vector_chunks WHERE namespace = 'user_pref' LIMIT 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(dim, Some(3)); +} + +#[tokio::test] +async fn vector_recall_excludes_other_model_signature() { + let tmp = TempDir::new().unwrap(); + + // Write under model A. + let emb_a = Arc::new(StubEmbedder { + name: "model-a", + vector: vec![1.0, 0.0, 0.0], + }); + { + let memory = UnifiedMemory::new(tmp.path(), emb_a.clone(), None).unwrap(); + memory + .upsert_document(pref_doc("p1", "formal tone for emails to my manager")) + .await + .unwrap(); + + // Same model → the vector is scored. + let chunks = memory.load_chunks_for_scope("user_pref").await.unwrap(); + let scores = memory + .query_vector_scores_from_chunks(&chunks, "email tone") + .await + .unwrap(); + assert!(!scores.is_empty(), "same-signature vectors must be scored"); + } + + // Reopen the same DB under a DIFFERENT model (swap), same dim + vector. + let emb_b = Arc::new(StubEmbedder { + name: "model-b", + vector: vec![1.0, 0.0, 0.0], + }); + let memory_b = UnifiedMemory::new(tmp.path(), emb_b, None).unwrap(); + let chunks = memory_b.load_chunks_for_scope("user_pref").await.unwrap(); + assert_eq!(chunks.len(), 1, "the chunk persists across reopen"); + let scores = memory_b + .query_vector_scores_from_chunks(&chunks, "email tone") + .await + .unwrap(); + assert!( + scores.is_empty(), + "vectors from a different embedding model must be excluded, not compared as garbage" + ); +} + +#[tokio::test] +async fn vector_recall_skips_dimension_mismatch_for_untagged_rows() { + let tmp = TempDir::new().unwrap(); + // Active model produces 4-dim vectors. + let emb = Arc::new(StubEmbedder { + name: "model-a", + vector: vec![1.0, 0.0, 0.0, 0.0], + }); + let memory = UnifiedMemory::new(tmp.path(), emb, None).unwrap(); + + // Insert a legacy chunk: NULL signature, 2-dim vector (a pre-tagging row left + // behind by a dimension-changing model swap). + let legacy_vec = UnifiedMemory::vec_to_bytes(&[1.0_f32, 0.0]); + memory + .conn + .lock() + .execute( + "INSERT INTO vector_chunks + (namespace, document_id, chunk_id, text, embedding, metadata_json, created_at, updated_at, model_signature, dim) + VALUES ('user_pref','legacy','legacy:0','old pref',?1,'{}',0,0,NULL,2)", + rusqlite::params![legacy_vec], + ) + .unwrap(); + + let chunks = memory.load_chunks_for_scope("user_pref").await.unwrap(); + assert_eq!(chunks.len(), 1); + assert!( + chunks[0].model_signature.is_none(), + "legacy row should have no signature" + ); + let scores = memory + .query_vector_scores_from_chunks(&chunks, "old pref") + .await + .unwrap(); + assert!( + scores.is_empty(), + "dimension-mismatched legacy vectors must be skipped, not scored 0" + ); +} + +// ── recall_relevant_by_vector — Lane B situational-pref relevance gate ───────── + +/// Embedder whose vector depends on keywords in the text, so a query can be +/// genuinely relevant (high cosine) or irrelevant (zero) to a stored pref. +struct KeywordEmbedder; + +#[async_trait] +impl EmbeddingProvider for KeywordEmbedder { + fn name(&self) -> &str { + "keyword-stub" + } + fn model_id(&self) -> &str { + "keyword-stub" + } + fn dimensions(&self) -> usize { + 2 + } + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(texts + .iter() + .map(|t| { + let lower = t.to_lowercase(); + vec![ + if lower.contains("rust") { 1.0 } else { 0.0 }, + if lower.contains("email") { 1.0 } else { 0.0 }, + ] + }) + .collect()) + } +} + +fn situational_doc(key: &str, content: &str) -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: "user_pref_situational".to_string(), + key: key.to_string(), + title: key.to_string(), + content: content.to_string(), + source_type: "pref".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + } +} + +#[tokio::test] +async fn recall_relevant_by_vector_gates_on_similarity() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(KeywordEmbedder), None).unwrap(); + + // Two situational prefs that embed onto orthogonal axes. + memory + .upsert_document(situational_doc( + "rust_style", + "When writing rust, prefer explicit error handling.", + )) + .await + .unwrap(); + memory + .upsert_document(situational_doc( + "email_tone", + "Be formal in email to my manager.", + )) + .await + .unwrap(); + + // A rust-related message recalls only the rust pref. + let hits = memory + .recall_relevant_by_vector("user_pref_situational", "help me with my rust code", 5, 0.5) + .await + .unwrap(); + assert_eq!(hits.len(), 1, "only the relevant pref should pass the gate"); + assert_eq!(hits[0].0, "rust_style"); + assert!(hits[0].1.contains("explicit error handling")); + + // An unrelated message clears the gate to nothing — no block injected. + let none = memory + .recall_relevant_by_vector("user_pref_situational", "what is the weather today", 5, 0.5) + .await + .unwrap(); + assert!( + none.is_empty(), + "an unrelated message must surface no situational preferences" + ); +} + +// ── Same-session self-echo exclusion (memory-search self-echo fix) ───────── +// +// Regression coverage for the workflow_builder self-echo bug: the harness +// auto-saves the user's own turn as a `[conversation]` document tagged with +// the live chat thread id, and without a filter a search issued mid-turn +// could retrieve that very request as its own top "relevant" result. See +// `UnifiedMemory::query_namespace_hits_excluding_session`. + +fn conversation_doc_with_session( + key: &str, + content: &str, + session_id: Option<&str>, +) -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: "global".to_string(), + key: key.to_string(), + title: key.to_string(), + content: content.to_string(), + source_type: "chat".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "conversation".to_string(), + session_id: session_id.map(str::to_string), + document_id: None, + taint: crate::MemoryTaint::Internal, + } +} + +#[tokio::test] +async fn excludes_same_session_document_but_keeps_unrelated_useful_doc() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + // (a) The current turn's own auto-saved request — tagged with the live + // session/thread id, exactly like `agent::harness::session::turn::core` + // tags the `user_msg:` autosave. + memory + .upsert_document(conversation_doc_with_session( + "user_msg:current-turn", + "Please look up Jordan Rivera's chat platform user ID for me.", + Some("thread-current"), + )) + .await + .unwrap(); + + // (b) An unrelated, genuinely useful fact from a prior turn/session that + // actually answers the query. + memory + .upsert_document(conversation_doc_with_session( + "fact:jordan-rivera-platform-id", + "Jordan Rivera's chat platform user ID is U0000042.", + Some("thread-other"), + )) + .await + .unwrap(); + + let query = "Jordan Rivera chat platform user ID"; + + // Sanity check: without exclusion, both documents are lexically relevant + // and both come back (this is the pre-fix, buggy shape). + let unfiltered = memory + .query_namespace_hits("global", query, 10) + .await + .unwrap(); + assert!( + unfiltered.iter().any(|h| h.key == "user_msg:current-turn"), + "sanity check: the self-request doc must be lexically relevant without a filter, got {unfiltered:#?}" + ); + + // With the current-session exclusion applied, the self-echo document is + // dropped and the useful fact survives. + let filtered = memory + .query_namespace_hits_excluding_session("global", query, 10, Some("thread-current")) + .await + .unwrap(); + + assert!( + !filtered.iter().any(|h| h.key == "user_msg:current-turn"), + "same-session self-request document must be excluded, got {filtered:#?}" + ); + assert!( + filtered + .iter() + .any(|h| h.key == "fact:jordan-rivera-platform-id"), + "unrelated useful document from another session must still be returned, got {filtered:#?}" + ); +} + +#[tokio::test] +async fn no_session_context_leaves_results_unchanged() { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + + memory + .upsert_document(conversation_doc_with_session( + "user_msg:current-turn", + "Please look up Jordan Rivera's chat platform user ID for me.", + Some("thread-current"), + )) + .await + .unwrap(); + memory + .upsert_document(conversation_doc_with_session( + "fact:jordan-rivera-platform-id", + "Jordan Rivera's chat platform user ID is U0000042.", + Some("thread-other"), + )) + .await + .unwrap(); + + let query = "Jordan Rivera chat platform user ID"; + + let baseline = memory + .query_namespace_hits("global", query, 10) + .await + .unwrap(); + + // `None` exclusion (no ambient session context — cron, CLI, standalone, + // or a caller with no `exclude_session_id` to give) must behave + // identically to the pre-existing `query_namespace_hits` entry point: + // same hit count, same keys, in the same order. + let explicit_none = memory + .query_namespace_hits_excluding_session("global", query, 10, None) + .await + .unwrap(); + + let baseline_keys: Vec<&str> = baseline.iter().map(|h| h.key.as_str()).collect(); + let explicit_none_keys: Vec<&str> = explicit_none.iter().map(|h| h.key.as_str()).collect(); + assert_eq!( + baseline_keys, explicit_none_keys, + "no session context must be a no-op vs. the unfiltered entry point" + ); + assert!( + explicit_none_keys.contains(&"user_msg:current-turn"), + "without any exclusion, the self-request document must still be present" + ); + + // An empty/whitespace exclude id must also be treated as "no filter", + // not accidentally matched against a document with `session_id: None`. + let empty_string = memory + .query_namespace_hits_excluding_session("global", query, 10, Some(" ")) + .await + .unwrap(); + let empty_string_keys: Vec<&str> = empty_string.iter().map(|h| h.key.as_str()).collect(); + assert_eq!( + baseline_keys, empty_string_keys, + "a blank exclude_session_id must not filter anything" + ); +} diff --git a/core/src/store/namespace_store/segments.rs b/core/src/store/namespace_store/segments.rs new file mode 100644 index 0000000..04a1dc7 --- /dev/null +++ b/core/src/store/namespace_store/segments.rs @@ -0,0 +1,631 @@ +//! Conversation segmentation — groups consecutive episodic turns into +//! coherent "segments" using lightweight heuristic boundary detection. +//! +//! Inspired by EverMemOS MemCells: instead of indexing raw turns individually, +//! segments capture a topic-coherent block of conversation that can be +//! summarised, searched, and used for downstream extraction (events, profile). + +use parking_lot::Mutex; +use rusqlite::{params, Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +/// SQL to create the conversation_segments table. Called during UnifiedMemory init. +pub const SEGMENTS_INIT_SQL: &str = r#" +CREATE TABLE IF NOT EXISTS conversation_segments ( + segment_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + namespace TEXT NOT NULL DEFAULT 'global', + start_episodic_id INTEGER NOT NULL, + end_episodic_id INTEGER, + start_timestamp REAL NOT NULL, + end_timestamp REAL, + turn_count INTEGER NOT NULL DEFAULT 0, + summary TEXT, + embedding BLOB, + topic_keywords TEXT, + status TEXT NOT NULL DEFAULT 'open', + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + -- Per-session sequence numbers from tinycortex::memory::archivist::store, populated + -- alongside start_episodic_id / end_episodic_id during the FTS5 -> md + -- migration. Once STM recall switches its segment-span dedup to use + -- (session_id, seq) the legacy episodic_id columns can be dropped. + start_seq INTEGER, + end_seq INTEGER +); + +CREATE INDEX IF NOT EXISTS idx_segments_session + ON conversation_segments(session_id, start_timestamp); + +CREATE INDEX IF NOT EXISTS idx_segments_namespace + ON conversation_segments(namespace, updated_at DESC); + +CREATE INDEX IF NOT EXISTS idx_segments_status + ON conversation_segments(status, session_id); + +-- Per-model segment embeddings for #1574. The legacy +-- `conversation_segments.embedding` column stays in place during staged +-- migration; this table lets provider/model switches become query-time +-- filters instead of destructive rewrites. +CREATE TABLE IF NOT EXISTS segment_embeddings ( + segment_id TEXT NOT NULL REFERENCES conversation_segments(segment_id) ON DELETE CASCADE, + model_signature TEXT NOT NULL, + vector BLOB NOT NULL, + dim INTEGER NOT NULL, + created_at REAL NOT NULL, + PRIMARY KEY (segment_id, model_signature) +); + +CREATE INDEX IF NOT EXISTS idx_segment_embeddings_model + ON segment_embeddings(model_signature); +"#; + +/// Segment status lifecycle: open → closed → summarised. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SegmentStatus { + Open, + Closed, + Summarised, +} + +impl SegmentStatus { + /// Stable lowercase identifier persisted in the `conversation_segments` table. + pub fn as_str(&self) -> &'static str { + match self { + Self::Open => "open", + Self::Closed => "closed", + Self::Summarised => "summarised", + } + } + + /// Parse a stored string back to a `SegmentStatus`; unknown values fall + /// back to `Open`. + pub fn parse_or_default(s: &str) -> Self { + match s { + "closed" => Self::Closed, + "summarised" => Self::Summarised, + _ => Self::Open, + } + } +} + +/// A conversation segment record. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConversationSegment { + pub segment_id: String, + pub session_id: String, + pub namespace: String, + pub start_episodic_id: i64, + pub end_episodic_id: Option, + pub start_timestamp: f64, + pub end_timestamp: Option, + pub turn_count: i32, + pub summary: Option, + pub embedding: Option>, + pub topic_keywords: Option, + pub status: SegmentStatus, + pub created_at: f64, + pub updated_at: f64, + /// Per-session seq number assigned by the TinyCortex archivist store. + /// for the user turn that opened this segment. `None` on legacy rows + /// written before the FTS5 -> md migration began. + #[serde(default)] + pub start_seq: Option, + /// Per-session seq for the latest turn appended to this segment. + /// `None` while the segment has no appended turns OR on legacy rows. + #[serde(default)] + pub end_seq: Option, +} + +/// Idempotent migrations applied alongside [`SEGMENTS_INIT_SQL`] for +/// databases created before the `(start_seq, end_seq)` columns existed. +/// Each statement either applies cleanly or fails with "duplicate column", +/// both of which are safe to swallow. +pub const SEGMENTS_MIGRATIONS_SQL: &[&str] = &[ + "ALTER TABLE conversation_segments ADD COLUMN start_seq INTEGER", + "ALTER TABLE conversation_segments ADD COLUMN end_seq INTEGER", +]; + +/// Boundary detection configuration. +#[derive(Debug, Clone)] +pub struct BoundaryConfig { + /// Maximum time gap (seconds) between turns before forcing a new segment. + pub max_time_gap_secs: f64, + /// Minimum cosine similarity between turn embedding and segment centroid. + /// Below this threshold, a boundary is detected. + pub min_cosine_similarity: f32, + /// Maximum turns per segment before forcing a boundary. + pub max_turns_per_segment: i32, +} + +impl Default for BoundaryConfig { + fn default() -> Self { + Self { + max_time_gap_secs: 600.0, // 10 minutes + min_cosine_similarity: 0.4, + max_turns_per_segment: 20, + } + } +} + +/// Result of boundary detection for a new turn. +#[derive(Debug, Clone)] +pub enum BoundaryDecision { + /// Continue accumulating into the current segment. + Continue, + /// Close the current segment and start a new one. + Boundary(BoundaryReason), +} + +/// Reason a new segment boundary was triggered. +#[derive(Debug, Clone)] +pub enum BoundaryReason { + TimeGap, + EmbeddingDrift, + ExplicitMarker, + TurnCountExceeded, +} + +impl std::fmt::Display for BoundaryReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TimeGap => write!(f, "time_gap"), + Self::EmbeddingDrift => write!(f, "embedding_drift"), + Self::ExplicitMarker => write!(f, "explicit_marker"), + Self::TurnCountExceeded => write!(f, "turn_count_exceeded"), + } + } +} + +/// Regex patterns that signal an explicit topic change. +const TOPIC_CHANGE_MARKERS: &[&str] = &[ + "now let's", + "now lets", + "switching to", + "different topic", + "moving on to", + "let's move on", + "lets move on", + "can you help me with", + "new question", + "unrelated but", + "changing subject", + "on another note", + "anyway,", + "by the way,", + "btw,", +]; + +/// Create a new open segment. +/// +/// Each parameter names a distinct column of the row being inserted; grouping +/// them into a struct would just move the same 8 fields one level out without +/// reducing the information the caller has to supply. +#[allow(clippy::too_many_arguments)] +pub fn segment_create( + conn: &Arc>, + segment_id: &str, + session_id: &str, + namespace: &str, + start_episodic_id: i64, + start_seq: Option, + start_timestamp: f64, + now: f64, +) -> anyhow::Result<()> { + let conn = conn.lock(); + conn.execute( + "INSERT INTO conversation_segments + (segment_id, session_id, namespace, start_episodic_id, start_seq, + start_timestamp, turn_count, status, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 1, 'open', ?7, ?7)", + params![ + segment_id, + session_id, + namespace, + start_episodic_id, + start_seq, + start_timestamp, + now + ], + )?; + tracing::debug!("[segments] created segment {segment_id} for session={session_id}"); + Ok(()) +} + +/// Increment turn count and update the latest episodic ID + seq + timestamp. +pub fn segment_append_turn( + conn: &Arc>, + segment_id: &str, + episodic_id: i64, + end_seq: Option, + timestamp: f64, + now: f64, +) -> anyhow::Result<()> { + let conn = conn.lock(); + conn.execute( + "UPDATE conversation_segments + SET turn_count = turn_count + 1, + end_episodic_id = ?2, + end_seq = ?3, + end_timestamp = ?4, + updated_at = ?5 + WHERE segment_id = ?1", + params![segment_id, episodic_id, end_seq, timestamp, now], + )?; + Ok(()) +} + +/// Close a segment (transition from open → closed). +pub fn segment_close( + conn: &Arc>, + segment_id: &str, + now: f64, +) -> anyhow::Result<()> { + let conn = conn.lock(); + conn.execute( + "UPDATE conversation_segments + SET status = 'closed', updated_at = ?2 + WHERE segment_id = ?1 AND status = 'open'", + params![segment_id, now], + )?; + tracing::debug!("[segments] closed segment {segment_id}"); + Ok(()) +} + +/// Update a segment's summary and mark as summarised. +pub fn segment_set_summary( + conn: &Arc>, + segment_id: &str, + summary: &str, + now: f64, +) -> anyhow::Result<()> { + let conn = conn.lock(); + conn.execute( + "UPDATE conversation_segments + SET summary = ?2, status = 'summarised', updated_at = ?3 + WHERE segment_id = ?1", + params![segment_id, summary, now], + )?; + Ok(()) +} + +/// Store the segment-level embedding. +pub fn segment_set_embedding( + conn: &Arc>, + segment_id: &str, + embedding: &[f32], + now: f64, +) -> anyhow::Result<()> { + let bytes = vec_to_bytes(embedding); + let conn = conn.lock(); + conn.execute( + "UPDATE conversation_segments SET embedding = ?2, updated_at = ?3 WHERE segment_id = ?1", + params![segment_id, bytes, now], + )?; + Ok(()) +} + +/// Store a segment embedding for a specific provider/model/dimension signature. +/// +/// This writes only the per-model table introduced for #1574. The legacy +/// `conversation_segments.embedding` column remains available for dual-read +/// fallback while query paths migrate. +pub fn segment_embedding_upsert( + conn: &Arc>, + segment_id: &str, + model_signature: &str, + embedding: &[f32], + created_at: f64, +) -> anyhow::Result<()> { + let bytes = vec_to_bytes(embedding); + let dim = i64::try_from(embedding.len())?; + let conn = conn.lock(); + conn.execute( + "INSERT INTO segment_embeddings (segment_id, model_signature, vector, dim, created_at) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(segment_id, model_signature) DO UPDATE SET + vector = excluded.vector, + dim = excluded.dim, + created_at = excluded.created_at", + params![segment_id, model_signature, bytes, dim, created_at], + )?; + Ok(()) +} + +/// Fetch a segment embedding for exactly one provider/model/dimension signature. +pub fn segment_embedding_get( + conn: &Arc>, + segment_id: &str, + model_signature: &str, +) -> anyhow::Result>> { + let conn = conn.lock(); + let row: Option<(Vec, i64)> = conn + .query_row( + "SELECT vector, dim + FROM segment_embeddings + WHERE segment_id = ?1 AND model_signature = ?2", + params![segment_id, model_signature], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()?; + match row { + None => Ok(None), + Some((bytes, dim)) => decode_embedding_row(&bytes, dim), + } +} + +/// Store topic keywords for the segment. +pub fn segment_set_keywords( + conn: &Arc>, + segment_id: &str, + keywords: &str, + now: f64, +) -> anyhow::Result<()> { + let conn = conn.lock(); + conn.execute( + "UPDATE conversation_segments SET topic_keywords = ?2, updated_at = ?3 WHERE segment_id = ?1", + params![segment_id, keywords, now], + )?; + Ok(()) +} + +/// Get the currently open segment for a session (if any). +pub fn open_segment_for_session( + conn: &Arc>, + session_id: &str, +) -> anyhow::Result> { + let conn = conn.lock(); + let row = conn + .query_row( + "SELECT segment_id, session_id, namespace, start_episodic_id, end_episodic_id, + start_timestamp, end_timestamp, turn_count, summary, embedding, + topic_keywords, status, created_at, updated_at, + start_seq, end_seq + FROM conversation_segments + WHERE session_id = ?1 AND status = 'open' + ORDER BY created_at DESC + LIMIT 1", + params![session_id], + row_to_segment, + ) + .optional()?; + Ok(row) +} + +/// List segments for a namespace (most recent first). +pub fn segments_by_namespace( + conn: &Arc>, + namespace: &str, + limit: usize, +) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT segment_id, session_id, namespace, start_episodic_id, end_episodic_id, + start_timestamp, end_timestamp, turn_count, summary, embedding, + topic_keywords, status, created_at, updated_at, + start_seq, end_seq + FROM conversation_segments + WHERE namespace = ?1 + ORDER BY updated_at DESC + LIMIT ?2", + )?; + let rows = stmt + .query_map(params![namespace, limit as i64], row_to_segment)? + .collect::, _>>()?; + Ok(rows) +} + +/// Get a specific segment by ID. +pub fn segment_get( + conn: &Arc>, + segment_id: &str, +) -> anyhow::Result> { + let conn = conn.lock(); + let row = conn + .query_row( + "SELECT segment_id, session_id, namespace, start_episodic_id, end_episodic_id, + start_timestamp, end_timestamp, turn_count, summary, embedding, + topic_keywords, status, created_at, updated_at, + start_seq, end_seq + FROM conversation_segments + WHERE segment_id = ?1", + params![segment_id], + row_to_segment, + ) + .optional()?; + Ok(row) +} + +/// Get all closed (unsummarised) segments that need summary generation. +pub fn segments_pending_summary( + conn: &Arc>, + limit: usize, +) -> anyhow::Result> { + let conn = conn.lock(); + let mut stmt = conn.prepare( + "SELECT segment_id, session_id, namespace, start_episodic_id, end_episodic_id, + start_timestamp, end_timestamp, turn_count, summary, embedding, + topic_keywords, status, created_at, updated_at, + start_seq, end_seq + FROM conversation_segments + WHERE status = 'closed' + ORDER BY created_at ASC + LIMIT ?1", + )?; + let rows = stmt + .query_map(params![limit as i64], row_to_segment)? + .collect::, _>>()?; + Ok(rows) +} + +/// Detect whether a boundary should be created based on heuristics. +pub fn detect_boundary( + config: &BoundaryConfig, + current_segment: &ConversationSegment, + new_turn_timestamp: f64, + new_turn_content: &str, + new_turn_embedding: Option<&[f32]>, +) -> BoundaryDecision { + // 1. Turn count exceeded. + if current_segment.turn_count >= config.max_turns_per_segment { + tracing::debug!( + "[segments] boundary: turn count {} >= {}", + current_segment.turn_count, + config.max_turns_per_segment + ); + return BoundaryDecision::Boundary(BoundaryReason::TurnCountExceeded); + } + + // 2. Time gap check. + let last_timestamp = current_segment + .end_timestamp + .unwrap_or(current_segment.start_timestamp); + let gap = new_turn_timestamp - last_timestamp; + if gap > config.max_time_gap_secs { + tracing::debug!( + "[segments] boundary: time gap {gap:.0}s > {}s", + config.max_time_gap_secs + ); + return BoundaryDecision::Boundary(BoundaryReason::TimeGap); + } + + // 3. Explicit topic-change markers. + let content_lower = new_turn_content.to_lowercase(); + for marker in TOPIC_CHANGE_MARKERS { + if content_lower.contains(marker) { + tracing::debug!("[segments] boundary: explicit marker '{marker}'"); + return BoundaryDecision::Boundary(BoundaryReason::ExplicitMarker); + } + } + + // 4. Embedding drift (cosine similarity). + if let (Some(segment_emb), Some(turn_emb)) = + (current_segment.embedding.as_ref(), new_turn_embedding) + { + if !segment_emb.is_empty() && segment_emb.len() == turn_emb.len() { + let similarity = cosine_similarity_f32(segment_emb, turn_emb); + if similarity < config.min_cosine_similarity { + tracing::debug!( + "[segments] boundary: embedding drift (sim={similarity:.3} < {})", + config.min_cosine_similarity + ); + return BoundaryDecision::Boundary(BoundaryReason::EmbeddingDrift); + } + } + } + + BoundaryDecision::Continue +} + +/// Compute mean embedding from an existing centroid and a new vector. +/// Returns a new centroid that is the incremental mean. +pub fn incremental_mean_embedding( + current_centroid: &[f32], + new_embedding: &[f32], + count: usize, +) -> Vec { + if current_centroid.is_empty() || current_centroid.len() != new_embedding.len() { + return new_embedding.to_vec(); + } + current_centroid + .iter() + .zip(new_embedding.iter()) + .map(|(c, n)| c + (n - c) / (count as f32 + 1.0)) + .collect() +} + +/// Build a fallback summary from first and last turn content. +pub fn fallback_summary(first_content: &str, last_content: &str, turn_count: i32) -> String { + let first_truncated = truncate_utf8_safe(first_content, 200); + let last_truncated = truncate_utf8_safe(last_content, 200); + format!( + "Conversation segment ({turn_count} turns). Started with: {first_truncated} | Ended with: {last_truncated}" + ) +} + +/// Truncate a string at a safe UTF-8 char boundary. +fn truncate_utf8_safe(s: &str, max_chars: usize) -> String { + match s.char_indices().nth(max_chars) { + Some((byte_idx, _)) => format!("{}...", &s[..byte_idx]), + None => s.to_string(), + } +} + +// ── helpers ── + +fn row_to_segment(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let embedding_blob: Option> = row.get(9)?; + let status_str: String = row.get(11)?; + Ok(ConversationSegment { + segment_id: row.get(0)?, + session_id: row.get(1)?, + namespace: row.get(2)?, + start_episodic_id: row.get(3)?, + end_episodic_id: row.get(4)?, + start_timestamp: row.get(5)?, + end_timestamp: row.get(6)?, + turn_count: row.get(7)?, + summary: row.get(8)?, + embedding: embedding_blob.as_deref().map(bytes_to_vec), + topic_keywords: row.get(10)?, + status: SegmentStatus::parse_or_default(&status_str), + created_at: row.get(12)?, + updated_at: row.get(13)?, + start_seq: row.get::<_, Option>(14)?.map(|v| v.max(0) as u32), + end_seq: row.get::<_, Option>(15)?.map(|v| v.max(0) as u32), + }) +} + +fn cosine_similarity_f32(a: &[f32], b: &[f32]) -> f32 { + let mut dot = 0.0_f32; + let mut norm_a = 0.0_f32; + let mut norm_b = 0.0_f32; + for (x, y) in a.iter().zip(b.iter()) { + dot += x * y; + norm_a += x * x; + norm_b += y * y; + } + let denom = norm_a.sqrt() * norm_b.sqrt(); + if denom < f32::EPSILON { + 0.0 + } else { + (dot / denom).clamp(-1.0, 1.0) + } +} + +fn vec_to_bytes(v: &[f32]) -> Vec { + v.iter().flat_map(|f| f.to_le_bytes()).collect() +} + +fn bytes_to_vec(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(4) + .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) + .collect() +} + +fn decode_embedding_row(bytes: &[u8], dim: i64) -> anyhow::Result>> { + if dim < 0 { + anyhow::bail!("segment embedding has negative dimension {dim}"); + } + if bytes.len() % 4 != 0 { + anyhow::bail!( + "segment embedding blob length {} not a multiple of 4", + bytes.len() + ); + } + let floats = bytes_to_vec(bytes); + if floats.len() != dim as usize { + anyhow::bail!( + "segment embedding dimension mismatch: dim column says {dim}, blob contains {} floats", + floats.len() + ); + } + Ok(Some(floats)) +} + +#[cfg(test)] +#[path = "segments_tests.rs"] +mod tests; diff --git a/core/src/store/namespace_store/segments_tests.rs b/core/src/store/namespace_store/segments_tests.rs new file mode 100644 index 0000000..ae7a640 --- /dev/null +++ b/core/src/store/namespace_store/segments_tests.rs @@ -0,0 +1,393 @@ +//! Tests for the `segments` module — boundary detection and segment lifecycle. + +use super::*; + +fn setup_db() -> Arc> { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(SEGMENTS_INIT_SQL).unwrap(); + // Also need episodic tables for integration. + conn.execute_batch(super::super::fts5::EPISODIC_INIT_SQL) + .unwrap(); + Arc::new(Mutex::new(conn)) +} + +#[test] +fn create_and_get_segment() { + let conn = setup_db(); + segment_create(&conn, "seg-1", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + let seg = segment_get(&conn, "seg-1").unwrap().unwrap(); + assert_eq!(seg.session_id, "s1"); + assert_eq!(seg.turn_count, 1); + assert_eq!(seg.status, SegmentStatus::Open); +} + +#[test] +fn segment_embeddings_are_scoped_by_model_signature() { + let conn = setup_db(); + segment_create(&conn, "seg-embed", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + + segment_embedding_upsert( + &conn, + "seg-embed", + "openai/text-embedding-3-small@1536", + &[0.1, 0.2], + 1001.0, + ) + .unwrap(); + segment_embedding_upsert( + &conn, + "seg-embed", + "local/bge-small@384", + &[0.3, 0.4, 0.5], + 1002.0, + ) + .unwrap(); + + assert_eq!( + segment_embedding_get(&conn, "seg-embed", "openai/text-embedding-3-small@1536").unwrap(), + Some(vec![0.1, 0.2]) + ); + assert_eq!( + segment_embedding_get(&conn, "seg-embed", "local/bge-small@384").unwrap(), + Some(vec![0.3, 0.4, 0.5]) + ); + assert!(segment_embedding_get(&conn, "seg-embed", "missing/model@1") + .unwrap() + .is_none()); + + let legacy_segment = segment_get(&conn, "seg-embed").unwrap().unwrap(); + assert!(legacy_segment.embedding.is_none()); +} + +#[test] +fn append_and_close_segment() { + let conn = setup_db(); + segment_create(&conn, "seg-2", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + segment_append_turn(&conn, "seg-2", 2, None, 1005.0, 1005.0).unwrap(); + segment_append_turn(&conn, "seg-2", 3, None, 1010.0, 1010.0).unwrap(); + + let seg = segment_get(&conn, "seg-2").unwrap().unwrap(); + assert_eq!(seg.turn_count, 3); + assert_eq!(seg.end_episodic_id, Some(3)); + + segment_close(&conn, "seg-2", 1010.0).unwrap(); + let seg = segment_get(&conn, "seg-2").unwrap().unwrap(); + assert_eq!(seg.status, SegmentStatus::Closed); +} + +#[test] +fn open_segment_for_session_returns_latest() { + let conn = setup_db(); + segment_create(&conn, "seg-a", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + segment_close(&conn, "seg-a", 1001.0).unwrap(); + segment_create(&conn, "seg-b", "s1", "global", 5, None, 1010.0, 1010.0).unwrap(); + + let open = open_segment_for_session(&conn, "s1").unwrap(); + assert!(open.is_some()); + assert_eq!(open.unwrap().segment_id, "seg-b"); + + // Different session has none. + let none = open_segment_for_session(&conn, "s2").unwrap(); + assert!(none.is_none()); +} + +#[test] +fn boundary_detection_time_gap() { + let config = BoundaryConfig::default(); + let seg = ConversationSegment { + segment_id: "s1".into(), + session_id: "sess".into(), + namespace: "global".into(), + start_episodic_id: 1, + end_episodic_id: Some(5), + start_timestamp: 1000.0, + end_timestamp: Some(1050.0), + turn_count: 5, + summary: None, + embedding: None, + topic_keywords: None, + status: SegmentStatus::Open, + created_at: 1000.0, + updated_at: 1050.0, + start_seq: None, + end_seq: None, + }; + + // Within time gap — continue. + let decision = detect_boundary(&config, &seg, 1100.0, "hello", None); + assert!(matches!(decision, BoundaryDecision::Continue)); + + // Exceeds time gap — boundary. + let decision = detect_boundary(&config, &seg, 1700.0, "hello", None); + assert!(matches!( + decision, + BoundaryDecision::Boundary(BoundaryReason::TimeGap) + )); +} + +#[test] +fn boundary_detection_explicit_marker() { + let config = BoundaryConfig::default(); + let seg = ConversationSegment { + segment_id: "s1".into(), + session_id: "sess".into(), + namespace: "global".into(), + start_episodic_id: 1, + end_episodic_id: None, + start_timestamp: 1000.0, + end_timestamp: None, + turn_count: 2, + summary: None, + embedding: None, + topic_keywords: None, + status: SegmentStatus::Open, + created_at: 1000.0, + updated_at: 1000.0, + start_seq: None, + end_seq: None, + }; + + let decision = detect_boundary( + &config, + &seg, + 1005.0, + "Switching to a different topic", + None, + ); + assert!(matches!( + decision, + BoundaryDecision::Boundary(BoundaryReason::ExplicitMarker) + )); +} + +#[test] +fn boundary_detection_turn_count() { + let config = BoundaryConfig { + max_turns_per_segment: 5, + ..Default::default() + }; + let seg = ConversationSegment { + segment_id: "s1".into(), + session_id: "sess".into(), + namespace: "global".into(), + start_episodic_id: 1, + end_episodic_id: Some(5), + start_timestamp: 1000.0, + end_timestamp: Some(1010.0), + turn_count: 5, + summary: None, + embedding: None, + topic_keywords: None, + status: SegmentStatus::Open, + created_at: 1000.0, + updated_at: 1010.0, + start_seq: None, + end_seq: None, + }; + + let decision = detect_boundary(&config, &seg, 1011.0, "next", None); + assert!(matches!( + decision, + BoundaryDecision::Boundary(BoundaryReason::TurnCountExceeded) + )); +} + +#[test] +fn boundary_detection_embedding_drift() { + let config = BoundaryConfig::default(); + let seg = ConversationSegment { + segment_id: "s1".into(), + session_id: "sess".into(), + namespace: "global".into(), + start_episodic_id: 1, + end_episodic_id: None, + start_timestamp: 1000.0, + end_timestamp: None, + turn_count: 3, + summary: None, + embedding: Some(vec![1.0, 0.0, 0.0]), + topic_keywords: None, + status: SegmentStatus::Open, + created_at: 1000.0, + updated_at: 1000.0, + start_seq: None, + end_seq: None, + }; + + // Similar direction — continue. + let decision = detect_boundary(&config, &seg, 1005.0, "hello", Some(&[0.9, 0.1, 0.0])); + assert!(matches!(decision, BoundaryDecision::Continue)); + + // Orthogonal direction — boundary. + let decision = detect_boundary(&config, &seg, 1005.0, "hello", Some(&[0.0, 1.0, 0.0])); + assert!(matches!( + decision, + BoundaryDecision::Boundary(BoundaryReason::EmbeddingDrift) + )); +} + +#[test] +fn incremental_mean_embedding_works() { + let centroid = vec![1.0, 0.0]; + let new = vec![0.0, 1.0]; + let result = incremental_mean_embedding(¢roid, &new, 1); + // After 2 vectors: mean should be [0.5, 0.5] + assert!((result[0] - 0.5).abs() < 0.01); + assert!((result[1] - 0.5).abs() < 0.01); +} + +#[test] +fn summary_set_and_read() { + let conn = setup_db(); + segment_create(&conn, "seg-s", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + segment_close(&conn, "seg-s", 1001.0).unwrap(); + segment_set_summary(&conn, "seg-s", "Discussed deployment strategy", 1002.0).unwrap(); + let seg = segment_get(&conn, "seg-s").unwrap().unwrap(); + assert_eq!(seg.status, SegmentStatus::Summarised); + assert_eq!( + seg.summary.as_deref(), + Some("Discussed deployment strategy") + ); +} + +#[test] +fn segments_by_namespace_returns_most_recent_first() { + let conn = setup_db(); + // Create three segments with different updated_at timestamps. + segment_create(&conn, "seg-ns-1", "s1", "myns", 1, None, 1000.0, 1000.0).unwrap(); + segment_create(&conn, "seg-ns-2", "s1", "myns", 5, None, 2000.0, 2000.0).unwrap(); + segment_create(&conn, "seg-ns-3", "s1", "myns", 10, None, 3000.0, 3000.0).unwrap(); + + // Append a turn to seg-ns-1 with a later timestamp to bump its updated_at. + // Leave seg-ns-3 as the most recently created (highest updated_at). + let segs = segments_by_namespace(&conn, "myns", 10).unwrap(); + assert_eq!(segs.len(), 3, "Expected 3 segments in namespace"); + + // Most recently updated segment should come first (DESC order on updated_at). + assert_eq!(segs[0].segment_id, "seg-ns-3"); + assert_eq!(segs[1].segment_id, "seg-ns-2"); + assert_eq!(segs[2].segment_id, "seg-ns-1"); + + // Bump seg-ns-1's updated_at by appending a turn. + segment_append_turn(&conn, "seg-ns-1", 2, None, 9000.0, 9000.0).unwrap(); + let segs = segments_by_namespace(&conn, "myns", 10).unwrap(); + assert_eq!(segs[0].segment_id, "seg-ns-1"); +} + +#[test] +fn segments_pending_summary_only_returns_closed() { + let conn = setup_db(); + // Open segment — should NOT appear. + segment_create(&conn, "seg-open", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + + // Closed segment — SHOULD appear. + segment_create(&conn, "seg-closed", "s2", "global", 5, None, 2000.0, 2000.0).unwrap(); + segment_close(&conn, "seg-closed", 2001.0).unwrap(); + + // Summarised segment — should NOT appear (only status='closed' is pending). + segment_create(&conn, "seg-summ", "s3", "global", 10, None, 3000.0, 3000.0).unwrap(); + segment_close(&conn, "seg-summ", 3001.0).unwrap(); + segment_set_summary(&conn, "seg-summ", "A summary", 3002.0).unwrap(); + + let pending = segments_pending_summary(&conn, 20).unwrap(); + assert_eq!( + pending.len(), + 1, + "Only the closed segment should be pending" + ); + assert_eq!(pending[0].segment_id, "seg-closed"); + assert_eq!(pending[0].status, SegmentStatus::Closed); +} + +#[test] +fn segment_set_embedding_roundtrip() { + let conn = setup_db(); + segment_create(&conn, "seg-emb", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + + let embedding = vec![0.1_f32, 0.2, 0.3, 0.4, 0.5]; + segment_set_embedding(&conn, "seg-emb", &embedding, 1001.0).unwrap(); + + let seg = segment_get(&conn, "seg-emb").unwrap().unwrap(); + let stored = seg.embedding.expect("embedding should be stored"); + assert_eq!(stored.len(), embedding.len()); + for (stored_val, expected_val) in stored.iter().zip(embedding.iter()) { + assert!( + (stored_val - expected_val).abs() < 1e-6, + "Embedding value mismatch: got {stored_val}, expected {expected_val}" + ); + } +} + +#[test] +fn segment_set_keywords_stores_and_reads() { + let conn = setup_db(); + segment_create(&conn, "seg-kw", "s1", "global", 1, None, 1000.0, 1000.0).unwrap(); + + let keywords = "rust,memory,performance"; + segment_set_keywords(&conn, "seg-kw", keywords, 1001.0).unwrap(); + + let seg = segment_get(&conn, "seg-kw").unwrap().unwrap(); + assert_eq!( + seg.topic_keywords.as_deref(), + Some("rust,memory,performance"), + "Keywords should round-trip correctly" + ); +} + +#[test] +fn boundary_no_false_positive_on_short_messages() { + let config = BoundaryConfig::default(); + let seg = ConversationSegment { + segment_id: "s1".into(), + session_id: "sess".into(), + namespace: "global".into(), + start_episodic_id: 1, + end_episodic_id: Some(3), + start_timestamp: 1000.0, + end_timestamp: Some(1010.0), + turn_count: 3, + summary: None, + embedding: None, + topic_keywords: None, + status: SegmentStatus::Open, + created_at: 1000.0, + updated_at: 1010.0, + start_seq: None, + end_seq: None, + }; + + // Short single-word messages must not trigger explicit marker detection. + for short_msg in &["yes", "ok", "no", "sure", "thanks", "great"] { + let decision = detect_boundary(&config, &seg, 1011.0, short_msg, None); + assert!( + matches!(decision, BoundaryDecision::Continue), + "Short message '{short_msg}' incorrectly triggered a boundary" + ); + } +} + +#[test] +fn fallback_summary_truncates_long_content() { + let long = "a".repeat(300); + let short = "brief ending"; + let summary = fallback_summary(&long, short, 5); + + // The truncated first content should end with "..." and be capped at 203 chars + // (200 chars + "..."). + assert!( + summary.contains("..."), + "Long content should be truncated with ellipsis" + ); + assert!( + !summary.contains(&long), + "Full long content should not appear verbatim in summary" + ); + // The summary should still reference the short last content. + assert!( + summary.contains(short), + "Last content should appear in summary" + ); + // Verify exact truncation: first 200 chars of `long` followed by "...". + let truncated_first = format!("{}...", &long[..200]); + assert!(summary.contains(&truncated_first)); +} diff --git a/core/src/store/profile_store.rs b/core/src/store/profile_store.rs new file mode 100644 index 0000000..9441d9a --- /dev/null +++ b/core/src/store/profile_store.rs @@ -0,0 +1,168 @@ +//! `ProfileStore` — the only typed door onto the `user_profile` table. +//! +//! Before this type existed, `MemoryClient::profile_conn()` handed a raw +//! `Arc>` to three domains outside the memory +//! family (`agent/learning/*`, `memory/sync/composio/providers/profile.rs`), +//! two of which wrote SQL inline at the call site. Every SQL statement against +//! profile/facet rows now lives either here or in +//! [`super::namespace_store::profile`], both inside `crate`; +//! callers outside the family hold this handle and never a `Connection`. +//! +//! **This is not a guard win.** The profile/facet tables have no capability +//! family in the `tinycortex_api` contract, so reads and writes through this +//! type still run beneath `crate::guard::MemoryGuard`'s +//! seven policy steps: no tier check, no source-scope predicate, no taint +//! stamping, no redaction, no budget, no audit event. What changed is the shape +//! of the door — raw SQLite reachable from three domains became one typed store +//! whose confinement the compiler enforces. + +use parking_lot::Mutex; +use rusqlite::{params, Connection}; +use std::sync::Arc; + +use super::namespace_store::profile::{self, FacetType, ProfileFacet, UserState}; + +/// Typed access to the `user_profile` table. +/// +/// Cheap to clone — it is an `Arc` over the same connection `MemoryClient` +/// owns, so clones share one lock. +#[derive(Clone)] +pub struct ProfileStore { + conn: Arc>, +} + +impl ProfileStore { + /// The single production construction site is + /// [`super::MemoryClient::profile_store`]. + pub(crate) fn from_conn(conn: Arc>) -> Self { + Self { conn } + } + + /// Test-only: build a store over a caller-owned in-memory database. + /// + /// Not a hole — the caller already holds the `Connection`, so this hands + /// out nothing a [`super::MemoryClient`] owns. Confinement is about not + /// *extracting* the client's connection, and `profile_conn()` stays + /// `pub(in crate)`. + /// + /// Deliberately **not** `#[cfg(test)]`: integration tests under `tests/` + /// link the lib compiled without `cfg(test)`, so a test-gated constructor + /// is invisible to them — which is exactly how + /// `tests/learning_phase4_integration_test.rs` was left uncompilable when + /// `FacetCache::new` changed shape. `#[doc(hidden)]` keeps it off the + /// public docs without hiding it from the linker. + #[doc(hidden)] + pub fn for_tests(conn: Arc>) -> Self { + Self { conn } + } + + // ── Facet-cache surface ─────────────────────────────────────────────── + + /// List all facets with `state = 'active'`, ordered by stability descending. + pub fn list_active(&self) -> anyhow::Result> { + profile::profile_select_active(&self.conn) + } + + /// List all facets (all states), ordered by stability descending. + pub fn list_all(&self) -> anyhow::Result> { + profile::profile_select_all(&self.conn) + } + + /// Fetch a single facet by its full key (e.g. `"style/verbosity"`). + pub fn get(&self, key: &str) -> anyhow::Result> { + profile::profile_get_by_key(&self.conn, key) + } + + /// Upsert a fully-formed facet row (rebuild path). + pub fn upsert_full(&self, facet: &ProfileFacet) -> anyhow::Result<()> { + profile::profile_upsert_full(&self.conn, facet) + } + + /// Override the `user_state` of a facet. `Ok(true)` if a row was updated. + pub fn set_user_state(&self, key: &str, user_state: UserState) -> anyhow::Result { + profile::profile_set_user_state(&self.conn, key, user_state) + } + + /// Delete a facet by key. Returns `true` if a row was removed. + pub fn delete(&self, key: &str) -> anyhow::Result { + profile::profile_delete_by_key(&self.conn, key) + } + + /// Delete all `Dropped`-state facets whose stability is below `threshold`. + pub fn drop_below_threshold(&self, threshold: f64) -> anyhow::Result { + profile::profile_delete_below_threshold(&self.conn, threshold) + } + + // ── Provider-identity surface ───────────────────────────────────────── + + /// Confidence-aware upsert of one provider-sourced facet row. + #[allow(clippy::too_many_arguments)] + pub fn upsert_provider_facet( + &self, + facet_id: &str, + facet_type: &FacetType, + key: &str, + value: &str, + confidence: f64, + segment_id: Option<&str>, + now: f64, + ) -> anyhow::Result<()> { + profile::profile_upsert( + &self.conn, facet_id, facet_type, key, value, confidence, segment_id, now, + ) + } + + /// Load every facet of `facet_type`, ordered by evidence count descending. + pub fn facets_by_type(&self, facet_type: &FacetType) -> anyhow::Result> { + profile::profile_facets_by_type(&self.conn, facet_type) + } + + /// True if any [`FacetType::Workflow`] (`"skill"`) row's key matches + /// `key_pattern` (a SQL `LIKE` pattern) with exactly `canonical_value`. + /// + /// Encapsulates the two hand-rolled `SELECT 1 … LIKE` queries the composio + /// provider used to write inline. Deliberately infallible: the callers are + /// "is this row the user?" predicates whose only sane answer on a database + /// error is "no", which is what the raw `.is_ok()` gave before. + pub fn skill_identity_matches(&self, key_pattern: &str, canonical_value: &str) -> bool { + let conn = self.conn.lock(); + let matched = conn + .query_row( + "SELECT 1 FROM user_profile + WHERE facet_type = ?1 + AND key LIKE ?2 + AND value = ?3 + LIMIT 1", + params![FacetType::Workflow.as_str(), key_pattern, canonical_value], + |_| Ok(()), + ) + .is_ok(); + // Facet values are user PII (emails, phone numbers, handles) — log the + // pattern and the verdict, never the value. + tracing::debug!( + pattern = %key_pattern, + matched, + "[memory::profile_store] skill_identity_matches" + ); + matched + } + + /// Delete exactly one row by `facet_id`. `Ok(true)` if a row was removed. + pub fn delete_by_facet_id(&self, facet_id: &str) -> anyhow::Result { + let conn = self.conn.lock(); + let removed = conn.execute( + "DELETE FROM user_profile WHERE facet_id = ?1", + params![facet_id], + )?; + tracing::debug!( + facet_id = %facet_id, + removed, + "[memory::profile_store] delete_by_facet_id" + ); + Ok(removed > 0) + } +} + +#[cfg(test)] +#[path = "profile_store_tests.rs"] +mod tests; diff --git a/core/src/store/profile_store_tests.rs b/core/src/store/profile_store_tests.rs new file mode 100644 index 0000000..3f5f1b6 --- /dev/null +++ b/core/src/store/profile_store_tests.rs @@ -0,0 +1,120 @@ +//! Tests for [`ProfileStore`]. +//! +//! The two interesting methods are the ones that replaced hand-rolled SQL in +//! `memory/sync/composio/providers/profile.rs`. A subtly wrong reimplementation +//! of `skill_identity_matches` makes the entity matcher stop recognising the +//! user, which degrades silently rather than erroring — so the oracle here is +//! the literal SQL that was replaced, executed against the same connection, +//! rather than my reading of it. + +use super::*; +use crate::store::profile::PROFILE_INIT_SQL; + +fn seeded_store() -> ProfileStore { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + let store = ProfileStore::for_tests(Arc::new(Mutex::new(conn))); + + let rows = [ + ( + "skill-gmail-default-email", + "skill:gmail:default:email", + "user@example.com", + ), + ( + "skill-slack-c123-handle", + "skill:slack:c123:handle", + "userhandle", + ), + ( + "skill-slack-c123-email", + "skill:slack:c123:email", + "work@example.com", + ), + ]; + for (facet_id, key, value) in rows { + store + .upsert_provider_facet( + facet_id, + &FacetType::Workflow, + key, + value, + 0.9, + None, + 1000.0, + ) + .unwrap(); + } + store +} + +/// The exact query string from the pre-refactor +/// `is_self_identity` / `is_self_identity_any_toolkit`, run here so the +/// assertion compares against the code that was replaced. +fn legacy_like_query(store: &ProfileStore, key_pattern: &str, canonical: &str) -> bool { + let conn = store.conn.lock(); + conn.query_row( + "SELECT 1 FROM user_profile + WHERE facet_type = 'skill' + AND key LIKE ?1 + AND value = ?2 + LIMIT 1", + params![key_pattern, canonical], + |_| Ok(()), + ) + .is_ok() +} + +#[test] +fn skill_identity_matches_agrees_with_the_legacy_like_query() { + let store = seeded_store(); + let cases = [ + ("skill:gmail:%:email", "user@example.com"), // exact toolkit hit + ("skill:slack:%:email", "user@example.com"), // wrong toolkit + ("skill:%:%:email", "user@example.com"), // cross-toolkit hit + ("skill:%:%:email", "other@example.com"), // value miss + ("skill:%:%:phone", "user@example.com"), // kind miss + ("skill:gmail:%:handle", ""), // empty value + ("skill:slack:%:handle", "userhandle"), // second toolkit hit + ]; + for (pattern, value) in cases { + let legacy = legacy_like_query(&store, pattern, value); + assert_eq!( + store.skill_identity_matches(pattern, value), + legacy, + "divergence for pattern={pattern:?} value={value:?}" + ); + } + // Non-vacuity: at least one case must actually be a hit, or the loop above + // would pass with a method that always returns false. + assert!(store.skill_identity_matches("skill:%:%:email", "user@example.com")); +} + +#[test] +fn delete_by_facet_id_removes_exactly_one_row() { + let store = seeded_store(); + assert_eq!(store.facets_by_type(&FacetType::Workflow).unwrap().len(), 3); + + assert!(store.delete_by_facet_id("skill-slack-c123-email").unwrap()); + + let survivors = store.facets_by_type(&FacetType::Workflow).unwrap(); + let ids: Vec<&str> = survivors.iter().map(|f| f.facet_id.as_str()).collect(); + assert_eq!(survivors.len(), 2, "deleted more than one row: {ids:?}"); + assert!(ids.contains(&"skill-gmail-default-email"), "{ids:?}"); + assert!(ids.contains(&"skill-slack-c123-handle"), "{ids:?}"); + + assert!( + !store.delete_by_facet_id("skill-does-not-exist").unwrap(), + "deleting an unknown facet_id must report false" + ); +} + +#[test] +fn facet_cache_surface_round_trips_through_the_store() { + let store = seeded_store(); + let facet = store.get("skill:gmail:default:email").unwrap(); + assert_eq!(facet.map(|f| f.value).as_deref(), Some("user@example.com")); + assert_eq!(store.list_all().unwrap().len(), 3); + assert!(store.delete("skill:gmail:default:email").unwrap()); + assert_eq!(store.list_all().unwrap().len(), 2); +} diff --git a/core/src/store/recall_policy.rs b/core/src/store/recall_policy.rs new file mode 100644 index 0000000..69582b2 --- /dev/null +++ b/core/src/store/recall_policy.rs @@ -0,0 +1,87 @@ +//! Host recall policy — the *product* rules that decide what a recall must not +//! return, kept out of the storage engine. +//! +//! # Why this module exists +//! +//! `UnifiedMemory` is a candidate to move into the `tinycortex` crate. A memory +//! *engine* answers "which rows rank highest for this query"; it must not read +//! the host's execution context to do it. Until this module existed, the +//! `Memory::recall` implementation reached directly into +//! `crate::thread_context` — an agent-harness +//! task-local — from inside the persistence layer. Shipping that into a +//! persistence crate would have baked an OpenHuman chat-turn concept into a +//! storage engine, which is very hard to undo afterwards. +//! +//! The engine now takes the exclusion as an explicit parameter +//! ([`UnifiedMemory::recall_excluding_session`]); this module is the single +//! place that resolves it from ambient host state, and it is deliberately +//! **not** part of the `namespace_store` move set. +//! +//! # The self-echo guard +//! +//! When a recall runs inside a live chat turn, the harness has an ambient +//! "current thread" id (set by the web channel around `agent.run_single`, see +//! `web_chat::run_task`) and the turn's own user message was *just* auto-saved +//! as a `[conversation]` document tagged with that same id (see +//! `agent::harness::session::turn::core`). Without this guard the agent's own +//! `memory_recall` surfaces the very request that triggered it as the top +//! "relevant" memory and echoes the user's text back at them. +//! +//! Outside a chat turn — cron, CLI, tests, standalone — the ambient id is +//! `None`, no exclusion applies, and recall behaves exactly as it would with no +//! guard at all. +//! +//! # Known limitation (why the read is still ambient) +//! +//! Ideally the exclusion would be threaded down from the caller that knows the +//! session id, so nothing anywhere reads a task-local. That is not reachable +//! today: both [`crate::Memory`] and +//! [`crate::RecallOpts`] are re-exported verbatim from the +//! vendored `tinycortex` crate, `RecallOpts` has exactly five fields and none of +//! them is an exclusion, and the trait method takes no further argument. Every +//! in-turn caller (`memory_recall` tool, memory loader, channel context, flow +//! memory tools) holds an `Arc`, so it has no channel to push an +//! exclusion through even though it could resolve one. Widening `RecallOpts` +//! with an `exclude_session_id` field upstream in `tinycortex` is the change +//! that unblocks the rest of this hoist; at that point this module keeps the +//! resolution and the call sites populate the field. + +/// Resolve the self-echo exclusion for a recall from ambient host state. +/// +/// Returns the current chat thread id when this call runs inside a live agent +/// turn, and `None` everywhere else. Callers pass the result straight into +/// [`UnifiedMemory::recall_excluding_session`]. +/// +/// [`UnifiedMemory::recall_excluding_session`]: +/// crate::store::UnifiedMemory::recall_excluding_session +pub(crate) fn current_self_echo_exclusion() -> Option { + let exclusion = crate::thread_context::current_thread_id(); + if let Some(ref session_id) = exclusion { + tracing::debug!( + exclude_session_id = %session_id, + "[memory:recall_policy] resolved same-session self-echo exclusion from ambient turn" + ); + } else { + tracing::trace!( + "[memory:recall_policy] no ambient chat turn; recall runs without a self-echo exclusion" + ); + } + exclusion +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::thread_context::with_thread_id; + + #[tokio::test] + async fn resolves_the_ambient_thread_id_inside_a_turn() { + let resolved = with_thread_id("thread-xyz", async { current_self_echo_exclusion() }).await; + assert_eq!(resolved.as_deref(), Some("thread-xyz")); + } + + #[tokio::test] + async fn resolves_to_none_outside_any_turn() { + assert_eq!(current_self_echo_exclusion(), None); + } +} diff --git a/core/src/store/retrieval/mod.rs b/core/src/store/retrieval/mod.rs new file mode 100644 index 0000000..8331948 --- /dev/null +++ b/core/src/store/retrieval/mod.rs @@ -0,0 +1,403 @@ +//! Unified retrieval facade over the memory_store backends. +//! +//! `memory_store` owns four distinct retrieval modalities, each implemented in +//! a different submodule today: +//! +//! 1. **tree-walk** — BFS over sealed summary nodes (delegates to the +//! existing drill_down logic in `memory_tree::retrieval::drill_down`). +//! 2. **vector search** — embedding-similarity ranking over namespace docs +//! (delegates to `UnifiedMemory::query_namespace_hits`). +//! 3. **keyword search** — FTS5/keyword overlap, same hybrid entry point as +//! vector (the hybrid scorer already blends both signals). +//! 4. **param/tag search** — structured filters over chunk metadata + content +//! store tags (delegates to `chunks::store::list_chunks` and +//! `content::tags`). +//! +//! The facade is a thin aggregation layer: it does NOT reimplement any +//! scoring or storage logic. It exists so callers have a single import surface +//! (`memory_store::retrieval::RetrievalFacade`) instead of reaching into four +//! different submodules. +//! +//! Layering note: `tree_walk` calls `memory_tree::retrieval::drill_down`, which is +//! a reverse dependency from `memory_store` up into `memory`. This is +//! intentional and bounded — `drill_down` is "tree walk over stored trees" and +//! conceptually belongs in `memory_store`, but moving it is out of scope for +//! the storage-extraction refactor. Revisit when drill_down's policy bits +//! (entity hits, source-vs-summary precedence) can be cleanly split from the +//! pure tree traversal. + +use anyhow::Result; +use std::sync::Arc; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::store::chunks::store::list_chunks; +use crate::store::chunks::types::{Chunk, SourceKind}; +use crate::store::types::NamespaceMemoryHit; +use crate::store::UnifiedMemory; +use crate::tree::retrieval::types::RetrievalHit; +use crate::Config; + +/// Optional filter set for `param_tag_search`. All `Some` fields are AND-ed +/// together; `None` fields are unconstrained. +#[derive(Debug, Default, Clone)] +pub struct ParamTagFilters { + pub source_kind: Option, + pub source_id: Option, + pub owner: Option, + /// Inclusive lower bound on chunk `timestamp_ms`. + pub since_ms: Option, + /// Inclusive upper bound on chunk `timestamp_ms`. + pub until_ms: Option, + /// If `Some`, post-filter to chunks whose `tags` contains every listed tag. + pub tags_all_of: Option>, + /// Max rows to return (default 100 when `None`). + pub limit: Option, +} + +/// Unified retrieval entry point. Construct with an `Arc` for +/// vector/keyword ops; tree-walk and param/tag ops only need `&Config`. +#[derive(Clone)] +pub struct RetrievalFacade { + unified: Arc, +} + +impl RetrievalFacade { + pub fn new(unified: Arc) -> Self { + Self { unified } + } + + /// BFS walk from `node_id` down to `max_depth`. When `query` is `Some`, + /// hits are reranked by cosine similarity to the query embedding. + /// + /// See `memory_tree::retrieval::drill_down::drill_down` for the full contract. + pub async fn tree_walk( + &self, + config: &Config, + node_id: &str, + max_depth: u32, + query: Option<&str>, + limit: Option, + ) -> Result> { + crate::tree::retrieval::drill_down::drill_down(config, node_id, max_depth, query, limit) + .await + } + + /// Hybrid vector + graph + freshness retrieval. Same underlying scorer as + /// `keyword_search`; the difference is purely semantic intent at the call + /// site (callers using this entry point are saying "I have an embeddable + /// query"). Returns the full ranked hit list. + pub async fn vector_search( + &self, + namespace: &str, + query: &str, + limit: u32, + ) -> Result, String> { + self.unified + .query_namespace_hits(namespace, query, limit) + .await + } + + /// Same hybrid scorer as `vector_search` — the underlying retrieval plan + /// blends keyword overlap and vector similarity in one pass. Exposed as a + /// separate method so callers that only want lexical matching have an + /// honest name; the result set is identical for any given query. + pub async fn keyword_search( + &self, + namespace: &str, + query: &str, + limit: u32, + ) -> Result, String> { + self.unified + .query_namespace_hits(namespace, query, limit) + .await + } + + /// Structured chunk search by source/owner/time/tag filters. Bypasses the + /// ranking pipeline entirely — results are timestamp-DESC ordered. Use + /// when the caller knows the exact subset of chunks it wants. + pub fn param_tag_search( + &self, + config: &Config, + filters: &ParamTagFilters, + ) -> Result> { + let query = crate::store::chunks::store::ListChunksQuery { + source_kind: filters.source_kind, + source_id: filters.source_id.clone(), + owner: filters.owner.clone(), + since_ms: filters.since_ms, + until_ms: filters.until_ms, + limit: filters.limit, + offset: None, + source_scope: None, + exclude_dropped: false, + }; + let rows = list_chunks(config, &query)?; + let Some(required) = filters.tags_all_of.as_ref() else { + return Ok(rows); + }; + if required.is_empty() { + return Ok(rows); + } + Ok(rows + .into_iter() + .filter(|c| { + required + .iter() + .all(|t| c.metadata.tags.iter().any(|ct| ct == t)) + }) + .collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::chunks::store::upsert_chunks; + use crate::store::chunks::types::{Chunk, Metadata}; + use chrono::{TimeZone, Utc}; + use tempfile::TempDir; + use tinymemory_api::host::NoopEmbedding; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + fn test_facade(tmp: &TempDir) -> RetrievalFacade { + let unified = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + RetrievalFacade::new(Arc::new(unified)) + } + + fn chunk( + id: &str, + source_kind: SourceKind, + source_id: &str, + owner: &str, + tags: &[&str], + ) -> Chunk { + chunk_at(id, source_kind, source_id, owner, tags, Utc::now()) + } + + fn chunk_at( + id: &str, + source_kind: SourceKind, + source_id: &str, + owner: &str, + tags: &[&str], + ts: chrono::DateTime, + ) -> Chunk { + Chunk { + id: id.into(), + content: format!("content for {id}"), + metadata: Metadata { + source_kind, + source_id: source_id.into(), + owner: owner.into(), + timestamp: ts, + time_range: (ts, ts), + tags: tags.iter().map(|s| (*s).to_string()).collect(), + source_ref: None, + path_scope: None, + }, + token_count: 3, + seq_in_source: 0, + created_at: ts, + partial_message: false, + } + } + + #[test] + fn param_tag_filters_default_to_no_constraints() { + let filters = ParamTagFilters::default(); + assert!(filters.source_kind.is_none()); + assert!(filters.source_id.is_none()); + assert!(filters.owner.is_none()); + assert!(filters.since_ms.is_none()); + assert!(filters.until_ms.is_none()); + assert!(filters.tags_all_of.is_none()); + assert!(filters.limit.is_none()); + } + + #[test] + fn param_tag_search_filters_by_tags_all_of() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[ + chunk( + "c1", + SourceKind::Chat, + "slack:#eng", + "alice", + &["person:alice", "deploy"], + ), + chunk( + "c2", + SourceKind::Chat, + "slack:#eng", + "alice", + &["person:alice"], + ), + chunk( + "c3", + SourceKind::Email, + "gmail:thread-1", + "bob", + &["deploy"], + ), + ], + ) + .unwrap(); + + let filters = ParamTagFilters { + tags_all_of: Some(vec!["person:alice".into(), "deploy".into()]), + ..ParamTagFilters::default() + }; + let hits = facade.param_tag_search(&cfg, &filters).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "c1"); + } + + #[test] + fn param_tag_search_respects_source_kind_filter() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[ + chunk("c1", SourceKind::Chat, "slack:#eng", "alice", &[]), + chunk("c2", SourceKind::Email, "gmail:thread-1", "alice", &[]), + ], + ) + .unwrap(); + + let filters = ParamTagFilters { + source_kind: Some(SourceKind::Email), + ..ParamTagFilters::default() + }; + let hits = facade.param_tag_search(&cfg, &filters).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "c2"); + } + + #[test] + fn param_tag_search_respects_source_id_owner_and_limit() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[ + chunk("c1", SourceKind::Chat, "slack:#eng", "alice", &[]), + chunk("c2", SourceKind::Chat, "slack:#eng", "bob", &[]), + chunk("c3", SourceKind::Chat, "slack:#ops", "alice", &[]), + ], + ) + .unwrap(); + + let filters = ParamTagFilters { + source_id: Some("slack:#eng".into()), + owner: Some("alice".into()), + limit: Some(1), + ..ParamTagFilters::default() + }; + let hits = facade.param_tag_search(&cfg, &filters).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "c1"); + assert_eq!(hits[0].metadata.source_id, "slack:#eng"); + assert_eq!(hits[0].metadata.owner, "alice"); + } + + #[test] + fn param_tag_search_empty_required_tags_is_noop() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[ + chunk("c1", SourceKind::Chat, "slack:#eng", "alice", &["deploy"]), + chunk( + "c2", + SourceKind::Email, + "gmail:thread-1", + "bob", + &["person:bob"], + ), + ], + ) + .unwrap(); + + let hits = facade + .param_tag_search( + &cfg, + &ParamTagFilters { + tags_all_of: Some(vec![]), + ..ParamTagFilters::default() + }, + ) + .unwrap(); + assert_eq!(hits.len(), 2); + } + + #[test] + fn param_tag_search_respects_since_and_until_bounds() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + let older = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + let newer = Utc.timestamp_millis_opt(1_700_100_000_000).unwrap(); + upsert_chunks( + &cfg, + &[ + chunk_at("c1", SourceKind::Chat, "slack:#eng", "alice", &[], older), + chunk_at("c2", SourceKind::Chat, "slack:#eng", "alice", &[], newer), + ], + ) + .unwrap(); + + let hits = facade + .param_tag_search( + &cfg, + &ParamTagFilters { + since_ms: Some(newer.timestamp_millis()), + until_ms: Some(newer.timestamp_millis()), + ..ParamTagFilters::default() + }, + ) + .unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "c2"); + } + + #[test] + fn param_tag_search_returns_empty_when_required_tag_is_missing() { + let (tmp, cfg) = test_config(); + let facade = test_facade(&tmp); + upsert_chunks( + &cfg, + &[chunk( + "c1", + SourceKind::Chat, + "slack:#eng", + "alice", + &["deploy"], + )], + ) + .unwrap(); + + let hits = facade + .param_tag_search( + &cfg, + &ParamTagFilters { + tags_all_of: Some(vec!["person:bob".into()]), + ..ParamTagFilters::default() + }, + ) + .unwrap(); + assert!(hits.is_empty()); + } +} diff --git a/core/src/store/safety/mod.rs b/core/src/store/safety/mod.rs new file mode 100644 index 0000000..bf48ad4 --- /dev/null +++ b/core/src/store/safety/mod.rs @@ -0,0 +1,340 @@ +//! Secret-detection and redaction for memory writes — thin host shim over +//! `tinycortex::memory::store::safety` (W3). +//! +//! The conservative secret + PII scrubbers (`has_likely_secret`, +//! `has_likely_pii`, `sanitize_text`, `sanitize_json`) + the +//! `SanitizationReport`/`Sanitized` types are the crate's — now including the +//! full multilingual national-ID PII module (ported into the crate so the crate +//! `sanitize_text` matches this host's byte-for-byte). The host keeps only +//! [`sanitize_document_input`], which scrubs the host-specific +//! [`NamespaceDocumentInput`] shape by delegating each field to the crate +//! scrubbers. The retained test suite doubles as a byte-parity guard: it asserts +//! the crate scrubber still redacts every secret/PII pattern the host relied on. + +pub mod pii; + +use crate::store::types::NamespaceDocumentInput; + +pub use tinycortex::memory::store::safety::{ + has_likely_pii, has_likely_secret, sanitize_json, sanitize_text, SanitizationReport, Sanitized, +}; + +/// Canonical storage form of a caller-supplied memory **identifier** — a +/// namespace, a document key, or a KV key. +/// +/// An identifier is an address, not content: whatever this returns is what the +/// row is stored under, so every read / update / delete that addresses a row by +/// identifier has to canonicalize through this same function, or it looks up a +/// row the write never created (#5164). +/// +/// Two properties make that safe, and both follow the split the crate's PII +/// module documents between its **strict boundary predicate** and its **lenient +/// content scrubber**: +/// +/// * **Strict gating.** Only identifiers that trip [`has_likely_pii`] — +/// formatted / keyword-gated national IDs (`ssn-123-45-6789`, +/// `cliente-RFC-VECJ880326XK4`, `cuit-20-11111111-2`) — are rewritten. +/// `redact_pii` on its own also rewrites bare digit-run shapes, and the +/// scanners legitimately build identifiers out of those: WhatsApp JIDs +/// (`12025551234-1543890267@g.us`), iMessage `+1…` chat ids, millisecond +/// timestamps, padded counters. Rewriting those maps two distinct contacts +/// onto one `(namespace, key)`, where the upsert's `ON CONFLICT … DO UPDATE` +/// has one contact's document silently overwrite the other's. +/// * **Idempotence.** The `[REDACTED_PII_*]` placeholders carry no PII pattern +/// of their own, so canonicalizing an already-canonical identifier is a +/// no-op — which is what lets read paths canonicalize unconditionally. +pub fn canonical_identifier(value: &str) -> String { + if !has_likely_pii(value) { + return value.to_string(); + } + pii::redact_pii(value).value +} + +/// Canonical storage form of a document key: the exact transform +/// `upsert_document` / `upsert_document_metadata_only` apply before writing the +/// `memory_docs.key` column (trim, then [`canonical_identifier`]). +/// +/// Single-sourced so the by-key read paths (`Memory::get`, `Memory::forget`) +/// cannot drift from the write path. Drift there is invisible — the lookup +/// simply misses, the caller treats the row as absent and writes again, which +/// is the unthrottled loop #5164 was reported for. +pub fn canonical_document_key(key: &str) -> String { + canonical_identifier(key.trim()) +} + +/// Scrub a namespace-document input, field by field, via the crate scrubbers. +/// +/// Sanitization is content-cleaning only; provenance `taint` survives untouched +/// so the write gate's taint check still sees the real source signal. +pub fn sanitize_document_input(input: NamespaceDocumentInput) -> Sanitized { + let mut report = SanitizationReport::default(); + + let title = sanitize_text(&input.title); + report = report.merge(title.report); + let content = sanitize_text(&input.content); + report = report.merge(content.report); + + let mut tags = Vec::with_capacity(input.tags.len()); + for tag in input.tags { + let sanitized = sanitize_text(&tag); + report = report.merge(sanitized.report); + tags.push(sanitized.value); + } + + let metadata = sanitize_json(&input.metadata); + report = report.merge(metadata.report); + + Sanitized { + value: NamespaceDocumentInput { + namespace: input.namespace, + key: input.key, + title: title.value, + content: content.value, + source_type: input.source_type, + priority: input.priority, + tags, + metadata: metadata.value, + category: input.category, + session_id: input.session_id, + document_id: input.document_id, + taint: input.taint, + }, + report, + } +} + +#[cfg(test)] +mod tests { + //! Byte-parity guard over the crate scrubber: every secret/PII pattern the + //! host used to redact must still be redacted after the port. + use super::*; + use serde_json::json; + + const REDACTED_SECRET: &str = "[REDACTED_SECRET]"; + const REDACTED_PRIVATE_KEY: &str = "[REDACTED_PRIVATE_KEY]"; + const MAX_JSON_SANITIZE_DEPTH: usize = 128; + + #[test] + fn sanitize_text_redacts_bearer_and_openai_key() { + let input = "Authorization: Bearer abcdefghijklmnop and sk-1234567890123456789012345"; + let sanitized = sanitize_text(input); + assert!(sanitized.value.contains("Bearer [REDACTED]")); + assert!(!sanitized.value.contains("sk-1234567890123456789012345")); + assert!(sanitized.report.text_redactions >= 2); + } + + #[test] + fn sanitize_text_blocks_private_key_blocks() { + let input = "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----"; + let sanitized = sanitize_text(input); + assert!(sanitized.value.contains(REDACTED_PRIVATE_KEY)); + assert!(sanitized.report.blocked_secret_hits >= 1); + } + + #[test] + fn sanitize_json_redacts_sensitive_keys_and_nested_strings() { + let input = json!({ + "token": "abc123", + "nested": { "notes": "Bearer supersecretvalue", "ok": "hello" }, + "arr": ["sk-1234567890123456789012345", "safe"] + }); + let sanitized = sanitize_json(&input); + assert_eq!(sanitized.value["token"], json!(REDACTED_SECRET)); + assert_eq!(sanitized.value["nested"]["ok"], json!("hello")); + assert!(sanitized.value["nested"]["notes"] + .as_str() + .unwrap_or_default() + .contains("[REDACTED]")); + assert!(sanitized.report.key_redactions >= 1); + assert!(sanitized.report.text_redactions >= 2); + } + + #[test] + fn sanitize_json_redacts_common_sensitive_key_variants() { + let input = json!({ + "db_password": "p@ss", "secret_key": "abc123", + "api_secret": "def456", "monkey": "banana" + }); + let sanitized = sanitize_json(&input); + assert_eq!(sanitized.value["db_password"], json!(REDACTED_SECRET)); + assert_eq!(sanitized.value["secret_key"], json!(REDACTED_SECRET)); + assert_eq!(sanitized.value["api_secret"], json!(REDACTED_SECRET)); + assert_eq!(sanitized.value["monkey"], json!(REDACTED_SECRET)); + assert!(sanitized.report.key_redactions >= 4); + } + + #[test] + fn has_likely_secret_detects_common_patterns() { + assert!(has_likely_secret("api_key=abc123")); + assert!(has_likely_secret("Bearer abcdefghijklmnopqrstuvwxyz")); + assert!(has_likely_secret("xoxb-1234567890-abcdef-ghijklmnop")); + assert!(has_likely_secret("glpat-aaaaaaaaaaaaaaaaaaaa")); + assert!(has_likely_secret("SG.aaaaaaaaaaaaaaaa.bbbbbbbbbbbbbbbb")); + assert!(!has_likely_secret("I prefer rust")); + } + + #[test] + fn sanitize_text_redacts_more_provider_secrets() { + let input = "auth=Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ== stripe=sk_live_12345678901234567890 npm=npm_abcdefghijklmnopqrstuvwxyz"; + let sanitized = sanitize_text(input); + assert!(!sanitized.value.contains("sk_live_12345678901234567890")); + assert!(!sanitized.value.contains("npm_abcdefghijklmnopqrstuvwxyz")); + assert!(sanitized.value.contains("[REDACTED]")); + assert!(sanitized.report.text_redactions >= 2); + } + + #[test] + fn sanitize_text_redacts_oauth_url_style_params() { + let input = "https://example.com/callback?access_token=abcd1234&refresh_token=efgh5678&id_token=jwt"; + let sanitized = sanitize_text(input); + assert!(!sanitized.value.contains("abcd1234")); + assert!(!sanitized.value.contains("efgh5678")); + assert!(!sanitized.value.contains("id_token=jwt")); + assert!(sanitized.report.text_redactions >= 3); + } + + #[test] + fn sanitize_text_redacts_multiline_private_key_blocks() { + let input = "BEGIN\n-----BEGIN OPENSSH PRIVATE KEY-----\nline1\nline2\n-----END OPENSSH PRIVATE KEY-----\nEND"; + let sanitized = sanitize_text(input); + assert!(!sanitized.value.contains("OPENSSH PRIVATE KEY")); + assert!(sanitized.value.contains(REDACTED_PRIVATE_KEY)); + assert!(sanitized.report.blocked_secret_hits >= 1); + } + + #[test] + fn sanitize_text_also_redacts_pii_after_secrets() { + let input = "Token sk-abcdefghijklmnopqrstuvwxyz; CPF 111.444.777-35; phone +15551234567"; + let sanitized = sanitize_text(input); + assert!(!sanitized.value.contains("sk-abcdefghijklmnopqrstuvwxyz")); + assert!(!sanitized.value.contains("111.444.777-35")); + assert!(!sanitized.value.contains("+15551234567")); + assert!(sanitized.value.contains("[REDACTED_PII_CPF]")); + assert!(sanitized.value.contains("[REDACTED_PII_PHONE]")); + assert!(sanitized.report.text_redactions >= 1); + assert_eq!(sanitized.report.pii_redactions, 2); + } + + #[test] + fn sanitize_json_propagates_pii_redaction_into_nested_strings() { + let input = json!({ + "note": "Cliente RFC VECJ880326XK4 confirmado", + "meta": { "cuit": "20-11111111-2" } + }); + let sanitized = sanitize_json(&input); + assert!(sanitized.value["note"] + .as_str() + .unwrap_or_default() + .contains("[REDACTED_PII_RFC]")); + assert!(sanitized.value["meta"]["cuit"] + .as_str() + .unwrap_or_default() + .contains("[REDACTED_PII_CUIT]")); + assert!(sanitized.report.pii_redactions >= 2); + } + + #[test] + fn sanitize_json_redacts_values_beyond_max_depth() { + let mut nested = json!("leaf"); + for _ in 0..(MAX_JSON_SANITIZE_DEPTH + 2) { + nested = json!({ "nested": nested }); + } + let sanitized = sanitize_json(&nested); + assert!(sanitized.report.depth_redactions >= 1); + assert!(sanitized + .value + .to_string() + .contains(&format!("\"{REDACTED_SECRET}\""))); + } + + /// #5164: identifiers are storage addresses, so canonicalization follows + /// the **strict** boundary predicate. Formatted / keyword-gated national IDs + /// are rewritten; the bare digit-run shapes the scanners build identifiers + /// out of are left alone (rewriting those maps distinct contacts onto one + /// `(namespace, key)` and the upsert silently overwrites). + #[test] + fn canonical_identifier_rewrites_only_strict_pii() { + for identifier in [ + "ssn-123-45-6789", + "cliente-RFC-VECJ880326XK4", + "cuit-20-11111111-2", + "user/111.444.777-35", + ] { + let canonical = canonical_identifier(identifier); + assert_ne!( + canonical, identifier, + "strict PII identifier must be canonicalized: {identifier}" + ); + assert!( + canonical.contains("[REDACTED_PII_"), + "expected a redaction placeholder, got: {canonical}" + ); + } + + for identifier in [ + // WhatsApp group JID / 1:1 JID / broadcast, iMessage E.164 chat id, + // telegram numeric peer id, padded ms timestamp, plain namespaces. + "12025551234-1543890267@g.us:2026-05-30", + "12025551234@c.us:2026-05-30", + "imessage:+12025551234:2026-05-30", + "4123456789:2026-05-30", + "accepted:000001747729035001", + "memory/global/preferences", + "skill-gmail", + ] { + assert_eq!( + canonical_identifier(identifier), + identifier, + "scanner-built identifier must keep its identity: {identifier}" + ); + } + } + + /// Read paths canonicalize unconditionally, so the transform has to be a + /// fixed point on its own output. + #[test] + fn canonical_identifier_is_idempotent() { + for identifier in ["ssn-123-45-6789", "cliente-RFC-VECJ880326XK4", "safe-key"] { + let once = canonical_identifier(identifier); + assert_eq!(canonical_identifier(&once), once, "not idempotent: {once}"); + } + } + + /// `canonical_document_key` single-sources the write-path transform, trim + /// included — otherwise `Memory::get` would address an untrimmed key that + /// `upsert_document` never wrote. + #[test] + fn canonical_document_key_trims_before_canonicalizing() { + assert_eq!(canonical_document_key(" doc-a "), "doc-a"); + assert_eq!( + canonical_document_key(" ssn-123-45-6789 "), + canonical_identifier("ssn-123-45-6789") + ); + assert_eq!(canonical_document_key(" "), ""); + } + + #[test] + fn sanitize_document_input_preserves_taint() { + let input = NamespaceDocumentInput { + namespace: "ns".into(), + key: "k".into(), + title: "Bearer secret123456789 visible title".into(), + content: "content with sk-abcdefghijklmnopqrstuvwxyz".into(), + source_type: "sync".into(), + priority: "normal".into(), + tags: vec!["tag1".into()], + metadata: json!({"safe": "value"}), + category: "core".into(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::ExternalSync, + }; + let sanitized = sanitize_document_input(input); + assert_eq!( + sanitized.value.taint, + crate::MemoryTaint::ExternalSync, + "taint must survive sanitization unchanged" + ); + assert!(sanitized.report.text_redactions >= 1); + } +} diff --git a/core/src/store/safety/pii.rs b/core/src/store/safety/pii.rs new file mode 100644 index 0000000..9a9a74b --- /dev/null +++ b/core/src/store/safety/pii.rs @@ -0,0 +1,8 @@ +//! Personal-PII detection — thin host re-export of the crate scrubber (W3). +//! +//! The full multilingual national-ID PII module (checksum-gated patterns + +//! Unicode normalization) now lives in `tinycortex::memory::store::safety::pii`; +//! content scrubbing runs inside the crate `sanitize_text`. Host consumers keep +//! their `safety::pii::has_likely_pii` import path. + +pub use tinycortex::memory::store::safety::pii::{has_likely_pii, redact_pii}; diff --git a/core/src/store/traits.rs b/core/src/store/traits.rs new file mode 100644 index 0000000..858fbde --- /dev/null +++ b/core/src/store/traits.rs @@ -0,0 +1,312 @@ +//! Storage compatibility traits. +//! +//! Every stored memory kind must answer two questions: +//! +//! 1. **Can it be embedded into a vector?** — yes, via [`VectorEmbeddable`]. +//! The trait provides the canonical embeddable string for the object so a +//! single embedding pipeline can index any kind uniformly. +//! 2. **Can it be represented as an Obsidian-compatible markdown file?** — +//! yes, via [`ObsidianRepresentable`]. The trait yields a relative vault +//! path and a fully-formed markdown body (YAML front-matter + content) +//! that can be written into the content store and opened by Obsidian +//! without further processing. +//! +//! Together these two traits are the contract that makes "everything in +//! memory_store is vector and obsidian compatible" a checkable property +//! rather than a slogan — the compiler enforces it for every new storage +//! kind that gets added. + +use std::path::PathBuf; + +use crate::people::types::Person; +use crate::store::chunks::types::Chunk; +use crate::store::kinds::MemoryKind; +use crate::store::trees::{SummaryNode, Tree}; + +/// A rendered Obsidian markdown file: where it lives in the vault and what +/// bytes to write. Vault path is relative to the content-store root. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObsidianFile { + pub relative_path: PathBuf, + pub markdown: String, +} + +/// Objects that can produce a canonical string for embedding into the +/// vector store. The returned text should be deterministic and stable across +/// calls so re-embedding produces consistent vectors. +pub trait VectorEmbeddable { + /// The MemoryKind this value belongs to. Used by the embedding pipeline + /// to route vectors into per-kind namespaces. + fn memory_kind(&self) -> MemoryKind; + + /// Canonical UTF-8 text fed to the embedding model. Strip front-matter, + /// markdown formatting noise, and anything not semantically meaningful. + fn embeddable_text(&self) -> String; +} + +/// Objects that can be rendered as an Obsidian-compatible markdown file. +/// The file should round-trip through the content store unchanged so vault +/// edits stay idempotent. +pub trait ObsidianRepresentable { + fn to_obsidian(&self) -> ObsidianFile; +} + +// ---- impls: Chunk ---------------------------------------------------------- + +impl VectorEmbeddable for Chunk { + fn memory_kind(&self) -> MemoryKind { + MemoryKind::Chunk + } + + fn embeddable_text(&self) -> String { + self.content.clone() + } +} + +impl ObsidianRepresentable for Chunk { + fn to_obsidian(&self) -> ObsidianFile { + let tags_yaml = if self.metadata.tags.is_empty() { + String::new() + } else { + let lines: Vec = self + .metadata + .tags + .iter() + .map(|t| format!(" - {}", t)) + .collect(); + format!("tags:\n{}\n", lines.join("\n")) + }; + let markdown = format!( + "---\nid: {}\nsource_kind: {}\nsource_id: {}\nseq: {}\n{}---\n\n{}\n", + self.id, + self.metadata.source_kind.as_str(), + self.metadata.source_id, + self.seq_in_source, + tags_yaml, + self.content + ); + ObsidianFile { + relative_path: PathBuf::from("chunks").join(format!("{}.md", self.id)), + markdown, + } + } +} + +// ---- impls: Tree + SummaryNode -------------------------------------------- + +impl VectorEmbeddable for SummaryNode { + fn memory_kind(&self) -> MemoryKind { + MemoryKind::Tree + } + + fn embeddable_text(&self) -> String { + self.content.clone() + } +} + +impl ObsidianRepresentable for SummaryNode { + fn to_obsidian(&self) -> ObsidianFile { + let markdown = format!( + "---\nid: {}\ntree_id: {}\nlevel: {}\n---\n\n{}\n", + self.id, self.tree_id, self.level, self.content + ); + ObsidianFile { + relative_path: PathBuf::from("summaries").join(format!("{}.md", self.id)), + markdown, + } + } +} + +impl ObsidianRepresentable for Tree { + fn to_obsidian(&self) -> ObsidianFile { + let markdown = format!( + "---\nid: {}\nkind: {:?}\nstatus: {:?}\n---\n\nTree {} ({:?})\n", + self.id, self.kind, self.status, self.id, self.kind + ); + ObsidianFile { + relative_path: PathBuf::from("trees").join(format!("{}.md", self.id)), + markdown, + } + } +} + +// ---- impls: Contact (Person) ---------------------------------------------- + +impl VectorEmbeddable for Person { + fn memory_kind(&self) -> MemoryKind { + MemoryKind::Contact + } + + fn embeddable_text(&self) -> String { + // Embed the display name plus primary email — both carry useful + // disambiguation signal. Handles are routing keys, not semantic + // content, and intentionally excluded. + let mut parts: Vec = Vec::new(); + if let Some(name) = self.display_name.as_deref() { + parts.push(name.to_string()); + } + if let Some(email) = self.primary_email.as_deref() { + parts.push(email.to_string()); + } + parts.join("\n") + } +} + +impl ObsidianRepresentable for Person { + fn to_obsidian(&self) -> ObsidianFile { + let display = self.display_name.as_deref().unwrap_or("Unknown"); + let email = self.primary_email.as_deref().unwrap_or(""); + let markdown = format!( + "---\nperson_id: {}\n---\n\n# {}\n\nEmail: {}\n", + self.id, display, email + ); + ObsidianFile { + relative_path: PathBuf::from("contacts").join(format!("{}.md", self.id)), + markdown, + } + } +} + +// Documents are no longer a first-class MemoryKind — the md backend +// (`content/`) is the canonical persistence for any document body. Anything +// that historically used `StoredMemoryDocument` should land its body as a +// raw md file and reference it via path. + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::chunks::types::{Metadata, SourceKind}; + use chrono::Utc; + + fn sample_chunk() -> Chunk { + let ts = Utc::now(); + Chunk { + id: "chunk-1".into(), + content: "hello world".into(), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: "slack:#eng".into(), + timestamp: ts, + time_range: (ts, ts), + owner: "alice".into(), + source_ref: None, + tags: vec!["person:alice".into()], + path_scope: None, + }, + seq_in_source: 7, + token_count: 2, + created_at: ts, + partial_message: false, + } + } + + #[test] + fn chunk_traits_render_expected_kind_and_obsidian_path() { + let chunk = sample_chunk(); + assert_eq!(chunk.memory_kind(), MemoryKind::Chunk); + assert_eq!(chunk.embeddable_text(), "hello world"); + + let obsidian = chunk.to_obsidian(); + assert_eq!(obsidian.relative_path, PathBuf::from("chunks/chunk-1.md")); + assert!(obsidian.markdown.contains("source_kind: chat")); + assert!(obsidian.markdown.contains("source_id: slack:#eng")); + assert!(obsidian.markdown.contains("hello world")); + } + + #[test] + fn summary_node_traits_render_expected_kind_and_path() { + let node = SummaryNode { + id: "summary-1".into(), + tree_id: "tree-1".into(), + tree_kind: crate::store::trees::TreeKind::Source, + level: 1, + parent_id: None, + child_ids: vec!["chunk-1".into()], + content: "summary body".into(), + token_count: 3, + entities: vec![], + topics: vec![], + time_range_start: Utc::now(), + time_range_end: Utc::now(), + score: 0.5, + sealed_at: Utc::now(), + deleted: false, + embedding: None, + doc_id: None, + version_ms: None, + }; + assert_eq!(node.memory_kind(), MemoryKind::Tree); + assert_eq!(node.embeddable_text(), "summary body"); + let obsidian = node.to_obsidian(); + assert_eq!( + obsidian.relative_path, + PathBuf::from("summaries/summary-1.md") + ); + assert!(obsidian.markdown.contains("tree_id: tree-1")); + assert!(obsidian.markdown.contains("summary body")); + } + + #[test] + fn tree_traits_render_obsidian_metadata() { + let tree = Tree { + id: "tree-1".into(), + kind: crate::store::trees::TreeKind::Topic, + scope: "topic:phoenix".into(), + ask: None, + root_id: Some("summary-root".into()), + max_level: 2, + status: crate::store::trees::TreeStatus::Active, + created_at: Utc::now(), + last_sealed_at: None, + }; + let obsidian = tree.to_obsidian(); + assert_eq!(obsidian.relative_path, PathBuf::from("trees/tree-1.md")); + assert!(obsidian.markdown.contains("id: tree-1")); + assert!(obsidian.markdown.contains("Tree tree-1")); + assert!(obsidian.markdown.contains("Topic")); + } + + #[test] + fn person_traits_render_name_and_email_when_present() { + let now = Utc::now(); + let person = Person { + id: crate::people::types::PersonId::new(), + display_name: Some("Alice Example".into()), + primary_email: Some("alice@example.com".into()), + primary_phone: Some("+1 555 0100".into()), + handles: vec![ + crate::people::types::Handle::DisplayName("Alice Example".into()), + crate::people::types::Handle::Email("alice@example.com".into()), + ], + created_at: now, + updated_at: now, + }; + assert_eq!(person.memory_kind(), MemoryKind::Contact); + assert_eq!(person.embeddable_text(), "Alice Example\nalice@example.com"); + let obsidian = person.to_obsidian(); + assert_eq!( + obsidian.relative_path, + PathBuf::from("contacts").join(format!("{}.md", person.id)) + ); + assert!(obsidian.markdown.contains("# Alice Example")); + assert!(obsidian.markdown.contains("Email: alice@example.com")); + } + + #[test] + fn person_traits_fall_back_when_fields_are_missing() { + let now = Utc::now(); + let person = Person { + id: crate::people::types::PersonId::new(), + display_name: None, + primary_email: None, + primary_phone: None, + handles: vec![], + created_at: now, + updated_at: now, + }; + assert_eq!(person.embeddable_text(), ""); + let obsidian = person.to_obsidian(); + assert!(obsidian.markdown.contains("# Unknown")); + assert!(obsidian.markdown.contains("Email: ")); + } +} diff --git a/core/src/store/trees/hotness.rs b/core/src/store/trees/hotness.rs new file mode 100644 index 0000000..5fc3ab4 --- /dev/null +++ b/core/src/store/trees/hotness.rs @@ -0,0 +1,30 @@ +//! `Config` adapters for tinycortex entity-hotness persistence. + +use anyhow::Result; + +use crate::store::trees::types::HotnessCounters; +use crate::tinycortex::engine_config; +use crate::Config; + +pub fn get(config: &Config, entity_id: &str) -> Result> { + tinycortex::memory::tree::store::hotness::get(&engine_config(config), entity_id) +} + +pub fn get_or_fresh(config: &Config, entity_id: &str) -> Result { + tinycortex::memory::tree::store::hotness::get_or_fresh(&engine_config(config), entity_id) +} + +pub fn upsert(config: &Config, counters: &HotnessCounters) -> Result<()> { + tinycortex::memory::tree::store::hotness::upsert(&engine_config(config), counters) +} + +pub fn distinct_sources_for(config: &Config, entity_id: &str) -> Result { + tinycortex::memory::tree::store::hotness::distinct_sources_for( + &engine_config(config), + entity_id, + ) +} + +pub fn count(config: &Config) -> Result { + tinycortex::memory::tree::store::hotness::count(&engine_config(config)) +} diff --git a/core/src/store/trees/mod.rs b/core/src/store/trees/mod.rs new file mode 100644 index 0000000..2625e40 --- /dev/null +++ b/core/src/store/trees/mod.rs @@ -0,0 +1,46 @@ +//! Tree persistence — source trees in `mem_tree_trees` keyed by [`TreeKind`]. +//! +//! (The global/topic kinds were removed; their variants survive only as +//! inert serialization plumbing for the one-shot purge migration.) This +//! module hosts: +//! - `store` — generic CRUD over the trees + summaries + buffers tables. +//! - `types` — Tree, SummaryNode, TreeKind, TreeStatus, Buffer, and the +//! entity-hotness types ([`HotnessCounters`], thresholds). +//! - `registry` — generic list / archive helpers. +//! - `hotness` — entity-hotness side-table (now a read-only subconscious +//! signal; the topic curator that wrote it was removed). +//! +//! Tree _logic_ (bucket_seal, flush, generic registry, source policy) stays +//! in `memory_tree`. + +pub mod hotness; +pub mod registry; +pub mod store; +pub mod types; + +pub use registry::{archive_tree, list_trees_by_kind}; +pub use store::{get_summary_embedding, set_summary_embedding}; +pub use types::{ + Buffer, EntityIndexStats, HotnessCounters, SummaryNode, Tree, TreeKind, TreeStatus, + INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, SUMMARY_FANOUT, TOPIC_ARCHIVE_THRESHOLD, + TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tree_module_reexports_expected_constants() { + assert_eq!(INPUT_TOKEN_BUDGET, 50_000); + assert_eq!(OUTPUT_TOKEN_BUDGET, 5_000); + assert_eq!(SUMMARY_FANOUT, 10); + // Compile-time guardrails: both sides are constants, so evaluate the + // invariant at build time rather than asserting a folded literal. + const _: () = assert!( + TOPIC_CREATION_THRESHOLD > TOPIC_ARCHIVE_THRESHOLD, + "topics must be created before they can be archived" + ); + const _: () = assert!(TOPIC_RECHECK_EVERY > 0); + } +} diff --git a/core/src/store/trees/registry.rs b/core/src/store/trees/registry.rs new file mode 100644 index 0000000..3ce8e1b --- /dev/null +++ b/core/src/store/trees/registry.rs @@ -0,0 +1,16 @@ +//! `Config` adapters for tinycortex's tree registry. + +use anyhow::Result; + +use crate::store::trees::types::{Tree, TreeKind}; +use crate::tinycortex::engine_config; +use crate::Config; + +pub fn list_trees_by_kind(config: &Config, kind: TreeKind) -> Result> { + tinycortex::memory::tree::store::list_trees_by_kind(&engine_config(config), kind) +} + +pub fn archive_tree(config: &Config, tree_id: &str) -> Result<()> { + log::debug!("[memory:trees] archive tree_id={tree_id}"); + tinycortex::memory::tree::store::archive_tree(&engine_config(config), tree_id) +} diff --git a/core/src/store/trees/store.rs b/core/src/store/trees/store.rs new file mode 100644 index 0000000..2508757 --- /dev/null +++ b/core/src/store/trees/store.rs @@ -0,0 +1,205 @@ +//! `Config` and transaction adapters for tinycortex tree persistence. + +use std::collections::HashMap; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use rusqlite::{Connection, Transaction}; + +use crate::store::content::StagedSummary; +use crate::store::trees::types::{Buffer, SummaryNode, Tree, TreeKind}; +use crate::tinycortex::engine_config; +use crate::Config; + +pub fn insert_tree(config: &Config, tree: &Tree) -> Result<()> { + tinycortex::memory::tree::store::insert_tree(&engine_config(config), tree) +} + +pub fn get_tree_by_scope(config: &Config, kind: TreeKind, scope: &str) -> Result> { + tinycortex::memory::tree::store::get_tree_by_scope(&engine_config(config), kind, scope) +} + +pub fn get_tree(config: &Config, id: &str) -> Result> { + tinycortex::memory::tree::store::get_tree(&engine_config(config), id) +} + +pub fn get_trees_batch(config: &Config, ids: &[String]) -> Result> { + tinycortex::memory::tree::store::get_trees_batch(&engine_config(config), ids) +} + +pub fn list_trees_by_kind(config: &Config, kind: TreeKind) -> Result> { + tinycortex::memory::tree::store::list_trees_by_kind(&engine_config(config), kind) +} + +pub fn update_tree_after_seal_tx( + tx: &Transaction<'_>, + tree_id: &str, + root_id: &str, + max_level: u32, + sealed_at: DateTime, +) -> Result<()> { + tinycortex::memory::tree::store::update_tree_after_seal_tx( + tx, tree_id, root_id, max_level, sealed_at, + ) +} + +pub fn insert_summary_tx( + tx: &Transaction<'_>, + node: &SummaryNode, + staged: Option<&StagedSummary>, + model_signature: &str, +) -> Result<()> { + tinycortex::memory::tree::store::insert_staged_summary_tx(tx, node, staged, model_signature) +} + +pub fn set_summary_embedding( + config: &Config, + summary_id: &str, + embedding: &[f32], +) -> Result { + tinycortex::memory::tree::store::set_summary_embedding( + &engine_config(config), + summary_id, + embedding, + )?; + Ok(1) +} + +pub fn get_summary_embedding(config: &Config, summary_id: &str) -> Result>> { + tinycortex::memory::tree::store::get_summary_embedding(&engine_config(config), summary_id) +} + +pub fn set_summary_embedding_for_signature( + config: &Config, + summary_id: &str, + signature: &str, + embedding: &[f32], +) -> Result<()> { + tinycortex::memory::tree::store::set_summary_embedding_for_signature( + &engine_config(config), + summary_id, + signature, + embedding, + ) +} + +pub fn mark_summary_reembed_skipped( + config: &Config, + summary_id: &str, + signature: &str, + reason: &str, +) -> Result<()> { + tinycortex::memory::chunks::mark_summary_reembed_skipped( + &engine_config(config), + summary_id, + signature, + reason, + ) +} + +pub fn clear_summary_reembed_skipped( + config: &Config, + summary_id: &str, + signature: &str, +) -> Result<()> { + tinycortex::memory::chunks::clear_summary_reembed_skipped( + &engine_config(config), + summary_id, + signature, + ) +} + +pub(crate) fn set_summary_embedding_for_signature_tx( + tx: &Transaction<'_>, + summary_id: &str, + signature: &str, + embedding: &[f32], +) -> Result<()> { + tinycortex::memory::chunks::set_summary_embedding_for_signature_tx( + tx, summary_id, signature, embedding, + ) +} + +pub fn get_summary_embedding_for_signature( + config: &Config, + summary_id: &str, + signature: &str, +) -> Result>> { + tinycortex::memory::tree::store::get_summary_embedding_for_signature( + &engine_config(config), + summary_id, + signature, + ) +} + +pub fn get_summary_embeddings_for_signature_batch( + config: &Config, + ids: &[String], + signature: &str, +) -> Result>> { + tinycortex::memory::tree::store::get_summary_embeddings_for_signature_batch( + &engine_config(config), + ids, + signature, + ) +} + +pub fn get_summary_embeddings_batch( + config: &Config, + ids: &[String], +) -> Result>> { + tinycortex::memory::tree::store::get_summary_embeddings_batch(&engine_config(config), ids) +} + +pub fn get_summary(config: &Config, id: &str) -> Result> { + tinycortex::memory::tree::store::get_summary(&engine_config(config), id) +} + +pub fn get_summaries_batch( + config: &Config, + ids: &[String], +) -> Result> { + tinycortex::memory::tree::store::get_summaries_batch(&engine_config(config), ids) +} + +pub fn list_summaries_at_level( + config: &Config, + tree_id: &str, + level: u32, +) -> Result> { + tinycortex::memory::tree::store::list_summaries_at_level(&engine_config(config), tree_id, level) +} + +pub fn list_summaries_in_window( + config: &Config, + tree_id: &str, + since_ms: i64, + until_ms: i64, +) -> Result> { + tinycortex::memory::tree::store::list_summaries_in_window( + &engine_config(config), + tree_id, + since_ms, + until_ms, + ) +} + +pub fn count_summaries(config: &Config, tree_id: &str) -> Result { + tinycortex::memory::tree::store::count_summaries(&engine_config(config), tree_id) +} + +pub fn get_buffer(config: &Config, tree_id: &str, level: u32) -> Result { + tinycortex::memory::tree::store::get_buffer(&engine_config(config), tree_id, level) +} + +pub(crate) fn get_buffer_conn(conn: &Connection, tree_id: &str, level: u32) -> Result { + tinycortex::memory::tree::store::get_buffer_conn(conn, tree_id, level) +} + +pub fn upsert_buffer_tx(tx: &Transaction<'_>, buffer: &Buffer) -> Result<()> { + tinycortex::memory::tree::store::upsert_buffer_tx(tx, buffer) +} + +pub fn list_stale_buffers(config: &Config, older_than: DateTime) -> Result> { + tinycortex::memory::tree::store::list_stale_buffers(&engine_config(config), older_than) +} diff --git a/core/src/store/trees/store_tests.rs b/core/src/store/trees/store_tests.rs new file mode 100644 index 0000000..d470951 --- /dev/null +++ b/core/src/store/trees/store_tests.rs @@ -0,0 +1,587 @@ +//! Unit tests for [`super::store`] — round-trip tree / summary / buffer +//! persistence including embedding blob handling and stale-buffer queries. + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use super::*; +use tempfile::TempDir; + +fn test_config() -> (TempDir, TestHostConfig) { + + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir() = tmp.path().to_path_buf(); + (tmp, cfg) +} + +fn sample_tree(id: &str, scope: &str) -> Tree { + Tree { + id: id.to_string(), + kind: TreeKind::Source, + scope: scope.to_string(), + ask: None, + root_id: None, + max_level: 0, + status: TreeStatus::Active, + created_at: Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(), + last_sealed_at: None, + } +} + +fn sample_summary(id: &str, tree_id: &str, level: u32) -> SummaryNode { + let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + SummaryNode { + id: id.to_string(), + tree_id: tree_id.to_string(), + tree_kind: TreeKind::Source, + level, + parent_id: None, + child_ids: vec!["leaf-a".into(), "leaf-b".into()], + content: "seal content".into(), + token_count: 100, + entities: vec!["entity:alice".into()], + topics: vec!["#launch".into()], + time_range_start: ts, + time_range_end: ts, + score: 0.75, + sealed_at: ts, + deleted: false, + embedding: None, + doc_id: None, + version_ms: None, + } +} + +#[test] +fn tree_round_trip() { + let (_tmp, cfg) = test_config(); + let t = sample_tree("tree-1", "slack:#eng"); + insert_tree(&cfg, &t).unwrap(); + let got = get_tree(&cfg, "tree-1").unwrap().unwrap(); + assert_eq!(got, t); + let by_scope = get_tree_by_scope(&cfg, TreeKind::Source, "slack:#eng") + .unwrap() + .unwrap(); + assert_eq!(by_scope.id, "tree-1"); +} + +#[test] +fn duplicate_scope_fails() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("t1", "slack:#eng")).unwrap(); + let dup = sample_tree("t2", "slack:#eng"); + assert!(insert_tree(&cfg, &dup).is_err()); +} + +#[test] +fn summary_insert_and_fetch() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let node = sample_summary("sum-1", "tree-1", 1); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &node, None, "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + let got = get_summary(&cfg, "sum-1").unwrap().unwrap(); + assert_eq!(got, node); + let at_level = list_summaries_at_level(&cfg, "tree-1", 1).unwrap(); + assert_eq!(at_level.len(), 1); + assert_eq!(count_summaries(&cfg, "tree-1").unwrap(), 1); +} + +#[test] +fn list_summaries_in_window_keeps_only_fully_contained() { + // The cover's eligibility filter: a summary is returned only when its + // ENTIRE envelope falls inside [since, until]. A node that straddles the + // window edge (starts before `since`) must be excluded — using it would + // drag in out-of-window content. + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + + let mk = |id: &str, start_ms: i64, end_ms: i64| { + let mut n = sample_summary(id, "tree-1", 1); + n.time_range_start = Utc.timestamp_millis_opt(start_ms).unwrap(); + n.time_range_end = Utc.timestamp_millis_opt(end_ms).unwrap(); + n + }; + // window = [1000, 2000] + let inside = mk("inside", 1100, 1900); // fully contained → eligible + let straddle_start = mk("straddle", 900, 1500); // begins before since → excluded + let straddle_end = mk("overrun", 1500, 2100); // ends after until → excluded + let outside = mk("outside", 3000, 3500); // wholly after → excluded + + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + for n in [&inside, &straddle_start, &straddle_end, &outside] { + insert_summary_tx(&tx, n, None, "test")?; + } + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let eligible = list_summaries_in_window(&cfg, "tree-1", 1000, 2000).unwrap(); + let ids: Vec<&str> = eligible.iter().map(|s| s.id.as_str()).collect(); + assert_eq!( + ids, + vec!["inside"], + "only the fully-contained summary is eligible" + ); +} + +#[test] +fn list_summaries_in_window_includes_exact_boundaries() { + // The window is inclusive on both ends: a summary whose envelope touches + // `since`/`until` exactly is still fully contained, so it must be eligible. + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + + let mk = |id: &str, start_ms: i64, end_ms: i64| { + let mut n = sample_summary(id, "tree-1", 1); + n.time_range_start = Utc.timestamp_millis_opt(start_ms).unwrap(); + n.time_range_end = Utc.timestamp_millis_opt(end_ms).unwrap(); + n + }; + // window = [1000, 2000] + let start_on_edge = mk("start-edge", 1000, 1500); // starts exactly at `since` + let end_on_edge = mk("end-edge", 1500, 2000); // ends exactly at `until` + let both_edges = mk("both-edges", 1000, 2000); // spans the whole window + + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + for n in [&start_on_edge, &end_on_edge, &both_edges] { + insert_summary_tx(&tx, n, None, "test")?; + } + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let eligible = list_summaries_in_window(&cfg, "tree-1", 1000, 2000).unwrap(); + let mut ids: Vec<&str> = eligible.iter().map(|s| s.id.as_str()).collect(); + ids.sort_unstable(); + assert_eq!( + ids, + vec!["both-edges", "end-edge", "start-edge"], + "summaries touching the inclusive window edges are eligible" + ); +} + +#[test] +fn list_summaries_in_window_excludes_deleted() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let mut node = sample_summary("sum-1", "tree-1", 1); + node.time_range_start = Utc.timestamp_millis_opt(1100).unwrap(); + node.time_range_end = Utc.timestamp_millis_opt(1900).unwrap(); + node.deleted = true; + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &node, None, "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + assert!( + list_summaries_in_window(&cfg, "tree-1", 1000, 2000) + .unwrap() + .is_empty(), + "tombstoned summaries are never eligible" + ); +} + +#[test] +fn summary_insert_is_idempotent_on_id() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let node = sample_summary("sum-1", "tree-1", 1); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &node, None, "test")?; + insert_summary_tx(&tx, &node, None, "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + assert_eq!(count_summaries(&cfg, "tree-1").unwrap(), 1); +} + +#[test] +fn summary_embeddings_are_scoped_by_model_signature() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let node = sample_summary("sum-embed", "tree-1", 1); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &node, None, "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + set_summary_embedding_for_signature( + &cfg, + "sum-embed", + "openai/text-embedding-3-small@1536", + &[0.1, 0.2], + ) + .unwrap(); + set_summary_embedding_for_signature(&cfg, "sum-embed", "local/bge-small@384", &[0.3, 0.4, 0.5]) + .unwrap(); + + assert_eq!( + get_summary_embedding_for_signature( + &cfg, + "sum-embed", + "openai/text-embedding-3-small@1536", + ) + .unwrap(), + Some(vec![0.1, 0.2]) + ); + assert_eq!( + get_summary_embedding_for_signature(&cfg, "sum-embed", "local/bge-small@384").unwrap(), + Some(vec![0.3, 0.4, 0.5]) + ); + assert!( + get_summary_embedding_for_signature(&cfg, "sum-embed", "missing/model@1") + .unwrap() + .is_none() + ); + + // #1574 cutover: the public `get_summary_embedding` now reads the sidecar + // at the *active* signature (not the legacy column). Nothing is written + // there yet → absent; never a cross-space read of the rows above. + assert!(get_summary_embedding(&cfg, "sum-embed").unwrap().is_none()); + + // The public setter targets the active signature and round-trips through + // the public getter — proves the cutover wiring end to end. + set_summary_embedding(&cfg, "sum-embed", &[0.7, 0.8]).unwrap(); + assert_eq!( + get_summary_embedding(&cfg, "sum-embed").unwrap(), + Some(vec![0.7, 0.8]) + ); + + // ...and the earlier per-signature rows remain independently scoped. + assert_eq!( + get_summary_embedding_for_signature(&cfg, "sum-embed", "local/bge-small@384").unwrap(), + Some(vec![0.3, 0.4, 0.5]) + ); +} + +#[test] +fn buffer_upsert_and_clear() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + let buf = Buffer { + tree_id: "tree-1".into(), + level: 0, + item_ids: vec!["leaf-a".into(), "leaf-b".into()], + token_sum: 500, + oldest_at: Some(ts), + }; + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + upsert_buffer_tx(&tx, &buf)?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + let got = get_buffer(&cfg, "tree-1", 0).unwrap(); + assert_eq!(got, buf); + + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + tinycortex::memory::tree::store::clear_buffer_tx(&tx, "tree-1", 0)?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + let cleared = get_buffer(&cfg, "tree-1", 0).unwrap(); + assert!(cleared.is_empty()); + assert_eq!(cleared.token_sum, 0); + assert!(cleared.oldest_at.is_none()); +} + +#[test] +fn get_buffer_returns_empty_when_missing() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let got = get_buffer(&cfg, "tree-1", 0).unwrap(); + assert!(got.is_empty()); + assert_eq!(got.tree_id, "tree-1"); +} + +#[test] +fn update_tree_after_seal_persists() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let sealed_at = Utc.timestamp_millis_opt(1_700_000_123_000).unwrap(); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + update_tree_after_seal_tx(&tx, "tree-1", "sum-1", 1, sealed_at)?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + let got = get_tree(&cfg, "tree-1").unwrap().unwrap(); + assert_eq!(got.root_id.as_deref(), Some("sum-1")); + assert_eq!(got.max_level, 1); + assert_eq!(got.last_sealed_at, Some(sealed_at)); +} + +#[test] +fn list_stale_buffers_orders_by_age() { + // Two L0 buffers across two trees, plus an L1 stale buffer that must + // be excluded — `list_stale_buffers` returns only L0 rows so flush + // cannot force-seal an under-fanout upper buffer (which would create + // a degenerate 1-child summary and collapse the tree into a chain). + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + insert_tree(&cfg, &sample_tree("tree-2", "slack:#ops")).unwrap(); + let t0 = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + let t1 = Utc.timestamp_millis_opt(1_700_000_010_000).unwrap(); + let t_l1 = Utc.timestamp_millis_opt(1_700_000_005_000).unwrap(); + let t2 = Utc.timestamp_millis_opt(1_700_000_020_000).unwrap(); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + upsert_buffer_tx( + &tx, + &Buffer { + tree_id: "tree-1".into(), + level: 0, + item_ids: vec!["a".into()], + token_sum: 10, + oldest_at: Some(t0), + }, + )?; + upsert_buffer_tx( + &tx, + &Buffer { + tree_id: "tree-1".into(), + level: 1, + item_ids: vec!["upper".into()], + token_sum: 5, + oldest_at: Some(t_l1), + }, + )?; + upsert_buffer_tx( + &tx, + &Buffer { + tree_id: "tree-2".into(), + level: 0, + item_ids: vec!["b".into()], + token_sum: 20, + oldest_at: Some(t1), + }, + )?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + let stale = list_stale_buffers(&cfg, t2).unwrap(); + assert_eq!(stale.len(), 2, "L1 stale buffer must be filtered out"); + assert!(stale.iter().all(|b| b.level == 0)); + assert_eq!(stale[0].oldest_at, Some(t0)); + assert_eq!(stale[1].oldest_at, Some(t1)); + // Tighter cutoff at t0 excludes tree-2's t1 buffer; only tree-1's + // L0 buffer (oldest_at == t0) remains. + let only_oldest = list_stale_buffers(&cfg, t0).unwrap(); + assert_eq!(only_oldest.len(), 1); + assert_eq!(only_oldest[0].level, 0); + assert_eq!(only_oldest[0].tree_id, "tree-1"); +} + +// ── get_trees_batch ──────────────────────────────────────────────────── +// +// Same shape as `chunks::store::get_chunks_batch` / +// `score::store::get_scores_batch`: present ids decode through the same +// `row_to_tree` path as the per-id `get_tree` and land in a `HashMap` +// keyed by id; missing ids are silently absent so the +// `flush_stale_buffers` orphan-buffer warn-and-skip path keeps working +// without an extra Ok(None) sentinel per id. + +#[test] +fn get_trees_batch_returns_present_ids_in_map() { + let (_tmp, cfg) = test_config(); + let a = sample_tree("tree-a", "slack:#eng"); + let b = sample_tree("tree-b", "slack:#design"); + insert_tree(&cfg, &a).unwrap(); + insert_tree(&cfg, &b).unwrap(); + + let ids = vec!["tree-a".to_string(), "tree-b".to_string()]; + let map = get_trees_batch(&cfg, &ids).unwrap(); + assert_eq!(map.len(), 2); + // Each decoded row must match the per-id `get_tree` path bit-for-bit + // — same `row_to_tree` decoder under the hood, so the structs are + // equal including the parsed `kind` / `status` enums. + assert_eq!(map.get("tree-a").unwrap(), &a); + assert_eq!(map.get("tree-b").unwrap(), &b); +} + +#[test] +fn get_trees_batch_empty_input_and_missing_ids() { + // Empty input: empty map (no SQL issued). + let (_tmp, cfg) = test_config(); + let empty = get_trees_batch(&cfg, &[]).unwrap(); + assert!(empty.is_empty()); + + // Missing ids: silently absent so `flush_stale_buffers` can warn + // + skip without an extra `Ok(None)` sentinel per id. + let a = sample_tree("tree-a", "slack:#eng"); + insert_tree(&cfg, &a).unwrap(); + let ids = vec!["tree-a".to_string(), "ghost:no-such".to_string()]; + let map = get_trees_batch(&cfg, &ids).unwrap(); + assert_eq!(map.len(), 1); + assert_eq!(map.get("tree-a").unwrap(), &a); + assert!(map.get("ghost:no-such").is_none()); +} + +// ── get_summaries_batch ──────────────────────────────────────────────── +// +// Same shape as `chunks::store::get_chunks_batch` / +// `score::store::get_scores_batch`: present ids decode through the same +// `row_to_summary` path as the per-id `get_summary` and land in a +// `HashMap` keyed by id; missing ids are silently absent so the +// `hydrate_summary_inputs` "missing row → warn + skip" contract keeps +// working without an extra Ok(None) sentinel. + +#[test] +fn get_summaries_batch_returns_present_ids_in_map() { + let (_tmp, cfg) = test_config(); + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let a = sample_summary("sum-a", "tree-1", 1); + let b = sample_summary("sum-b", "tree-1", 1); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &a, None, "test")?; + insert_summary_tx(&tx, &b, None, "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let ids = vec!["sum-a".to_string(), "sum-b".to_string()]; + let map = get_summaries_batch(&cfg, &ids).unwrap(); + assert_eq!(map.len(), 2); + // Each decoded row must match the per-id `get_summary` path bit-for-bit + // — same `row_to_summary` decoder under the hood, so the structs are + // equal including the deserialised JSON columns. + assert_eq!(map.get("sum-a").unwrap(), &a); + assert_eq!(map.get("sum-b").unwrap(), &b); +} + +#[test] +fn get_summaries_batch_empty_input_and_missing_ids() { + // Empty input: empty map (no SQL issued). + let (_tmp, cfg) = test_config(); + let empty = get_summaries_batch(&cfg, &[]).unwrap(); + assert!(empty.is_empty()); + + // Missing ids: silently absent so `hydrate_summary_inputs` can warn + // + skip without an extra `Ok(None)` sentinel per id. + insert_tree(&cfg, &sample_tree("tree-1", "slack:#eng")).unwrap(); + let a = sample_summary("sum-a", "tree-1", 1); + with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &a, None, "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); + + let ids = vec!["sum-a".to_string(), "ghost:no-such".to_string()]; + let map = get_summaries_batch(&cfg, &ids).unwrap(); + assert_eq!(map.len(), 1); + assert_eq!(map.get("sum-a").unwrap(), &a); + assert!(map.get("ghost:no-such").is_none()); +} + +// ---------- get_summary_embeddings_for_signature_batch ---------- +// +// Contract mirror of the chunks-side batch helper: equivalent to looping +// `get_summary_embedding_for_signature` per id, but in +// O(ceil(n / MAX_EMBEDDING_BATCH)) round-trips instead of O(n). The map +// contains only ids that have a non-null vector under the requested +// signature; absent rows (no sidecar entry, or sidecar entry with NULL +// vector) are silently dropped (same as the per-row helper returning +// Ok(None)). Chunking-window behaviour is covered on the chunks side +// (`batch_embedding_lookup_splits_id_list_above_per_batch_threshold`); +// the implementations share the same `chunks(MAX_EMBEDDING_BATCH)` loop +// shape so re-validating it here would be pure duplication. + +fn seed_summary(cfg: &Config, tree_id: &str, summary_id: &str) { + insert_tree(cfg, &sample_tree(tree_id, &format!("scope:{tree_id}"))).ok(); + let node = sample_summary(summary_id, tree_id, 1); + with_connection(cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &node, None, "test")?; + tx.commit()?; + Ok(()) + }) + .unwrap(); +} + +#[test] +fn summary_batch_embedding_lookup_returns_only_signature_scoped_rows() { + let (_tmp, cfg) = test_config(); + seed_summary(&cfg, "tree-1", "sum-1"); + seed_summary(&cfg, "tree-1", "sum-2"); + seed_summary(&cfg, "tree-1", "sum-3"); + + let sig_a = "openai/text-embedding-3-small@1536"; + let sig_b = "local/bge-small@384"; + set_summary_embedding_for_signature(&cfg, "sum-1", sig_a, &[0.1, 0.2]).unwrap(); + set_summary_embedding_for_signature(&cfg, "sum-2", sig_a, &[0.3, 0.4]).unwrap(); + set_summary_embedding_for_signature(&cfg, "sum-3", sig_b, &[0.5, 0.6, 0.7]).unwrap(); + + let ids = vec![ + "sum-1".to_string(), + "sum-2".to_string(), + "sum-3".to_string(), + ]; + let map_a = get_summary_embeddings_for_signature_batch(&cfg, &ids, sig_a).unwrap(); + assert_eq!(map_a.len(), 2, "only sum-1 and sum-2 are under sig_a"); + assert_eq!(map_a.get("sum-1").cloned(), Some(vec![0.1, 0.2])); + assert_eq!(map_a.get("sum-2").cloned(), Some(vec![0.3, 0.4])); + assert!(map_a.get("sum-3").is_none(), "sum-3 has only sig_b"); + + let map_b = get_summary_embeddings_for_signature_batch(&cfg, &ids, sig_b).unwrap(); + assert_eq!(map_b.len(), 1); + assert_eq!(map_b.get("sum-3").cloned(), Some(vec![0.5, 0.6, 0.7])); +} + +#[test] +fn summary_batch_embedding_lookup_empty_input_returns_empty_map() { + let (_tmp, cfg) = test_config(); + let map = get_summary_embeddings_for_signature_batch(&cfg, &[], "any/sig@1").unwrap(); + assert!(map.is_empty()); +} + +#[test] +fn summary_batch_embedding_lookup_unknown_ids_absent_from_map() { + // Pre-batch contract: per-row helper returned Ok(None) for missing + // summaries OR for summaries whose sidecar row has a NULL vector + // (pending re-embed). The batch helper must mirror that — missing + // ids absent from the map, present ids carry their vector. The + // retrieval rerank path depends on this so absent rows get the + // (NEG_INFINITY, false) sink-to-bottom treatment. + let (_tmp, cfg) = test_config(); + seed_summary(&cfg, "tree-1", "sum-1"); + let sig = "openai/text-embedding-3-small@1536"; + set_summary_embedding_for_signature(&cfg, "sum-1", sig, &[0.1]).unwrap(); + + let ids = vec![ + "sum-1".to_string(), + "ghost:no-such-summary-1".to_string(), + "ghost:no-such-summary-2".to_string(), + ]; + let map = get_summary_embeddings_for_signature_batch(&cfg, &ids, sig).unwrap(); + assert_eq!(map.len(), 1); + assert_eq!(map.get("sum-1").cloned(), Some(vec![0.1])); +} diff --git a/core/src/store/trees/types.rs b/core/src/store/trees/types.rs new file mode 100644 index 0000000..114fb97 --- /dev/null +++ b/core/src/store/trees/types.rs @@ -0,0 +1,7 @@ +//! Compatibility exports for tinycortex summary-tree persistence types. + +pub use tinycortex::memory::tree::store::{ + Buffer, EntityIndexStats, HotnessCounters, SummaryNode, Tree, TreeKind, TreeStatus, + DEFAULT_FLUSH_AGE_SECS, INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, SUMMARY_FANOUT, + TOPIC_ARCHIVE_THRESHOLD, TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, +}; diff --git a/core/src/store/types.rs b/core/src/store/types.rs new file mode 100644 index 0000000..d72101e --- /dev/null +++ b/core/src/store/types.rs @@ -0,0 +1,9 @@ +//! Stable host path for tinycortex-owned namespace memory contracts. + +pub use tinycortex::memory::{ + GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput, + NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown, + StoredMemoryDocument, +}; + +pub(crate) use tinycortex::memory::types::GLOBAL_NAMESPACE; diff --git a/core/src/store/write_gate.rs b/core/src/store/write_gate.rs new file mode 100644 index 0000000..1f204c3 --- /dev/null +++ b/core/src/store/write_gate.rs @@ -0,0 +1,171 @@ +//! Host write policy for memory documents — the secret/PII gate that runs +//! **before** the storage driver sees a document. +//! +//! # Why this module exists +//! +//! Redaction is host product policy, not persistence. Until this module +//! existed, `UnifiedMemory::upsert_document` ran the whole gate *inside* the +//! driver call: a caller handed it raw content and the SQL layer decided what +//! to scrub. `UnifiedMemory` is a candidate to move into the `tinycortex` +//! crate, and a persistence crate that owns "which substrings of a user's +//! document are secrets" is a policy decision shipped somewhere it cannot be +//! revisited per host. +//! +//! So the gate lives here and the driver methods +//! ([`UnifiedMemory::upsert_document_presanitized`] and +//! [`UnifiedMemory::upsert_document_metadata_only_presanitized`]) now take +//! already-sanitized input. This module re-declares `upsert_document` / +//! `upsert_document_metadata_only` as inherent methods on `UnifiedMemory` with +//! the **same names and signatures they always had**, so every existing caller +//! is routed through the gate without a single call-site edit — which is also +//! what makes the "no bypass" claim below checkable rather than hopeful. +//! +//! # The gate, in order +//! +//! The three steps are one ordered policy unit and were hoisted together; +//! running the redactor over a key that has not been canonicalized first would +//! scrub a different string than the one the row is addressed by. +//! +//! 1. **Reject** a namespace or key that looks like a secret. The identifier is +//! the row's address and is echoed in logs, so a credential there is not +//! something to redact-and-continue. +//! 2. **Canonicalize** a PII-bearing key rather than rejecting the write +//! (#5164): rejection returned `Err` on every attempt and callers retry, so +//! one such key produced an unthrottled error loop (3,055 Sentry events from +//! a single user). `safety::canonical_document_key` is strict-gated so +//! scanner-built identifiers (WhatsApp JIDs, `+1…` chat ids, timestamps) +//! keep their identity, and the by-key read paths (`Memory::get` / +//! `Memory::forget`) canonicalize through the same helper, so a rewritten +//! identifier stays addressable instead of reading back as a missing row. +//! 3. **Redact** secret/PII content out of every field via +//! `safety::sanitize_document_input`. +//! +//! Provenance `taint` is deliberately untouched by all three — sanitization is +//! content cleaning, and the taint is the signal the subconscious gate reads. +//! +//! # No bypass +//! +//! `upsert_document_presanitized` / `upsert_document_metadata_only_presanitized` +//! are `pub(crate)` and newly named, and this module holds their only call +//! sites — verify with: +//! +//! ```text +//! rg 'upsert_document(_metadata_only)?_presanitized' src/ +//! ``` +//! +//! Every other writer in the tree (`Memory::store_with_taint`, `MemoryClient`, +//! the ingestion queue, the RPC handlers, tests) calls the unsuffixed names and +//! is therefore gated. `write_gate_tests.rs` pins both halves: the gate +//! redacts, and the raw driver method does not. + +use crate::store::safety; +use crate::store::types::NamespaceDocumentInput; + +use super::namespace_store::UnifiedMemory; + +/// Outcome of running the host write gate over a caller-supplied document. +enum GateOutcome { + /// The (possibly rewritten) input the driver may persist. + Admit(Box), + /// The write is refused; the string is the caller-facing error. + Reject(String), +} + +/// Run the secret/PII gate over `input`. +/// +/// `flow` is a short grep tag naming the write path (`"document"` / +/// `"metadata-only"`) so the two callers' log lines stay distinguishable. +fn gate(input: NamespaceDocumentInput, flow: &str) -> GateOutcome { + // 1. Reject a secret-like address outright. + if safety::has_likely_secret(&input.namespace) || safety::has_likely_secret(&input.key) { + log::warn!( + "[memory:write_gate] {flow} write rejected due to secret-like namespace/key \ + namespace_chars={} key_chars={}", + input.namespace.chars().count(), + input.key.chars().count() + ); + return GateOutcome::Reject("document namespace/key cannot contain secrets".to_string()); + } + + // 2. Canonicalize a PII-bearing key rather than rejecting the write (#5164). + let input = { + let key = safety::canonical_document_key(&input.key); + if key != input.key { + log::info!( + "[memory:write_gate] {flow} write canonicalized PII-like key key_chars={}", + input.key.chars().count() + ); + } + NamespaceDocumentInput { key, ..input } + }; + + // 3. Redact secret/PII content out of every field. + let sanitized = safety::sanitize_document_input(input); + let input = sanitized.value; + if sanitized.report.changed() { + log::warn!( + "[memory:write_gate] {flow} write sanitized namespace_chars={} key_chars={} \ + text_redactions={} key_redactions={} blocked_secret_hits={} depth_redactions={} \ + pii_redactions={}", + input.namespace.chars().count(), + input.key.chars().count(), + sanitized.report.text_redactions, + sanitized.report.key_redactions, + sanitized.report.blocked_secret_hits, + sanitized.report.depth_redactions, + sanitized.report.pii_redactions + ); + } else { + log::trace!("[memory:write_gate] {flow} write passed the gate unchanged"); + } + + GateOutcome::Admit(Box::new(input)) +} + +impl UnifiedMemory { + /// Insert or update a document by `(namespace, key)`, applying the host + /// secret/PII write gate first. + /// + /// This is the entry point every writer should use. It runs the gate + /// documented at the module level and then delegates to + /// `Self::upsert_document_presanitized`, which does the persistence + /// (markdown sidecar, `memory_docs` upsert, chunking, embedding). + /// + /// # Errors + /// + /// Returns `Err` when the namespace or key looks like a credential, when + /// the key is empty, or on any storage/embedding failure. + pub async fn upsert_document(&self, input: NamespaceDocumentInput) -> Result { + match gate(input, "document") { + GateOutcome::Reject(err) => Err(err), + GateOutcome::Admit(input) => self.upsert_document_presanitized(*input).await, + } + } + + /// Store a document without chunking, embedding, or graph extraction, + /// applying the host secret/PII write gate first. + /// + /// Same gate as [`Self::upsert_document`]; suitable for high-frequency, + /// low-value writes (e.g. transient sync checkpoints) where the full + /// ingestion pipeline would be too expensive. + /// + /// # Errors + /// + /// Same failure modes as [`Self::upsert_document`]. + pub async fn upsert_document_metadata_only( + &self, + input: NamespaceDocumentInput, + ) -> Result { + match gate(input, "metadata-only") { + GateOutcome::Reject(err) => Err(err), + GateOutcome::Admit(input) => { + self.upsert_document_metadata_only_presanitized(*input) + .await + } + } + } +} + +#[cfg(test)] +#[path = "write_gate_tests.rs"] +mod tests; diff --git a/core/src/store/write_gate_tests.rs b/core/src/store/write_gate_tests.rs new file mode 100644 index 0000000..9048d6f --- /dev/null +++ b/core/src/store/write_gate_tests.rs @@ -0,0 +1,176 @@ +//! Tests for the host secret/PII write gate. +//! +//! The gate was hoisted out of `namespace_store::documents` (H0, piece 2) so +//! the storage driver never decides what counts as a secret. These tests pin +//! **both** halves of that split, which is what makes the hoist provable rather +//! than cosmetic: +//! +//! * the gated entry points (`upsert_document`, +//! `upsert_document_metadata_only`) still redact — the pre-existing +//! behaviour, unchanged; +//! * the raw driver methods (`*_presanitized`) do **not** — they persist what +//! they are handed, which is only safe because the gate is the sole caller. +//! +//! If someone folds redaction back into the driver, the second half fails. + +use std::sync::Arc; + +use serde_json::json; +use tempfile::TempDir; + +use crate::store::{NamespaceDocumentInput, UnifiedMemory}; +use tinymemory_api::host::NoopEmbedding; + +/// A private key body, split so this source file does not itself contain a +/// scanner-tripping literal in one piece. +const PRIVATE_KEY_BODY: &str = + "-----BEGIN PRIVATE KEY-----\nMIIBVgIBADANBgkq\n-----END PRIVATE KEY-----"; + +fn secret_doc(key: &str) -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: "safe".to_string(), + key: key.to_string(), + title: "Bearer abcdefghijklmnop".to_string(), + content: PRIVATE_KEY_BODY.to_string(), + source_type: "doc".to_string(), + priority: "medium".to_string(), + tags: vec![], + metadata: json!({}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: crate::MemoryTaint::Internal, + } +} + +fn fresh() -> (TempDir, UnifiedMemory) { + let tmp = TempDir::new().unwrap(); + let memory = UnifiedMemory::new(tmp.path(), Arc::new(NoopEmbedding), None).unwrap(); + (tmp, memory) +} + +#[tokio::test] +async fn gated_upsert_document_redacts_before_the_driver_persists() { + let (_tmp, memory) = fresh(); + + memory.upsert_document(secret_doc("note")).await.unwrap(); + + let docs = memory.load_documents_for_scope("safe").await.unwrap(); + assert_eq!(docs.len(), 1); + assert!( + !docs[0].content.contains("BEGIN PRIVATE KEY"), + "the gated entry point must redact private-key material, got {:?}", + docs[0].content + ); + assert!( + !docs[0].title.contains("abcdefghijklmnop"), + "the gated entry point must redact a bearer token in the title, got {:?}", + docs[0].title + ); +} + +#[tokio::test] +async fn gated_metadata_only_upsert_redacts_before_the_driver_persists() { + let (_tmp, memory) = fresh(); + + memory + .upsert_document_metadata_only(secret_doc("note")) + .await + .unwrap(); + + let docs = memory.load_documents_for_scope("safe").await.unwrap(); + assert_eq!(docs.len(), 1); + assert!( + !docs[0].content.contains("BEGIN PRIVATE KEY"), + "the gated metadata-only entry point must redact, got {:?}", + docs[0].content + ); +} + +#[tokio::test] +async fn raw_driver_upsert_does_not_redact_so_the_gate_is_the_only_thing_doing_it() { + let (_tmp, memory) = fresh(); + + // Bypassing the gate deliberately: this is what proves redaction now lives + // in `write_gate` and not inside the persistence call. + memory + .upsert_document_presanitized(secret_doc("note")) + .await + .unwrap(); + + let docs = memory.load_documents_for_scope("safe").await.unwrap(); + assert_eq!(docs.len(), 1); + assert!( + docs[0].content.contains("BEGIN PRIVATE KEY"), + "the raw driver method must persist its input verbatim — if this fails, redaction has \ + been folded back into the storage layer, got {:?}", + docs[0].content + ); +} + +#[tokio::test] +async fn raw_metadata_only_driver_upsert_does_not_redact() { + let (_tmp, memory) = fresh(); + + memory + .upsert_document_metadata_only_presanitized(secret_doc("note")) + .await + .unwrap(); + + let docs = memory.load_documents_for_scope("safe").await.unwrap(); + assert_eq!(docs.len(), 1); + assert!( + docs[0].content.contains("BEGIN PRIVATE KEY"), + "the raw metadata-only driver method must persist its input verbatim, got {:?}", + docs[0].content + ); +} + +#[tokio::test] +async fn gate_rejects_a_secret_like_namespace_or_key_before_touching_the_driver() { + let (_tmp, memory) = fresh(); + + let mut secret_key = secret_doc("sk-1234567890123456789012345"); + secret_key.namespace = "safe".to_string(); + let err = memory.upsert_document(secret_key).await.unwrap_err(); + assert!( + err.contains("cannot contain secrets"), + "secret-like key must be refused, got {err:?}" + ); + + let mut secret_ns = secret_doc("note"); + secret_ns.namespace = "sk-1234567890123456789012345".to_string(); + let err = memory + .upsert_document_metadata_only(secret_ns) + .await + .unwrap_err(); + assert!( + err.contains("cannot contain secrets"), + "secret-like namespace must be refused, got {err:?}" + ); + + // Nothing reached storage. + assert!(memory + .load_documents_for_scope("safe") + .await + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn gate_canonicalizes_a_pii_like_key_and_keeps_the_row_addressable() { + let (_tmp, memory) = fresh(); + + memory + .upsert_document(secret_doc("ssn-123-45-6789")) + .await + .unwrap(); + + let docs = memory.load_documents_for_scope("safe").await.unwrap(); + assert_eq!(docs.len(), 1); + assert!( + !docs[0].key.contains("123-45-6789"), + "a PII-like key must be canonicalized rather than stored raw, got {:?}", + docs[0].key + ); +} diff --git a/core/src/sync/README.md b/core/src/sync/README.md new file mode 100644 index 0000000..73c92dd --- /dev/null +++ b/core/src/sync/README.md @@ -0,0 +1,28 @@ +# memory_sync + +OpenHuman orchestration and product policy around the TinyCortex sync engine. + +TinyCortex owns generic Composio provider fetch/pagination, canonical memory +records, sync budgets/state, workspace reconciliation, and persistence traits. +The live host path calls it through `src/openhuman/memory/tinycortex/sync.rs` from +`composio::run_connection_sync` and the default provider `sync()` method. + +OpenHuman retains: + +- periodic scheduling and connection selection; +- credentials and Composio action execution; +- source-scope and redaction policy; +- translation into host `DomainEvent`s; +- JSON-RPC/status/connect surfaces; +- agent-facing action tools and result post-processing; +- product task/profile projections for GitHub, Notion, Linear, and ClickUp; +- local workspace watching and MCP orchestration. + +The provider directories therefore are not alternate sync engines. Their +remaining `provider.rs`, `tools.rs`, `normalization.rs`, profile, catalog, and +post-processing files implement host product surfaces over the crate-backed +sync path. New generic parsing or persistence behavior belongs in +`vendor/tinycortex/src/memory/sync/`. + +D4.1-D4.4 are closed in `docs/tinycortex-drift-ledger.md`. Gmail's bounded +25-message page is crate-owned and prevents Composio 413 responses. diff --git a/core/src/sync/composio/mod.rs b/core/src/sync/composio/mod.rs new file mode 100644 index 0000000..37e2721 --- /dev/null +++ b/core/src/sync/composio/mod.rs @@ -0,0 +1,212 @@ +//! Composio-backed sync pipelines. +//! +//! This module owns the "pull upstream provider data into memory" side of +//! Composio integrations: +//! +//! - provider sync implementations (`providers/*/provider.rs`, `sync.rs`) +//! - periodic scheduler (`periodic.rs`) +//! - trigger / connection-created event subscribers (`bus.rs`) +//! - sync-state persistence and profile-to-memory shaping +//! +//! The host's sibling `integrations::composio` domain still owns auth, +//! connection management, action execution, and general Composio RPC/tool +//! surfaces. This submodule is specifically the memory-sync half of that +//! integration boundary. + +use std::sync::Arc; +pub mod periodic; +pub mod providers; + +use crate::composio_host::{self, ComposioConnection}; +use crate::Config; + +pub use periodic::{record_sync_success, start_periodic_sync}; +pub use providers::{ + all_providers as all_composio_sync_providers, get_provider as get_composio_sync_provider, + init_default_providers as init_default_composio_sync_providers, ComposioProvider, + ComposioUsage, ProviderContext, ProviderUserProfile, SyncOutcome, SyncReason, +}; + +/// One provider-backed connection that the memory sync layer can execute. +#[derive(Debug, Clone)] +pub struct SyncTarget { + pub toolkit: String, + pub connection_id: String, +} + +/// List active Composio connections that have a native memory-sync provider. +/// +/// When memory_sources entries exist with `kind=composio` and `enabled=true`, +/// those are used as the authoritative source list (user curated). When no +/// memory_sources composio entries exist, falls back to scanning all active +/// Composio connections (legacy behavior). +pub async fn list_sync_targets(config: &Config) -> Result, String> { + init_default_composio_sync_providers(); + + // Try memory_sources registry first (user-curated list). + let registry_sources = + crate::sources::list_enabled_by_kind(crate::sources::SourceKind::Composio) + .await + .unwrap_or_default(); + + if !registry_sources.is_empty() { + let from_registry: Vec = registry_sources + .into_iter() + .filter_map(|s| { + let toolkit = s.toolkit?; + let connection_id = s.connection_id?; + get_composio_sync_provider(&toolkit).map(|_| SyncTarget { + toolkit, + connection_id, + }) + }) + .collect(); + if !from_registry.is_empty() { + tracing::debug!( + count = from_registry.len(), + "[composio:sync] using memory_sources registry for sync targets" + ); + return Ok(from_registry); + } + // Registry has entries but none yielded a valid target (missing + // fields or unregistered toolkit). Fall through to a fresh scan + // rather than reporting an empty target list — otherwise newly + // connected integrations stay invisible until reconcile runs. + tracing::debug!( + "[composio:sync] registry yielded zero valid targets; falling back to connection scan" + ); + } else { + tracing::debug!( + "[composio:sync] no memory_sources entries; falling back to connection scan" + ); + } + + scan_active_sync_targets(config).await +} + +/// Scan all active Composio connections that have a native memory-sync +/// provider. Always hits Composio directly — does not consult the +/// memory_sources registry. Used by reconciliation to seed the registry. +pub async fn scan_active_sync_targets(config: &Config) -> Result, String> { + init_default_composio_sync_providers(); + + // Mode dispatch lives in the host's `ComposioHost` impl: backend mode + // walks the tinyhumans tenant, direct mode the user's own Composio v3 + // tenant. Either way this side gets one flat list. + let connections = composio_host::list_connections(config).await?; + + Ok(connections + .into_iter() + .filter_map(connection_to_sync_target) + .collect()) +} + +/// Run one provider-backed sync end-to-end in-process. +/// +/// Returns the provider's [`SyncOutcome`] together with the +/// [`ComposioUsage`] tally (billable action count + actual USD cost) +/// accumulated at the `execute` chokepoint during this run, so the +/// sync-audit caller can record Composio API-call cost alongside the LLM +/// summarisation cost (#3111). +pub async fn run_connection_sync( + config: Arc, + connection_id: &str, + reason: SyncReason, +) -> Result<(SyncOutcome, ComposioUsage), (String, ComposioUsage)> { + init_default_composio_sync_providers(); + + let no_usage = |e: String| (e, ComposioUsage::default()); + + let target = list_sync_targets(&*config) + .await + .map_err(no_usage)? + .into_iter() + .find(|target| target.connection_id == connection_id) + .ok_or_else(|| { + no_usage(format!( + "no provider-backed active sync target for connection_id={connection_id}", + )) + })?; + + let provider = get_composio_sync_provider(&target.toolkit).ok_or_else(|| { + no_usage(format!( + "no native memory sync provider registered for toolkit '{}'", + target.toolkit, + )) + })?; + + // Look up the source entry to obtain any user-configured caps. + // Non-fatal: if the registry read fails we proceed uncapped. + let (src_max_items, src_sync_depth_days) = { + let registry_sources = + crate::sources::list_enabled_by_kind(crate::sources::SourceKind::Composio) + .await + .unwrap_or_default(); + registry_sources + .iter() + .find(|s| s.connection_id.as_deref() == Some(&target.connection_id)) + .map(|s| (s.max_items, s.sync_depth_days)) + .unwrap_or((None, None)) + }; + + tracing::debug!( + connection_id = %target.connection_id, + max_items = ?src_max_items, + sync_depth_days = ?src_sync_depth_days, + "[composio:sync] run_connection_sync: caps from registry" + ); + + let _ = (provider, src_max_items, src_sync_depth_days); + let started_at_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + match crate::tinycortex::run_composio_connection( + &target.toolkit, + &target.connection_id, + &*config, + ) + .await + { + Ok(outcome) => { + let usage = ComposioUsage { + actions_called: outcome.actions_called, + cost_usd: outcome.provider_cost_usd, + }; + Ok(( + SyncOutcome { + toolkit: target.toolkit, + connection_id: Some(target.connection_id), + reason: reason.as_str().to_string(), + items_ingested: outcome.records_ingested as usize, + started_at_ms, + finished_at_ms: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64, + summary: outcome.note.unwrap_or_else(|| "sync completed".to_string()), + details: serde_json::json!({ "more_pending": outcome.more_pending }), + }, + usage, + )) + } + Err(error) => Err(( + error.to_string(), + ComposioUsage { + actions_called: error.actions_called, + cost_usd: error.provider_cost_usd, + }, + )), + } +} + +fn connection_to_sync_target(connection: ComposioConnection) -> Option { + if !connection.is_active() { + return None; + } + let toolkit = connection.normalized_toolkit(); + get_composio_sync_provider(&toolkit).map(|_| SyncTarget { + toolkit, + connection_id: connection.id, + }) +} diff --git a/core/src/sync/composio/periodic.rs b/core/src/sync/composio/periodic.rs new file mode 100644 index 0000000..22cfa71 --- /dev/null +++ b/core/src/sync/composio/periodic.rs @@ -0,0 +1,1263 @@ +//! Periodic sync scheduler for the Composio domain. +//! +//! Spawned once at startup. The scheduler walks every active Composio +//! connection on a fixed tick, looks up the matching native provider, +//! and dispatches the matching tinycortex pipeline if enough time +//! has elapsed since that connection's last sync (per the provider's +//! `sync_interval_secs`). +//! +//! ## Direct mode (`[composio-direct]`) +//! +//! As of #1710 Wave 1, the scheduler is **mode-aware**: it resolves the +//! client via `create_composio_client` each tick so a direct-mode +//! user's personal Composio v3 tenant gets walked (via +//! `direct_list_connections`) instead of returning an empty list from +//! the tinyhumans tenant. The per-connection sync calls go through +//! `ProviderContext::execute` which is itself mode-aware. +//! +//! Real-time trigger webhooks (`composio:trigger` socket.io events +//! fanned out from `wss://api.tinyhumans.ai`) still do not reach the +//! core when `config.composio().mode == "direct"`, because the backend +//! HMAC-verifies the Composio webhook and pushes it down a per-user +//! socket — direct-mode users see synchronous tool execution and +//! periodic poll-based sync, but not async trigger pushes in this +//! release. See the `composio.direct_mode_triggers_gap` capability +//! entry in `about_app/catalog.rs` for the user-visible status. +//! +//! Design notes: +//! +//! * One global tick (5min) drives every provider — we don't spawn a +//! task per connection, because the number of connections per user +//! is small and a single tick keeps the bookkeeping trivial. +//! * Per-connection state (last sync timestamp) lives in a +//! process-global `Arc>` keyed by `(toolkit, +//! connection_id)`. The map is shared with event-driven sync paths +//! (bus subscribers, `on_connection_created`) via +//! [`record_sync_success`] so a recent non-periodic sync prevents +//! the scheduler from redundantly re-firing. The map is rebuilt on +//! restart; to keep a user-configured cadence (e.g. "Sync every 24h", +//! #3302) from re-firing on every cold start, the due-check falls back +//! to the **persisted** sync-audit timestamp (`read_audit_log`) when +//! the in-memory record is absent — see `persisted_since_last_sync`. +//! * Errors are logged and swallowed; the scheduler must never panic +//! out of its loop or periodic sync stops silently for the rest of +//! the process lifetime. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +use tokio::time::interval; + +use crate::config_loader as config_rpc; +use crate::scheduler_gate::PauseReason; +use crate::scheduler_gate::{current_policy, resume_notify}; +use crate::sources::{memory_sync_defaults_for_toolkit, MemorySourceEntry, SourceKind}; +use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; + +use super::providers::{get_provider, ComposioUsage}; +use crate::composio_host; +use crate::tinycortex::{append_audit_entry, try_read_audit_log, SyncAuditEntry}; +use chrono::{DateTime, Utc}; + +/// How often the scheduler wakes up to look for due syncs. Independent +/// from per-provider `sync_interval_secs` — this just bounds how long +/// past a provider's interval we might fire. +/// +/// 20 min trades a little staleness for noticeably less foreground load: +/// each tick triggers an HTTP fetch + DB write per due connection, and +/// for users with several connected providers the old 60s cadence kept +/// the laptop visibly busy. Per-provider `sync_interval_secs` still +/// caps the *minimum* delay between actual syncs — this only loosens +/// the upper bound. +const TICK_SECONDS: u64 = 1200; + +/// Process-wide guard so the scheduler is only started once even +/// when both `start_channels` and `bootstrap_core_runtime` call into +/// us during startup. Without this we'd end up with two parallel tick +/// loops competing for the same connections. +static SCHEDULER_STARTED: OnceLock<()> = OnceLock::new(); + +/// Process-wide map of `(toolkit, connection_id) → last successful sync +/// instant`. Shared between the periodic scheduler loop and event-driven +/// sync paths (e.g. `ComposioConnectionCreatedSubscriber`, +/// `on_connection_created`) so that a recent non-periodic sync prevents +/// the scheduler from firing immediately on the next tick. +type SyncTimestampMap = Arc>>; + +static LAST_SYNC_AT: OnceLock = OnceLock::new(); + +/// Get (or lazily initialise) the shared last-sync-at map. +fn last_sync_map() -> SyncTimestampMap { + LAST_SYNC_AT + .get_or_init(|| Arc::new(Mutex::new(HashMap::new()))) + .clone() +} + +/// Record a successful sync for the given `(toolkit, connection_id)` key. +/// Called by the periodic scheduler after a successful sync and by +/// event-driven paths (bus subscribers, `on_connection_created`) so the +/// periodic ticker respects recent non-periodic syncs. +pub fn record_sync_success(toolkit: &str, connection_id: &str) { + if let Ok(mut map) = last_sync_map().lock() { + map.insert( + (toolkit.to_string(), connection_id.to_string()), + Instant::now(), + ); + } +} + +/// Resolve the effective periodic sync interval (seconds) for one connection, +/// combining the provider's own default with the user's global +/// memory-sync cadence ([`Config::memory_sync_interval_secs`], #3302). +/// +/// - `global == Some(0)` → `None`: "Manual only" — the scheduler skips this +/// source entirely (manual sync still works). +/// - `global == Some(n)` → `Some(max(n, provider_default))`: the user's +/// cadence overrides the provider default but is floored at it, so we never +/// sync *more* often than the provider intended. +/// - `global == None` → `Some(max(DEFAULT, provider_default))`: no explicit +/// user choice, so fall back to the 24h default cadence (also floored at the +/// provider default). +pub(crate) fn effective_interval_secs(provider_default: u64, global: Option) -> Option { + match global { + Some(0) => None, + Some(n) => Some(n.max(provider_default)), + None => Some(DEFAULT_MEMORY_SYNC_INTERVAL_SECS.max(provider_default)), + } +} + +/// Decide whether a connection is due for a periodic sync right now, given the +/// effective interval and how long ago it last synced this run. +/// +/// `since_last_sync == None` means we have no record of a sync this process +/// lifetime, so we fire immediately (the restart-recovery path). Kept pure so +/// the due-check can be simulated without driving the real `Instant` clock. +pub(crate) fn connection_is_due(interval_secs: u64, since_last_sync: Option) -> bool { + match since_last_sync { + Some(elapsed) => elapsed >= Duration::from_secs(interval_secs), + None => true, + } +} + +/// Build an index of `connection_id → most recent successful Composio sync +/// timestamp` from the persisted sync audit log (#3302). +/// +/// The periodic loop writes audit entries with `source_id = connection_id` and +/// `scope = "{toolkit}:{connection_id}"`; we accept either shape and key by the +/// connection id. Only successful syncs count — matching the in-memory +/// [`record_sync_success`] semantics, which never records a failed tick so the +/// next tick retries. This is the wall-clock record that lets the cadence +/// survive restarts (the in-memory monotonic map cannot). +fn index_last_success_by_connection(entries: &[SyncAuditEntry]) -> HashMap> { + let mut idx: HashMap> = HashMap::new(); + for e in entries { + if e.source_kind != "composio" || !e.success { + continue; + } + let connection_id = e + .scope + .rsplit_once(':') + .map(|(_, c)| c.to_string()) + .filter(|c| !c.is_empty()) + .unwrap_or_else(|| e.source_id.clone()); + idx.entry(connection_id) + .and_modify(|t| { + if e.timestamp > *t { + *t = e.timestamp; + } + }) + .or_insert(e.timestamp); + } + idx +} + +/// Wall-clock elapsed since a connection's last persisted successful sync, if +/// any. Saturates at zero for a future timestamp (clock skew), so a skewed +/// record never reads as "wildly overdue". Returns `None` when the connection +/// has no persisted sync — letting the caller treat it as never-synced. +fn persisted_since_last_sync( + idx: &HashMap>, + connection_id: &str, + now: DateTime, +) -> Option { + idx.get(connection_id).map(|ts| { + let secs = (now - *ts).num_seconds().max(0) as u64; + Duration::from_secs(secs) + }) +} + +/// Outcome of consulting the per-source registry for one Composio connection +/// during a periodic tick (#2831). +#[derive(Debug, PartialEq, Eq)] +enum PeriodicSourceDecision { + /// The user toggled this source **off** — skip background sync entirely. + /// Manual `memory_sources_sync` still works (it has its own `enabled` + /// guard); only the automatic loop honours this here. + Skip, + /// Sync this connection with the given caps (`None` = uncapped for that + /// dimension). + Sync { + max_items: Option, + sync_depth_days: Option, + }, +} + +/// Decide whether — and with what caps — to periodically sync one connection, +/// honouring the per-source `enabled` toggle (#2831). Pure so the three +/// branches can be unit-tested without async/registry I/O. +/// +/// - **disabled row** → [`PeriodicSourceDecision::Skip`]: the background loop +/// must not sync a source the user switched off (this was the row-2 leak — +/// previously the loop only read the registry for caps and synced disabled +/// sources anyway, uncapped). +/// - **enabled row** → `Sync` with the row's caps. +/// - **no row yet** → `Sync` with conservative per-toolkit defaults +/// ([`memory_sync_defaults_for_toolkit`]). `reconcile` normally backfills an +/// enabled, capped row for every connection; this covers the brief +/// pre-reconcile window. The no-match path defaults to *sync-bounded*, never +/// *skip* and never *uncapped*, so a missing/mismatched row degrades safely +/// (data keeps flowing, just capped) instead of silently going dark. +fn decide_periodic_source( + source: Option<&MemorySourceEntry>, + toolkit: &str, +) -> PeriodicSourceDecision { + match source { + Some(s) if !s.enabled => PeriodicSourceDecision::Skip, + Some(s) => PeriodicSourceDecision::Sync { + max_items: s.max_items, + sync_depth_days: s.sync_depth_days, + }, + None => { + let (max_items, sync_depth_days) = memory_sync_defaults_for_toolkit(toolkit); + PeriodicSourceDecision::Sync { + max_items, + sync_depth_days, + } + } + } +} + +/// Spawn the periodic sync background task. Idempotent: only the +/// first call actually spawns the loop, every subsequent call is a +/// cheap no-op (logged at `debug` so it's visible during startup +/// tracing without spamming `info`). +pub fn start_periodic_sync() { + if SCHEDULER_STARTED.get().is_some() { + tracing::debug!("[composio:periodic] scheduler already running, skipping start"); + return; + } + // Race-safe: only the thread that wins `set` runs the spawn body. + if SCHEDULER_STARTED.set(()).is_err() { + tracing::debug!("[composio:periodic] scheduler already running (race), skipping start"); + return; + } + + tokio::spawn(async move { + tracing::info!( + tick_seconds = TICK_SECONDS, + "[composio:periodic] scheduler starting" + ); + run_loop().await; + // run_loop only returns on a fatal error in the bus — log it + // so the silent stop is at least visible in the trace. + tracing::error!("[composio:periodic] scheduler loop exited"); + }); +} + +/// Inner loop, broken out so it's easy to mock-replace in tests if we +/// ever want to drive ticks deterministically. +/// +/// Each iteration waits on whichever comes first (#2831): +/// * the 20-min `ticker` — the steady-state cadence, or +/// * the scheduler-gate **resume** notify — fired when the user toggles +/// Memory Tree back on or signs back in. +/// +/// On a resume wake we run a tick **immediately** (so sync restarts within +/// seconds, not at the next ≤20-min boundary) and `reset()` the ticker so the +/// *next* scheduled tick is a full `TICK_SECONDS` out. The reset is what stops +/// rapid off-on toggling from bunch-firing: many wakes collapse into at most +/// one extra tick (the `Notify` stores a single permit), and the cadence +/// re-bases from the last actual tick. +async fn run_loop() { + let mut ticker = interval(Duration::from_secs(TICK_SECONDS)); + let resume = resume_notify(); + // Skip the immediate-fire tick so startup isn't slammed before the + // user even has time to sign in. + ticker.tick().await; + + loop { + tokio::select! { + _ = ticker.tick() => {} + _ = resume.notified() => { + // Woke early on a resume transition. Re-base the cadence so the + // next scheduled tick is TICK_SECONDS from now, then fall + // through and run the tick immediately. + ticker.reset(); + } + } + if let Err(e) = run_one_tick().await { + tracing::warn!( + error = %e, + "[composio:periodic] tick failed (continuing)" + ); + } + } +} + +/// Inspect the scheduler-gate policy and decide whether this tick should +/// fire at all. Returns `Some(reason)` for paused states so the caller can +/// log a single, attributable line instead of doing the work and discovering +/// per-LLM-call later that everything's gated. +/// +/// Covers two reasons the memory subsystem treats as "do no background +/// work": +/// - [`PauseReason::UserDisabled`] — user flipped the Memory Tree toggle off +/// in Settings (#1856 Part 1). The 20-min Composio fetch loop honouring +/// this flag is the explicit follow-up listed in the #2719 PR body. +/// - [`PauseReason::SignedOut`] — no live session; periodic work would just +/// 401-loop against the backend. +/// +/// Other [`PauseReason`] variants: +/// - `OnBattery` / `CpuPressure` (future, per #1073) — intentionally **not** +/// gated here; periodic Composio fetch is network-light, so battery / CPU +/// pressure shouldn't stop the user's data flowing in. Those signals +/// already throttle LLM-bound work through the regular gate. +/// - `Unknown` — documented in `scheduler_gate::policy` as a safe fallback; +/// `Policy::pause_reason()` returns it only when the gate state is in a +/// transitional / not-yet-resolved condition. Letting the tick proceed +/// here keeps periodic sync running through brief transitions instead of +/// pausing on stale unresolved state. +pub(crate) fn periodic_pause_reason() -> Option { + // Delegate the `Policy::Paused { .. }` → `PauseReason` extraction to + // the existing `Policy::pause_reason()` helper (avoids re-implementing + // the same destructure twice). The allow-list below is the only thing + // this site has to own — future `PauseReason` variants stay opt-in. + let reason = current_policy().pause_reason()?; + matches!(reason, PauseReason::UserDisabled | PauseReason::SignedOut).then_some(reason) +} + +/// Process-level "was the last tick paused?" tracker for transition logging. +/// +/// We want `info!` *once* when the periodic loop crosses the pause boundary +/// (so fleet operators investigating "why is Composio not syncing?" see a +/// breadcrumb at default log level), without spamming `info` every 20 min +/// while the user has the toggle off. `Relaxed` ordering is fine because +/// the only consumer is the inside of `run_one_tick`, which is serialised +/// by the singleton scheduler loop. +static LAST_TICK_WAS_PAUSED: AtomicBool = AtomicBool::new(false); + +/// Run a single scheduler tick. Public-ish (`pub(crate)`) so the test +/// module can drive ticks without spinning up the real `interval`. +pub(crate) async fn run_one_tick() -> Result<(), String> { + // Step 0: scheduler-gate check. When the user has paused Memory Tree + // via the Settings toggle, every subsequent tick should be a cheap + // no-op — no `list_connections` call, no provider walk, no API budget + // burn. The check runs **before** config load + auth-client build so + // a paused session never even resolves the API token. + // + // Transition logging: emit `info!` once when the loop crosses the + // pause boundary in either direction; stay at `debug!` for the + // already-paused / already-running steady state. Without this, fleet + // operators investigating "why is Composio not syncing?" see nothing + // at default log level. + if let Some(reason) = periodic_pause_reason() { + let was_paused = LAST_TICK_WAS_PAUSED.swap(true, Ordering::Relaxed); + if was_paused { + tracing::debug!( + reason = reason.as_str(), + "[composio:periodic] scheduler-gate paused — skipping tick" + ); + } else { + tracing::info!( + reason = reason.as_str(), + "[composio:periodic] scheduler-gate paused — pausing periodic Composio sync" + ); + } + return Ok(()); + } else { + let was_paused = LAST_TICK_WAS_PAUSED.swap(false, Ordering::Relaxed); + if was_paused { + tracing::info!( + "[composio:periodic] scheduler-gate resumed — periodic Composio sync re-enabled" + ); + } + } + + // Step 1: load config (also gives us the auth token via the + // shared integrations client builder). + let config = config_rpc::load_config_with_timeout() + .await + .map_err(|e| format!("load_config: {e}"))?; + + // Step 2: list active connections — mode-aware. Backend mode walks + // the tinyhumans tenant; direct mode walks the user's personal + // Composio v3 tenant. Mirrors `ops::composio_list_connections` so + // direct-mode users get periodic sync against their own connections + // instead of seeing an empty list (#1710). + // Mode dispatch lives in the host's `ComposioHost` impl, and so does the + // 401 classification that used to happen here: the direct-mode v3 + // `/connected_accounts` 401 shape is a property of the client, not of this + // loop, and the host reports it through the same observability classifier + // the UI poll uses. A failure here means "not signed in / no direct key" as + // often as it means a real fault, so the tick skips rather than erroring. + let connections = match composio_host::list_connections(&*config).await { + Ok(connections) => connections, + Err(e) => { + tracing::debug!( + error = %e, + "[composio:periodic] no connections (not signed in? no direct key?), skipping tick" + ); + return Ok(()); + } + }; + + let sync_map = last_sync_map(); + + // Global, user-configurable memory-sync cadence (#3302). Applied to every + // opted-in source as a floor/override over the provider's own default; a + // value of `Some(0)` disables periodic auto-sync ("Manual only"). + let global_interval = config.memory_sync_interval_secs(); + + // Persisted last-sync fallback (#3302). The in-memory `LAST_SYNC_AT` map is + // rebuilt empty on every launch, so without this a cold start would re-fire + // every connection on the first tick — silently breaking the configured + // "Sync every 24h" gap across app restarts. We index the persisted sync + // audit log (wall-clock timestamps that survive restarts) and use it as the + // due-check fallback whenever the in-memory monotonic record is absent. + let (audit_index, audit_available) = composio_audit_state(try_read_audit_log(&*config)); + if !audit_available { + tracing::warn!( + "[memory_sync:periodic] audit unavailable; sources without in-memory cadence will be skipped" + ); + } + let now = Utc::now(); + + // Per-source registry snapshot (#2831). The periodic loop gates on the + // per-source `enabled` toggle so a source the user switched off stops + // syncing in the background — matching the manual paths + // (`memory_sources::sync_source`, `memory_sources_sync_all`), which already + // early-return on `!enabled`. Index every Composio source (enabled and + // disabled) by connection id; the per-connection branch below resolves + // skip/caps via `decide_periodic_source`. + // + // Built from the **already-loaded** `config` snapshot (Step 1), not a second + // `list_sources()` read. A separate read whose error we swallowed to an + // empty map would make every disabled source fall through to the + // `decide_periodic_source(None, ..)` default-caps path — silently + // re-enabling background sync for sources the user switched off on a + // transient config-read failure. Reusing the tick's snapshot is fail-closed + // (a disabled row stays disabled) and avoids the extra read entirely. + let composio_sources: HashMap = + crate::sources::decode_memory_sources(&*config) + .iter() + .filter(|s| s.kind == SourceKind::Composio) + .filter_map(|s| s.connection_id.clone().map(|id| (id, s.clone()))) + .collect(); + + let mut considered = 0usize; + let mut fired = 0usize; + for conn in connections { + considered += 1; + + // Skip connections that aren't actually live yet. + if !conn.is_active() { + continue; + } + + let toolkit = conn.normalized_toolkit(); + let Some(provider) = get_provider(&toolkit) else { + // No provider registered for this toolkit — that's fine, + // we just don't have native code for it. Tools still work + // through `composio_execute`. + continue; + }; + + let Some(provider_default) = provider.sync_interval_secs() else { + // Provider opted out of periodic sync entirely. + continue; + }; + + let Some(interval_secs) = effective_interval_secs(provider_default, global_interval) else { + // User selected "Manual only" — skip auto-sync for this source. + // Manual `memory_sources_sync` still works. + tracing::debug!( + toolkit = %toolkit, + connection_id = %conn.id, + "[composio:periodic] manual-only mode — skipping periodic sync" + ); + continue; + }; + + let key = (toolkit.clone(), conn.id.clone()); + // Prefer the in-memory monotonic record (most accurate within this run); + // fall back to the persisted audit timestamp so the configured cadence + // is honoured across restarts instead of re-firing on every cold start. + let in_memory_since = { + let map = sync_map.lock().unwrap_or_else(|e| e.into_inner()); + map.get(&key).map(|when| when.elapsed()) + }; + let Some(since_last_sync) = cadence_from_audit( + in_memory_since, + audit_available, + persisted_since_last_sync(&audit_index, &conn.id, now), + ) else { + tracing::debug!( + toolkit = %toolkit, + "[composio:periodic] source has unknown cadence while audit is unavailable; skipping" + ); + continue; + }; + if !connection_is_due(interval_secs, since_last_sync) { + continue; + } + + // Per-source gate + caps from the memory_sources registry (#2831). + // A disabled source is skipped here (the background-sync half of the + // toggle); enabled sources sync with their caps; a connection with no + // registry row yet syncs with conservative per-toolkit defaults. + let (src_max_items, src_sync_depth_days) = + match decide_periodic_source(composio_sources.get(&conn.id), &toolkit) { + PeriodicSourceDecision::Skip => { + tracing::debug!( + toolkit = %toolkit, + connection_id = %conn.id, + "[composio:periodic] source disabled — skipping periodic sync" + ); + continue; + } + PeriodicSourceDecision::Sync { + max_items, + sync_depth_days, + } => (max_items, sync_depth_days), + }; + + tracing::debug!( + toolkit = %toolkit, + connection_id = %conn.id, + max_items = ?src_max_items, + sync_depth_days = ?src_sync_depth_days, + "[composio:periodic] caps from registry" + ); + + let mut source = composio_sources + .get(&conn.id) + .cloned() + .unwrap_or_else(|| periodic_source(&toolkit, &conn.id)); + source.max_items = src_max_items; + source.sync_depth_days = src_sync_depth_days; + + tracing::debug!( + toolkit = %conn.toolkit, + connection_id = %conn.id, + interval_secs, + "[composio:periodic] firing sync" + ); + let sync_started = Instant::now(); + let result = crate::tinycortex::run_source_pipeline(&source, &*config).await; + let duration_ms = sync_started.elapsed().as_millis() as u64; + + match result { + Ok(outcome) => { + let usage = ComposioUsage { + actions_called: outcome.actions_called, + cost_usd: outcome.provider_cost_usd, + }; + tracing::debug!( + toolkit = %conn.toolkit, + connection_id = %conn.id, + items = outcome.records_ingested, + composio_actions = usage.actions_called, + "[composio:periodic] sync ok" + ); + let entry = build_periodic_audit_entry( + &toolkit, + &conn.id, + &usage, + outcome.records_ingested as usize, + duration_ms, + None, + ); + append_audit_entry(&*config, &entry); + record_sync_success(&conn.toolkit, &conn.id); + fired += 1; + } + Err(e) => { + let usage = ComposioUsage { + actions_called: e.actions_called, + cost_usd: e.provider_cost_usd, + }; + tracing::warn!( + toolkit = %conn.toolkit, + connection_id = %conn.id, + error = %e, + "[composio:periodic] sync failed (will retry next tick)" + ); + // A failed tick may still have fired billable fetch actions + // before erroring — audit the partial cost so it isn't lost. + let entry = build_periodic_audit_entry( + &toolkit, + &conn.id, + &usage, + 0, + duration_ms, + Some(e.to_string()), + ); + append_audit_entry(&*config, &entry); + // Intentionally do NOT update last_sync_at on failure + // so the next tick retries immediately. + } + } + } + + tracing::debug!(considered, fired, "[composio:periodic] tick complete"); + Ok(()) +} + +fn composio_audit_state( + read: anyhow::Result>, +) -> (HashMap>, bool) { + match read { + Ok(entries) => (index_last_success_by_connection(&entries), true), + Err(error) => { + tracing::warn!(%error, "[memory_sync:periodic] audit read failed"); + (HashMap::new(), false) + } + } +} + +fn cadence_from_audit( + in_memory_since: Option, + audit_available: bool, + persisted_since: Option, +) -> Option> { + match in_memory_since { + Some(since) => Some(Some(since)), + None if audit_available => Some(persisted_since), + None => None, + } +} + +fn periodic_source(toolkit: &str, connection_id: &str) -> MemorySourceEntry { + MemorySourceEntry { + id: format!("composio:{connection_id}"), + kind: SourceKind::Composio, + label: toolkit.to_string(), + enabled: true, + toolkit: Some(toolkit.to_string()), + connection_id: Some(connection_id.to_string()), + path: None, + glob: None, + url: None, + branch: None, + paths: Vec::new(), + max_commits: None, + max_issues: None, + max_prs: None, + query: None, + since_days: None, + max_items: None, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days: None, + } +} + +/// Build a [`SyncAuditEntry`] for one periodic Composio sync tick (#3111 +/// follow-up). +/// +/// Periodic syncs only fetch + ingest; summarisation runs later in the async +/// job worker, so the LLM-cost columns (tokens, estimated / actual charge) +/// are zero here. The meaningful spend is the Composio billable actions the +/// fetch fired, carried in `usage`. `scope` is `{toolkit}:{connection_id}` to +/// match the owner shape the per-source memory-tree ingest uses, and +/// `source_kind` is `"composio"` so the Sync History panel groups periodic +/// rows alongside the manual-sync rows the dispatcher already writes. +fn build_periodic_audit_entry( + toolkit: &str, + connection_id: &str, + usage: &ComposioUsage, + items_ingested: usize, + duration_ms: u64, + error: Option, +) -> SyncAuditEntry { + SyncAuditEntry { + timestamp: chrono::Utc::now(), + source_id: connection_id.to_string(), + source_kind: "composio".to_string(), + scope: format!("{toolkit}:{connection_id}"), + items_fetched: items_ingested as u32, + batches: 0, + input_tokens: 0, + output_tokens: 0, + estimated_cost_usd: 0.0, + composio_actions_called: usage.actions_called, + composio_cost_usd: usage.cost_usd, + actual_charged_usd: None, + duration_ms, + success: error.is_none(), + error, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_env_lock::TEST_ENV_LOCK as ENV_LOCK; + use tempfile::tempdir; + + #[test] + fn tick_seconds_is_sane_default() { + // Sanity check: don't accidentally ship a 1-second tick. + const _: () = assert!(TICK_SECONDS >= 30); + const _: () = assert!(TICK_SECONDS <= 3600); + } + + #[test] + fn effective_interval_none_falls_back_to_default() { + // No user choice → 24h default, floored at the provider default. + assert_eq!( + effective_interval_secs(15 * 60, None), + Some(DEFAULT_MEMORY_SYNC_INTERVAL_SECS) + ); + } + + #[test] + fn effective_interval_manual_disables_sync() { + // Some(0) is the "Manual only" sentinel — periodic sync is skipped. + assert_eq!(effective_interval_secs(15 * 60, Some(0)), None); + } + + #[test] + fn effective_interval_override_is_floored_at_provider_default() { + // A user cadence longer than the provider default is honoured as-is. + assert_eq!( + effective_interval_secs(15 * 60, Some(4 * 3600)), + Some(4 * 3600) + ); + // A user cadence shorter than the provider default is clamped up to it + // so we never sync more often than the provider intends. + assert_eq!(effective_interval_secs(30 * 60, Some(60)), Some(30 * 60)); + // Exactly equal stays equal. + assert_eq!(effective_interval_secs(1800, Some(1800)), Some(1800)); + } + + #[test] + fn effective_interval_default_is_floored_at_a_longer_provider_default() { + // If a provider ever defaults to longer than 24h, that wins under None. + let long = DEFAULT_MEMORY_SYNC_INTERVAL_SECS + 3600; + assert_eq!(effective_interval_secs(long, None), Some(long)); + } + + #[test] + fn connection_is_due_compares_elapsed_against_interval() { + let interval = 4 * 3600; + // Never synced this run → always due. + assert!(connection_is_due(interval, None)); + // Synced more recently than the interval → not due. + assert!(!connection_is_due( + interval, + Some(Duration::from_secs(3600)) + )); + // Synced exactly at the interval boundary → due. + assert!(connection_is_due( + interval, + Some(Duration::from_secs(interval)) + )); + // Synced longer ago than the interval → due. + assert!(connection_is_due( + interval, + Some(Duration::from_secs(interval + 1)) + )); + } + + /// Build a minimal Composio `MemorySourceEntry` for the per-source gate + /// tests — only the fields `decide_periodic_source` reads are meaningful. + fn composio_source( + enabled: bool, + max_items: Option, + sync_depth_days: Option, + ) -> MemorySourceEntry { + MemorySourceEntry { + id: "src_test".to_string(), + kind: SourceKind::Composio, + label: "test".to_string(), + enabled, + toolkit: Some("gmail".to_string()), + connection_id: Some("cmp-1".to_string()), + path: None, + glob: None, + url: None, + branch: None, + paths: Vec::new(), + max_commits: None, + max_issues: None, + max_prs: None, + query: None, + since_days: None, + max_items, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days, + } + } + + /// #2831 row 2: a source explicitly toggled **off** must be skipped by the + /// background loop — this is the leak the gate closes. + #[test] + fn decide_periodic_source_skips_disabled_source() { + let src = composio_source(false, Some(100), Some(30)); + assert_eq!( + decide_periodic_source(Some(&src), "gmail"), + PeriodicSourceDecision::Skip + ); + } + + /// An enabled source syncs with exactly its configured caps (no defaulting). + #[test] + fn decide_periodic_source_uses_enabled_source_caps() { + let src = composio_source(true, Some(42), Some(7)); + assert_eq!( + decide_periodic_source(Some(&src), "gmail"), + PeriodicSourceDecision::Sync { + max_items: Some(42), + sync_depth_days: Some(7), + } + ); + } + + /// A connection with no registry row yet (pre-reconcile window) syncs with + /// the conservative per-toolkit defaults — **bounded**, never uncapped, and + /// never skipped. This is the safe-direction fallback for a missing match. + #[test] + fn decide_periodic_source_defaults_caps_when_no_row() { + let (want_items, want_depth) = memory_sync_defaults_for_toolkit("gmail"); + assert_eq!( + decide_periodic_source(None, "gmail"), + PeriodicSourceDecision::Sync { + max_items: want_items, + sync_depth_days: want_depth, + } + ); + // The defaults are bounded for a known toolkit (regression guard against + // an accidental return to uncapped background fetches). + assert!(want_items.is_some()); + } + + /// Multi-account regression (#3443 added multiple account connections per + /// toolkit): two live connections of the *same* toolkit must be gated + /// **independently** by their own per-`connection_id` source rows. This + /// pins the loop's connection-id keying — re-keying the lookup by toolkit + /// would collapse the two accounts and is the regression this guards. + #[test] + fn per_connection_gate_is_independent_across_accounts_of_same_toolkit() { + // gmail account A: enabled with caps; gmail account B: disabled. + let mut a = composio_source(true, Some(10), Some(5)); + a.connection_id = Some("conn-A".to_string()); + a.toolkit = Some("gmail".to_string()); + let mut b = composio_source(false, Some(99), Some(99)); + b.connection_id = Some("conn-B".to_string()); + b.toolkit = Some("gmail".to_string()); + + // Build the same connection_id → entry index the live tick builds. + let index: HashMap = [a, b] + .into_iter() + .filter_map(|s| s.connection_id.clone().map(|id| (id, s))) + .collect(); + + // Account A (enabled) syncs with its own caps... + assert_eq!( + decide_periodic_source(index.get("conn-A"), "gmail"), + PeriodicSourceDecision::Sync { + max_items: Some(10), + sync_depth_days: Some(5), + } + ); + // ...account B (disabled) is skipped, even though it shares the toolkit. + assert_eq!( + decide_periodic_source(index.get("conn-B"), "gmail"), + PeriodicSourceDecision::Skip + ); + // A third, not-yet-registered account of the same toolkit falls back to + // bounded defaults (never skipped, never uncapped). + let (def_items, def_depth) = memory_sync_defaults_for_toolkit("gmail"); + assert_eq!( + decide_periodic_source(index.get("conn-C"), "gmail"), + PeriodicSourceDecision::Sync { + max_items: def_items, + sync_depth_days: def_depth, + } + ); + } + + /// End-to-end simulation of the scheduler's per-connection decision: prove + /// that **changing the global setting changes when the next sync fires** + /// (issue #3302 acceptance criterion). We drive the same two pure helpers + /// the live tick uses (`effective_interval_secs` → `connection_is_due`) + /// across realistic last-sync ages, so no clock or network is needed. + #[test] + fn scheduler_decision_honors_the_global_setting() { + // A chatty provider that natively wants to sync every 15 minutes. + let provider_default = 15 * 60; + + // Helper mirroring the live loop: returns whether the connection would + // fire right now, or `None` for "Manual only" (skipped entirely). + let decide = |global: Option, since: Option| -> Option { + effective_interval_secs(provider_default, global) + .map(|interval| connection_is_due(interval, since)) + }; + + let one_hour_ago = Some(Duration::from_secs(3600)); + let five_hours_ago = Some(Duration::from_secs(5 * 3600)); + + // Baseline (no global override): with only the 15m provider default, a + // connection synced an hour ago is already overdue and WOULD fire. + // (This is the behavior the feature is reining in.) + assert!(connection_is_due(provider_default, one_hour_ago)); + + // User picks "every 4h": now that same hour-old connection must NOT + // fire — the global cadence (not the 15m default) governs the gap… + assert_eq!(decide(Some(4 * 3600), one_hour_ago), Some(false)); + // …but once 5h have passed it fires again. + assert_eq!(decide(Some(4 * 3600), five_hours_ago), Some(true)); + + // User picks "Manual only" (0): never auto-fires, no matter how stale. + assert_eq!(decide(Some(0), five_hours_ago), None); + assert_eq!(decide(Some(0), None), None); + + // Unset (None) → 24h default: the hour-old connection is not yet due, + // confirming the default is far more conservative than the 15m native + // cadence. + assert_eq!(decide(None, one_hour_ago), Some(false)); + assert_eq!( + decide(None, Some(Duration::from_secs(25 * 3600))), + Some(true) + ); + + // A never-synced connection fires on any non-manual setting (the + // restart-recovery path). + assert_eq!(decide(Some(4 * 3600), None), Some(true)); + } + + fn audit_entry( + connection_id: &str, + scope: &str, + success: bool, + ts: DateTime, + ) -> SyncAuditEntry { + SyncAuditEntry { + timestamp: ts, + source_id: connection_id.to_string(), + source_kind: "composio".to_string(), + scope: scope.to_string(), + items_fetched: 1, + batches: 0, + input_tokens: 0, + output_tokens: 0, + estimated_cost_usd: 0.0, + composio_actions_called: 1, + composio_cost_usd: 0.0, + actual_charged_usd: None, + duration_ms: 10, + success, + error: None, + } + } + + #[test] + fn index_last_success_keeps_latest_success_and_ignores_failures() { + let now = Utc::now(); + let older = now - chrono::Duration::hours(6); + let newer = now - chrono::Duration::hours(1); + let entries = vec![ + audit_entry("cmp-1", "gmail:cmp-1", true, older), + audit_entry("cmp-1", "gmail:cmp-1", true, newer), // newer success wins + audit_entry("cmp-1", "gmail:cmp-1", false, now), // failure ignored + audit_entry("cmp-2", "slack:cmp-2", false, now), // only-failure → absent + ]; + let idx = index_last_success_by_connection(&entries); + assert_eq!(idx.get("cmp-1"), Some(&newer)); + assert!( + !idx.contains_key("cmp-2"), + "a connection with only failed syncs is not indexed" + ); + } + + #[test] + fn index_last_success_falls_back_to_source_id_without_scope_suffix() { + let now = Utc::now(); + // A non-composio kind is skipped entirely. + let entries = vec![ + SyncAuditEntry { + source_kind: "github_repo".to_string(), + ..audit_entry("ignored", "github:org/repo", true, now) + }, + // Composio entry whose scope has no ':' → key by source_id. + audit_entry("cmp-3", "noscope", true, now), + ]; + let idx = index_last_success_by_connection(&entries); + assert!(idx.contains_key("cmp-3")); + assert!(!idx.contains_key("ignored")); + } + + #[test] + fn persisted_since_last_sync_computes_and_saturates() { + let now = Utc::now(); + let mut idx = HashMap::new(); + idx.insert("cmp-1".to_string(), now - chrono::Duration::hours(3)); + idx.insert("future".to_string(), now + chrono::Duration::hours(2)); + + let elapsed = persisted_since_last_sync(&idx, "cmp-1", now).unwrap(); + // ~3h, allow a small window for test execution time. + assert!(elapsed >= Duration::from_secs(3 * 3600 - 5)); + assert!(elapsed <= Duration::from_secs(3 * 3600 + 5)); + // Clock skew (future timestamp) saturates to zero, not a huge value. + assert_eq!( + persisted_since_last_sync(&idx, "future", now), + Some(Duration::ZERO) + ); + // Unknown connection → None (treated as never synced). + assert_eq!(persisted_since_last_sync(&idx, "unknown", now), None); + } + + /// The cadence must survive a restart: with the in-memory map cold, the + /// persisted audit timestamp drives the due-check so a connection synced + /// 1h ago does NOT re-fire under a 4h setting, but one synced 5h ago does. + #[test] + fn cadence_survives_restart_via_persisted_audit() { + let now = Utc::now(); + let mut idx = HashMap::new(); + idx.insert("cmp-1".to_string(), now - chrono::Duration::hours(1)); + idx.insert("cmp-2".to_string(), now - chrono::Duration::hours(5)); + + let interval = effective_interval_secs(15 * 60, Some(4 * 3600)).unwrap(); + + // cmp-1 (synced 1h ago) — in-memory cold, persisted fallback says NOT due. + let cmp1 = None.or_else(|| persisted_since_last_sync(&idx, "cmp-1", now)); + assert!(!connection_is_due(interval, cmp1)); + + // cmp-2 (synced 5h ago) — persisted fallback says due. + let cmp2 = None.or_else(|| persisted_since_last_sync(&idx, "cmp-2", now)); + assert!(connection_is_due(interval, cmp2)); + + // A connection with no persisted record still fires (truly fresh). + let fresh = None.or_else(|| persisted_since_last_sync(&idx, "cmp-new", now)); + assert!(connection_is_due(interval, fresh)); + } + + #[test] + fn audit_failure_is_unavailable_and_unknown_cadence_is_skipped() { + let (index, available) = + composio_audit_state(Err(anyhow::anyhow!("simulated audit I/O failure"))); + assert!(index.is_empty()); + assert!(!available); + assert_eq!(cadence_from_audit(None, available, None), None); + + let known = Duration::from_secs(60); + assert_eq!( + cadence_from_audit(Some(known), available, None), + Some(Some(known)) + ); + } + + #[test] + fn readable_empty_audit_preserves_first_sync_behavior() { + let (index, available) = composio_audit_state(Ok(Vec::new())); + assert!(index.is_empty()); + assert!(available); + + let cadence = cadence_from_audit(None, available, None) + .expect("readable empty audit keeps the source eligible"); + assert!(connection_is_due(3600, cadence)); + } + + /// A successful periodic tick produces a Composio-kind audit entry that + /// carries the billable-action tally + cost and zeroes the LLM-cost + /// columns (summarisation happens later in the job worker). Pins the + /// shape the Sync History panel reads (#3111 follow-up). + #[test] + fn periodic_audit_entry_records_composio_cost_on_success() { + let usage = ComposioUsage { + actions_called: 3, + cost_usd: 0.042, + }; + let entry = build_periodic_audit_entry("gmail", "cmp-123", &usage, 17, 1234, None); + + assert_eq!(entry.source_kind, "composio"); + assert_eq!(entry.source_id, "cmp-123"); + assert_eq!(entry.scope, "gmail:cmp-123"); + assert_eq!(entry.items_fetched, 17); + assert_eq!(entry.composio_actions_called, 3); + assert!((entry.composio_cost_usd - 0.042).abs() < f64::EPSILON); + assert!(entry.success); + assert!(entry.error.is_none()); + // Periodic fetch does no summarisation — LLM cost columns stay zero, + // and the Composio spend is the whole combined cost. + assert_eq!(entry.input_tokens, 0); + assert_eq!(entry.estimated_cost_usd, 0.0); + assert!((entry.combined_cost_usd() - 0.042).abs() < f64::EPSILON); + } + + /// A failed periodic tick still records the partial billable cost it + /// incurred before erroring (the fetch may have fired actions), with + /// `success = false` and the error message preserved. + #[test] + fn periodic_audit_entry_preserves_partial_cost_on_failure() { + let usage = ComposioUsage { + actions_called: 1, + cost_usd: 0.01, + }; + let entry = build_periodic_audit_entry( + "notion", + "cmp-9", + &usage, + 0, + 500, + Some("fetch timed out".to_string()), + ); + + assert!(!entry.success); + assert_eq!(entry.error.as_deref(), Some("fetch timed out")); + assert_eq!(entry.items_fetched, 0); + // The billable action it managed to fire before failing is still + // recorded so cost isn't under-reported on failures. + assert_eq!(entry.composio_actions_called, 1); + assert!((entry.composio_cost_usd - 0.01).abs() < f64::EPSILON); + } + + #[test] + fn record_sync_success_stores_timestamp_keyed_by_toolkit_and_connection() { + // Use unique keys so this test doesn't collide with other tests + // writing into the process-wide map. + let toolkit = "test_periodic_toolkit_a"; + let conn = "test-conn-a"; + record_sync_success(toolkit, conn); + let map = last_sync_map(); + let guard = map.lock().expect("lock"); + let ts = guard + .get(&(toolkit.to_string(), conn.to_string())) + .expect("entry recorded"); + // Just-recorded timestamps should be very recent. + assert!(ts.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn record_sync_success_overwrites_previous_timestamp() { + let toolkit = "test_periodic_toolkit_b"; + let conn = "test-conn-b"; + record_sync_success(toolkit, conn); + let first = last_sync_map() + .lock() + .expect("lock") + .get(&(toolkit.to_string(), conn.to_string())) + .copied() + .expect("first entry"); + // Second call must replace (not keep the older) timestamp. + std::thread::sleep(Duration::from_millis(5)); + record_sync_success(toolkit, conn); + let second = last_sync_map() + .lock() + .expect("lock") + .get(&(toolkit.to_string(), conn.to_string())) + .copied() + .expect("second entry"); + assert!( + second >= first, + "record_sync_success should advance the stored Instant" + ); + } + + // The `_guard` below is held deliberately across the `run_one_tick().await` + // in this test: it's a std::sync::Mutex used purely as a test-isolation + // gate around the process-global `OPENHUMAN_WORKSPACE` env var, not an + // async resource lock guarding shared runtime state. Dropping it before + // the await would let a sibling test mutate the env var mid-tick, + // defeating the isolation this guard exists to provide. + #[allow(clippy::await_holding_lock)] + #[tokio::test] + async fn run_one_tick_returns_ok_when_no_client() { + // Isolate the workspace/env so config loading doesn't contend with + // sibling tests mutating OPENHUMAN_WORKSPACE in parallel. + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let tmp = tempdir().expect("tempdir"); + unsafe { + std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); + } + + // With no session stored in the isolated workspace, + // `build_composio_client` returns None and the tick should + // silently skip (returning Ok). This covers the early-return + // path that's otherwise only hit in production. + let inner = tokio::time::timeout(Duration::from_secs(5), run_one_tick()) + .await + .expect("run_one_tick should not hang indefinitely during tests"); + assert!( + inner.is_ok(), + "run_one_tick should return Ok when no client is available: {inner:?}" + ); + + unsafe { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } + + #[tokio::test] + async fn start_periodic_sync_is_idempotent() { + // First call installs the scheduler via the OnceLock; subsequent + // calls must be cheap no-ops without panicking. `tokio::spawn` + // needs an ambient runtime, so this test runs under `tokio::test`. + start_periodic_sync(); + start_periodic_sync(); + assert!(SCHEDULER_STARTED.get().is_some()); + } + + #[test] + fn record_sync_success_distinguishes_connections() { + let toolkit = "test_periodic_toolkit_c"; + record_sync_success(toolkit, "conn-1"); + record_sync_success(toolkit, "conn-2"); + let map = last_sync_map(); + let guard = map.lock().expect("lock"); + assert!(guard + .get(&(toolkit.to_string(), "conn-1".to_string())) + .is_some()); + assert!(guard + .get(&(toolkit.to_string(), "conn-2".to_string())) + .is_some()); + // Unrelated key should be absent. + assert!(guard + .get(&(toolkit.to_string(), "conn-3".to_string())) + .is_none()); + } + + /// In unit tests `scheduler_gate::STATE` is never initialised, so + /// `current_policy()` returns `Policy::Normal` and the helper must + /// return `None` — i.e. the tick is allowed to proceed. This pins the + /// happy-path wiring; an accidental "always pause" regression in the + /// helper would break every `run_one_tick`-driven test that follows it. + /// + /// (The redundant "does-not-short-circuit" tick-level test that was + /// here in the first review pass was dropped per @oxoxDev's + /// [#2825 review](https://github.com/tinyhumansai/openhuman/pull/2825): + /// it duplicated `run_one_tick_returns_ok_when_no_client` because + /// both exited at the same `create_composio_client` no-client branch, + /// so neither actually proved the new gate-check arm fired in the + /// right direction. Asserting log-line absence via `tracing-test` + /// would prove it but adds a new dev-dependency for one assertion — + /// the helper-level test below already pins the wiring.) + #[test] + fn periodic_pause_reason_returns_none_when_gate_not_initialised() { + // Calling without `scheduler_gate::init_global(...)` exercises the + // OnceLock-uninitialised branch in `current_policy`, which is the + // realistic test-environment state. + assert!( + periodic_pause_reason().is_none(), + "expected None (i.e. tick proceeds) when scheduler_gate is in default Normal state, \ + got {:?}", + periodic_pause_reason() + ); + } +} diff --git a/core/src/sync/composio/providers/catalogs.rs b/core/src/sync/composio/providers/catalogs.rs new file mode 100644 index 0000000..dfd0d81 --- /dev/null +++ b/core/src/sync/composio/providers/catalogs.rs @@ -0,0 +1,38 @@ +//! Curated catalogs for Composio toolkits that don't (yet) have a +//! native [`super::ComposioProvider`] implementation. +//! +//! These slices are consulted by [`super::catalog_for_toolkit`] alongside +//! provider-supplied catalogs (gmail, notion, github), so the meta-tool +//! layer applies the same whitelist + scope filtering. +//! +//! Slugs sourced from `https://docs.composio.dev/toolkits/.md` — +//! best-effort. Slugs that don't exist on the backend simply never +//! appear in `composio_list_tools`, so extras are harmless. +//! +//! Data is split into category submodules: +//! - `catalogs_messaging` — Slack, Discord, Telegram, WhatsApp, MS Teams +//! - `catalogs_google` — GoogleCalendar, GoogleDrive, GoogleDocs, GoogleSheets +//! - `catalogs_microsoft` — OneDrive, Excel +//! - `catalogs_productivity` — Outlook, Linear, Jira, Trello, Asana, Dropbox, Todoist +//! - `catalogs_social_media` — Twitter, Spotify, YouTube +//! - `catalogs_business` — Shopify, Stripe, HubSpot, Salesforce, Airtable, Figma + +pub use super::catalogs_business::{ + AIRTABLE_CURATED, FIGMA_CURATED, HUBSPOT_CURATED, SALESFORCE_CURATED, SHOPIFY_CURATED, + STRIPE_CURATED, +}; +pub use super::catalogs_google::{ + GOOGLECALENDAR_CURATED, GOOGLEDOCS_CURATED, GOOGLEDRIVE_CURATED, GOOGLESHEETS_CURATED, +}; +pub use super::catalogs_messaging::{ + DISCORD_CURATED, MICROSOFT_TEAMS_CURATED, SLACK_CURATED, TELEGRAM_CURATED, WHATSAPP_CURATED, +}; +pub use super::catalogs_microsoft::{EXCEL_CURATED, ONE_DRIVE_CURATED}; +pub use super::catalogs_productivity::{ + ASANA_CURATED, DROPBOX_CURATED, JIRA_CURATED, OUTLOOK_CURATED, TODOIST_CURATED, TRELLO_CURATED, +}; +// `LINEAR_CURATED` moved into `super::linear::LINEAR_CURATED` alongside +// the native LinearProvider impl. `catalog_for_toolkit("linear")` now +// routes there directly. Removing the re-export keeps a single source +// of truth and matches how `gmail` / `notion` / `clickup` are wired. +pub use super::catalogs_social_media::{SPOTIFY_CURATED, TWITTER_CURATED, YOUTUBE_CURATED}; diff --git a/core/src/sync/composio/providers/catalogs_business.rs b/core/src/sync/composio/providers/catalogs_business.rs new file mode 100644 index 0000000..0033aec --- /dev/null +++ b/core/src/sync/composio/providers/catalogs_business.rs @@ -0,0 +1,524 @@ +//! Curated catalogs — business toolkits: Shopify, Stripe, HubSpot, +//! Salesforce, Airtable, Figma. + +use super::tool_scope::{CuratedTool, ToolScope}; + +// ── shopify ───────────────────────────────────────────────────────── +pub const SHOPIFY_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "SHOPIFY_BULK_QUERY_OPERATION", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SHOPIFY_COUNT_PRODUCTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SHOPIFY_COUNT_ORDERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SHOPIFY_COUNT_FULFILLMENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SHOPIFY_COUNT_CUSTOMERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SHOPIFY_CREATE_ORDER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_CREATE_PRODUCT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_CREATE_DRAFT_ORDER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_CREATE_FULFILLMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_CREATE_CUSTOMER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_CREATE_PRICE_RULE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_ADJUST_INVENTORY_LEVEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_CREATE_DISCOUNT_CODE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_UPDATE_PRODUCT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_CREATE_CUSTOM_COLLECTION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SHOPIFY_CANCEL_ORDER", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SHOPIFY_CANCEL_FULFILLMENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SHOPIFY_DELETE_PRODUCT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SHOPIFY_BULK_DELETE_CUSTOMER_ADDRESSES", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SHOPIFY_BULK_DELETE_METAFIELDS", + scope: ToolScope::Admin, + }, +]; + +// ── stripe ────────────────────────────────────────────────────────── +pub const STRIPE_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "STRIPE_GET_PAYMENT_INTENT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "STRIPE_LIST_INVOICES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "STRIPE_GET_CUSTOMER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "STRIPE_LIST_CHARGES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "STRIPE_GET_SUBSCRIPTION", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "STRIPE_CREATE_PAYMENT_INTENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "STRIPE_CREATE_INVOICE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "STRIPE_CREATE_CUSTOMER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "STRIPE_CREATE_CUSTOMER_SUBSCRIPTION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "STRIPE_CREATE_CHECKOUT_SESSION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "STRIPE_CONFIRM_PAYMENT_INTENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "STRIPE_CAPTURE_PAYMENT_INTENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "STRIPE_ATTACH_PAYMENT_METHOD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "STRIPE_CANCEL_SUBSCRIPTION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "STRIPE_CANCEL_PAYMENT_INTENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "STRIPE_CREATE_CHARGE_REFUND", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "STRIPE_CLOSE_DISPUTE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "STRIPE_CANCEL_SETUP_INTENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "STRIPE_ARCHIVE_BILLING_ALERT", + scope: ToolScope::Admin, + }, +]; + +// ── hubspot ───────────────────────────────────────────────────────── +pub const HUBSPOT_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "HUBSPOT_GET_CONTACTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "HUBSPOT_SEARCH_CONTACTS_BY_CRITERIA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "HUBSPOT_LIST_CONTACTS_PAGE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "HUBSPOT_GET_COMPANIES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "HUBSPOT_GET_DEALS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "HUBSPOT_GET_CRM_OBJECT_BY_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "HUBSPOT_BATCH_READ_COMPANIES_BY_PROPERTIES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "HUBSPOT_CREATE_CONTACT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_CREATE_COMPANY", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_CREATE_DEAL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_CREATE_CONTACTS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_UPDATE_CONTACT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_UPDATE_COMPANY", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_CREATE_OBJECT_ASSOCIATION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_CREATE_A_NEW_MARKETING_EMAIL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_CREATE_BATCH_OF_OBJECTS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_BATCH_UPDATE_QUOTES", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "HUBSPOT_ARCHIVE_CONTACT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "HUBSPOT_ARCHIVE_COMPANY", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "HUBSPOT_ARCHIVE_DEAL", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "HUBSPOT_ARCHIVE_CONTACTS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "HUBSPOT_ARCHIVE_COMPANIES", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "HUBSPOT_ARCHIVE_DEALS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "HUBSPOT_ARCHIVE_CRM_OBJECT_BY_ID", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "HUBSPOT_ARCHIVE_PROPERTY_BY_OBJECT_TYPE_AND_NAME", + scope: ToolScope::Admin, + }, +]; + +// ── salesforce ────────────────────────────────────────────────────── +pub const SALESFORCE_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "SALESFORCE_RUN_SOQL_QUERY", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SALESFORCE_EXECUTE_SOSL_SEARCH", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SALESFORCE_GET_ACCOUNT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SALESFORCE_GET_CAMPAIGN", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SALESFORCE_GET_ALL_FIELDS_FOR_OBJECT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SALESFORCE_GET_ALL_CUSTOM_OBJECTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SALESFORCE_CREATE_ACCOUNT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_CREATE_CONTACT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_CREATE_LEAD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_CREATE_OPPORTUNITY", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_CREATE_CAMPAIGN", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_CREATE_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_UPDATE_ACCOUNT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_UPDATE_CONTACT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_UPDATE_OPPORTUNITY", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_ADD_OPPORTUNITY_LINE_ITEM", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_ADD_CONTACT_TO_CAMPAIGN", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_ADD_LEAD_TO_CAMPAIGN", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_ASSOCIATE_CONTACT_TO_ACCOUNT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_CLONE_OPPORTUNITY_WITH_PRODUCTS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SALESFORCE_DELETE_ACCOUNT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SALESFORCE_DELETE_CONTACT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SALESFORCE_DELETE_LEAD", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SALESFORCE_DELETE_OPPORTUNITY", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SALESFORCE_DELETE_CAMPAIGN", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SALESFORCE_DELETE_SOBJECT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SALESFORCE_DELETE_SOBJECT_COLLECTIONS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SALESFORCE_CREATE_CUSTOM_FIELD", + scope: ToolScope::Admin, + }, +]; + +// ── airtable ──────────────────────────────────────────────────────── +pub const AIRTABLE_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "AIRTABLE_LIST_RECORDS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "AIRTABLE_GET_RECORD", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "AIRTABLE_GET_BASE_SCHEMA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "AIRTABLE_LIST_BASES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "AIRTABLE_LIST_COMMENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "AIRTABLE_CREATE_RECORDS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_UPDATE_RECORD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_UPDATE_MULTIPLE_RECORDS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_CREATE_FIELD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_CREATE_TABLE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_CREATE_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_UPLOAD_ATTACHMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_UPDATE_FIELD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_UPDATE_TABLE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "AIRTABLE_DELETE_RECORD", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "AIRTABLE_DELETE_MULTIPLE_RECORDS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "AIRTABLE_DELETE_COMMENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "AIRTABLE_CREATE_BASE", + scope: ToolScope::Admin, + }, +]; + +// ── figma ─────────────────────────────────────────────────────────── +pub const FIGMA_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "FIGMA_GET_FILE_JSON", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "FIGMA_GET_FILE_NODES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "FIGMA_GET_COMMENTS_IN_A_FILE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "FIGMA_GET_CURRENT_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "FIGMA_DISCOVER_FIGMA_RESOURCES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "FIGMA_GET_FILE_COMPONENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "FIGMA_GET_LOCAL_VARIABLES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "FIGMA_EXTRACT_DESIGN_TOKENS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "FIGMA_ADD_A_COMMENT_TO_A_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "FIGMA_CREATE_DEV_RESOURCES", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "FIGMA_CREATE_MODIFY_DELETE_VARIABLES", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "FIGMA_DELETE_A_COMMENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "FIGMA_DELETE_A_WEBHOOK", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "FIGMA_DELETE_DEV_RESOURCE", + scope: ToolScope::Admin, + }, +]; diff --git a/core/src/sync/composio/providers/catalogs_google.rs b/core/src/sync/composio/providers/catalogs_google.rs new file mode 100644 index 0000000..be25c83 --- /dev/null +++ b/core/src/sync/composio/providers/catalogs_google.rs @@ -0,0 +1,360 @@ +//! Curated catalogs — Google toolkits: GoogleCalendar, GoogleDrive, +//! GoogleDocs, GoogleSheets. + +use super::tool_scope::{CuratedTool, ToolScope}; + +// ── googlecalendar ────────────────────────────────────────────────── +pub const GOOGLECALENDAR_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "GOOGLECALENDAR_EVENTS_LIST", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLECALENDAR_FIND_EVENT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLECALENDAR_LIST_CALENDARS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLECALENDAR_EVENTS_GET", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLECALENDAR_FIND_FREE_SLOTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLECALENDAR_GET_CALENDAR", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLECALENDAR_EVENTS_LIST_ALL_CALENDARS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLECALENDAR_CREATE_EVENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLECALENDAR_UPDATE_EVENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLECALENDAR_PATCH_EVENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLECALENDAR_QUICK_ADD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLECALENDAR_EVENTS_MOVE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLECALENDAR_REMOVE_ATTENDEE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLECALENDAR_EVENTS_IMPORT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLECALENDAR_DELETE_EVENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLECALENDAR_CLEAR_CALENDAR", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLECALENDAR_CALENDARS_DELETE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLECALENDAR_DUPLICATE_CALENDAR", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLECALENDAR_PATCH_CALENDAR", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLECALENDAR_ACL_INSERT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLECALENDAR_ACL_DELETE", + scope: ToolScope::Admin, + }, +]; + +// ── googledrive ───────────────────────────────────────────────────── +pub const GOOGLEDRIVE_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "GOOGLEDRIVE_FIND_FILE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDRIVE_LIST_FILES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDRIVE_GET_FILE_METADATA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDRIVE_DOWNLOAD_FILE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDRIVE_LIST_PERMISSIONS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDRIVE_FIND_FOLDER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDRIVE_GET_ABOUT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDRIVE_CREATE_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDRIVE_CREATE_FOLDER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDRIVE_UPLOAD_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDRIVE_CREATE_FILE_FROM_TEXT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDRIVE_COPY_FILE_ADVANCED", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDRIVE_MOVE_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDRIVE_EDIT_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDRIVE_RENAME_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDRIVE_CREATE_PERMISSION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDRIVE_DELETE_PERMISSION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDRIVE_UPDATE_PERMISSION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDRIVE_DELETE_FILE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDRIVE_GOOGLE_DRIVE_DELETE_FOLDER_OR_FILE_ACTION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDRIVE_EMPTY_TRASH", + scope: ToolScope::Admin, + }, +]; + +// ── googledocs ────────────────────────────────────────────────────── +pub const GOOGLEDOCS_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "GOOGLEDOCS_GET_DOCUMENT_BY_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDOCS_SEARCH_DOCUMENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLEDOCS_CREATE_DOCUMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_CREATE_DOCUMENT_MARKDOWN", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_INSERT_TEXT_ACTION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_INSERT_TABLE_ACTION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_INSERT_INLINE_IMAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_UPDATE_EXISTING_DOCUMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_UPDATE_DOCUMENT_MARKDOWN", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_UPDATE_DOCUMENT_SECTION_MARKDOWN", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_REPLACE_ALL_TEXT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_COPY_DOCUMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_CREATE_HEADER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_CREATE_FOOTER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLEDOCS_DELETE_CONTENT_RANGE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDOCS_DELETE_HEADER", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDOCS_DELETE_FOOTER", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDOCS_DELETE_NAMED_RANGE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDOCS_DELETE_TABLE_ROW", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLEDOCS_DELETE_TABLE_COLUMN", + scope: ToolScope::Admin, + }, +]; + +// ── googlesheets ──────────────────────────────────────────────────── +pub const GOOGLESHEETS_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "GOOGLESHEETS_BATCH_GET", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLESHEETS_VALUES_GET", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLESHEETS_LOOKUP_SPREADSHEET_ROW", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLESHEETS_GET_SPREADSHEET_INFO", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLESHEETS_GET_SHEET_NAMES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLESHEETS_SEARCH_SPREADSHEETS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GOOGLESHEETS_VALUES_UPDATE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_UPDATE_VALUES_BATCH", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_SPREADSHEETS_VALUES_APPEND", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_UPSERT_ROWS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_CREATE_GOOGLE_SHEET1", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_ADD_SHEET", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_CREATE_SPREADSHEET_ROW", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_CREATE_SPREADSHEET_COLUMN", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_FIND_REPLACE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_FORMAT_CELL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_SET_DATA_VALIDATION_RULE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GOOGLESHEETS_SPREADSHEETS_VALUES_BATCH_CLEAR", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLESHEETS_DELETE_SHEET", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLESHEETS_DELETE_DIMENSION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLESHEETS_UPDATE_SHEET_PROPERTIES", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GOOGLESHEETS_UPDATE_SPREADSHEET_PROPERTIES", + scope: ToolScope::Admin, + }, +]; diff --git a/core/src/sync/composio/providers/catalogs_messaging.rs b/core/src/sync/composio/providers/catalogs_messaging.rs new file mode 100644 index 0000000..33d2975 --- /dev/null +++ b/core/src/sync/composio/providers/catalogs_messaging.rs @@ -0,0 +1,386 @@ +//! Curated catalogs — messaging toolkits: Slack, Discord, Telegram, +//! WhatsApp, Microsoft Teams. + +use super::tool_scope::{CuratedTool, ToolScope}; + +// ── slack ─────────────────────────────────────────────────────────── +pub const SLACK_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "SLACK_FIND_CHANNELS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_FIND_USERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_FETCH_CONVERSATION_HISTORY", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_FETCH_MESSAGE_THREAD_FROM_A_CONVERSATION", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_LIST_ALL_CHANNELS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_LIST_ALL_USERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_LIST_CONVERSATIONS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_FETCH_TEAM_INFO", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_GET_USER_PRESENCE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_ASSISTANT_SEARCH_CONTEXT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SLACK_SEND_MESSAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_POST_MESSAGE_TO_CHANNEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_SEND_MESSAGE_TO_CHANNEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_CREATE_CHANNEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_INVITE_USERS_TO_A_SLACK_CHANNEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_ADD_REACTION_TO_AN_ITEM", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_UPLOAD_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_CREATE_A_REMINDER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_CREATE_USER_GROUP", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SLACK_DELETE_CHANNEL", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SLACK_ARCHIVE_CONVERSATION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SLACK_DELETE_FILE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SLACK_DELETES_A_MESSAGE_FROM_A_CHAT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SLACK_DELETE_REMINDER", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SLACK_LEAVE_CONVERSATION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SLACK_INVITE_USER_TO_WORKSPACE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SLACK_CONVERT_CHANNEL_TO_PRIVATE", + scope: ToolScope::Admin, + }, +]; + +// ── discord ───────────────────────────────────────────────────────── +pub const DISCORD_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "DISCORD_GET_MY_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DISCORD_GET_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DISCORD_LIST_MY_GUILDS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DISCORD_GET_MY_GUILD_MEMBER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DISCORD_INVITE_RESOLVE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DISCORD_GET_GUILD_WIDGET", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DISCORD_LIST_MY_CONNECTIONS", + scope: ToolScope::Read, + }, + // NOTE: guild-channel and channel-message actions are intentionally NOT + // listed here. Composio's `discord` toolkit is OAuth2 / user-scoped and does + // not expose them (its full action set is user/account-scoped: my user, my + // guilds, my member, invites, …). `DISCORD_LIST_GUILD_CHANNELS`, + // `DISCORD_GET_CHANNEL`, `DISCORD_SEND_MESSAGE`, and `DISCORD_CREATE_MESSAGE` + // were whitelisted here (#3085/#3144) but no such slugs exist on this + // toolkit, so Composio never returned them — the whitelist entries were + // inert and misleadingly implied channel access was possible over OAuth. + // Guild-channel / message reads live in Composio's SEPARATE `discordbot` + // toolkit (bot-token auth, `DISCORDBOT_*` slugs, e.g. + // `DISCORDBOT_FETCH_MESSAGES_FROM_CHANNEL`). Those pass the visibility + // filter via `classify_unknown` once a `discordbot` connection exists; do + // NOT hand-list `DISCORDBOT_*` slugs here from guesses — a wrong slug makes + // `find_curated` drop the real tool (worse than the pass-through default). +]; + +// ── telegram ──────────────────────────────────────────────────────── +pub const TELEGRAM_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "TELEGRAM_GET_UPDATES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TELEGRAM_GET_CHAT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TELEGRAM_GET_CHAT_HISTORY", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TELEGRAM_GET_CHAT_MEMBER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TELEGRAM_GET_CHAT_MEMBERS_COUNT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TELEGRAM_GET_CHAT_ADMINISTRATORS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TELEGRAM_GET_ME", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TELEGRAM_SEND_MESSAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TELEGRAM_SEND_PHOTO", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TELEGRAM_SEND_DOCUMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TELEGRAM_SEND_LOCATION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TELEGRAM_SEND_POLL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TELEGRAM_FORWARD_MESSAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TELEGRAM_EDIT_MESSAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TELEGRAM_ANSWER_CALLBACK_QUERY", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TELEGRAM_DELETE_MESSAGE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TELEGRAM_CREATE_CHAT_INVITE_LINK", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TELEGRAM_SET_MY_COMMANDS", + scope: ToolScope::Admin, + }, +]; + +// ── whatsapp ──────────────────────────────────────────────────────── +pub const WHATSAPP_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "WHATSAPP_GET_PHONE_NUMBERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "WHATSAPP_GET_MESSAGE_TEMPLATES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "WHATSAPP_GET_PHONE_NUMBER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "WHATSAPP_GET_BUSINESS_PROFILE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "WHATSAPP_GET_TEMPLATE_STATUS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "WHATSAPP_GET_MEDIA_INFO", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "WHATSAPP_SEND_MESSAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "WHATSAPP_SEND_TEMPLATE_MESSAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "WHATSAPP_SEND_MEDIA", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "WHATSAPP_SEND_MEDIA_BY_ID", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "WHATSAPP_SEND_INTERACTIVE_BUTTONS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "WHATSAPP_SEND_INTERACTIVE_LIST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "WHATSAPP_UPLOAD_MEDIA", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "WHATSAPP_CREATE_MESSAGE_TEMPLATE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "WHATSAPP_DELETE_MESSAGE_TEMPLATE", + scope: ToolScope::Admin, + }, +]; + +// ── microsoft_teams ───────────────────────────────────────────────── +pub const MICROSOFT_TEAMS_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "MICROSOFT_TEAMS_GET_CHAT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_GET_CHANNEL", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_GET_TEAM_FROM_GROUP", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_CHATS_GET_ALL_CHATS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_GET_PRESENCE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_GET_ONLINE_MEETING", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_GET_SCHEDULE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_CREATE_CHANNEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_CREATE_TEAM", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_CREATE_MEETING", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_ADD_TEAM_MEMBER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_ADD_CHAT_MEMBER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_CREATE_SHIFT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_CREATE_TIME_OFF_REQUEST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_DELETE_TEAM", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_DELETE_CHANNEL", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_ARCHIVE_TEAM", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_ARCHIVE_CHANNEL", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_DELETE_TAB", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "MICROSOFT_TEAMS_DELETE_TIME_OFF", + scope: ToolScope::Admin, + }, +]; diff --git a/core/src/sync/composio/providers/catalogs_microsoft.rs b/core/src/sync/composio/providers/catalogs_microsoft.rs new file mode 100644 index 0000000..cd54ede --- /dev/null +++ b/core/src/sync/composio/providers/catalogs_microsoft.rs @@ -0,0 +1,207 @@ +//! Curated catalogs — Microsoft personal-productivity toolkits: +//! OneDrive (files) and Excel (spreadsheets). +//! +//! These toolkits are catalog-only: they don't ship a native +//! [`super::ComposioProvider`] implementation, so they have no +//! user-profile fetch, no initial/periodic sync, no trigger webhooks, +//! and no memory ingestion. Connecting them via the UI lets the agent +//! invoke the listed actions through Composio's API, but their data +//! is not pre-ingested into OpenHuman's memory tree. +//! +//! Action slugs are sourced best-effort from +//! `https://docs.composio.dev/toolkits/.md`. Slugs that don't +//! exist on the backend simply never appear in `composio_list_tools`, +//! so over-shooting is harmless. + +use super::tool_scope::{CuratedTool, ToolScope}; + +// ── onedrive ──────────────────────────────────────────────────────── +pub const ONE_DRIVE_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "ONE_DRIVE_GET_FILE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ONE_DRIVE_GET_FILE_METADATA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ONE_DRIVE_LIST_FILES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ONE_DRIVE_LIST_CHILDREN", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ONE_DRIVE_SEARCH_FILES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ONE_DRIVE_DOWNLOAD_FILE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ONE_DRIVE_GET_DRIVE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ONE_DRIVE_UPLOAD_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ONE_DRIVE_CREATE_FOLDER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ONE_DRIVE_COPY_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ONE_DRIVE_MOVE_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ONE_DRIVE_UPDATE_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ONE_DRIVE_CREATE_SHARE_LINK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ONE_DRIVE_DELETE_FILE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "ONE_DRIVE_DELETE_FOLDER", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "ONE_DRIVE_RESTORE_FILE", + scope: ToolScope::Admin, + }, +]; + +// ── excel ─────────────────────────────────────────────────────────── +pub const EXCEL_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "EXCEL_GET_WORKBOOK", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "EXCEL_LIST_WORKSHEETS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "EXCEL_GET_WORKSHEET", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "EXCEL_GET_RANGE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "EXCEL_GET_USED_RANGE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "EXCEL_LIST_TABLES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "EXCEL_GET_TABLE_ROWS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "EXCEL_CREATE_WORKSHEET", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "EXCEL_UPDATE_RANGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "EXCEL_APPEND_ROWS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "EXCEL_INSERT_TABLE_ROW", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "EXCEL_UPDATE_TABLE_ROW", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "EXCEL_CREATE_TABLE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "EXCEL_FORMAT_RANGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "EXCEL_DELETE_WORKSHEET", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "EXCEL_DELETE_TABLE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "EXCEL_DELETE_TABLE_ROW", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "EXCEL_CLEAR_RANGE", + scope: ToolScope::Admin, + }, +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn one_drive_catalog_is_non_empty_and_unique() { + assert!(!ONE_DRIVE_CURATED.is_empty()); + let mut slugs: Vec<&'static str> = ONE_DRIVE_CURATED.iter().map(|t| t.slug).collect(); + slugs.sort_unstable(); + slugs.dedup(); + assert_eq!(slugs.len(), ONE_DRIVE_CURATED.len()); + for tool in ONE_DRIVE_CURATED { + assert!(tool.slug.starts_with("ONE_DRIVE_")); + } + } + + #[test] + fn excel_catalog_is_non_empty_and_unique() { + assert!(!EXCEL_CURATED.is_empty()); + let mut slugs: Vec<&'static str> = EXCEL_CURATED.iter().map(|t| t.slug).collect(); + slugs.sort_unstable(); + slugs.dedup(); + assert_eq!(slugs.len(), EXCEL_CURATED.len()); + for tool in EXCEL_CURATED { + assert!(tool.slug.starts_with("EXCEL_")); + } + } + + #[test] + fn one_drive_catalog_covers_all_three_scopes() { + assert!(ONE_DRIVE_CURATED.iter().any(|t| t.scope == ToolScope::Read)); + assert!(ONE_DRIVE_CURATED + .iter() + .any(|t| t.scope == ToolScope::Write)); + assert!(ONE_DRIVE_CURATED + .iter() + .any(|t| t.scope == ToolScope::Admin)); + } + + #[test] + fn excel_catalog_covers_all_three_scopes() { + assert!(EXCEL_CURATED.iter().any(|t| t.scope == ToolScope::Read)); + assert!(EXCEL_CURATED.iter().any(|t| t.scope == ToolScope::Write)); + assert!(EXCEL_CURATED.iter().any(|t| t.scope == ToolScope::Admin)); + } +} diff --git a/core/src/sync/composio/providers/catalogs_productivity.rs b/core/src/sync/composio/providers/catalogs_productivity.rs new file mode 100644 index 0000000..424efde --- /dev/null +++ b/core/src/sync/composio/providers/catalogs_productivity.rs @@ -0,0 +1,600 @@ +//! Curated catalogs — productivity toolkits: Outlook, Linear, Jira, +//! Trello, Asana, Dropbox, Todoist. +//! +//! Catalog-only toolkits (Linear, Jira, Trello, Asana, Dropbox, +//! Todoist) don't ship a native [`super::ComposioProvider`] — they +//! have no user-profile fetch, no initial/periodic sync, no trigger +//! webhooks, and no memory ingestion. The agent invokes their actions +//! through Composio's API, but their data is not pre-ingested into +//! OpenHuman's memory tree. + +use super::tool_scope::{CuratedTool, ToolScope}; + +// ── outlook ───────────────────────────────────────────────────────── +pub const OUTLOOK_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "OUTLOOK_GET_MESSAGE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "OUTLOOK_LIST_MESSAGES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "OUTLOOK_SEARCH_MESSAGES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "OUTLOOK_LIST_CALENDARS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "OUTLOOK_LIST_CALENDAR_EVENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "OUTLOOK_GET_CALENDAR_EVENT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "OUTLOOK_LIST_CONTACTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "OUTLOOK_LIST_MAIL_FOLDERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "OUTLOOK_SEND_EMAIL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "OUTLOOK_CREATE_DRAFT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "OUTLOOK_SEND_DRAFT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "OUTLOOK_CREATE_DRAFT_REPLY", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "OUTLOOK_CREATE_ME_FORWARD_DRAFT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "OUTLOOK_CALENDAR_CREATE_EVENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "OUTLOOK_CREATE_CONTACT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "OUTLOOK_CREATE_MAIL_FOLDER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "OUTLOOK_DELETE_MESSAGE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "OUTLOOK_BATCH_MOVE_MESSAGES", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "OUTLOOK_BATCH_UPDATE_MESSAGES", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "OUTLOOK_ACCEPT_EVENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "OUTLOOK_CANCEL_EVENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "OUTLOOK_CREATE_ME_CALENDAR_PERMISSION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "OUTLOOK_CREATE_EMAIL_RULE", + scope: ToolScope::Admin, + }, +]; + +// ── linear ────────────────────────────────────────────────────────── +// +// `LINEAR_CURATED` lives in `super::linear::tools` alongside the native +// `LinearProvider` impl (per-issue #2400). `catalog_for_toolkit("linear")` +// in `super::mod` routes through that constant directly. Removing the +// catalog-only declaration here keeps a single source of truth and +// matches how `gmail` / `notion` / `clickup` are wired. + +// ── jira ──────────────────────────────────────────────────────────── +pub const JIRA_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "JIRA_GET_ISSUE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_GET_ALL_PROJECTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_FETCH_BULK_ISSUES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_GET_ISSUE_TYPES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_GET_PROJECT_ROLES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_FIND_USERS2", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_GET_FIELDS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_GET_ISSUE_EDIT_METADATA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_GET_PROJECT_VERSIONS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "JIRA_CREATE_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_BULK_CREATE_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_EDIT_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_ADD_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_ASSIGN_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_ADD_ATTACHMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_CREATE_ISSUE_LINK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_ADD_WORKLOG", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_TRANSITION_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "JIRA_DELETE_ISSUE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "JIRA_DELETE_COMMENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "JIRA_DELETE_VERSION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "JIRA_DELETE_WORKLOG", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "JIRA_CREATE_PROJECT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "JIRA_ADD_USERS_TO_PROJECT_ROLE", + scope: ToolScope::Admin, + }, +]; + +// ── trello ────────────────────────────────────────────────────────── +pub const TRELLO_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "TRELLO_GET_BOARDS_BY_ID_BOARD", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TRELLO_GET_ACTIONS_BY_ID_ACTION", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TRELLO_GET_BATCH", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TRELLO_GET_BOARDS_ACTIONS_BY_ID_BOARD", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TRELLO_GET_MEMBERS_BOARDS_BY_ID_MEMBER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TRELLO_ADD_CARDS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_ADD_BOARDS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_ADD_LISTS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_ADD_CARDS_ACTIONS_COMMENTS_BY_ID_CARD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_ADD_MEMBER_TO_CARD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_CREATE_CARD_LABEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_ADD_CARDS_ATTACHMENTS_BY_ID_CARD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_ADD_CARDS_CHECKLISTS_BY_ID_CARD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_CREATE_WEBHOOK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TRELLO_DELETE_CARDS_BY_ID_CARD", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TRELLO_DELETE_BOARD", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TRELLO_DELETE_CHECKLISTS_BY_ID_CHECKLIST", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TRELLO_ARCHIVE_ALL_LIST_CARDS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TRELLO_DELETE_CARD_COMMENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TRELLO_DELETE_LABELS_BY_ID_LABEL", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TRELLO_DELETE_ORGANIZATIONS_BY_ID_ORG", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TRELLO_DELETE_WEBHOOKS_BY_ID_WEBHOOK", + scope: ToolScope::Admin, + }, +]; + +// ── asana ─────────────────────────────────────────────────────────── +pub const ASANA_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "ASANA_GET_A_TASK", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_GET_A_PROJECT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_GET_MULTIPLE_TASKS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_GET_MULTIPLE_PROJECTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_GET_CURRENT_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_GET_MULTIPLE_WORKSPACES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_GET_PORTFOLIO", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_GET_GOALS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_GET_CUSTOM_FIELDS_FOR_WORKSPACE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "ASANA_CREATE_A_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_CREATE_A_PROJECT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_CREATE_SUBTASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_CREATE_TASK_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_UPDATE_A_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_ADD_FOLLOWERS_TO_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_ADD_TAG_TO_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_ADD_PROJECT_FOR_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_ADD_TASK_DEPENDENCIES", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_CREATE_ATTACHMENT_FOR_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "ASANA_DELETE_TASK", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "ASANA_DELETE_PROJECT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "ASANA_DELETE_SECTION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "ASANA_DELETE_TAG", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "ASANA_DELETE_CUSTOM_FIELD", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "ASANA_DELETE_MEMBERSHIP", + scope: ToolScope::Admin, + }, +]; + +// ── dropbox ───────────────────────────────────────────────────────── +pub const DROPBOX_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "DROPBOX_GET_METADATA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DROPBOX_FILES_SEARCH", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DROPBOX_LIST_FILE_MEMBERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DROPBOX_GET_SHARED_LINK_METADATA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DROPBOX_GET_ABOUT_ME", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DROPBOX_GET_SPACE_USAGE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "DROPBOX_ALPHA_UPLOAD_FILE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "DROPBOX_CREATE_FOLDER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "DROPBOX_COPY_FILE_OR_FOLDER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "DROPBOX_CREATE_SHARED_LINK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "DROPBOX_ADD_FILE_MEMBER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "DROPBOX_DELETE_FILE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "DROPBOX_DELETE_BATCH", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "DROPBOX_ADD_TEAM_MEMBERS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "DROPBOX_CREATE_TEAM_FOLDER", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "DROPBOX_ARCHIVE_TEAM_FOLDER", + scope: ToolScope::Admin, + }, +]; + +// ── todoist ───────────────────────────────────────────────────────── +pub const TODOIST_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "TODOIST_GET_TASK", + scope: ToolScope::Read, + }, + CuratedTool { + // Composio's catalog has no `TODOIST_GET_ACTIVE_TASKS`; the real + // incomplete-tasks slug is `TODOIST_GET_ALL_TASKS` (docs.composio.dev/ + // toolkits/todoist). The old slug was rejected as an unknown action. + slug: "TODOIST_GET_ALL_TASKS", + scope: ToolScope::Read, + }, + CuratedTool { + // Real completed-tasks slug; `TODOIST_GET_COMPLETED_TASKS` does not + // exist in Composio's catalog. + slug: "TODOIST_LIST_COMPLETED_TASKS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TODOIST_GET_PROJECTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TODOIST_GET_PROJECT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TODOIST_GET_SECTIONS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TODOIST_GET_LABELS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TODOIST_GET_COMMENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TODOIST_CREATE_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_UPDATE_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_CLOSE_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_REOPEN_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_CREATE_PROJECT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_UPDATE_PROJECT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_CREATE_SECTION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_CREATE_LABEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_CREATE_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TODOIST_DELETE_TASK", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TODOIST_DELETE_PROJECT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TODOIST_DELETE_SECTION", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TODOIST_DELETE_LABEL", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TODOIST_DELETE_COMMENT", + scope: ToolScope::Admin, + }, +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn todoist_catalog_is_non_empty_and_unique() { + assert!(!TODOIST_CURATED.is_empty()); + let mut slugs: Vec<&'static str> = TODOIST_CURATED.iter().map(|t| t.slug).collect(); + slugs.sort_unstable(); + slugs.dedup(); + assert_eq!(slugs.len(), TODOIST_CURATED.len()); + for tool in TODOIST_CURATED { + assert!(tool.slug.starts_with("TODOIST_")); + } + } + + #[test] + fn todoist_catalog_covers_all_three_scopes() { + assert!(TODOIST_CURATED.iter().any(|t| t.scope == ToolScope::Read)); + assert!(TODOIST_CURATED.iter().any(|t| t.scope == ToolScope::Write)); + assert!(TODOIST_CURATED.iter().any(|t| t.scope == ToolScope::Admin)); + } +} diff --git a/core/src/sync/composio/providers/catalogs_social_media.rs b/core/src/sync/composio/providers/catalogs_social_media.rs new file mode 100644 index 0000000..a05c2b4 --- /dev/null +++ b/core/src/sync/composio/providers/catalogs_social_media.rs @@ -0,0 +1,248 @@ +//! Curated catalogs — social media / entertainment toolkits: Twitter, +//! Spotify, YouTube. + +use super::tool_scope::{CuratedTool, ToolScope}; + +// ── twitter ───────────────────────────────────────────────────────── +pub const TWITTER_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "TWITTER_RECENT_SEARCH", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TWITTER_GET_USER_BY_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TWITTER_POST_LOOKUP_BY_POST_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TWITTER_FOLLOWERS_BY_USER_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TWITTER_FOLLOWING_BY_USER_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TWITTER_BOOKMARKS_BY_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TWITTER_GET_LIST_MEMBERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TWITTER_FULL_ARCHIVE_SEARCH", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "TWITTER_CREATION_OF_A_POST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TWITTER_RETWEET_POST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TWITTER_ADD_POST_TO_BOOKMARKS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TWITTER_FOLLOW_USER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TWITTER_MUTE_USER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TWITTER_CREATE_DM_CONVERSATION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TWITTER_CREATE_LIST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TWITTER_ADD_LIST_MEMBER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "TWITTER_POST_DELETE_BY_POST_ID", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TWITTER_DELETE_LIST", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TWITTER_REMOVE_LIST_MEMBER", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TWITTER_DELETE_DM", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "TWITTER_REMOVE_POST_FROM_BOOKMARKS", + scope: ToolScope::Admin, + }, +]; + +// ── spotify ───────────────────────────────────────────────────────── +pub const SPOTIFY_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "SPOTIFY_GET_CURRENT_USER_S_PROFILE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SPOTIFY_GET_USER_S_TOP_TRACKS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SPOTIFY_GET_PLAYLIST", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SPOTIFY_GET_PLAYLIST_ITEMS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SPOTIFY_GET_RECENTLY_PLAYED_TRACKS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SPOTIFY_GET_USER_S_SAVED_TRACKS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SPOTIFY_SEARCH_FOR_ITEM", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SPOTIFY_GET_AVAILABLE_DEVICES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "SPOTIFY_ADD_ITEMS_TO_PLAYLIST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SPOTIFY_CREATE_PLAYLIST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SPOTIFY_SAVE_TRACKS_FOR_CURRENT_USER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SPOTIFY_PAUSE_PLAYBACK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SPOTIFY_ADD_ITEM_TO_PLAYBACK_QUEUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SPOTIFY_CHANGE_PLAYLIST_DETAILS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "SPOTIFY_REMOVE_PLAYLIST_ITEMS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SPOTIFY_REMOVE_USER_S_SAVED_TRACKS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SPOTIFY_UNFOLLOW_ARTISTS_OR_USERS", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "SPOTIFY_REMOVE_USER_S_SAVED_ALBUMS", + scope: ToolScope::Admin, + }, +]; + +// ── youtube ───────────────────────────────────────────────────────── +pub const YOUTUBE_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "YOUTUBE_SEARCH_YOU_TUBE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "YOUTUBE_LIST_CHANNEL_VIDEOS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "YOUTUBE_GET_CHANNEL_STATISTICS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "YOUTUBE_LIST_COMMENT_THREADS2", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "YOUTUBE_LIST_COMMENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "YOUTUBE_GET_VIDEO_DETAILS_BATCH", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "YOUTUBE_LIST_USER_PLAYLISTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "YOUTUBE_LIST_PLAYLIST_ITEMS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "YOUTUBE_UPLOAD_VIDEO", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "YOUTUBE_UPDATE_VIDEO", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "YOUTUBE_CREATE_PLAYLIST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "YOUTUBE_ADD_VIDEO_TO_PLAYLIST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "YOUTUBE_POST_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "YOUTUBE_RATE_VIDEO", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "YOUTUBE_UPDATE_PLAYLIST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "YOUTUBE_DELETE_VIDEO", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "YOUTUBE_DELETE_PLAYLIST", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "YOUTUBE_DELETE_COMMENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "YOUTUBE_DELETE_PLAYLIST_ITEM", + scope: ToolScope::Admin, + }, +]; diff --git a/core/src/sync/composio/providers/clickup/mod.rs b/core/src/sync/composio/providers/clickup/mod.rs new file mode 100644 index 0000000..bbf3908 --- /dev/null +++ b/core/src/sync/composio/providers/clickup/mod.rs @@ -0,0 +1,26 @@ +//! ClickUp Composio provider — incremental Memory Tree ingest for +//! tasks owned by (or assigned to) the connected user. +//! +//! Mirrors the [`crate::sync::composio::providers::notion`] layout +//! so anyone familiar with Notion/Slack ingestion can read this without +//! re-learning a new shape: +//! +//! - `provider.rs` — `impl ComposioProvider for ClickUpProvider` +//! - `normalization` — payload-shape helpers, now `tinycortex::…::normalize::clickup` +//! - `ingest.rs` — memory_tree document ingest (issue #2885) +//! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions +//! - `tests.rs` — unit tests for the helpers + trait metadata +//! +//! Issue: #2288 (introduction); #2885 (memory_tree migration). + +// The payload normalisers moved to tinycortex (they are pure Value +// transforms, i.e. driver-side). Aliased under the old module name so +// every `normalization::extract_*` call site below stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::clickup as normalization; +mod provider; +#[cfg(test)] +mod tests; +pub mod tools; + +pub use provider::ClickUpProvider; +pub use tools::CLICKUP_CURATED; diff --git a/core/src/sync/composio/providers/clickup/provider.rs b/core/src/sync/composio/providers/clickup/provider.rs new file mode 100644 index 0000000..916caaa --- /dev/null +++ b/core/src/sync/composio/providers/clickup/provider.rs @@ -0,0 +1,271 @@ +//! ClickUp provider — incremental sync of tasks assigned to the +//! authenticated user, with per-item persistence into the Memory Tree. +//! +//! On each sync pass: +//! +//! 1. Load persistent [`SyncState`] from the KV store. +//! 2. Check the daily request budget — bail early if exhausted. +//! 3. If we don't yet know the user's numeric ID, call +//! `CLICKUP_GET_AUTHORIZED_USER` and cache the result in memory +//! (it doesn't change for the lifetime of the connection). +//! 4. If we don't yet know which workspaces (teams) the connection +//! can see, call `CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES` and +//! cache the list. +//! 5. For each workspace, page through +//! `CLICKUP_GET_FILTERED_TEAM_TASKS` filtered to the user as +//! assignee, sorted by `date_updated` descending. Stop a workspace +//! early once we hit tasks older than the cursor. +//! 6. For each task, persist as a single memory document if it's new +//! *or* edited since the last sync. +//! 7. Advance the cursor to the newest `date_updated` seen and save. +//! +//! Privacy posture: we only pull tasks the user is assigned to, never +//! the whole workspace's task graph. This mirrors the +//! "fetch-what-the-user-sees" model `gmail` / `notion` already follow +//! and avoids accidentally ingesting other teammates' private tasks. + +use async_trait::async_trait; +use serde_json::json; + +use super::normalization; +use crate::sync::composio::providers::{ + first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, + CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind, +}; + +pub(crate) const ACTION_GET_AUTHORIZED_USER: &str = "CLICKUP_GET_AUTHORIZED_USER"; +pub(crate) const ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES: &str = + "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES"; +pub(crate) const ACTION_GET_FILTERED_TEAM_TASKS: &str = "CLICKUP_GET_FILTERED_TEAM_TASKS"; + +/// Paths for extracting a task's unique ID. Composio sometimes wraps +/// the upstream payload under `data`, so we check both shapes. +pub(super) const TASK_ID_PATHS: &[&str] = &["id", "data.id", "task_id", "data.task_id"]; + +pub struct ClickUpProvider; + +impl ClickUpProvider { + pub fn new() -> Self { + Self + } +} + +impl Default for ClickUpProvider { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ComposioProvider for ClickUpProvider { + fn toolkit_slug(&self) -> &'static str { + "clickup" + } + + fn curated_tools(&self) -> Option<&'static [CuratedTool]> { + Some(super::tools::CLICKUP_CURATED) + } + + fn sync_interval_secs(&self) -> Option { + // 30 minutes — same cadence as Notion. ClickUp tasks change + // more slowly than chat but faster than email, so this is in + // the middle. + Some(resolve_sync_interval_secs("clickup", 30 * 60)) + } + + async fn fetch_user_profile( + &self, + ctx: &ProviderContext, + ) -> Result { + tracing::debug!( + connection_id = ?ctx.connection_id, + "[composio:clickup] fetch_user_profile via {ACTION_GET_AUTHORIZED_USER}" + ); + + let resp = ctx + .execute(ACTION_GET_AUTHORIZED_USER, Some(json!({}))) + .await + .map_err(|e| { + format!("[composio:clickup] {ACTION_GET_AUTHORIZED_USER} failed: {e:#}") + })?; + + if !resp.successful { + let err = resp + .error + .clone() + .unwrap_or_else(|| "provider reported failure".to_string()); + return Err(format!( + "[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {err}" + )); + } + + // Composio's wrapping puts ClickUp's `{user: {…}}` payload at + // `data` or `data.user`. We probe both — `pick_str` walks dotted + // paths so `user.username` and `data.user.username` both work. + let data = &resp.data; + let display_name = pick_str(data, &["user.username", "data.user.username", "username"]); + let email = pick_str(data, &["user.email", "data.user.email", "email"]); + let username = normalization::extract_user_id(data); + let avatar_url = pick_str( + data, + &[ + "user.profilePicture", + "data.user.profilePicture", + "profilePicture", + ], + ); + let profile_url = None; + + Ok(ProviderUserProfile { + toolkit: "clickup".to_string(), + connection_id: ctx.connection_id.clone(), + display_name, + email, + username, + avatar_url, + profile_url, + extras: data.clone(), + }) + } + + /// Incremental sync via the generic + /// `orchestrator`: + /// user/workspace resolution, the per-workspace page loop, dedup, the + /// `max_items` cap, the epoch-ms `sync_depth_days` window, and cursor + /// handling live in `run_sync`; the ClickUp-specific primitives live in + /// `super::source`. + async fn fetch_tasks( + &self, + ctx: &ProviderContext, + filter: &TaskFetchFilter, + ) -> Result, String> { + let max = filter.effective_max(); + tracing::debug!( + connection_id = ?ctx.connection_id, + max, + team_id = ?filter.team_id, + assignee_is_me = filter.assignee_is_me, + "[composio:clickup] fetch_tasks" + ); + + // Resolve which workspaces (teams) to query. An explicit + // `team_id` from the filter wins; otherwise enumerate every + // workspace the connection can see. + let workspaces = match &filter.team_id { + Some(team) if !team.trim().is_empty() => vec![team.trim().to_string()], + _ => { + let resp = ctx + .execute(ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES, Some(json!({}))) + .await + .map_err(|e| { + format!( + "[composio:clickup] {ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES}: {e:#}" + ) + })?; + if !resp.successful { + return Err(format!( + "[composio:clickup] {ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES}: {}", + resp.error.unwrap_or_else(|| "provider failure".into()) + )); + } + normalization::extract_workspace_ids(&resp.data) + } + }; + + // Resolve the current user id only when the filter scopes to + // "assigned to me". + let assignees: Vec = if filter.assignee_is_me { + let resp = ctx + .execute(ACTION_GET_AUTHORIZED_USER, Some(json!({}))) + .await + .map_err(|e| format!("[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {e:#}"))?; + // Fail closed: if we can't resolve the user, error rather than + // silently dropping the assignee filter and fetching the whole + // workspace's tasks. + if !resp.successful { + return Err(format!( + "[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {}", + resp.error.unwrap_or_else(|| "provider failure".into()) + )); + } + let id = normalization::extract_user_id(&resp.data).ok_or_else(|| { + "[composio:clickup] CLICKUP_GET_AUTHORIZED_USER returned no user.id".to_string() + })?; + vec![id] + } else { + Vec::new() + }; + + let mut out: Vec = Vec::new(); + 'workspaces: for workspace_id in &workspaces { + let mut args = json!({ + "team_id": workspace_id, + "order_by": "updated", + "reverse": true, + "page": 0, + "page_size": max.min(100) as u32, + "subtasks": true, + }); + if !assignees.is_empty() { + args["assignees"] = json!(assignees); + } + if let Some(list_id) = filter.list_id.as_deref().filter(|s| !s.trim().is_empty()) { + args["list_ids"] = json!([list_id]); + } + merge_extra(&mut args, &filter.extra); + + let resp = ctx + .execute(ACTION_GET_FILTERED_TEAM_TASKS, Some(args)) + .await + .map_err(|e| { + format!("[composio:clickup] {ACTION_GET_FILTERED_TEAM_TASKS} ws={workspace_id}: {e:#}") + })?; + if !resp.successful { + return Err(format!( + "[composio:clickup] {ACTION_GET_FILTERED_TEAM_TASKS} ws={workspace_id}: {}", + resp.error.unwrap_or_else(|| "provider failure".into()) + )); + } + + for task in normalization::extract_tasks(&resp.data) { + if out.len() >= max { + break 'workspaces; + } + if let Some(nt) = normalize_clickup_task(&task) { + out.push(nt); + } + } + } + + tracing::debug!(count = out.len(), "[composio:clickup] fetch_tasks complete"); + Ok(out) + } +} + +/// Map a raw ClickUp task payload into a [`NormalizedTask`]. Returns +/// `None` only when the task has no extractable id (unroutable). +fn normalize_clickup_task(task: &serde_json::Value) -> Option { + let external_id = pick_str(task, TASK_ID_PATHS)?; + let title = normalization::extract_task_name(task) + .unwrap_or_else(|| format!("ClickUp task {external_id}")); + Some(NormalizedTask { + external_id, + source_id: String::new(), + provider: "clickup".to_string(), + kind: TaskKind::Generic, + title, + body: pick_str(task, &["description", "data.description", "text_content"]), + url: pick_str(task, &["url", "data.url"]), + status: pick_str(task, &["status.status", "data.status.status", "status"]), + assignee: first_array_str( + task, + &["assignees", "data.assignees"], + &["username", "email"], + ), + due: pick_str(task, &["due_date", "data.due_date"]), + labels: Vec::new(), + priority: pick_str(task, &["priority.priority", "data.priority.priority"]), + updated_at: normalization::extract_task_updated(task), + raw: task.clone(), + }) +} diff --git a/core/src/sync/composio/providers/clickup/tests.rs b/core/src/sync/composio/providers/clickup/tests.rs new file mode 100644 index 0000000..5294aa0 --- /dev/null +++ b/core/src/sync/composio/providers/clickup/tests.rs @@ -0,0 +1,155 @@ +//! Unit tests for the ClickUp provider. + +use super::normalization::{ + extract_task_name, extract_task_updated, extract_tasks, extract_user_id, extract_workspace_ids, +}; +use super::ClickUpProvider; +use crate::sync::composio::providers::ComposioProvider; +use serde_json::json; + +#[test] +fn extract_tasks_walks_common_shapes() { + let v1 = json!({ "data": { "tasks": [{"id": "t1"}] } }); + let v2 = json!({ "tasks": [{"id": "t2"}, {"id": "t3"}] }); + let v3 = json!({ "data": {} }); + assert_eq!(extract_tasks(&v1).len(), 1); + assert_eq!(extract_tasks(&v2).len(), 2); + assert_eq!(extract_tasks(&v3).len(), 0); +} + +#[test] +fn extract_task_name_finds_name_field() { + let task = json!({ "id": "abc", "name": "Build feature X" }); + assert_eq!(extract_task_name(&task), Some("Build feature X".into())); +} + +#[test] +fn extract_task_name_falls_back_to_wrapped_data() { + let task = json!({ "data": { "name": "Wrapped" } }); + assert_eq!(extract_task_name(&task), Some("Wrapped".into())); +} + +#[test] +fn extract_task_name_returns_none_when_missing() { + let task = json!({ "id": "abc" }); + assert!(extract_task_name(&task).is_none()); +} + +#[test] +fn extract_task_updated_handles_string_form() { + let task = json!({ "date_updated": "1733412345678" }); + assert_eq!( + extract_task_updated(&task), + Some("1733412345678".to_string()) + ); +} + +#[test] +fn extract_task_updated_handles_nested_data() { + let task = json!({ "data": { "dateUpdated": "1700000000000" } }); + assert_eq!( + extract_task_updated(&task), + Some("1700000000000".to_string()) + ); +} + +#[test] +fn extract_task_updated_returns_none_when_missing() { + let task = json!({ "id": "abc" }); + assert!(extract_task_updated(&task).is_none()); +} + +#[test] +fn extract_user_id_handles_numeric_id() { + let data = json!({ "user": { "id": 12345 } }); + assert_eq!(extract_user_id(&data), Some("12345".to_string())); +} + +#[test] +fn extract_user_id_handles_wrapped_payload() { + let data = json!({ "data": { "user": { "id": "777" } } }); + assert_eq!(extract_user_id(&data), Some("777".to_string())); +} + +#[test] +fn extract_user_id_none_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_user_id(&data).is_none()); +} + +#[test] +fn extract_workspace_ids_from_teams_array() { + let data = json!({ + "teams": [ + { "id": "ws1", "name": "Personal" }, + { "id": "ws2", "name": "Acme" }, + ] + }); + assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2"]); +} + +#[test] +fn extract_workspace_ids_handles_wrapped_payload() { + let data = json!({ + "data": { + "teams": [ + { "id": "ws1" }, + { "id": "ws2" }, + { "id": "ws3" }, + ] + } + }); + assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2", "ws3"]); +} + +#[test] +fn extract_workspace_ids_empty_when_no_teams() { + let data = json!({ "foo": "bar" }); + assert!(extract_workspace_ids(&data).is_empty()); +} + +#[test] +fn extract_workspace_ids_skips_entries_without_id() { + let data = json!({ + "teams": [ + { "name": "Anonymous" }, + { "id": "ws1", "name": "Real" }, + ] + }); + assert_eq!(extract_workspace_ids(&data), vec!["ws1"]); +} + +#[test] +fn provider_metadata_is_stable() { + let p = ClickUpProvider::new(); + assert_eq!(p.toolkit_slug(), "clickup"); + assert_eq!(p.sync_interval_secs(), Some(30 * 60)); + assert!(p.curated_tools().is_some()); +} + +#[test] +fn curated_tools_contains_core_read_surface() { + let p = ClickUpProvider::new(); + let curated = p.curated_tools().expect("CLICKUP_CURATED is registered"); + let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); + // The three actions the sync path depends on must be advertised. + assert!(slugs.contains(&"CLICKUP_GET_AUTHORIZED_USER")); + assert!(slugs.contains(&"CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES")); + assert!(slugs.contains(&"CLICKUP_GET_FILTERED_TEAM_TASKS")); +} + +#[test] +fn default_impl_matches_new() { + // `ClickUpProvider` is a unit struct, so we compare observable + // trait surface instead of deriving `PartialEq`. This catches a + // future regression where `new()` and `default()` drift apart + // (e.g. one is given an extra field but the other is forgotten). + let a = ClickUpProvider::new(); + let b = ::default(); + assert_eq!(a.toolkit_slug(), b.toolkit_slug()); + assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); + assert_eq!( + a.curated_tools().map(<[_]>::len), + b.curated_tools().map(<[_]>::len), + ); +} diff --git a/core/src/sync/composio/providers/clickup/tools.rs b/core/src/sync/composio/providers/clickup/tools.rs new file mode 100644 index 0000000..d0c89fe --- /dev/null +++ b/core/src/sync/composio/providers/clickup/tools.rs @@ -0,0 +1,124 @@ +//! Curated catalog of ClickUp Composio actions exposed to the agent. +//! +//! Slugs match Composio's naming convention (`_`) for +//! the ClickUp REST surface. See +//! for the canonical action list; the entries here are the read-oriented +//! subset the periodic Memory Tree sync relies on, plus the most common +//! task-write surface the agent already uses through generic tool-calling. + +use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; + +pub const CLICKUP_CURATED: &[CuratedTool] = &[ + // ── Read: identity ───────────────────────────────────────────── + CuratedTool { + slug: "CLICKUP_GET_AUTHORIZED_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES", + scope: ToolScope::Read, + }, + // ── Read: structure (workspace → space → folder → list) ────── + CuratedTool { + slug: "CLICKUP_GET_SPACES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_FOLDERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_LISTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_FOLDERLESS_LISTS", + scope: ToolScope::Read, + }, + // ── Read: tasks (the main memory ingest surface) ────────────── + CuratedTool { + slug: "CLICKUP_GET_FILTERED_TEAM_TASKS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_TASKS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_TASK", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_TASK_COMMENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_LIST_COMMENTS", + scope: ToolScope::Read, + }, + // ── Read: docs / views / time tracking ──────────────────────── + CuratedTool { + slug: "CLICKUP_SEARCH_DOCS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_DOC_PAGES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_VIEW_TASKS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_TIME_ENTRIES_WITHIN_A_DATE_RANGE", + scope: ToolScope::Read, + }, + // ── Read: members ───────────────────────────────────────────── + CuratedTool { + slug: "CLICKUP_GET_WORKSPACE_MEMBERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "CLICKUP_GET_TASK_MEMBERS", + scope: ToolScope::Read, + }, + // ── Write: create / update tasks ────────────────────────────── + CuratedTool { + slug: "CLICKUP_CREATE_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "CLICKUP_UPDATE_TASK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "CLICKUP_CREATE_TASK_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "CLICKUP_UPDATE_COMMENT", + scope: ToolScope::Write, + }, + // ── Write: structure ────────────────────────────────────────── + CuratedTool { + slug: "CLICKUP_CREATE_LIST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "CLICKUP_UPDATE_LIST", + scope: ToolScope::Write, + }, + // ── Admin: destructive ──────────────────────────────────────── + CuratedTool { + slug: "CLICKUP_DELETE_TASK", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "CLICKUP_DELETE_COMMENT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "CLICKUP_DELETE_LIST", + scope: ToolScope::Admin, + }, +]; diff --git a/core/src/sync/composio/providers/descriptions.rs b/core/src/sync/composio/providers/descriptions.rs new file mode 100644 index 0000000..0946844 --- /dev/null +++ b/core/src/sync/composio/providers/descriptions.rs @@ -0,0 +1,61 @@ +//! Human-readable capability summaries for Composio toolkit slugs. + +/// Human-readable capability summary for a Composio toolkit slug. +/// +/// Used by the prompt renderer to tell the orchestrator what each connected +/// integration can do. Covers the most common toolkits; unknown slugs get +/// a generic fallback so newly connected services still appear. +pub fn toolkit_description(slug: &str) -> &'static str { + match slug { + "gmail" => { + "Send, read, draft, reply, forward, and search emails; manage labels and threads" + } + "notion" => "Create, read, update, and search notion pages and notion databases", + "github" => { + "Manage repositories, issues, and pull requests on GitHub; sync \ + assigned issues into Memory Tree" + } + "slack" => "Send messages, read channels, manage threads, and post updates in Slack", + "discord" => "Send messages, manage channels, and interact with Discord servers", + "google_calendar" => "Create, update, and query calendar events; check availability", + "google_drive" => "Upload, download, search, and share files in Google Drive", + "google_docs" => "Create, read, and edit Google Docs documents", + "google_sheets" => "Read, write, and manage Google Sheets spreadsheets", + "outlook" => "Send, read, and manage emails in Microsoft Outlook", + "microsoft_teams" => "Send messages and manage channels in Microsoft Teams", + "larksuite" => { + "Connect Lark / Feishu workspace chat, docs, wiki, and meetings via Composio" + } + "linear" => { + "Create, read, and manage issues, projects, and cycles in Linear; sync \ + assigned issues into Memory Tree" + } + "jira" => "Create and manage issues, projects, and sprints in Jira", + "trello" => "Create and manage cards, lists, and boards in Trello", + "asana" => "Create and manage tasks, projects, and sections in Asana", + "clickup" => { + "Create, read, and manage tasks, lists, and docs in ClickUp; sync \ + assigned tasks into Memory Tree" + } + "dropbox" => "Upload, download, and share files in Dropbox", + "twitter" => "Post tweets, read timelines, and manage Twitter interactions", + "spotify" => "Control playback, search music, and manage playlists on Spotify", + "telegram" => "Send and receive messages via Telegram", + "whatsapp" => "Send and receive messages via WhatsApp", + "twilio" => "Send SMS, make calls, and manage communications via Twilio", + "shopify" => "Manage products, orders, and customers in Shopify", + "stripe" => "Manage payments, subscriptions, and customers in Stripe", + "hubspot" => "Manage contacts, deals, and marketing in HubSpot", + "salesforce" => "Manage contacts, leads, and opportunities in Salesforce", + "airtable" => "Read and write records in Airtable bases", + "figma" => "Access and manage Figma design files and components", + "youtube" => "Search videos, manage playlists, and interact with YouTube", + "calendar" => "Create, update, and query calendar events", + "one_drive" | "onedrive" => { + "Upload, download, search, and share files in Microsoft OneDrive" + } + "excel" => "Read, write, and manage workbooks, worksheets, and tables in Microsoft Excel", + "todoist" => "Create and manage tasks, projects, sections, and labels in Todoist", + _ => "Interact with this connected service via its available actions", + } +} diff --git a/core/src/sync/composio/providers/github/mod.rs b/core/src/sync/composio/providers/github/mod.rs new file mode 100644 index 0000000..691b6e2 --- /dev/null +++ b/core/src/sync/composio/providers/github/mod.rs @@ -0,0 +1,25 @@ +//! GitHub Composio provider — incremental Memory Tree ingest for issues and +//! pull requests involving the connected user. +//! +//! Mirrors the [`crate::sync::composio::providers::clickup`] layout so +//! anyone familiar with ClickUp/Notion ingestion can read this without +//! re-learning a new shape: +//! +//! - `provider.rs` — `impl ComposioProvider for GitHubProvider` +//! - `normalization` — payload-shape helpers, now `tinycortex::…::normalize::github` +//! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions +//! - `tests.rs` — unit tests for the helpers + trait metadata +//! +//! Issue: #2408. + +// The payload normalisers moved to tinycortex (they are pure Value +// transforms, i.e. driver-side). Aliased under the old module name so +// every `normalization::extract_*` call site below stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::github as normalization; +mod provider; +#[cfg(test)] +mod tests; +pub mod tools; + +pub use provider::GitHubProvider; +pub use tools::GITHUB_CURATED; diff --git a/core/src/sync/composio/providers/github/provider.rs b/core/src/sync/composio/providers/github/provider.rs new file mode 100644 index 0000000..e01992a --- /dev/null +++ b/core/src/sync/composio/providers/github/provider.rs @@ -0,0 +1,547 @@ +//! GitHub provider — incremental sync of issues and pull requests involving +//! the authenticated user, with per-item persistence into the Memory Tree. +//! +//! On each sync pass: +//! +//! 1. Load persistent [`SyncState`] from the KV store. +//! 2. Check the daily request budget — bail early if exhausted. +//! 3. Resolve the authenticated user's GitHub login (used in the search +//! query); cached cheaply across re-fetches. +//! 4. Search for issues and PRs involving the user via +//! `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` with `involves:{login}`, filtered to items +//! updated since the cursor (when available). +//! 5. For each result, persist as a single memory document if it's new +//! *or* edited since the last sync. +//! 6. Advance the cursor to the newest `updated_at` seen and save. +//! +//! Privacy posture: the `involves:` search qualifier returns only items the +//! user created, was assigned to, mentioned in, or commented on — it never +//! surfaces private repos the user can't access. This mirrors the +//! "fetch-what-the-user-sees" model gmail / notion already follow. + +use async_trait::async_trait; +use serde_json::{json, Value}; +use std::time::Duration; + +use super::normalization; +use crate::sync::composio::providers::{ + merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, + GithubFetchMode, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, + TaskKind, +}; + +pub(crate) const ACTION_GET_AUTHENTICATED_USER: &str = "GITHUB_GET_THE_AUTHENTICATED_USER"; +pub(crate) const ACTION_SEARCH_ISSUES: &str = "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS"; + +const GH_CLI_TIMEOUT: Duration = Duration::from_secs(30); +const GITHUB_TASK_SEARCH_TIMEOUT: Duration = Duration::from_secs(20); + +pub struct GitHubProvider; + +impl GitHubProvider { + pub fn new() -> Self { + Self + } +} + +impl Default for GitHubProvider { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ComposioProvider for GitHubProvider { + fn toolkit_slug(&self) -> &'static str { + "github" + } + + fn curated_tools(&self) -> Option<&'static [CuratedTool]> { + Some(super::tools::GITHUB_CURATED) + } + + fn sync_interval_secs(&self) -> Option { + // 30 minutes — GitHub issues change less frequently than Slack + // messages, so a half-hour cadence keeps the memory fresh without + // hammering the search API. + Some(resolve_sync_interval_secs("github", 30 * 60)) + } + + async fn fetch_user_profile( + &self, + ctx: &ProviderContext, + ) -> Result { + tracing::debug!( + connection_id = ?ctx.connection_id, + "[composio:github] fetch_user_profile via {ACTION_GET_AUTHENTICATED_USER}" + ); + + let resp = ctx + .execute(ACTION_GET_AUTHENTICATED_USER, Some(json!({}))) + .await + .map_err(|e| { + format!("[composio:github] {ACTION_GET_AUTHENTICATED_USER} failed: {e:#}") + })?; + + if !resp.successful { + let err = resp + .error + .clone() + .unwrap_or_else(|| "provider reported failure".to_string()); + return Err(format!( + "[composio:github] {ACTION_GET_AUTHENTICATED_USER}: {err}" + )); + } + + let data = &resp.data; + let login = normalization::extract_user_login(data); + let display_name = pick_str(data, &["name", "data.name"]).or_else(|| login.clone()); + let email = pick_str(data, &["email", "data.email"]); + let avatar_url = pick_str(data, &["avatar_url", "data.avatar_url"]); + let profile_url = pick_str(data, &["html_url", "data.html_url"]); + + Ok(ProviderUserProfile { + toolkit: "github".to_string(), + connection_id: ctx.connection_id.clone(), + display_name, + email, + username: login, + avatar_url, + profile_url, + extras: data.clone(), + }) + } + + /// Incremental sync via the generic + /// `orchestrator`: + /// login resolution, pagination, dedup, the `max_items` cap, and cursor + /// handling live in `run_sync`; the GitHub-specific primitives — including + /// the **server-side** `sync_depth_days` window — live in `super::source`. + async fn fetch_tasks( + &self, + ctx: &ProviderContext, + filter: &TaskFetchFilter, + ) -> Result, String> { + let max = filter.effective_max(); + let query = build_fetch_query(filter); + tracing::debug!( + connection_id = ?ctx.connection_id, + max, + mode = ?filter.github_fetch_mode, + query = %query, + "[composio:github] fetch_tasks" + ); + + // Select the data source by the user-configured fetch mode. `Auto` + // (the default) keeps the shipped Composio path as primary and treats + // local `gh`/REST as a true fallback — only used when the Composio + // round-trip errors or is unavailable. `Composio` / `Local` force one + // path. Normalization happens ONCE below regardless of source. + let data = match filter.github_fetch_mode { + GithubFetchMode::Composio => { + fetch_github_tasks_composio(ctx, &query, max, &filter.extra).await? + } + GithubFetchMode::Local => fetch_github_tasks_local(&query, max, &filter.extra).await?, + GithubFetchMode::Auto => { + match fetch_github_tasks_composio(ctx, &query, max, &filter.extra).await { + Ok(d) => d, + Err(e) => { + tracing::info!( + error = %e, + "[composio:github] Composio fetch unavailable; falling back to local gh/REST" + ); + fetch_github_tasks_local(&query, max, &filter.extra).await? + } + } + } + }; + + let mut out: Vec = Vec::new(); + for issue in normalization::extract_issues(&data) { + if out.len() >= max { + break; + } + if let Some(nt) = normalize_github_issue(&issue) { + out.push(nt); + } + } + tracing::debug!(count = out.len(), "[composio:github] fetch_tasks complete"); + Ok(out) + } +} + +/// Fetch GitHub issues/PRs through the connected Composio account. +/// +/// This is the original shipped `fetch_tasks` data path: it builds the +/// `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` search args, merges any advanced +/// `extra` query fragment, fires the action through the mode-aware +/// `ctx.execute` chokepoint, and returns the raw response `data` for the +/// shared normalization loop. Kept as a sibling of +/// [`fetch_github_tasks_local`] so `fetch_tasks` can select between them by +/// [`GithubFetchMode`]. +async fn fetch_github_tasks_composio( + ctx: &ProviderContext, + query: &str, + max: usize, + extra: &Value, +) -> Result { + let mut args = json!({ + "q": query, + "sort": "updated", + "order": "desc", + "per_page": max.min(100) as u32, + "page": 1, + }); + merge_extra(&mut args, extra); + + let resp = ctx + .execute(ACTION_SEARCH_ISSUES, Some(args)) + .await + .map_err(|e| format!("[composio:github] {ACTION_SEARCH_ISSUES}: {e:#}"))?; + if !resp.successful { + return Err(format!( + "[composio:github] {ACTION_SEARCH_ISSUES}: {}", + resp.error.unwrap_or_else(|| "provider failure".into()) + )); + } + Ok(resp.data) +} + +async fn fetch_github_tasks_local(query: &str, max: usize, extra: &Value) -> Result { + let mut args = json!({ + "q": query, + "sort": "updated", + "order": "desc", + "per_page": max.min(100) as u32, + "page": 1, + }); + merge_extra(&mut args, extra); + expand_me_in_github_search_args(&mut args).await; + + match gh_search_issues(&args).await { + Ok(data) => Ok(data), + Err(gh_err) => { + tracing::debug!( + error = %gh_err, + "[task_sources:github] gh api search failed, falling back to REST" + ); + rest_search_issues(&args).await.map_err(|rest_err| { + format!("[task_sources:github] local GitHub search failed: gh: {gh_err}; REST: {rest_err}") + }) + } + } +} + +async fn gh_search_issues(args: &Value) -> Result { + let mut cmd = tokio::process::Command::new("gh"); + cmd.arg("api") + .arg("--method") + .arg("GET") + .arg("search/issues"); + for (key, value) in github_search_arg_pairs(args)? { + cmd.arg("-f").arg(format!("{key}={value}")); + } + + let output = tokio::time::timeout(GH_CLI_TIMEOUT, cmd.output()) + .await + .map_err(|_| format!("gh command timed out after {}s", GH_CLI_TIMEOUT.as_secs()))? + .map_err(|e| format!("gh command failed: {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("gh exited {}: {stderr}", output.status)); + } + + let stdout = + String::from_utf8(output.stdout).map_err(|e| format!("gh output not utf8: {e}"))?; + serde_json::from_str(&stdout).map_err(|e| format!("parse gh search response: {e}")) +} + +async fn rest_search_issues(args: &Value) -> Result { + let client = reqwest::Client::builder() + .timeout(GITHUB_TASK_SEARCH_TIMEOUT) + .build() + .map_err(|e| format!("failed to build GitHub client: {e}"))?; + + let mut request = client + .get("https://api.github.com/search/issues") + .header("User-Agent", "openhuman") + .header("Accept", "application/vnd.github+json"); + + if let Some(token) = github_env_token() { + request = request.header("Authorization", format!("Bearer {token}")); + } + + let pairs = github_search_arg_pairs(args)?; + let resp = request + .query(&pairs) + .send() + .await + .map_err(|e| format!("GitHub API request failed: {e}"))?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(format!("GitHub API returned {status}: {body}")); + } + + resp.json::() + .await + .map_err(|e| format!("parse GitHub API response: {e}")) +} + +pub(super) fn github_env_token() -> Option { + std::env::var("GH_TOKEN") + .or_else(|_| std::env::var("GITHUB_TOKEN")) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +async fn expand_me_in_github_search_args(args: &mut Value) { + let Some(query) = args.get("q").and_then(Value::as_str).map(str::to_string) else { + return; + }; + if !query.contains("@me") { + return; + } + let Some(login) = resolve_github_login().await else { + return; + }; + if let Some(obj) = args.as_object_mut() { + obj.insert("q".to_string(), Value::String(query.replace("@me", &login))); + } +} + +async fn resolve_github_login() -> Option { + if let Some(login) = resolve_github_login_with_gh().await { + return Some(login); + } + resolve_github_login_with_rest().await +} + +async fn resolve_github_login_with_gh() -> Option { + let output = tokio::time::timeout( + GH_CLI_TIMEOUT, + tokio::process::Command::new("gh") + .arg("api") + .arg("user") + .arg("--jq") + .arg(".login") + .output(), + ) + .await + .ok()? + .ok()?; + if !output.status.success() { + return None; + } + String::from_utf8(output.stdout) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +async fn resolve_github_login_with_rest() -> Option { + let token = github_env_token()?; + let client = reqwest::Client::builder() + .timeout(GITHUB_TASK_SEARCH_TIMEOUT) + .build() + .ok()?; + let resp = client + .get("https://api.github.com/user") + .header("User-Agent", "openhuman") + .header("Accept", "application/vnd.github+json") + .header("Authorization", format!("Bearer {token}")) + .send() + .await + .ok()?; + if !resp.status().is_success() { + return None; + } + resp.json::() + .await + .ok() + .and_then(|value| { + value + .get("login") + .and_then(Value::as_str) + .map(str::to_string) + }) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +pub(super) fn github_search_arg_pairs(args: &Value) -> Result, String> { + let obj = args + .as_object() + .ok_or_else(|| "GitHub search args must be a JSON object".to_string())?; + let mut out = Vec::with_capacity(obj.len()); + for (key, value) in obj { + let rendered = match value { + Value::String(s) => s.trim().to_string(), + Value::Number(n) => n.to_string(), + Value::Bool(b) => b.to_string(), + Value::Null => continue, + other => other.to_string(), + }; + if !rendered.is_empty() { + out.push((key.clone(), rendered)); + } + } + Ok(out) +} + +/// Build a GitHub Search-Issues query from a [`TaskFetchFilter`]. +/// +/// Combines repo / label / state / assignee qualifiers. When the filter +/// carries no scoping constraints at all we fall back to `involves:@me` so a +/// task source never accidentally pulls the entire public issue universe. +/// +/// State bias: when the filter sets no explicit `state`, we append `is:open` +/// so closed issues and merged/closed PRs aren't fetched in the first place +/// (the unconditional skip in `normalize_github_issue` is the hard guarantee; +/// this is the fetch-side optimization). An explicit `state` is respected and +/// `is:open` is not double-added. +pub(super) fn build_fetch_query(filter: &TaskFetchFilter) -> String { + let mut parts: Vec = Vec::new(); + if let Some(repo) = filter + .repo + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(normalize_github_repo_filter) + { + parts.push(format!("repo:{repo}")); + } + for label in filter + .labels + .iter() + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + { + parts.push(format!("label:\"{label}\"")); + } + if filter.assignee_is_me { + parts.push("assignee:@me".to_string()); + } + // If no repo/label/assignee scoping was supplied, fall back to + // `involves:@me` (plus the open bias) rather than the whole issue universe. + if parts.is_empty() { + parts.push("involves:@me".to_string()); + } + let explicit_state = filter + .state + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()); + match explicit_state { + // Caller pinned a state — respect it verbatim, don't add `is:open`. + Some(state) => parts.push(format!("state:{state}")), + // No explicit state — bias the fetch toward open items. + None => parts.push("is:open".to_string()), + } + parts.join(" ") +} + +pub(super) fn normalize_github_repo_filter(raw: &str) -> String { + let trimmed = raw.trim(); + let without_scheme = trimmed + .strip_prefix("https://github.com/") + .or_else(|| trimmed.strip_prefix("http://github.com/")) + .or_else(|| trimmed.strip_prefix("git@github.com:")) + .unwrap_or(trimmed); + let cleaned = without_scheme + .trim_start_matches('/') + .trim_end_matches('/') + .trim_end_matches(".git"); + let mut parts = cleaned.split('/').filter(|part| !part.is_empty()); + match (parts.next(), parts.next()) { + (Some(owner), Some(repo)) => { + let repo = repo.trim_end_matches(".git"); + if owner.is_empty() || repo.is_empty() { + trimmed.to_string() + } else { + format!("{owner}/{repo}") + } + } + _ => trimmed.to_string(), + } +} + +/// Map a raw GitHub issue/PR payload into a [`NormalizedTask`]. +/// +/// GitHub's search-issues-and-PRs endpoint returns both shapes; a hit is a +/// pull request iff it carries a `pull_request` object. We tag the kind here +/// so enrichment can phrase the objective as "review" vs "resolve". +/// +/// Returns `None` when the item's state is `"closed"` — a merged/closed PR +/// and a closed issue both report `state == "closed"`, and there is no point +/// ingesting work that is already done. This skip is unconditional (it does +/// not depend on the fetch query), so even if a `closed` item slips through +/// the query bias it is dropped here. +pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option { + let external_id = normalization::extract_issue_id(issue)?; + let status = pick_str(issue, &["state", "data.state"]); + if status + .as_deref() + .map(|s| s.eq_ignore_ascii_case("closed")) + .unwrap_or(false) + { + tracing::debug!( + external_id = %external_id, + "[composio:github] normalize_github_issue: skipping closed item (merged PR / closed issue)" + ); + return None; + } + let title = normalization::extract_issue_title(issue) + .unwrap_or_else(|| format!("GitHub issue {external_id}")); + let kind = if is_pull_request(issue) { + TaskKind::PullRequest + } else { + TaskKind::Issue + }; + Some(NormalizedTask { + external_id, + source_id: String::new(), + provider: "github".to_string(), + kind, + title, + body: pick_str(issue, &["body", "data.body"]), + url: pick_str(issue, &["html_url", "data.html_url"]), + status, + assignee: pick_str(issue, &["assignee.login", "data.assignee.login"]), + due: None, + labels: extract_github_labels(issue), + priority: None, + updated_at: normalization::extract_issue_updated_at(issue), + raw: issue.clone(), + }) +} + +/// A GitHub search hit is a pull request iff it carries a non-null +/// `pull_request` object (issues never do). Tolerant of the Composio `data` +/// wrapper. +fn is_pull_request(issue: &serde_json::Value) -> bool { + let pr = issue + .get("pull_request") + .or_else(|| issue.get("data").and_then(|d| d.get("pull_request"))); + matches!(pr, Some(v) if !v.is_null()) +} + +/// Extract label names from a GitHub issue payload (`labels` is an array +/// of `{ name }` objects). Tolerant of the Composio `data` wrapper. +fn extract_github_labels(issue: &serde_json::Value) -> Vec { + let arr = issue + .get("labels") + .or_else(|| issue.get("data").and_then(|d| d.get("labels"))) + .and_then(|v| v.as_array()); + match arr { + Some(items) => items + .iter() + .filter_map(|l| l.get("name").and_then(|n| n.as_str())) + .map(|s| s.to_string()) + .collect(), + None => Vec::new(), + } +} diff --git a/core/src/sync/composio/providers/github/tests.rs b/core/src/sync/composio/providers/github/tests.rs new file mode 100644 index 0000000..10ee962 --- /dev/null +++ b/core/src/sync/composio/providers/github/tests.rs @@ -0,0 +1,611 @@ +//! Unit tests for the GitHub Composio provider. + +use super::normalization::{ + extract_issue_id, extract_issue_title, extract_issue_updated_at, extract_issues, + extract_user_login, +}; +use super::provider::github_env_token; +use super::provider::{ + build_fetch_query, github_search_arg_pairs, normalize_github_issue, + normalize_github_repo_filter, ACTION_GET_AUTHENTICATED_USER, ACTION_SEARCH_ISSUES, +}; +use super::tools::GITHUB_CURATED; +use super::GitHubProvider; +use crate::sync::composio::providers::ComposioProvider; +use crate::sync::composio::providers::{GithubFetchMode, TaskFetchFilter, TaskKind}; +use serde_json::json; + +// ── extract_issues ─────────────────────────────────────────────────────────── + +#[test] +fn extract_issues_walks_data_items_shape() { + let data = json!({ "data": { "items": [{"id": 1u64}] } }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_walks_top_level_items_shape() { + let data = json!({ "items": [{"id": 1u64}, {"id": 2u64}] }); + assert_eq!(extract_issues(&data).len(), 2); +} + +#[test] +fn extract_issues_returns_empty_when_no_items_key() { + let data = json!({ "foo": "bar" }); + assert!(extract_issues(&data).is_empty()); +} + +#[test] +fn extract_issues_handles_data_data_nesting() { + let data = json!({ "data": { "data": { "items": [{"id": 9u64}] } } }); + assert_eq!(extract_issues(&data).len(), 1); +} + +// ── extract_issue_id ───────────────────────────────────────────────────────── + +#[test] +fn extract_issue_id_from_numeric_id() { + let issue = json!({ "id": 123456789u64, "title": "Fix race" }); + assert_eq!(extract_issue_id(&issue), Some("123456789".to_string())); +} + +#[test] +fn extract_issue_id_from_wrapped_data() { + let issue = json!({ "data": { "id": 42u64 } }); + assert_eq!(extract_issue_id(&issue), Some("42".to_string())); +} + +#[test] +fn extract_issue_id_falls_back_to_html_url_path() { + let issue = json!({ + "html_url": "https://github.com/owner/repo/issues/7" + }); + assert_eq!(extract_issue_id(&issue), Some("owner/repo#7".to_string())); +} + +#[test] +fn extract_issue_id_none_when_no_id_or_url() { + let issue = json!({ "title": "orphan" }); + assert!(extract_issue_id(&issue).is_none()); +} + +// ── extract_issue_title ────────────────────────────────────────────────────── + +#[test] +fn extract_issue_title_builds_prefixed_title() { + let issue = json!({ + "id": 1u64, + "title": "Fix race condition", + "html_url": "https://github.com/acme/core/issues/99" + }); + assert_eq!( + extract_issue_title(&issue), + Some("GitHub: acme/core#99: Fix race condition".to_string()) + ); +} + +#[test] +fn extract_issue_title_pr_url_also_works() { + let issue = json!({ + "id": 2u64, + "title": "Add feature", + "html_url": "https://github.com/org/repo/pull/101" + }); + assert_eq!( + extract_issue_title(&issue), + Some("GitHub: org/repo#101: Add feature".to_string()) + ); +} + +#[test] +fn extract_issue_title_returns_raw_title_when_no_url() { + let issue = json!({ "title": "Bare title" }); + assert_eq!(extract_issue_title(&issue), Some("Bare title".to_string())); +} + +#[test] +fn extract_issue_title_none_when_no_title() { + let issue = json!({ "id": 1u64 }); + assert!(extract_issue_title(&issue).is_none()); +} + +// ── extract_issue_updated_at ───────────────────────────────────────────────── + +#[test] +fn extract_issue_updated_at_from_top_level() { + let issue = json!({ "updated_at": "2024-05-21T15:30:00Z" }); + assert_eq!( + extract_issue_updated_at(&issue), + Some("2024-05-21T15:30:00Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_at_from_data_wrapper() { + let issue = json!({ "data": { "updated_at": "2023-01-01T00:00:00Z" } }); + assert_eq!( + extract_issue_updated_at(&issue), + Some("2023-01-01T00:00:00Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_at_none_when_missing() { + let issue = json!({ "id": 1u64 }); + assert!(extract_issue_updated_at(&issue).is_none()); +} + +// ── extract_user_login ─────────────────────────────────────────────────────── + +#[test] +fn extract_user_login_from_top_level() { + let data = json!({ "login": "octocat" }); + assert_eq!(extract_user_login(&data), Some("octocat".to_string())); +} + +#[test] +fn extract_user_login_from_data_wrapper() { + let data = json!({ "data": { "login": "monalisa" } }); + assert_eq!(extract_user_login(&data), Some("monalisa".to_string())); +} + +#[test] +fn extract_user_login_none_when_missing() { + let data = json!({ "id": 1u64 }); + assert!(extract_user_login(&data).is_none()); +} + +// ── provider metadata ──────────────────────────────────────────────────────── + +#[test] +fn provider_metadata_is_stable() { + let p = GitHubProvider::new(); + assert_eq!(p.toolkit_slug(), "github"); + assert_eq!(p.sync_interval_secs(), Some(30 * 60)); + assert!(p.curated_tools().is_some()); +} + +#[test] +fn curated_tools_contains_core_actions() { + let p = GitHubProvider::new(); + let curated = p.curated_tools().expect("GITHUB_CURATED is registered"); + let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); + assert!(slugs.contains(&"GITHUB_GET_THE_AUTHENTICATED_USER")); + assert!(slugs.contains(&"GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS")); + assert!(slugs.contains(&"GITHUB_LIST_REPOSITORY_ISSUES")); + assert!(slugs.contains(&"GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER")); + assert!(slugs.contains(&"GITHUB_CREATE_A_REPOSITORY_FOR_THE_AUTHENTICATED_USER")); + // DELETE_A_REFERENCE replaces DELETE_A_BRANCH (Composio v3 rename). + assert!(slugs.contains(&"GITHUB_DELETE_A_REFERENCE")); + // CLOSE_AN_ISSUE was removed — callers must use UPDATE_AN_ISSUE with state:"closed". + assert!( + !slugs.contains(&"GITHUB_CLOSE_AN_ISSUE"), + "GITHUB_CLOSE_AN_ISSUE was removed — use GITHUB_UPDATE_AN_ISSUE with state:closed" + ); +} + +#[test] +fn default_impl_matches_new() { + let a = GitHubProvider::new(); + let b = ::default(); + assert_eq!(a.toolkit_slug(), b.toolkit_slug()); + assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); + assert_eq!( + a.curated_tools().map(<[_]>::len), + b.curated_tools().map(<[_]>::len), + ); +} + +#[test] +fn build_fetch_query_scopes_repo_labels_state_and_assignee() { + let query = build_fetch_query(&TaskFetchFilter { + repo: Some("tinyhumansai/openhuman".to_string()), + labels: vec!["bug".to_string(), "agent harness".to_string()], + state: Some("open".to_string()), + assignee_is_me: true, + ..Default::default() + }); + + assert_eq!( + query, + "repo:tinyhumansai/openhuman label:\"bug\" label:\"agent harness\" assignee:@me state:open" + ); +} + +#[test] +fn build_fetch_query_normalizes_github_repo_urls() { + let query = build_fetch_query(&TaskFetchFilter { + repo: Some("https://github.com/tinyhumansai/openhuman/pull/3267".to_string()), + state: Some("open".to_string()), + ..Default::default() + }); + + assert_eq!(query, "repo:tinyhumansai/openhuman state:open"); +} + +#[test] +fn normalize_github_repo_filter_accepts_common_repo_inputs() { + assert_eq!( + normalize_github_repo_filter("tinyhumansai/openhuman"), + "tinyhumansai/openhuman" + ); + assert_eq!( + normalize_github_repo_filter("https://github.com/tinyhumansai/openhuman.git"), + "tinyhumansai/openhuman" + ); + assert_eq!( + normalize_github_repo_filter("git@github.com:tinyhumansai/openhuman.git"), + "tinyhumansai/openhuman" + ); +} + +#[test] +fn build_fetch_query_falls_back_to_involves_me_when_unscoped() { + // No scoping and no explicit state: fall back to `involves:@me` and bias + // toward open items so closed issues / merged PRs aren't even fetched. + assert_eq!( + build_fetch_query(&TaskFetchFilter::default()), + "involves:@me is:open" + ); +} + +#[test] +fn build_fetch_query_appends_is_open_when_no_explicit_state() { + // Scoped by repo but no explicit state — `is:open` is appended. + let query = build_fetch_query(&TaskFetchFilter { + repo: Some("tinyhumansai/openhuman".to_string()), + ..Default::default() + }); + assert_eq!(query, "repo:tinyhumansai/openhuman is:open"); +} + +#[test] +fn build_fetch_query_respects_explicit_state_without_double_open() { + // Explicit `state` is respected verbatim and `is:open` is NOT added. + let query = build_fetch_query(&TaskFetchFilter { + repo: Some("tinyhumansai/openhuman".to_string()), + state: Some("closed".to_string()), + ..Default::default() + }); + assert_eq!(query, "repo:tinyhumansai/openhuman state:closed"); + assert!(!query.contains("is:open")); +} + +#[test] +fn github_search_arg_pairs_render_cli_and_rest_params() { + let args = json!({ + "q": "repo:tinyhumansai/openhuman state:open", + "sort": "updated", + "order": "desc", + "per_page": 25, + "page": 1, + "include_prs": true, + "skip": null + }); + + let pairs = github_search_arg_pairs(&args).expect("pairs"); + assert!(pairs.contains(&( + "q".to_string(), + "repo:tinyhumansai/openhuman state:open".to_string() + ))); + assert!(pairs.contains(&("per_page".to_string(), "25".to_string()))); + assert!(pairs.contains(&("include_prs".to_string(), "true".to_string()))); + assert!(!pairs.iter().any(|(key, _)| key == "skip")); +} + +// ── slug regression tests (#2768) ─────────────────────────────────────────── +// +// Guard the current Composio action slug values used by the GitHub provider. +// Outdated slugs (e.g. GITHUB_USERS_GET_AUTHENTICATED, GITHUB_LIST_REPOS, +// GITHUB_LIST_ISSUES) were previously scattered across tests; these assertions +// pin the correct values in one place so a slug rename is caught immediately. + +#[test] +fn action_get_authenticated_user_slug_is_current() { + // The Composio v3 slug is GITHUB_GET_THE_AUTHENTICATED_USER. + // Regression: was mistakenly referenced as GITHUB_USERS_GET_AUTHENTICATED + // in tests (see issue #2768). + assert_eq!( + ACTION_GET_AUTHENTICATED_USER, "GITHUB_GET_THE_AUTHENTICATED_USER", + "slug must match Composio v3 catalog; old slug GITHUB_USERS_GET_AUTHENTICATED is retired" + ); +} + +#[test] +fn action_search_issues_slug_is_current() { + assert_eq!( + ACTION_SEARCH_ISSUES, "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS", + "slug must match Composio v3 catalog" + ); +} + +#[test] +fn curated_list_does_not_contain_retired_slugs() { + // Guard against re-introducing removed slugs that no longer exist in the + // Composio v3 GitHub app catalog. + const RETIRED: &[&str] = &[ + "GITHUB_USERS_GET_AUTHENTICATED", // replaced by GITHUB_GET_THE_AUTHENTICATED_USER + "GITHUB_LIST_REPOS", // replaced by GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER + "GITHUB_LIST_ISSUES", // replaced by GITHUB_LIST_REPOSITORY_ISSUES + "GITHUB_COMMIT_MULTIPLE_FILES", // removed from Composio catalog + "GITHUB_CLOSE_AN_ISSUE", // removed; use GITHUB_UPDATE_AN_ISSUE with state=closed + "GITHUB_DELETE_A_BRANCH", // removed; use GITHUB_DELETE_A_REFERENCE + ]; + + let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); + for retired in RETIRED { + assert!( + !slugs.contains(retired), + "curated list must not contain retired slug {retired} (see #2768)" + ); + } +} + +#[test] +fn curated_list_contains_current_read_slugs() { + // Verify that the primary read-tier actions are present with their correct + // v3 slug names (not the old v1/v2 names). + let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); + let required = [ + "GITHUB_GET_THE_AUTHENTICATED_USER", + "GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER", + "GITHUB_LIST_REPOSITORY_ISSUES", + "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS", + "GITHUB_LIST_PULL_REQUESTS", + "GITHUB_GET_A_PULL_REQUEST", + ]; + for slug in required { + assert!( + slugs.contains(&slug), + "curated list must contain current slug {slug} (see #2768)" + ); + } +} + +#[test] +fn curated_list_contains_current_write_slugs() { + let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); + let required = [ + "GITHUB_CREATE_AN_ISSUE", + "GITHUB_UPDATE_AN_ISSUE", + "GITHUB_CREATE_A_PULL_REQUEST", + "GITHUB_MERGE_A_PULL_REQUEST", + ]; + for slug in required { + assert!( + slugs.contains(&slug), + "curated list must contain current write slug {slug} (see #2768)" + ); + } +} + +// ── GithubFetchMode (#3279) ───────────────────────────────────────────────── +// +// The fetch-mode selector makes the local `gh`/REST path a *true fallback* +// (default `Auto`) instead of a hard Composio replacement. These tests pin the +// default and the serde wire contract the UI persists into a source's +// `FilterSpec::Github { fetch_mode }`. + +#[test] +fn github_fetch_mode_defaults_to_auto() { + // `Auto` must be the default so shipped Composio users keep working and + // local/dev setups still get the fallback — neither side regresses. + assert_eq!(GithubFetchMode::default(), GithubFetchMode::Auto); +} + +#[test] +fn task_fetch_filter_default_uses_auto_fetch_mode() { + // A filter built with no explicit mode (the common path) carries `Auto`. + let filter = TaskFetchFilter::default(); + assert_eq!(filter.github_fetch_mode, GithubFetchMode::Auto); +} + +#[test] +fn github_fetch_mode_serializes_snake_case() { + assert_eq!( + serde_json::to_value(GithubFetchMode::Auto).expect("ser auto"), + json!("auto") + ); + assert_eq!( + serde_json::to_value(GithubFetchMode::Composio).expect("ser composio"), + json!("composio") + ); + assert_eq!( + serde_json::to_value(GithubFetchMode::Local).expect("ser local"), + json!("local") + ); +} + +#[test] +fn github_fetch_mode_deserializes_each_variant() { + let auto: GithubFetchMode = serde_json::from_value(json!("auto")).expect("de auto"); + let composio: GithubFetchMode = serde_json::from_value(json!("composio")).expect("de composio"); + let local: GithubFetchMode = serde_json::from_value(json!("local")).expect("de local"); + assert_eq!(auto, GithubFetchMode::Auto); + assert_eq!(composio, GithubFetchMode::Composio); + assert_eq!(local, GithubFetchMode::Local); +} + +#[test] +fn github_fetch_mode_round_trips_through_json() { + for mode in [ + GithubFetchMode::Auto, + GithubFetchMode::Composio, + GithubFetchMode::Local, + ] { + let json = serde_json::to_string(&mode).expect("ser"); + let back: GithubFetchMode = serde_json::from_str(&json).expect("de"); + assert_eq!(back, mode, "round-trip must preserve {mode:?}"); + } +} + +#[test] +fn github_fetch_mode_rejects_unknown_variant() { + let parsed: Result = serde_json::from_value(json!("remote")); + assert!(parsed.is_err(), "unknown mode strings must fail to parse"); +} + +// ── github_search_arg_pairs edge cases (#3279) ────────────────────────────── + +#[test] +fn github_search_arg_pairs_skips_null_and_empty_string_values() { + // Null values are dropped entirely; whitespace-only / empty strings are + // trimmed to empty and also dropped, so they never reach the gh CLI / REST + // query as blank params. + let args = json!({ + "q": "involves:@me", + "empty": "", + "blank": " ", + "missing": null, + "page": 1, + }); + let pairs = github_search_arg_pairs(&args).expect("pairs"); + assert!(pairs.contains(&("q".to_string(), "involves:@me".to_string()))); + assert!(pairs.contains(&("page".to_string(), "1".to_string()))); + assert!(!pairs.iter().any(|(k, _)| k == "empty")); + assert!(!pairs.iter().any(|(k, _)| k == "blank")); + assert!(!pairs.iter().any(|(k, _)| k == "missing")); +} + +#[test] +fn github_search_arg_pairs_errors_when_not_an_object() { + // A non-object value (array, scalar) is a programmer error — surface it + // rather than silently producing an empty arg set. + let err = github_search_arg_pairs(&json!(["not", "an", "object"])) + .expect_err("array args must error"); + assert!(err.contains("JSON object"), "got: {err}"); +} + +// ── github_env_token (#3279) ──────────────────────────────────────────────── + +#[test] +fn github_env_token_reads_env_and_is_none_when_unset() { + // Env-mutation test: this whole suite is the only reader of GH_TOKEN / + // GITHUB_TOKEN, but cargo runs tests in parallel threads sharing the + // process env. Hold a process-wide lock so concurrent token reads don't + // race, and restore the original values on exit. + use std::sync::Mutex; + static ENV_LOCK: Mutex<()> = Mutex::new(()); + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + + let prev_gh = std::env::var("GH_TOKEN").ok(); + let prev_github = std::env::var("GITHUB_TOKEN").ok(); + + // Neither set → None. + std::env::remove_var("GH_TOKEN"); + std::env::remove_var("GITHUB_TOKEN"); + assert_eq!(github_env_token(), None, "no token vars → None"); + + // GH_TOKEN takes precedence; surrounding whitespace is trimmed. + std::env::set_var("GH_TOKEN", " gh-pat-123 "); + assert_eq!(github_env_token().as_deref(), Some("gh-pat-123")); + + // Falls back to GITHUB_TOKEN when GH_TOKEN is absent. + std::env::remove_var("GH_TOKEN"); + std::env::set_var("GITHUB_TOKEN", "github-pat-456"); + assert_eq!(github_env_token().as_deref(), Some("github-pat-456")); + + // A blank token is treated as unset. + std::env::set_var("GH_TOKEN", " "); + std::env::remove_var("GITHUB_TOKEN"); + assert_eq!(github_env_token(), None, "blank token → None"); + + // Restore original env. + match prev_gh { + Some(v) => std::env::set_var("GH_TOKEN", v), + None => std::env::remove_var("GH_TOKEN"), + } + match prev_github { + Some(v) => std::env::set_var("GITHUB_TOKEN", v), + None => std::env::remove_var("GITHUB_TOKEN"), + } +} + +// ── issue vs pull-request kind detection ───────────────────────────────────── + +#[test] +fn normalize_tags_pull_request_when_pull_request_object_present() { + // GitHub's issues-and-PRs search marks a PR hit with a `pull_request` object. + let pr = json!({ + "id": 42, + "title": "Add retry to fetch", + "state": "open", + "html_url": "https://github.com/o/r/pull/42", + "pull_request": { "url": "https://api.github.com/repos/o/r/pulls/42" } + }); + let nt = normalize_github_issue(&pr).expect("normalizes"); + assert_eq!(nt.kind, TaskKind::PullRequest); +} + +#[test] +fn normalize_tags_issue_when_no_pull_request_object() { + let issue = json!({ + "id": 7, + "title": "Login throws on empty password", + "state": "open", + "html_url": "https://github.com/o/r/issues/7" + }); + let nt = normalize_github_issue(&issue).expect("normalizes"); + assert_eq!(nt.kind, TaskKind::Issue); +} + +#[test] +fn normalize_tags_issue_when_pull_request_is_null() { + // The REST issue payload carries `pull_request: null` for plain issues. + let issue = json!({ + "id": 8, + "title": "Docs typo", + "state": "open", + "html_url": "https://github.com/o/r/issues/8", + "pull_request": null + }); + let nt = normalize_github_issue(&issue).expect("normalizes"); + assert_eq!(nt.kind, TaskKind::Issue); +} + +// ── skip merged PRs / closed issues ────────────────────────────────────────── + +#[test] +fn normalize_skips_closed_issue() { + // A closed issue is already-done work — drop it. + let issue = json!({ + "id": 100, + "title": "Old bug", + "state": "closed", + "html_url": "https://github.com/o/r/issues/100" + }); + assert!( + normalize_github_issue(&issue).is_none(), + "closed issue must be skipped" + ); +} + +#[test] +fn normalize_skips_merged_or_closed_pull_request() { + // A merged/closed PR also reports `state == "closed"` — drop it too. + let pr = json!({ + "id": 101, + "title": "Shipped feature", + "state": "closed", + "html_url": "https://github.com/o/r/pull/101", + "pull_request": { "url": "https://api.github.com/repos/o/r/pulls/101" } + }); + assert!( + normalize_github_issue(&pr).is_none(), + "merged/closed PR must be skipped" + ); +} + +#[test] +fn normalize_keeps_open_item() { + // An open item is kept and tagged with its kind. + let issue = json!({ + "id": 102, + "title": "Active work", + "state": "open", + "html_url": "https://github.com/o/r/issues/102" + }); + let nt = normalize_github_issue(&issue).expect("open item is kept"); + assert_eq!(nt.kind, TaskKind::Issue); + assert_eq!(nt.status.as_deref(), Some("open")); +} diff --git a/core/src/sync/composio/providers/github/tools.rs b/core/src/sync/composio/providers/github/tools.rs new file mode 100644 index 0000000..40be913 --- /dev/null +++ b/core/src/sync/composio/providers/github/tools.rs @@ -0,0 +1,189 @@ +//! Curated catalog of GitHub Composio actions exposed to the agent. +//! +//! Composio publishes hundreds of GitHub actions; this hand-tuned slice +//! covers the day-to-day operations an AI assistant actually performs +//! (browsing repos, reading/writing issues + PRs, code search, basic +//! workflow control) and hides the long tail of admin endpoints. + +use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; + +pub const GITHUB_CURATED: &[CuratedTool] = &[ + // ── Read: user / repos ────────────────────────────────────────── + CuratedTool { + slug: "GITHUB_GET_THE_AUTHENTICATED_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_GET_A_REPOSITORY", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_LIST_REPOSITORY_COLLABORATORS", + scope: ToolScope::Read, + }, + // ── Read: search ──────────────────────────────────────────────── + CuratedTool { + slug: "GITHUB_SEARCH_REPOSITORIES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_SEARCH_CODE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_SEARCH_USERS", + scope: ToolScope::Read, + }, + // ── Read: issues ──────────────────────────────────────────────── + CuratedTool { + slug: "GITHUB_LIST_REPOSITORY_ISSUES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_GET_AN_ISSUE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_LIST_ISSUE_COMMENTS", + scope: ToolScope::Read, + }, + // ── Read: pull requests ───────────────────────────────────────── + CuratedTool { + slug: "GITHUB_LIST_PULL_REQUESTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_GET_A_PULL_REQUEST", + scope: ToolScope::Read, + }, + // CuratedTool { slug: "GITHUB_CHECK_IF_PULL_REQUEST_HAS_BEEN_MERGED", scope: ToolScope::Read }, + // ── Read: branches / commits ──────────────────────────────────── + CuratedTool { + slug: "GITHUB_LIST_BRANCHES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_GET_A_BRANCH", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_LIST_COMMITS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GITHUB_GET_A_COMMIT", + scope: ToolScope::Read, + }, + // CuratedTool { slug: "GITHUB_COMPARE_TWO_COMMITS", scope: ToolScope::Read }, + // // ── Read: contents / releases / gists ─────────────────────────── + // CuratedTool { slug: "GITHUB_GET_REPOSITORY_CONTENTS", scope: ToolScope::Read }, + // CuratedTool { slug: "GITHUB_LIST_RELEASES", scope: ToolScope::Read }, + // CuratedTool { slug: "GITHUB_LIST_GISTS", scope: ToolScope::Read }, + // // ── Read: workflows ───────────────────────────────────────────── + // CuratedTool { slug: "GITHUB_LIST_WORKFLOWS", scope: ToolScope::Read }, + // CuratedTool { slug: "GITHUB_LIST_WORKFLOW_RUNS", scope: ToolScope::Read }, + // ── Write: repos / contents ───────────────────────────────────── + CuratedTool { + slug: "GITHUB_CREATE_A_REPOSITORY_FOR_THE_AUTHENTICATED_USER", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_CREATE_OR_UPDATE_FILE_CONTENTS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_CREATE_A_COMMIT", + scope: ToolScope::Write, + }, + // GITHUB_COMMIT_MULTIPLE_FILES removed from Composio catalog + CuratedTool { + slug: "GITHUB_CREATE_A_COMMIT_COMMENT", + scope: ToolScope::Write, + }, + // ── Write: issues ─────────────────────────────────────────────── + CuratedTool { + slug: "GITHUB_CREATE_AN_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_UPDATE_AN_ISSUE", + scope: ToolScope::Write, + }, + // GITHUB_CLOSE_AN_ISSUE — removed: no dedicated Composio slug. + // Use GITHUB_UPDATE_AN_ISSUE with state:"closed" instead. + CuratedTool { + slug: "GITHUB_CREATE_AN_ISSUE_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_ADD_LABELS_TO_AN_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_ADD_ASSIGNEES_TO_AN_ISSUE", + scope: ToolScope::Write, + }, + // ── Write: pull requests ──────────────────────────────────────── + CuratedTool { + slug: "GITHUB_CREATE_A_PULL_REQUEST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_UPDATE_A_PULL_REQUEST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_MERGE_A_PULL_REQUEST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_CREATE_A_REVIEW_FOR_A_PULL_REQUEST", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GITHUB_CREATE_A_REVIEW_COMMENT_FOR_A_PULL_REQUEST", + scope: ToolScope::Write, + }, + // // ── Write: releases / gists / workflows ───────────────────────── + // CuratedTool { slug: "GITHUB_CREATE_A_RELEASE", scope: ToolScope::Write }, + CuratedTool { + slug: "GITHUB_CREATE_A_GIST", + scope: ToolScope::Write, + }, + // CuratedTool { slug: "GITHUB_CREATE_WORKFLOW_DISPATCH", scope: ToolScope::Write }, + // ── Admin: destructive / permission-changing ──────────────────── + CuratedTool { + slug: "GITHUB_DELETE_A_REPOSITORY", + scope: ToolScope::Admin, + }, + // DELETE_A_REFERENCE maps to DELETE /repos/{owner}/{repo}/git/refs/{ref}. + // The ref must be a full path (e.g. `refs/heads/branch-name` or + // `refs/tags/v1.0`) — passing a bare branch name deletes nothing (404). + // This replaces the old GITHUB_DELETE_A_BRANCH slug (Composio v3 rename); + // it is broader — it can delete tags too — so agents should always specify + // a `refs/heads/` prefix when the intent is branch deletion. + CuratedTool { + slug: "GITHUB_DELETE_A_REFERENCE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GITHUB_DELETE_A_FILE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GITHUB_ADD_A_REPOSITORY_COLLABORATOR", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GITHUB_CANCEL_A_WORKFLOW_RUN", + scope: ToolScope::Admin, + }, +]; diff --git a/core/src/sync/composio/providers/gmail/mod.rs b/core/src/sync/composio/providers/gmail/mod.rs new file mode 100644 index 0000000..fcdc23e --- /dev/null +++ b/core/src/sync/composio/providers/gmail/mod.rs @@ -0,0 +1,11 @@ +// The Gmail post-processor moved to tinycortex (a pure Value transform, i.e. +// driver-side). Aliased under the old module name so the single call site in +// `provider.rs` stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::gmail_post_process as post_process; +mod provider; +#[cfg(test)] +mod tests; +pub mod tools; + +pub use provider::GmailProvider; +pub use tools::GMAIL_CURATED; diff --git a/core/src/sync/composio/providers/gmail/provider.rs b/core/src/sync/composio/providers/gmail/provider.rs new file mode 100644 index 0000000..07b7f0c --- /dev/null +++ b/core/src/sync/composio/providers/gmail/provider.rs @@ -0,0 +1,175 @@ +//! Gmail provider — incremental sync into the memory tree. +//! +//! On each sync pass: +//! +//! 1. Load persistent [`SyncState`] from the KV store. +//! 2. Check the daily request budget — bail early if exhausted. +//! 3. Fetch a page of recent messages via `GMAIL_FETCH_EMAILS`, adding +//! a date filter when a cursor exists so only newer mail is returned. +//! 4. Run [`ComposioProvider::post_process_action_result`] (bounded +//! HTML→text, normalise, sanitise) on the page so the LLM-facing chunk +//! content is cleaned, not raw. +//! 5. Delegate incremental filtering and document ingestion to tinycortex. +//! 6. Paginate (up to budget) until no more results or all items in the +//! page are already synced. +//! 7. Advance the cursor and save state. +//! +//! Daily budget (`DEFAULT_DAILY_REQUEST_LIMIT`, default 500) caps the +//! number of `execute_tool` calls per calendar day, preventing runaway +//! API usage during large initial backfills. + +use async_trait::async_trait; +use serde_json::{json, Value}; + +use crate::sync::composio::providers::{ + pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, ProviderContext, + ProviderUserProfile, +}; + +pub(super) const ACTION_GET_PROFILE: &str = "GMAIL_GET_PROFILE"; + +pub struct GmailProvider; + +impl GmailProvider { + pub fn new() -> Self { + Self + } +} + +impl Default for GmailProvider { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ComposioProvider for GmailProvider { + fn toolkit_slug(&self) -> &'static str { + "gmail" + } + + fn curated_tools(&self) -> Option<&'static [CuratedTool]> { + Some(super::tools::GMAIL_CURATED) + } + + fn sync_interval_secs(&self) -> Option { + Some(resolve_sync_interval_secs("gmail", 15 * 60)) + } + + fn post_process_action_result( + &self, + slug: &str, + arguments: Option<&serde_json::Value>, + data: &mut serde_json::Value, + ) { + super::post_process::post_process(slug, arguments, data); + } + + async fn fetch_user_profile( + &self, + ctx: &ProviderContext, + ) -> Result { + tracing::debug!( + connection_id = ?ctx.connection_id, + "[composio:gmail] fetch_user_profile via {ACTION_GET_PROFILE}" + ); + + let resp = ctx + .execute(ACTION_GET_PROFILE, Some(json!({}))) + .await + .map_err(|e| format!("[composio:gmail] {ACTION_GET_PROFILE} failed: {e:#}"))?; + + if !resp.successful { + let err = resp + .error + .clone() + .unwrap_or_else(|| "provider reported failure".to_string()); + return Err(format!("[composio:gmail] {ACTION_GET_PROFILE}: {err}")); + } + + // `data` is the inner Composio payload — paths here are relative + // to it. (The previous `data.*` paths were dead — `pick_str` + // does dotted-path traversal, so `data.emailAddress` looked for + // a nested `data.data.emailAddress` that never exists.) + let data = &resp.data; + let email = pick_str(data, &["emailAddress", "email", "profile.emailAddress"]); + // Don't fall back to the email when no name is returned — that + // produces duplicated `display_name == email` rows in the + // identity registry (#1365). Gmail's `GMAIL_GET_PROFILE` action + // doesn't return a name today, so this stays None. + let display_name = pick_str(data, &["name", "profile.name", "displayName"]); + let profile_url = pick_str( + data, + &["display_url", "profileUrl", "profile_url", "profile.url"], + ); + + let profile = ProviderUserProfile { + toolkit: "gmail".to_string(), + connection_id: ctx.connection_id.clone(), + display_name, + email, + username: None, + avatar_url: None, + profile_url, + extras: data.clone(), + }; + let has_email = profile.email.is_some(); + let email_domain = profile + .email + .as_deref() + .and_then(|e| e.split('@').nth(1)) + .map(|d| d.to_string()); + tracing::info!( + connection_id = ?profile.connection_id, + has_email, + email_domain = ?email_domain, + "[composio:gmail] fetched user profile" + ); + Ok(profile) + } + + /// Incremental sync via the generic + /// `orchestrator`: + /// pagination, dedup, the `max_items` cap, and cursor handling live in + /// `run_sync`; the Gmail-specific primitives — the account-email preamble, + /// server-side `after:` depth window, adaptive page ceiling, all-synced + /// stop, and batch ingest — live in `super::source`. + async fn on_trigger( + &self, + ctx: &ProviderContext, + trigger: &str, + _payload: &Value, + ) -> Result<(), String> { + tracing::info!( + connection_id = ?ctx.connection_id, + trigger = %trigger, + "[composio:gmail] on_trigger" + ); + + if trigger.eq_ignore_ascii_case("GMAIL_NEW_GMAIL_MESSAGE") + || trigger.eq_ignore_ascii_case("GMAIL_NEW_MESSAGE") + { + let Some(connection_id) = ctx.connection_id.as_deref() else { + return Err("[composio:gmail] trigger missing connection_id".to_string()); + }; + if let Err(e) = crate::tinycortex::run_composio_connection( + "gmail", + connection_id, + ctx.config.as_ref(), + ) + .await + { + tracing::warn!( + error = %e, + "[composio:gmail] trigger-driven sync failed (non-fatal)" + ); + } + } + Ok(()) + } +} + +// Message fetching (the `GMAIL_FETCH_EMAILS` action, the search query, the +// `max_items` cap math and the `sync_depth_days` `after:` floor) is owned +// by `tinycortex::memory::sync::GmailSyncPipeline`. What stays here is the +// host-side provider surface: profile lookup and trigger dispatch. diff --git a/core/src/sync/composio/providers/gmail/tests.rs b/core/src/sync/composio/providers/gmail/tests.rs new file mode 100644 index 0000000..70b5e6b --- /dev/null +++ b/core/src/sync/composio/providers/gmail/tests.rs @@ -0,0 +1,29 @@ +//! Host-owned Gmail provider surface tests. +//! +//! Pagination, cursor, envelope parsing, and ingest behavior are owned and +//! tested by `tinycortex::memory::sync::GmailSyncPipeline`. + +use super::GmailProvider; +use crate::sync::composio::providers::ComposioProvider; + +#[test] +fn provider_metadata_is_stable() { + let provider = GmailProvider::new(); + assert_eq!(provider.toolkit_slug(), "gmail"); + assert_eq!(provider.sync_interval_secs(), Some(15 * 60)); +} + +#[test] +fn default_impl_matches_new() { + let _new = GmailProvider::new(); + let _default = ::default(); +} + +#[test] +fn provider_source_does_not_restrict_to_inbox() { + let source = include_str!("provider.rs"); + assert!( + !source.contains("\"in:inbox"), + "provider query must not exclude sent mail" + ); +} diff --git a/core/src/sync/composio/providers/gmail/tools.rs b/core/src/sync/composio/providers/gmail/tools.rs new file mode 100644 index 0000000..ac14e10 --- /dev/null +++ b/core/src/sync/composio/providers/gmail/tools.rs @@ -0,0 +1,145 @@ +//! Curated catalog of Gmail Composio actions exposed to the agent. +//! +//! Composio publishes 60+ Gmail actions; this hand-tuned slice covers +//! the cases the agent actually plans for (read, compose, manage) and +//! hides the long tail of edge-case admin endpoints. + +use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; + +pub const GMAIL_CURATED: &[CuratedTool] = &[ + // ── Read: messages & threads ──────────────────────────────────── + CuratedTool { + slug: "GMAIL_FETCH_EMAILS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_LIST_MESSAGES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_FETCH_MESSAGE_BY_MESSAGE_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_FETCH_MESSAGE_BY_THREAD_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_LIST_THREADS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_GET_ATTACHMENT", + scope: ToolScope::Read, + }, + // ── Read: profile & settings ──────────────────────────────────── + CuratedTool { + slug: "GMAIL_GET_PROFILE", + scope: ToolScope::Read, + }, + // CuratedTool { slug: "GMAIL_GET_LANGUAGE_SETTINGS", scope: ToolScope::Read }, + // CuratedTool { slug: "GMAIL_GET_VACATION_SETTINGS", scope: ToolScope::Read }, + // CuratedTool { slug: "GMAIL_GET_AUTO_FORWARDING", scope: ToolScope::Read }, + // ── Read: contacts & people ───────────────────────────────────── + CuratedTool { + slug: "GMAIL_GET_CONTACTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_GET_PEOPLE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_SEARCH_PEOPLE", + scope: ToolScope::Read, + }, + // ── Read: drafts & labels ─────────────────────────────────────── + CuratedTool { + slug: "GMAIL_LIST_DRAFTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_GET_DRAFT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_LIST_LABELS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "GMAIL_GET_LABEL", + scope: ToolScope::Read, + }, + // ── Write: send & compose ─────────────────────────────────────── + CuratedTool { + slug: "GMAIL_SEND_EMAIL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GMAIL_REPLY_TO_THREAD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GMAIL_FORWARD_MESSAGE", + scope: ToolScope::Write, + }, + // ── Write: drafts ─────────────────────────────────────────────── + CuratedTool { + slug: "GMAIL_CREATE_EMAIL_DRAFT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GMAIL_UPDATE_DRAFT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "GMAIL_SEND_DRAFT", + scope: ToolScope::Write, + }, + // ── Write: labels (create/update on user labels) ──────────────── + // CuratedTool { slug: "GMAIL_CREATE_LABEL", scope: ToolScope::Write }, + // CuratedTool { slug: "GMAIL_UPDATE_LABEL", scope: ToolScope::Write }, + // CuratedTool { slug: "GMAIL_PATCH_LABEL", scope: ToolScope::Write }, + CuratedTool { + slug: "GMAIL_ADD_LABEL_TO_EMAIL", + scope: ToolScope::Write, + }, + // ── Admin: destructive & permission-changing ──────────────────── + CuratedTool { + slug: "GMAIL_DELETE_MESSAGE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GMAIL_BATCH_DELETE_MESSAGES", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GMAIL_MOVE_TO_TRASH", + scope: ToolScope::Admin, + }, + // CuratedTool { slug: "GMAIL_UNTRASH_MESSAGE", scope: ToolScope::Admin }, + CuratedTool { + slug: "GMAIL_DELETE_THREAD", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GMAIL_MOVE_THREAD_TO_TRASH", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GMAIL_UNTRASH_THREAD", + scope: ToolScope::Admin, + }, + // CuratedTool { slug: "GMAIL_MODIFY_THREAD_LABELS", scope: ToolScope::Admin }, + // CuratedTool { slug: "GMAIL_BATCH_MODIFY_MESSAGES", scope: ToolScope::Admin }, + CuratedTool { + slug: "GMAIL_DELETE_DRAFT", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "GMAIL_DELETE_LABEL", + scope: ToolScope::Admin, + }, + // CuratedTool { slug: "GMAIL_PATCH_SEND_AS", scope: ToolScope::Admin }, + // CuratedTool { slug: "GMAIL_UPDATE_IMAP_SETTINGS", scope: ToolScope::Admin }, +]; diff --git a/core/src/sync/composio/providers/helpers.rs b/core/src/sync/composio/providers/helpers.rs new file mode 100644 index 0000000..6e1e2ba --- /dev/null +++ b/core/src/sync/composio/providers/helpers.rs @@ -0,0 +1,56 @@ +//! Shared helpers for Composio provider implementations. +//! +//! `pick_str` used to live here. It is a provider payload normaliser, so it +//! moved to `tinycortex::memory::sync::composio::providers::normalize::helpers` +//! and is re-exported from this module's parent. The helpers that remain are +//! request-building rather than normalisation, and stay host-side. + +use tinycortex::memory::sync::composio::providers::normalize::helpers::pick_str; + +/// Shallow-merge an `extra` JSON object into a (mutable) action-args +/// object. Only object-typed extras are merged; non-object `extra` +/// values are ignored. Backs the `task_sources` advanced free-form +/// filter escape hatch — provider `fetch_tasks` impls call this to fold +/// user-supplied provider-native query fragments into their request +/// arguments. +pub(crate) fn merge_extra(args: &mut serde_json::Value, extra: &serde_json::Value) { + if let (Some(args_obj), Some(extra_obj)) = (args.as_object_mut(), extra.as_object()) { + for (k, v) in extra_obj { + args_obj.insert(k.clone(), v.clone()); + } + } +} + +/// Resolve the first array found among `array_paths` (dotted object +/// paths), then return the first non-empty string at one of `fields` +/// on that array's first element. Complements [`pick_str`], which +/// cannot index into arrays. Used to pull e.g. the first assignee's +/// username out of an `assignees` array. +pub(crate) fn first_array_str( + value: &serde_json::Value, + array_paths: &[&str], + fields: &[&str], +) -> Option { + for path in array_paths { + let mut cur = value; + let mut ok = true; + for segment in path.split('.') { + match cur.get(segment) { + Some(next) => cur = next, + None => { + ok = false; + break; + } + } + } + if !ok { + continue; + } + if let Some(first) = cur.as_array().and_then(|a| a.first()) { + if let Some(found) = pick_str(first, fields) { + return Some(found); + } + } + } + None +} diff --git a/core/src/sync/composio/providers/linear/mod.rs b/core/src/sync/composio/providers/linear/mod.rs new file mode 100644 index 0000000..27ff7b7 --- /dev/null +++ b/core/src/sync/composio/providers/linear/mod.rs @@ -0,0 +1,16 @@ +//! Linear Composio provider — incremental Memory Tree ingest for +//! issues assigned to the connected user. +//! +//! Issue: #2400. + +// The payload normalisers moved to tinycortex (they are pure Value +// transforms, i.e. driver-side). Aliased under the old module name so +// every `normalization::extract_*` call site below stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::linear as normalization; +mod provider; +#[cfg(test)] +mod tests; +pub mod tools; + +pub use provider::LinearProvider; +pub use tools::LINEAR_CURATED; diff --git a/core/src/sync/composio/providers/linear/provider.rs b/core/src/sync/composio/providers/linear/provider.rs new file mode 100644 index 0000000..3ee9584 --- /dev/null +++ b/core/src/sync/composio/providers/linear/provider.rs @@ -0,0 +1,241 @@ +//! Linear provider — incremental sync of issues assigned to the +//! authenticated user, with per-issue memory_tree ingest. +//! +//! On each sync pass: +//! +//! 1. Load persistent [`SyncState`] from the KV store. +//! 2. Check the daily request budget — bail early if exhausted. +//! 3. Resolve the viewer ID via `LINEAR_LIST_LINEAR_USERS { isMe: true }`. +//! 4. Page through `LINEAR_LIST_LINEAR_ISSUES` filtered to the viewer as +//! assignee, ordered by `updatedAt` descending. Stop early once we hit +//! issues older than the cursor or a page without a next-page cursor. +//! 5. For each issue, ingest into memory_tree if it's new *or* edited +//! since the last sync. +//! 6. Advance the cursor to the newest `updatedAt` seen and save. +//! +//! Privacy posture: we only pull issues the user is assigned to, never +//! the whole workspace's issue graph. This mirrors the +//! "fetch-what-the-user-sees" model `gmail` / `notion` already follow +//! and avoids accidentally ingesting other teammates' private issues. + +use async_trait::async_trait; +use serde_json::json; + +use super::normalization; +use crate::sync::composio::providers::{ + merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, + NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind, +}; + +pub(super) const ACTION_LIST_USERS: &str = "LINEAR_LIST_LINEAR_USERS"; +pub(super) const ACTION_LIST_ISSUES: &str = "LINEAR_LIST_LINEAR_ISSUES"; + +/// Paths for extracting a Linear issue's unique ID. +pub(super) const ISSUE_ID_PATHS: &[&str] = &["id", "data.id", "identifier", "data.identifier"]; + +pub struct LinearProvider; + +impl LinearProvider { + pub fn new() -> Self { + Self + } +} + +impl Default for LinearProvider { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ComposioProvider for LinearProvider { + fn toolkit_slug(&self) -> &'static str { + "linear" + } + + fn curated_tools(&self) -> Option<&'static [CuratedTool]> { + Some(super::tools::LINEAR_CURATED) + } + + fn sync_interval_secs(&self) -> Option { + // 30 minutes — same cadence as ClickUp/Notion. Linear issues change + // more slowly than chat but faster than email. + Some(resolve_sync_interval_secs("linear", 30 * 60)) + } + + async fn fetch_user_profile( + &self, + ctx: &ProviderContext, + ) -> Result { + tracing::debug!( + connection_id = ?ctx.connection_id, + "[composio:linear] fetch_user_profile via {ACTION_LIST_USERS}" + ); + + let resp = ctx + .execute(ACTION_LIST_USERS, Some(json!({ "isMe": true }))) + .await + .map_err(|e| format!("[composio:linear] {ACTION_LIST_USERS} failed: {e:#}"))?; + + if !resp.successful { + let err = resp + .error + .clone() + .unwrap_or_else(|| "provider reported failure".to_string()); + return Err(format!("[composio:linear] {ACTION_LIST_USERS}: {err}")); + } + + let data = &resp.data; + let viewer = normalization::extract_viewer(data); + let viewer_ref = viewer.as_ref().unwrap_or(data); + + let display_name = pick_str(viewer_ref, &["name", "data.name", "displayName"]); + let email = pick_str(viewer_ref, &["email", "data.email"]); + let username = pick_str(viewer_ref, &["id", "data.id"]); + let avatar_url = pick_str(viewer_ref, &["avatarUrl", "data.avatarUrl"]); + let profile_url = pick_str(viewer_ref, &["url", "data.url"]); + + Ok(ProviderUserProfile { + toolkit: "linear".to_string(), + connection_id: ctx.connection_id.clone(), + display_name, + email, + username, + avatar_url, + profile_url, + extras: data.clone(), + }) + } + + /// Incremental sync via the generic + /// `orchestrator`: + /// viewer resolution, pagination, dedup, the `max_items` cap, the + /// `sync_depth_days` window, and cursor handling live in `run_sync`; the + /// Linear-specific primitives live in `super::source`. + async fn fetch_tasks( + &self, + ctx: &ProviderContext, + filter: &TaskFetchFilter, + ) -> Result, String> { + let max = filter.effective_max(); + tracing::debug!( + connection_id = ?ctx.connection_id, + max, + team_id = ?filter.team_id, + assignee_is_me = filter.assignee_is_me, + "[composio:linear] fetch_tasks" + ); + + let mut args = json!({ + "first": max.min(100) as u64, + "orderBy": "updatedAt", + }); + if filter.assignee_is_me { + let resp = ctx + .execute(ACTION_LIST_USERS, Some(json!({ "isMe": true }))) + .await + .map_err(|e| format!("[composio:linear] {ACTION_LIST_USERS}: {e:#}"))?; + // Fail closed: a failed viewer lookup must not silently widen + // the query beyond "assigned to me". + if !resp.successful { + return Err(format!( + "[composio:linear] {ACTION_LIST_USERS}: {}", + resp.error.unwrap_or_else(|| "provider failure".into()) + )); + } + let viewer_id = normalization::extract_viewer_id(&resp.data).ok_or_else(|| { + "[composio:linear] LINEAR_LIST_LINEAR_USERS returned no viewer id".to_string() + })?; + args["assigneeId"] = json!(viewer_id); + } + if let Some(team) = filter + .team_id + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + { + args["teamId"] = json!(team); + } + merge_extra(&mut args, &filter.extra); + + let resp = ctx + .execute(ACTION_LIST_ISSUES, Some(args)) + .await + .map_err(|e| format!("[composio:linear] {ACTION_LIST_ISSUES}: {e:#}"))?; + if !resp.successful { + return Err(format!( + "[composio:linear] {ACTION_LIST_ISSUES}: {}", + resp.error.unwrap_or_else(|| "provider failure".into()) + )); + } + + let want_state = filter + .state + .as_deref() + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty()); + + let mut out: Vec = Vec::new(); + for issue in normalization::extract_issues(&resp.data) { + if out.len() >= max { + break; + } + let Some(nt) = normalize_linear_issue(&issue) else { + continue; + }; + if let Some(ref want) = want_state { + let matches = nt + .status + .as_deref() + .map(|s| s.to_ascii_lowercase() == *want) + .unwrap_or(false); + if !matches { + continue; + } + } + out.push(nt); + } + tracing::debug!(count = out.len(), "[composio:linear] fetch_tasks complete"); + Ok(out) + } +} + +/// Map a raw Linear issue payload into a [`NormalizedTask`]. +fn normalize_linear_issue(issue: &serde_json::Value) -> Option { + let external_id = pick_str(issue, ISSUE_ID_PATHS)?; + let title = normalization::extract_issue_title(issue) + .unwrap_or_else(|| format!("Linear issue {external_id}")); + Some(NormalizedTask { + external_id, + source_id: String::new(), + provider: "linear".to_string(), + kind: TaskKind::Generic, + title, + body: pick_str(issue, &["description", "data.description"]), + url: pick_str(issue, &["url", "data.url"]), + status: pick_str(issue, &["state.name", "data.state.name", "state.type"]), + assignee: pick_str(issue, &["assignee.name", "data.assignee.name"]), + due: pick_str(issue, &["dueDate", "data.dueDate"]), + labels: extract_linear_labels(issue), + priority: pick_str(issue, &["priorityLabel", "data.priorityLabel"]), + updated_at: normalization::extract_issue_updated(issue), + raw: issue.clone(), + }) +} + +/// Extract label names from a Linear issue (`labels.nodes[].name`). +fn extract_linear_labels(issue: &serde_json::Value) -> Vec { + let arr = issue + .get("labels") + .or_else(|| issue.get("data").and_then(|d| d.get("labels"))) + .and_then(|l| l.get("nodes")) + .and_then(|v| v.as_array()); + match arr { + Some(items) => items + .iter() + .filter_map(|l| l.get("name").and_then(|n| n.as_str())) + .map(|s| s.to_string()) + .collect(), + None => Vec::new(), + } +} diff --git a/core/src/sync/composio/providers/linear/tests.rs b/core/src/sync/composio/providers/linear/tests.rs new file mode 100644 index 0000000..c84ace8 --- /dev/null +++ b/core/src/sync/composio/providers/linear/tests.rs @@ -0,0 +1,172 @@ +//! Unit tests for the Linear provider. + +use super::normalization::{ + extract_issue_title, extract_issue_updated, extract_issues, extract_pagination_cursor, + extract_viewer, extract_viewer_id, +}; +use super::LinearProvider; +use crate::sync::composio::providers::ComposioProvider; +use serde_json::json; + +// ── extract_issues ─────────────────────────────────────────────────── + +#[test] +fn extract_issues_walks_common_shapes() { + let v1 = json!({ "data": { "nodes": [{"id": "i1"}] } }); + let v2 = json!({ "nodes": [{"id": "i2"}, {"id": "i3"}] }); + let v3 = json!({ "data": { "issues": { "nodes": [{"id": "i4"}] } } }); + let v4 = json!({ "foo": "bar" }); + assert_eq!(extract_issues(&v1).len(), 1); + assert_eq!(extract_issues(&v2).len(), 2); + assert_eq!(extract_issues(&v3).len(), 1); + assert_eq!(extract_issues(&v4).len(), 0); +} + +// ── extract_issue_title ────────────────────────────────────────────── + +#[test] +fn extract_issue_title_finds_title_field() { + let issue = json!({ "id": "i1", "title": "Fix the login bug" }); + assert_eq!( + extract_issue_title(&issue), + Some("Fix the login bug".into()) + ); +} + +#[test] +fn extract_issue_title_falls_back_to_wrapped_data() { + let issue = json!({ "data": { "title": "Wrapped issue" } }); + assert_eq!(extract_issue_title(&issue), Some("Wrapped issue".into())); +} + +#[test] +fn extract_issue_title_falls_back_to_identifier() { + let issue = json!({ "identifier": "ENG-99" }); + assert_eq!(extract_issue_title(&issue), Some("ENG-99".into())); +} + +// ── extract_issue_updated ──────────────────────────────────────────── + +#[test] +fn extract_issue_updated_handles_camel_case() { + let issue = json!({ "updatedAt": "2026-03-01T12:00:00.000Z" }); + assert_eq!( + extract_issue_updated(&issue), + Some("2026-03-01T12:00:00.000Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_handles_wrapped_data() { + let issue = json!({ "data": { "updatedAt": "2026-01-15T08:30:00.000Z" } }); + assert_eq!( + extract_issue_updated(&issue), + Some("2026-01-15T08:30:00.000Z".to_string()) + ); +} + +// ── extract_viewer ─────────────────────────────────────────────────── + +#[test] +fn extract_viewer_finds_first_node() { + let data = json!({ "data": { "nodes": [{ "id": "usr_1", "email": "a@b.com" }] } }); + let v = extract_viewer(&data).expect("viewer found"); + assert_eq!(v["id"], "usr_1"); +} + +#[test] +fn extract_viewer_from_top_level_nodes() { + let data = json!({ "nodes": [{ "id": "usr_2" }] }); + let v = extract_viewer(&data).expect("viewer found"); + assert_eq!(v["id"], "usr_2"); +} + +#[test] +fn extract_viewer_fallback_direct_object() { + let data = json!({ "id": "usr_direct", "name": "Alice" }); + let v = extract_viewer(&data).expect("viewer found"); + assert_eq!(v["id"], "usr_direct"); +} + +#[test] +fn extract_viewer_returns_none_when_absent() { + let data = json!({ "foo": "bar" }); + assert!(extract_viewer(&data).is_none()); +} + +// ── extract_pagination_cursor ──────────────────────────────────────── + +#[test] +fn extract_pagination_cursor_returns_cursor_on_has_next_page() { + let data = json!({ + "data": { + "pageInfo": { "hasNextPage": true, "endCursor": "abc123" } + } + }); + assert_eq!(extract_pagination_cursor(&data), Some("abc123".to_string())); +} + +#[test] +fn extract_pagination_cursor_returns_none_on_last_page() { + let data = json!({ + "pageInfo": { "hasNextPage": false, "endCursor": "xyz" } + }); + assert!(extract_pagination_cursor(&data).is_none()); +} + +#[test] +fn extract_pagination_cursor_returns_none_when_absent() { + let data = json!({ "nodes": [{"id": "i1"}] }); + assert!(extract_pagination_cursor(&data).is_none()); +} + +// ── extract_viewer_id ──────────────────────────────────────────────── + +#[test] +fn extract_viewer_id_from_data_nodes() { + let data = json!({ "data": { "nodes": [{ "id": "usr_abc" }] } }); + assert_eq!(extract_viewer_id(&data), Some("usr_abc".to_string())); +} + +#[test] +fn extract_viewer_id_returns_none_when_absent() { + let data = json!({ "foo": "bar" }); + assert!(extract_viewer_id(&data).is_none()); +} + +// ── provider metadata ──────────────────────────────────────────────── + +#[test] +fn provider_metadata_is_stable() { + let p = LinearProvider::new(); + assert_eq!(p.toolkit_slug(), "linear"); + assert_eq!(p.sync_interval_secs(), Some(30 * 60)); + assert!(p.curated_tools().is_some()); +} + +#[test] +fn curated_tools_contains_core_sync_surface() { + let p = LinearProvider::new(); + let curated = p.curated_tools().expect("LINEAR_CURATED is registered"); + let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); + assert!( + slugs.contains(&"LINEAR_LIST_LINEAR_USERS"), + "LINEAR_LIST_LINEAR_USERS must be in curated catalog" + ); + assert!( + slugs.contains(&"LINEAR_LIST_LINEAR_ISSUES"), + "LINEAR_LIST_LINEAR_ISSUES must be in curated catalog" + ); +} + +#[test] +fn default_impl_matches_new() { + let a = LinearProvider::new(); + let b = ::default(); + assert_eq!(a.toolkit_slug(), b.toolkit_slug()); + assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); + assert_eq!( + a.curated_tools().map(<[_]>::len), + b.curated_tools().map(<[_]>::len), + ); +} diff --git a/core/src/sync/composio/providers/linear/tools.rs b/core/src/sync/composio/providers/linear/tools.rs new file mode 100644 index 0000000..9eb61b5 --- /dev/null +++ b/core/src/sync/composio/providers/linear/tools.rs @@ -0,0 +1,90 @@ +//! Curated catalog of Linear Composio actions. + +use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; + +pub const LINEAR_CURATED: &[CuratedTool] = &[ + CuratedTool { + slug: "LINEAR_LIST_LINEAR_USERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_LIST_LINEAR_ISSUES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_GET_LINEAR_ISSUE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_SEARCH_ISSUES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_LIST_LINEAR_TEAMS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_LIST_LINEAR_PROJECTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_GET_LINEAR_PROJECT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_LIST_LINEAR_STATES", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_GET_CYCLES_BY_TEAM_ID", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_LIST_LINEAR_LABELS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "LINEAR_CREATE_LINEAR_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_UPDATE_ISSUE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_CREATE_LINEAR_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_UPDATE_LINEAR_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_CREATE_ATTACHMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_CREATE_ISSUE_RELATION", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_CREATE_LINEAR_PROJECT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_UPDATE_LINEAR_PROJECT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_CREATE_LINEAR_LABEL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "LINEAR_DELETE_LINEAR_ISSUE", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "LINEAR_REMOVE_ISSUE_LABEL", + scope: ToolScope::Admin, + }, +]; diff --git a/core/src/sync/composio/providers/mod.rs b/core/src/sync/composio/providers/mod.rs new file mode 100644 index 0000000..0a86355 --- /dev/null +++ b/core/src/sync/composio/providers/mod.rs @@ -0,0 +1,615 @@ +//! Provider-specific code for Composio toolkits. +//! +//! Each Composio toolkit (gmail, notion, slack, …) can register a +//! [`ComposioProvider`] implementation that knows how to: +//! +//! * Fetch a normalized **user profile** for a connected account. +//! * Run an **initial / periodic sync** that pulls fresh data from the +//! upstream service via the backend-proxied +//! `ComposioClient`. +//! * React to **trigger webhooks** that arrive over the +//! `composio:trigger` Socket.IO bridge. +//! * React to **OAuth handoff completion** so the very first sync can +//! run as soon as a user connects an account. +//! +//! Providers are pure Rust — there is no JS sandbox involved. They are +//! the native counterpart to the QuickJS skill bundles in +//! `tinyhumansai/openhuman-skills`, but specialized for Composio's API +//! surface and run inside the core process directly. +//! +//! ## Registry & dispatch +//! +//! The [`registry`] module owns a process-global `HashMap>`. The composio event bus subscriber +//! (`super::bus::ComposioTriggerSubscriber`) and the periodic sync +//! task both look up providers by toolkit slug and call into them. +//! +//! ## Why a trait, not a giant `match` +//! +//! Each provider has provider-specific shapes (gmail returns +//! emailAddress + messagesTotal, notion returns workspaces + pages, …) +//! and a different idea of what "sync" means. A trait keeps each +//! provider's implementation isolated, individually testable, and +//! easy to add without touching the dispatch layer. + +mod descriptions; +pub(crate) mod helpers; +mod scope_lookup; +pub mod tool_scope; +mod traits; +mod types; +pub mod user_scopes; + +pub mod catalogs; +pub mod catalogs_business; +pub mod catalogs_google; +pub mod catalogs_messaging; +pub mod catalogs_microsoft; +pub mod catalogs_productivity; +pub mod catalogs_social_media; +pub mod clickup; +pub mod github; +pub mod gmail; +pub mod linear; +pub mod notion; +pub mod profile; +pub mod profile_md; +pub mod registry; +pub mod slack; +pub mod sync_state; + +use crate::composio_host::ComposioCapability; + +const CAPABILITY_TOOLKITS: &[&str] = &[ + "gmail", + "notion", + "slack", + "clickup", + "github", + "discord", + "googlecalendar", + "googledrive", + "googledocs", + "googlesheets", + "outlook", + "microsoft_teams", + "linear", + "jira", + "trello", + "asana", + "dropbox", + "twitter", + "spotify", + "telegram", + "whatsapp", + "shopify", + "stripe", + "hubspot", + "salesforce", + "airtable", + "figma", + "youtube", + "one_drive", + "excel", + "todoist", +]; + +fn native_provider_sync_interval(toolkit: &str) -> Option { + match toolkit { + "gmail" => Some(gmail::GmailProvider::new().sync_interval_secs()), + "notion" => Some(notion::NotionProvider::new().sync_interval_secs()), + "slack" => Some(slack::SlackProvider::new().sync_interval_secs()), + "clickup" => Some(clickup::ClickUpProvider::new().sync_interval_secs()), + "github" => Some(github::GitHubProvider::new().sync_interval_secs()), + "linear" => Some(linear::LinearProvider::new().sync_interval_secs()), + _ => None, + } + .flatten() +} + +fn has_native_provider(toolkit: &str) -> bool { + matches!( + toolkit, + "gmail" | "notion" | "slack" | "clickup" | "github" | "linear" + ) +} + +/// Static overview of the Composio integrations supported by this core build. +/// +/// This deliberately does not consult the live Composio backend/direct tenant: +/// it is an observability surface for OpenHuman's own capability tiers. Use +/// `composio_list_toolkits` / `composio_list_connections` when callers need +/// the currently signed-in user's allowlist or OAuth state. +pub fn capability_matrix() -> Vec { + CAPABILITY_TOOLKITS + .iter() + .map(|toolkit| { + let native_provider = has_native_provider(toolkit); + let catalog = catalog_for_toolkit(toolkit); + let sync_interval_secs = native_provider_sync_interval(toolkit); + ComposioCapability { + toolkit: (*toolkit).to_string(), + description: toolkit_description(toolkit).to_string(), + native_provider, + curated_tools: catalog.is_some(), + curated_tool_count: catalog.map_or(0, <[CuratedTool]>::len), + tool_execution: catalog.is_some(), + user_profile: native_provider, + initial_sync: native_provider, + periodic_sync: sync_interval_secs.is_some(), + sync_interval_secs, + trigger_webhooks: native_provider, + memory_ingest: native_provider, + } + }) + .collect() +} + +/// Static toolkit → curated catalog map. +/// +/// This is consulted by the meta-tool layer alongside any registered +/// provider's [`ComposioProvider::curated_tools`]. It lets toolkits +/// without a full native provider still benefit from curated +/// whitelisting. +/// +/// Lookup key is the lowercased prefix returned by +/// [`toolkit_from_slug`] applied to the action slug — e.g. +/// `GOOGLECALENDAR_CREATE_EVENT` → `"googlecalendar"`. Multi-segment +/// prefixes like `MICROSOFT_TEAMS_*` return their known toolkit slug. +/// Synchronous visibility check for a Composio action slug given a +/// pre-loaded user scope preference. +/// +/// Returns `true` if the action should appear in the agent's tool +/// surface — i.e. it's in the toolkit's curated whitelist (or the +/// toolkit has no curation) **and** the user's scope pref allows its +/// classification. Falls back to [`classify_unknown`] for un-curated +/// toolkits. +/// +/// Use this when the user pref has already been loaded for the +/// toolkit (typical inside a `for slug in toolkits {...}` loop where +/// awaiting once per toolkit is cheaper than once per action). +pub fn is_action_visible_with_pref(slug: &str, pref: &UserScopePref) -> bool { + let Some(toolkit) = toolkit_from_slug(slug) else { + return true; + }; + let catalog = get_provider(&toolkit) + .and_then(|p| p.curated_tools()) + .or_else(|| catalog_for_toolkit(&toolkit)); + match catalog { + Some(catalog) => match find_curated(catalog, slug) { + Some(curated) => pref.allows(curated.scope), + None => false, + }, + None => pref.allows(classify_unknown(slug)), + } +} + +pub fn catalog_for_toolkit(toolkit: &str) -> Option<&'static [CuratedTool]> { + match toolkit.trim().to_ascii_lowercase().as_str() { + // Native providers + "gmail" => Some(gmail::GMAIL_CURATED), + "notion" => Some(notion::NOTION_CURATED), + "github" => Some(github::GITHUB_CURATED), + "linear" => Some(linear::LINEAR_CURATED), + // Catalog-only toolkits + "slack" => Some(catalogs::SLACK_CURATED), + "discord" => Some(catalogs::DISCORD_CURATED), + "googlecalendar" | "google_calendar" => Some(catalogs::GOOGLECALENDAR_CURATED), + "googledrive" | "google_drive" => Some(catalogs::GOOGLEDRIVE_CURATED), + "googledocs" | "google_docs" => Some(catalogs::GOOGLEDOCS_CURATED), + "googlesheets" | "google_sheets" => Some(catalogs::GOOGLESHEETS_CURATED), + "outlook" => Some(catalogs::OUTLOOK_CURATED), + // Keep the legacy "microsoft" alias while toolkit_from_slug now + // returns the precise "microsoft_teams" slug for Teams actions. + "microsoft" | "microsoft_teams" => Some(catalogs::MICROSOFT_TEAMS_CURATED), + "jira" => Some(catalogs::JIRA_CURATED), + "trello" => Some(catalogs::TRELLO_CURATED), + "asana" => Some(catalogs::ASANA_CURATED), + "clickup" => Some(clickup::CLICKUP_CURATED), + "dropbox" => Some(catalogs::DROPBOX_CURATED), + "twitter" => Some(catalogs::TWITTER_CURATED), + "spotify" => Some(catalogs::SPOTIFY_CURATED), + "telegram" => Some(catalogs::TELEGRAM_CURATED), + "whatsapp" => Some(catalogs::WHATSAPP_CURATED), + "shopify" => Some(catalogs::SHOPIFY_CURATED), + "stripe" => Some(catalogs::STRIPE_CURATED), + "hubspot" => Some(catalogs::HUBSPOT_CURATED), + "salesforce" => Some(catalogs::SALESFORCE_CURATED), + "airtable" => Some(catalogs::AIRTABLE_CURATED), + "figma" => Some(catalogs::FIGMA_CURATED), + "youtube" => Some(catalogs::YOUTUBE_CURATED), + // ONE_DRIVE_* slugs extract to "one" via toolkit_from_slug; + // alias both the prefix and the canonical UI/backend slugs. + "one" | "one_drive" | "onedrive" => Some(catalogs::ONE_DRIVE_CURATED), + "excel" => Some(catalogs::EXCEL_CURATED), + "todoist" => Some(catalogs::TODOIST_CURATED), + _ => None, + } +} + +/// All toolkit slugs that have a curated agent-ready catalog. +/// +/// Source of truth for the UI "preview / agent integration coming +/// soon" badge: any connected toolkit whose slug is NOT in this list +/// can be authorized but lacks a curated tool surface, so the agent +/// can't use it productively. +/// +/// Returned in sorted order to keep the RPC response stable across +/// builds. +pub fn agent_ready_toolkits() -> Vec<&'static str> { + let mut slugs: Vec<&'static str> = vec![ + // Native providers + "gmail", + "notion", + "github", + // Catalog-only toolkits + "slack", + "discord", + "googlecalendar", + "googledrive", + "googledocs", + "googlesheets", + "outlook", + "microsoft_teams", + "linear", + "jira", + "trello", + "asana", + "dropbox", + "twitter", + "spotify", + "telegram", + "whatsapp", + "shopify", + "stripe", + "hubspot", + "salesforce", + "airtable", + "figma", + "youtube", + "one_drive", + "excel", + "todoist", + ]; + slugs.sort_unstable(); + slugs +} + +pub use descriptions::toolkit_description; +pub(crate) use helpers::{first_array_str, merge_extra}; +// `pick_str` is a provider payload normaliser and lives in tinycortex; it is +// re-exported here so the ~40 in-tree call sites keep resolving unchanged. +// Note this is deliberately NOT `providers::common::pick_str`, which coerces +// numbers to strings — see the doc comments on both definitions. +pub use registry::{ + all_providers, get_provider, init_default_providers, register_provider, ProviderArc, +}; +pub use scope_lookup::{curated_scope_for, toolkit_has_scope}; +pub(crate) use tinycortex::memory::sync::composio::providers::normalize::helpers::pick_str; +pub use tool_scope::{classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope}; +pub use traits::{resolve_sync_interval_secs, sync_interval_env_var, ComposioProvider}; +pub use types::{ + ComposioUsage, ComposioUsageHandle, GithubFetchMode, NormalizedTask, ProviderContext, + ProviderUserProfile, SyncOutcome, SyncReason, TaskContainer, TaskFetchFilter, TaskKind, +}; +pub use user_scopes::{load_or_default as load_user_scope_or_default, UserScopePref}; + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn pick_str_finds_first_non_empty_match() { + let v = json!({ + "data": { "user": { "email": " user@example.com ", "name": "" } }, + "fallback": "fallback@example.com" + }); + // first path empty -> falls through + assert_eq!( + pick_str(&v, &["data.user.name", "data.user.email"]), + Some("user@example.com".to_string()) + ); + // missing path -> falls through to fallback + assert_eq!( + pick_str(&v, &["data.missing", "fallback"]), + Some("fallback@example.com".to_string()) + ); + // nothing matches + assert_eq!(pick_str(&v, &["nope.nope"]), None); + } + + #[test] + fn sync_outcome_elapsed_ms_is_safe_when_finish_lt_start() { + let mut o = SyncOutcome { + started_at_ms: 100, + finished_at_ms: 50, + ..Default::default() + }; + assert_eq!(o.elapsed_ms(), 0); + o.finished_at_ms = 250; + assert_eq!(o.elapsed_ms(), 150); + } + + #[test] + fn pick_str_returns_none_for_non_string_values() { + let v = json!({ "count": 42, "flag": true, "empty": "", "whitespace": " " }); + assert_eq!(pick_str(&v, &["count"]), None); + assert_eq!(pick_str(&v, &["flag"]), None); + assert_eq!(pick_str(&v, &["empty"]), None); + assert_eq!(pick_str(&v, &["whitespace"]), None); + } + + #[test] + fn pick_str_respects_path_order() { + let v = json!({ "a": "first", "b": "second" }); + assert_eq!(pick_str(&v, &["a", "b"]), Some("first".into())); + assert_eq!(pick_str(&v, &["b", "a"]), Some("second".into())); + } + + #[test] + fn sync_reason_as_str_matches_enum_variant() { + assert_eq!(SyncReason::ConnectionCreated.as_str(), "connection_created"); + assert_eq!(SyncReason::Periodic.as_str(), "periodic"); + assert_eq!(SyncReason::Manual.as_str(), "manual"); + } + + #[test] + fn sync_reason_serde_is_snake_case() { + let s = serde_json::to_string(&SyncReason::ConnectionCreated).unwrap(); + assert_eq!(s, "\"connection_created\""); + let back: SyncReason = serde_json::from_str(&s).unwrap(); + assert_eq!(back, SyncReason::ConnectionCreated); + } + + // Note: `toolkit_has_scope` tests now live in `scope_lookup.rs` + // alongside the implementation. + + #[test] + fn catalog_for_toolkit_resolves_new_microsoft_and_todoist_slugs() { + // Newly added catalogs (#2283): OneDrive, Excel, Todoist must be + // discoverable both by their canonical UI slug AND by the + // prefix that `toolkit_from_slug` extracts from action slugs. + assert!(catalog_for_toolkit("one_drive").is_some()); + assert!(catalog_for_toolkit("onedrive").is_some()); + // ONE_DRIVE_GET_FILE → toolkit_from_slug() → "one" + assert!(catalog_for_toolkit("one").is_some()); + assert!(catalog_for_toolkit("excel").is_some()); + assert!(catalog_for_toolkit("todoist").is_some()); + } + + #[test] + fn agent_ready_toolkits_includes_new_catalogs_and_is_sorted() { + let slugs = agent_ready_toolkits(); + assert!(slugs.contains(&"one_drive")); + assert!(slugs.contains(&"excel")); + assert!(slugs.contains(&"todoist")); + // Spot-check legacy entries still present. + assert!(slugs.contains(&"gmail")); + assert!(slugs.contains(&"slack")); + // Uncurated toolkit must NOT appear — guarantees the UI badge + // logic can rely on this set to flag "preview" toolkits. + assert!(!slugs.contains(&"sharepoint")); + assert!(!slugs.contains(&"clickup")); + // Stable order across builds — the RPC consumer caches it. + let mut expected = slugs.clone(); + expected.sort_unstable(); + assert_eq!(slugs, expected); + } + + #[test] + fn capability_matrix_includes_new_catalog_only_toolkits() { + let matrix = capability_matrix(); + for slug in ["one_drive", "excel", "todoist"] { + let row = matrix + .iter() + .find(|entry| entry.toolkit == slug) + .unwrap_or_else(|| panic!("{slug} capability row missing")); + assert!(!row.native_provider, "{slug} should not be native"); + assert!(row.curated_tools, "{slug} should be catalogued"); + assert!( + row.curated_tool_count > 0, + "{slug} catalog should be non-empty" + ); + assert!( + row.tool_execution, + "{slug} tool execution should be enabled" + ); + // No profile/sync/memory ingest — catalog-only. + assert!(!row.user_profile); + assert!(!row.initial_sync); + assert!(!row.periodic_sync); + assert!(!row.memory_ingest); + } + } + + #[test] + fn capability_matrix_distinguishes_native_from_catalog_only_toolkits() { + let matrix = capability_matrix(); + + let gmail = matrix + .iter() + .find(|entry| entry.toolkit == "gmail") + .expect("gmail capability row"); + assert!(gmail.native_provider); + assert!(gmail.curated_tools); + assert!(gmail.curated_tool_count > 0); + assert!(gmail.user_profile); + assert!(gmail.initial_sync); + assert!(gmail.periodic_sync); + assert_eq!(gmail.sync_interval_secs, Some(15 * 60)); + assert!(gmail.trigger_webhooks); + assert!(gmail.memory_ingest); + + let google_calendar = matrix + .iter() + .find(|entry| entry.toolkit == "googlecalendar") + .expect("googlecalendar capability row"); + assert!(!google_calendar.native_provider); + assert!(google_calendar.curated_tools); + assert!(google_calendar.curated_tool_count > 0); + assert!(google_calendar.tool_execution); + assert!(!google_calendar.user_profile); + assert!(!google_calendar.initial_sync); + assert!(!google_calendar.periodic_sync); + assert_eq!(google_calendar.sync_interval_secs, None); + assert!(!google_calendar.memory_ingest); + } + + #[test] + fn capability_matrix_includes_clickup_as_native_memory_provider() { + // Locks in the per-issue #2288 registration: a ClickUp row must + // appear in the capability matrix with the same native-provider + // flags Gmail/Notion/Slack already carry (`memory_ingest`, + // `periodic_sync`, non-zero `sync_interval_secs`). If a future + // change drops one of the four registration touchpoints + // (CAPABILITY_TOOLKITS, has_native_provider, + // native_provider_sync_interval, catalog_for_toolkit) this test + // fails loud rather than silently degrading the provider to + // catalog-only status. + let matrix = capability_matrix(); + let clickup = matrix + .iter() + .find(|entry| entry.toolkit == "clickup") + .expect("clickup capability row"); + assert!(clickup.native_provider, "clickup must be native"); + assert!(clickup.curated_tools, "clickup must have a curated catalog"); + assert!( + clickup.curated_tool_count > 0, + "clickup catalog must be non-empty" + ); + assert!(clickup.user_profile); + assert!(clickup.initial_sync); + assert!(clickup.periodic_sync); + assert_eq!(clickup.sync_interval_secs, Some(30 * 60)); + assert!(clickup.memory_ingest); + } + + #[test] + fn capability_matrix_includes_linear_as_native_memory_provider() { + // Per-issue #2400 registration: a Linear row must appear in + // the capability matrix as a native memory-ingest provider, + // matching gmail / notion / slack / clickup. If a future + // change drops one of the five registration touchpoints + // (CAPABILITY_TOOLKITS, has_native_provider, + // native_provider_sync_interval, catalog_for_toolkit, + // toolkit_description) this test fails loud rather than + // silently degrading the provider to catalog-only status. + let matrix = capability_matrix(); + let linear = matrix + .iter() + .find(|entry| entry.toolkit == "linear") + .expect("linear capability row"); + assert!(linear.native_provider, "linear must be native"); + assert!(linear.curated_tools, "linear must have a curated catalog"); + assert!( + linear.curated_tool_count > 0, + "linear catalog must be non-empty" + ); + assert!(linear.user_profile); + assert!(linear.initial_sync); + assert!(linear.periodic_sync); + assert_eq!(linear.sync_interval_secs, Some(30 * 60)); + assert!(linear.memory_ingest); + } + + #[test] + fn capability_matrix_includes_github_as_native_memory_provider() { + let matrix = capability_matrix(); + let github = matrix + .iter() + .find(|entry| entry.toolkit == "github") + .expect("github capability row"); + assert!(github.native_provider, "github must be native"); + assert!(github.curated_tools, "github must have a curated catalog"); + assert!( + github.curated_tool_count > 0, + "github catalog must be non-empty" + ); + assert!(github.user_profile); + assert!(github.initial_sync); + assert!(github.periodic_sync); + assert_eq!(github.sync_interval_secs, Some(30 * 60)); + assert!(github.memory_ingest); + } + + #[test] + fn toolkit_description_known_slugs_are_distinct_and_non_empty() { + let known = [ + "gmail", + "notion", + "github", + "slack", + "discord", + "google_calendar", + "google_drive", + "google_docs", + "google_sheets", + "outlook", + "microsoft_teams", + "linear", + "jira", + "trello", + "asana", + "dropbox", + "twitter", + "spotify", + "telegram", + "whatsapp", + "twilio", + "shopify", + "stripe", + "hubspot", + "salesforce", + "airtable", + "figma", + "youtube", + "calendar", + ]; + let fallback = toolkit_description("__definitely_unknown_slug__"); + for slug in known { + let desc = toolkit_description(slug); + assert!(!desc.is_empty(), "{slug} description must not be empty"); + assert_ne!( + desc, fallback, + "known slug `{slug}` must not map to the generic fallback" + ); + } + } + + #[test] + fn toolkit_description_unknown_slug_uses_generic_fallback() { + assert_eq!( + toolkit_description("not_a_real_toolkit_123"), + "Interact with this connected service via its available actions" + ); + assert_eq!( + toolkit_description(""), + "Interact with this connected service via its available actions" + ); + } + + #[test] + fn toolkit_description_is_case_sensitive() { + // The match is lowercase-only by convention; an uppercase slug + // should fall through to the generic description. Explicitly + // documenting this guards against accidental case-insensitive + // matching sneaking in later. + let fallback = toolkit_description("__fallback__"); + assert_eq!(toolkit_description("GMAIL"), fallback); + assert_eq!(toolkit_description("Notion"), fallback); + } + + #[test] + fn provider_user_profile_default_is_empty() { + let p = ProviderUserProfile::default(); + assert!(p.toolkit.is_empty()); + assert!(p.connection_id.is_none()); + assert!(p.display_name.is_none()); + assert!(p.email.is_none()); + assert!(p.username.is_none()); + assert!(p.avatar_url.is_none()); + assert!(p.profile_url.is_none()); + assert!(p.extras.is_null()); + } +} diff --git a/core/src/sync/composio/providers/notion/mod.rs b/core/src/sync/composio/providers/notion/mod.rs new file mode 100644 index 0000000..5613ed0 --- /dev/null +++ b/core/src/sync/composio/providers/notion/mod.rs @@ -0,0 +1,11 @@ +// The payload normalisers moved to tinycortex (they are pure Value +// transforms, i.e. driver-side). Aliased under the old module name so +// every `normalization::extract_*` call site below stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::notion as normalization; +mod provider; +#[cfg(test)] +mod tests; +pub mod tools; + +pub use provider::NotionProvider; +pub use tools::NOTION_CURATED; diff --git a/core/src/sync/composio/providers/notion/provider.rs b/core/src/sync/composio/providers/notion/provider.rs new file mode 100644 index 0000000..63990fd --- /dev/null +++ b/core/src/sync/composio/providers/notion/provider.rs @@ -0,0 +1,406 @@ +//! Notion provider — incremental sync with per-item persistence. +//! +//! On each sync pass: +//! +//! 1. Load persistent [`SyncState`] from the KV store. +//! 2. Check the daily request budget — bail early if exhausted. +//! 3. Fetch a page of recently edited pages via `NOTION_FETCH_DATA`, +//! sorted by `last_edited_time` descending. When a cursor exists +//! we can stop as soon as we see pages older than the cursor. +//! 4. Deduplicate against `synced_ids` in the state. Pages that have +//! been *edited* since their last sync are re-persisted (the cursor +//! is based on `last_edited_time`, so an edited page appears again). +//! 5. Persist each **new or updated** page as its own memory document. +//! 6. Paginate (up to budget) until no more results or all items in the +//! page are older than the cursor. +//! 7. Advance the cursor and save state. + +use async_trait::async_trait; +use serde_json::{json, Value}; + +use super::normalization; +use crate::sync::composio::providers::{ + first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, + CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskContainer, + TaskFetchFilter, TaskKind, +}; + +pub(crate) const ACTION_GET_ABOUT_ME: &str = "NOTION_GET_ABOUT_ME"; +pub(crate) const ACTION_FETCH_DATA: &str = "NOTION_FETCH_DATA"; +pub(crate) const ACTION_QUERY_DATABASE: &str = "NOTION_QUERY_DATABASE"; +pub(crate) const ACTION_SEARCH_NOTION_PAGE: &str = "NOTION_SEARCH_NOTION_PAGE"; + +/// Paths for extracting a page's unique ID. +pub(crate) const PAGE_ID_PATHS: &[&str] = &["id", "data.id", "pageId", "data.pageId"]; + +/// Paths for extracting the `last_edited_time` used as sync cursor. +pub(crate) const PAGE_EDITED_PATHS: &[&str] = &[ + "last_edited_time", + "data.last_edited_time", + "lastEditedTime", + "data.lastEditedTime", +]; + +pub struct NotionProvider; + +impl NotionProvider { + pub fn new() -> Self { + Self + } +} + +impl Default for NotionProvider { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ComposioProvider for NotionProvider { + fn toolkit_slug(&self) -> &'static str { + "notion" + } + + fn curated_tools(&self) -> Option<&'static [CuratedTool]> { + Some(super::tools::NOTION_CURATED) + } + + fn sync_interval_secs(&self) -> Option { + Some(resolve_sync_interval_secs("notion", 30 * 60)) + } + + async fn fetch_user_profile( + &self, + ctx: &ProviderContext, + ) -> Result { + tracing::debug!( + connection_id = ?ctx.connection_id, + "[composio:notion] fetch_user_profile via {ACTION_GET_ABOUT_ME}" + ); + + let resp = ctx + .execute(ACTION_GET_ABOUT_ME, Some(json!({}))) + .await + .map_err(|e| format!("[composio:notion] {ACTION_GET_ABOUT_ME} failed: {e:#}"))?; + + if !resp.successful { + let err = resp + .error + .clone() + .unwrap_or_else(|| "provider reported failure".to_string()); + return Err(format!("[composio:notion] {ACTION_GET_ABOUT_ME}: {err}")); + } + + // `data` is already the inner Composio response payload — paths + // here are relative to it. For bot-token connections the + // top-level `name` is the *integration's* name (e.g. "Composio"), + // and the actual owning user lives at `bot.owner.user.*`. Probe + // the bot-owner paths first so identity reflects the user (#1365). + let data = &resp.data; + let display_name = pick_str(data, &["bot.owner.user.name", "user.name", "name"]); + let email = pick_str( + data, + &[ + "bot.owner.user.person.email", + "user.person.email", + "person.email", + "email", + ], + ); + let username = pick_str(data, &["bot.owner.user.id", "user.id", "id"]); + let avatar_url = pick_str( + data, + &["bot.owner.user.avatar_url", "user.avatar_url", "avatar_url"], + ); + let profile_url = pick_str(data, &["url", "profile_url", "profile.url"]); + + Ok(ProviderUserProfile { + toolkit: "notion".to_string(), + connection_id: ctx.connection_id.clone(), + display_name, + email, + username, + avatar_url, + profile_url, + extras: data.clone(), + }) + } + + /// Incremental sync. Notion was the first provider migrated to the generic + /// `orchestrator`: + /// the per-item loop, dedup, `max_items` cap, `sync_depth_days` window, and + /// cursor handling all live in `run_sync`; the Notion-specific primitives + /// (page fetch, dedup key, body fetch, ingest) live in `super::source`. + async fn fetch_tasks( + &self, + ctx: &ProviderContext, + filter: &TaskFetchFilter, + ) -> Result, String> { + let max = filter.effective_max(); + let database_id = filter + .database_id + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()); + + tracing::debug!( + connection_id = ?ctx.connection_id, + max, + has_database = database_id.is_some(), + "[composio:notion] fetch_tasks" + ); + + // A configured board (database) uses NOTION_QUERY_DATABASE; + // otherwise fall back to NOTION_FETCH_DATA (recent pages), the + // same action the periodic sync uses. + let (action, mut args) = match database_id { + Some(db) => ( + ACTION_QUERY_DATABASE, + json!({ + "database_id": db, + "page_size": max.min(100) as u32, + "sorts": [ { "timestamp": "last_edited_time", "direction": "descending" } ], + }), + ), + None => ( + ACTION_FETCH_DATA, + json!({ + "page_size": max.min(100) as u32, + "filter": { "value": "page", "property": "object" }, + "sort": { "direction": "descending", "timestamp": "last_edited_time" }, + }), + ), + }; + merge_extra(&mut args, &filter.extra); + + let resp = ctx + .execute(action, Some(args)) + .await + .map_err(|e| format!("[composio:notion] {action}: {e:#}"))?; + if !resp.successful { + return Err(format!( + "[composio:notion] {action}: {}", + resp.error.unwrap_or_else(|| "provider failure".into()) + )); + } + + // Optional client-side status filter — Notion status properties + // are user-defined, so we match on the normalized status rather + // than building a server-side property filter. + let want_status = filter + .status + .as_deref() + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty()); + + let mut out: Vec = Vec::new(); + for page in normalization::extract_results(&resp.data) { + if out.len() >= max { + break; + } + let Some(nt) = normalize_notion_page(&page) else { + continue; + }; + if let Some(ref want) = want_status { + let matches = nt + .status + .as_deref() + .map(|s| s.to_ascii_lowercase() == *want) + .unwrap_or(false); + if !matches { + continue; + } + } + out.push(nt); + } + tracing::debug!(count = out.len(), "[composio:notion] fetch_tasks complete"); + Ok(out) + } + + /// List the Notion databases (tables) the connected integration can see, + /// via `NOTION_SEARCH_NOTION_PAGE` filtered to database objects, so the + /// task-source UI can offer a picker for `database_id`. Only databases the + /// integration has been *shared with* in Notion are returned. + async fn list_databases(&self, ctx: &ProviderContext) -> Result, String> { + tracing::debug!( + connection_id = ?ctx.connection_id, + "[composio:notion] list_databases via {ACTION_SEARCH_NOTION_PAGE}" + ); + // Composio's NOTION_SEARCH_NOTION_PAGE *flattens* Notion's native + // `filter: { value, property }` into top-level `filter_value` / + // `filter_property` params and silently drops the nested form (which + // returned only pages). We send the flat params here; the nested + // `filter` is kept too as a belt-and-braces hint for any variant that + // honours it, and the parser still drops any stray `page` items. + let args = json!({ + "query": "", + "filter_value": "database", + "filter_property": "object", + "filter": { "value": "database", "property": "object" }, + "page_size": 100, + }); + let resp = ctx + .execute(ACTION_SEARCH_NOTION_PAGE, Some(args)) + .await + .map_err(|e| format!("[composio:notion] {ACTION_SEARCH_NOTION_PAGE}: {e:#}"))?; + if !resp.successful { + return Err(format!( + "[composio:notion] {ACTION_SEARCH_NOTION_PAGE}: {}", + resp.error.unwrap_or_else(|| "provider failure".into()) + )); + } + + tracing::info!( + successful = resp.successful, + data_is_array = resp.data.is_array(), + data_keys = ?resp.data.as_object().map(|o| o.keys().cloned().collect::>()), + "[composio:notion] list_databases raw response shape" + ); + let out = parse_database_results(&resp.data); + tracing::info!( + count = out.len(), + "[composio:notion] list_databases complete" + ); + Ok(out) + } + + async fn on_trigger( + &self, + ctx: &ProviderContext, + trigger: &str, + _payload: &Value, + ) -> Result<(), String> { + tracing::info!( + connection_id = ?ctx.connection_id, + trigger = %trigger, + "[composio:notion] on_trigger" + ); + let Some(connection_id) = ctx.connection_id.as_deref() else { + return Err("[composio:notion] trigger missing connection_id".to_string()); + }; + if let Err(e) = + crate::tinycortex::run_composio_connection("notion", connection_id, ctx.config.as_ref()) + .await + { + tracing::warn!( + error = %e, + "[composio:notion] trigger-driven sync failed (non-fatal)" + ); + } + Ok(()) + } +} + +/// Map a raw Notion page payload into a [`NormalizedTask`]. +/// +/// Notion databases are user-defined, so property extraction is +/// best-effort against common property names (`Status`, `Assignee`, +/// `Due`). Anything unmatched is simply left `None` — the raw payload is +/// preserved for enrichment. +fn normalize_notion_page(page: &serde_json::Value) -> Option { + let external_id = pick_str(page, PAGE_ID_PATHS)?; + let title = normalization::extract_page_title(page) + .unwrap_or_else(|| format!("Notion page {external_id}")); + Some(NormalizedTask { + external_id, + source_id: String::new(), + provider: "notion".to_string(), + kind: TaskKind::Generic, + title, + body: None, + url: pick_str(page, &["url", "data.url"]), + status: pick_str( + page, + &[ + "properties.Status.status.name", + "properties.Status.select.name", + "data.properties.Status.status.name", + ], + ), + assignee: first_array_str( + page, + &[ + "properties.Assignee.people", + "data.properties.Assignee.people", + ], + &["name"], + ), + due: pick_str( + page, + &[ + "properties.Due.date.start", + "data.properties.Due.date.start", + ], + ), + labels: Vec::new(), + priority: pick_str( + page, + &[ + "properties.Priority.select.name", + "data.properties.Priority.select.name", + ], + ), + updated_at: pick_str(page, PAGE_EDITED_PATHS), + raw: page.clone(), + }) +} + +/// Map a `NOTION_SEARCH_NOTION_PAGE` response into the database containers +/// the UI picker needs. +/// +/// We send a server-side `object: database` filter, so the response is +/// already scoped — we therefore *trust* it and only drop items explicitly +/// typed as `page`. This is intentional: Composio's response items don't +/// always carry a top-level `object` field, and an over-strict +/// "keep only object==database" check silently dropped every database. +/// Pure (no I/O) so it is unit-testable. +pub(super) fn parse_database_results(data: &serde_json::Value) -> Vec { + let results = normalization::extract_results(data); + let mut kinds: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut out: Vec = Vec::new(); + for item in &results { + let object = pick_str(item, &["object", "data.object"]); + *kinds + .entry(object.clone().unwrap_or_else(|| "".to_string())) + .or_default() += 1; + // Trust the server-side database filter: keep databases / data_sources + // *and* objectless items; only drop items explicitly typed as pages. + if object.as_deref() == Some("page") { + continue; + } + let Some(id) = pick_str(item, PAGE_ID_PATHS) else { + continue; + }; + let title = extract_database_title(item).unwrap_or_else(|| format!("Notion database {id}")); + out.push(TaskContainer { id, title }); + } + tracing::info!( + raw = results.len(), + kept = out.len(), + object_kinds = ?kinds, + "[composio:notion] parse_database_results" + ); + out +} + +/// Extract a Notion database's display title from its top-level `title` +/// rich-text array (`title[].plain_text`), tolerant of the Composio `data` +/// wrapper. Returns `None` for an untitled / shapeless database. +fn extract_database_title(db: &serde_json::Value) -> Option { + let arr = db + .get("title") + .or_else(|| db.get("data").and_then(|d| d.get("title"))) + .and_then(|v| v.as_array())?; + let text: String = arr + .iter() + .filter_map(|t| { + t.get("plain_text").and_then(|p| p.as_str()).or_else(|| { + t.get("text") + .and_then(|x| x.get("content")) + .and_then(|c| c.as_str()) + }) + }) + .collect(); + let text = text.trim(); + (!text.is_empty()).then(|| text.to_string()) +} diff --git a/core/src/sync/composio/providers/notion/tests.rs b/core/src/sync/composio/providers/notion/tests.rs new file mode 100644 index 0000000..0ddf92c --- /dev/null +++ b/core/src/sync/composio/providers/notion/tests.rs @@ -0,0 +1,119 @@ +//! Unit tests for the Notion provider. + +use super::normalization::{extract_notion_cursor, extract_page_title, extract_results}; +use super::NotionProvider; +use crate::sync::composio::providers::ComposioProvider; +use serde_json::json; + +#[test] +fn extract_results_walks_common_shapes() { + let v1 = json!({ "data": { "results": [{"id": "p1"}] } }); + let v2 = json!({ "results": [{"id": "p2"}, {"id": "p3"}] }); + let v3 = json!({ "data": {} }); + assert_eq!(extract_results(&v1).len(), 1); + assert_eq!(extract_results(&v2).len(), 2); + assert_eq!(extract_results(&v3).len(), 0); +} + +#[test] +fn extract_notion_cursor_finds_nested() { + let v = json!({ "data": { "next_cursor": "abc123" } }); + assert_eq!(extract_notion_cursor(&v), Some("abc123".to_string())); +} + +#[test] +fn extract_notion_cursor_none_when_missing() { + let v = json!({ "data": { "has_more": false } }); + assert_eq!(extract_notion_cursor(&v), None); +} + +#[test] +fn extract_page_title_from_properties() { + let page = json!({ + "id": "page-1", + "properties": { + "Name": { + "type": "title", + "title": [ + { "plain_text": "My " }, + { "plain_text": "Page Title" } + ] + } + } + }); + assert_eq!(extract_page_title(&page), Some("My Page Title".to_string())); +} + +#[test] +fn extract_page_title_fallback_to_top_level() { + let page = json!({ "title": "Fallback Title" }); + assert_eq!( + extract_page_title(&page), + Some("Fallback Title".to_string()) + ); +} + +#[test] +fn extract_page_title_returns_none_when_missing() { + let page = json!({ "id": "p1" }); + assert_eq!(extract_page_title(&page), None); +} + +#[test] +fn provider_metadata_is_stable() { + let p = NotionProvider::new(); + assert_eq!(p.toolkit_slug(), "notion"); + assert_eq!(p.sync_interval_secs(), Some(30 * 60)); +} + +#[test] +fn default_impl_matches_new() { + let _a = NotionProvider::new(); + let _b = ::default(); +} + +// ── parse_database_results (list_databases parser) ─────────────────────────── + +#[test] +fn parse_database_results_keeps_databases_and_extracts_title() { + use super::provider::parse_database_results; + let data = json!({ + "results": [ + { + "object": "database", + "id": "db-1", + "title": [{ "plain_text": "Engineering " }, { "plain_text": "Tasks" }] + }, + // A page hit must be filtered out — list_databases is databases only. + { "object": "page", "id": "pg-9", "title": [{ "plain_text": "Some page" }] }, + // Newest API exposes databases as `data_source`. + { "object": "data_source", "id": "db-2", "title": [{ "plain_text": "Roadmap" }] }, + // Untitled database falls back to a synthesized label. + { "object": "database", "id": "db-3", "title": [] } + ] + }); + let dbs = parse_database_results(&data); + assert_eq!( + dbs.len(), + 3, + "two named databases + one data_source, page dropped" + ); + assert_eq!(dbs[0].id, "db-1"); + assert_eq!(dbs[0].title, "Engineering Tasks"); + assert_eq!(dbs[1].id, "db-2"); + assert_eq!(dbs[1].title, "Roadmap"); + assert_eq!(dbs[2].title, "Notion database db-3"); +} + +#[test] +fn parse_database_results_handles_data_wrapper_and_empty() { + use super::provider::parse_database_results; + let wrapped = json!({ "data": { "results": [ + { "object": "database", "id": "x", "title": [{ "plain_text": "Wrapped" }] } + ] } }); + let dbs = parse_database_results(&wrapped); + assert_eq!(dbs.len(), 1); + assert_eq!(dbs[0].title, "Wrapped"); + + assert!(parse_database_results(&json!({ "results": [] })).is_empty()); +} diff --git a/core/src/sync/composio/providers/notion/tools.rs b/core/src/sync/composio/providers/notion/tools.rs new file mode 100644 index 0000000..89f4efc --- /dev/null +++ b/core/src/sync/composio/providers/notion/tools.rs @@ -0,0 +1,196 @@ +//! Curated catalog of Notion Composio actions exposed to the agent. + +use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; + +pub const NOTION_CURATED: &[CuratedTool] = &[ + // ── Read: search & fetch ──────────────────────────────────────── + CuratedTool { + slug: "NOTION_SEARCH_NOTION_PAGE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_FETCH_DATA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_FETCH_DATABASE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_FETCH_ROW", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_FETCH_BLOCK_METADATA", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_FETCH_BLOCK_CONTENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_FETCH_ALL_BLOCK_CONTENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_FETCH_COMMENTS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_GET_PAGE_MARKDOWN", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_GET_PAGE_PROPERTY_ACTION", + scope: ToolScope::Read, + }, + // ── Read: query & retrieve ────────────────────────────────────── + CuratedTool { + slug: "NOTION_QUERY_DATABASE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_QUERY_DATABASE_WITH_FILTER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_QUERY_DATA_SOURCE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_RETRIEVE_PAGE", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_RETRIEVE_COMMENT", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_RETRIEVE_DATABASE_PROPERTY", + scope: ToolScope::Read, + }, + // ── Read: profile / users / files ─────────────────────────────── + CuratedTool { + slug: "NOTION_LIST_USERS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_GET_ABOUT_USER", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_GET_ABOUT_ME", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_LIST_FILE_UPLOADS", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_RETRIEVE_FILE_UPLOAD", + scope: ToolScope::Read, + }, + CuratedTool { + slug: "NOTION_LIST_DATA_SOURCE_TEMPLATES", + scope: ToolScope::Read, + }, + // ── Write: create ─────────────────────────────────────────────── + CuratedTool { + slug: "NOTION_CREATE_NOTION_PAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_CREATE_DATABASE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_CREATE_COMMENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_CREATE_FILE_UPLOAD", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_SEND_FILE_UPLOAD", + scope: ToolScope::Write, + }, + // ── Write: update / append ────────────────────────────────────── + CuratedTool { + slug: "NOTION_UPDATE_PAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_UPDATE_BLOCK", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_UPDATE_ROW_DATABASE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_INSERT_ROW_DATABASE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_INSERT_ROW_FROM_NL", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_REPLACE_PAGE_CONTENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_ADD_PAGE_CONTENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_ADD_MULTIPLE_PAGE_CONTENT", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_APPEND_BLOCK_CHILDREN", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_APPEND_TEXT_BLOCKS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_APPEND_TASK_BLOCKS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_APPEND_CODE_BLOCKS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_APPEND_MEDIA_BLOCKS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_APPEND_LAYOUT_BLOCKS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_APPEND_TABLE_BLOCKS", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_DUPLICATE_PAGE", + scope: ToolScope::Write, + }, + CuratedTool { + slug: "NOTION_MOVE_PAGE", + scope: ToolScope::Write, + }, + // ── Admin: destructive ────────────────────────────────────────── + CuratedTool { + slug: "NOTION_DELETE_BLOCK", + scope: ToolScope::Admin, + }, + CuratedTool { + slug: "NOTION_ARCHIVE_NOTION_PAGE", + scope: ToolScope::Admin, + }, +]; diff --git a/core/src/sync/composio/providers/profile.rs b/core/src/sync/composio/providers/profile.rs new file mode 100644 index 0000000..baf9a22 --- /dev/null +++ b/core/src/sync/composio/providers/profile.rs @@ -0,0 +1,821 @@ +//! Profile persistence — maps [`ProviderUserProfile`] (and provider-specific +//! `extras`) into [`IdentityKind`]-tagged facet rows so the self-identity +//! matcher can join directly against the memory tree's `EntityKind` and the +//! structural sender field on chunks. +//! +//! Schema: `user_profile.facet_type='skill'`, +//! `key = "skill:{toolkit}:{conn_id}:{identity_kind}"`, `value` = +//! canonicalized identifier. Confidence is set per-kind so the matcher can +//! refuse to auto-promote weak signals (display_name) to `is_self`. +//! +//! One [`ProviderUserProfile`] expands to multiple rows — including +//! identifiers carried in `extras` that the previous fixed-fields shape +//! dropped on the floor (e.g. Slack screen-name handle). +//! +//! Callers invoke [`persist_provider_profile`] after every successful +//! `fetch_user_profile` call — from `on_connection_created`, periodic syncs, +//! and the `composio_get_user_profile` / `composio_refresh_all_identities` +//! RPC ops. + +use super::ProviderUserProfile; +use crate::learning_candidate::{ + self as learning_candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate, +}; +use crate::store::profile::FacetType; +use serde_json::Value; +use std::collections::BTreeMap; + +// ──────────────────────────────────────────────────────────────────────── +// IdentityKind — the matching axis +// ──────────────────────────────────────────────────────────────────────── + +/// Shape of an identifier persisted against a connection. Mirrors the +/// matching dimensions of the memory tree's +/// `crate::tree::score::extract::EntityKind` so the +/// self-check is a direct `(toolkit, kind, value)` lookup. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdentityKind { + /// Platform-canonical immutable id — Slack `U123ABC`, Notion UUID. + UserId, + Email, + /// `@`-style screen name, canonicalised without the leading `@`. + Handle, + /// E.164 phone number. + Phone, + /// Human display label. Weak signal — never auto-promotes to is_self. + DisplayName, + /// Not for matching; kept for UI / prompt rendering. + AvatarUrl, + /// Not for matching; kept for UI / prompt rendering. + ProfileUrl, +} + +impl IdentityKind { + pub fn as_str(self) -> &'static str { + match self { + Self::UserId => "user_id", + Self::Email => "email", + Self::Handle => "handle", + Self::Phone => "phone", + Self::DisplayName => "display_name", + Self::AvatarUrl => "avatar_url", + Self::ProfileUrl => "profile_url", + } + } + + pub fn parse(s: &str) -> Option { + Some(match s { + "user_id" => Self::UserId, + "email" => Self::Email, + "handle" => Self::Handle, + "phone" => Self::Phone, + "display_name" => Self::DisplayName, + "avatar_url" => Self::AvatarUrl, + "profile_url" => Self::ProfileUrl, + _ => return None, + }) + } + + /// Confidence the matcher records on the row. Hard kinds auto-promote + /// a chunk to `is_self`; weak kinds require corroboration. + pub fn confidence(self) -> f64 { + match self { + Self::UserId | Self::Phone => 1.00, + Self::Email => 0.95, + Self::Handle => 0.70, + Self::DisplayName => 0.40, + Self::AvatarUrl | Self::ProfileUrl => 0.50, + } + } + + /// True if this kind is a real identity signal worth running through + /// the matcher (vs. UI-only fields). + pub fn is_matchable(self) -> bool { + matches!( + self, + Self::UserId | Self::Email | Self::Handle | Self::Phone | Self::DisplayName + ) + } +} + +/// Canonicalize a raw value for storage and lookup. The same routine runs +/// on the entity side at match time, so equality of canonical forms is the +/// matcher's only test — no `COLLATE NOCASE`, no per-call lowercasing. +pub fn canonicalize(kind: IdentityKind, raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + Some(match kind { + IdentityKind::Email => trimmed.to_lowercase(), + IdentityKind::Handle => trimmed.trim_start_matches('@').to_lowercase(), + IdentityKind::Phone => trimmed + .chars() + .filter(|c| c.is_ascii_digit() || *c == '+') + .collect(), + IdentityKind::DisplayName => trimmed.split_whitespace().collect::>().join(" "), + IdentityKind::UserId | IdentityKind::AvatarUrl | IdentityKind::ProfileUrl => { + trimmed.to_string() + } + }) +} + +// ──────────────────────────────────────────────────────────────────────── +// Persist +// ──────────────────────────────────────────────────────────────────────── + +/// Persist a provider profile as one facet row per (kind, value). Returns +/// the number of rows written. Silently no-ops if the memory client isn't +/// ready (startup race / unauthenticated CLI). +pub fn persist_provider_profile(profile: &ProviderUserProfile) -> usize { + let Some(client) = crate::global::client_if_ready() else { + tracing::debug!( + toolkit = %profile.toolkit, + "[composio:profile] memory client not ready, skipping persist" + ); + return 0; + }; + let store = client.profile_store(); + + let now = now_secs(); + let toolkit = normalize_token(&profile.toolkit); + let identifier = profile + .connection_id + .as_deref() + .map(normalize_token) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| "default".to_string()); + + let rows = expand_identity_rows(&toolkit, profile); + + let mut written = 0usize; + for (kind, value) in rows { + let key = format!("skill:{toolkit}:{identifier}:{}", kind.as_str()); + let facet_id = format!("skill-{toolkit}-{identifier}-{}", kind.as_str()); + + if let Err(e) = store.upsert_provider_facet( + &facet_id, + &FacetType::Workflow, + &key, + &value, + kind.confidence(), + None, + now, + ) { + tracing::warn!( + toolkit = %toolkit, + identifier = %identifier, + kind = kind.as_str(), + error = %e, + "[composio:profile] profile_upsert failed (non-fatal)" + ); + continue; + } + + // Phase 3 (#566): also emit a LearningCandidate so the stability detector + // can score provider data alongside other evidence on the next rebuild. + // We use the `identity/` key prefix for provider identity fields. + if kind.is_matchable() { + let identity_key = format!("{}:{}", normalize_token(&toolkit), kind.as_str()); + let candidate = LearningCandidate { + class: FacetClass::Identity, + key: identity_key, + value: value.clone(), + cue_family: CueFamily::Structural, + evidence: EvidenceRef::Provider { + toolkit: toolkit.clone(), + connection_id: identifier.clone(), + field: kind.as_str().to_string(), + }, + initial_confidence: kind.confidence(), + observed_at: now, + }; + learning_candidate::global().push(candidate); + } + + written += 1; + } + + if written > 0 { + tracing::debug!( + toolkit = %toolkit, + identifier = %identifier, + rows_written = written, + "[composio:profile] persisted identity rows (+ emitted Identity candidates)" + ); + } + written +} + +/// Expand a [`ProviderUserProfile`] (and provider-specific `extras`) into +/// the canonical (kind, value) rows. **All per-toolkit quirks live here**; +/// the matcher only sees normalized tuples. +fn expand_identity_rows( + toolkit: &str, + profile: &ProviderUserProfile, +) -> Vec<(IdentityKind, String)> { + let mut rows: Vec<(IdentityKind, String)> = Vec::new(); + let mut push = |kind: IdentityKind, raw: Option<&str>| { + if let Some(v) = raw.and_then(|s| canonicalize(kind, s)) { + rows.push((kind, v)); + } + }; + + push(IdentityKind::DisplayName, profile.display_name.as_deref()); + push(IdentityKind::Email, profile.email.as_deref()); + push(IdentityKind::AvatarUrl, profile.avatar_url.as_deref()); + push(IdentityKind::ProfileUrl, profile.profile_url.as_deref()); + + match toolkit { + "slack" => { + // After the auth.test + users.info fix in slack/provider.rs: + // profile.username == Slack user_id (e.g. U123ABC) + // extras.handle == Slack screen_name (e.g. "cyrus") + // extras.team_* → workspace context, not identity + push(IdentityKind::UserId, profile.username.as_deref()); + push(IdentityKind::Handle, json_str(&profile.extras, "handle")); + } + "notion" => { + // Notion's `username` is the user UUID + // (`data.bot.owner.user.id` per notion/provider.rs). + push(IdentityKind::UserId, profile.username.as_deref()); + } + "gmail" => { + // Email + display_name only — no platform user_id worth matching. + } + _ => { + // Unknown toolkit: best-effort. If `username` is set treat it + // as a handle so weak-match logic (medium confidence) applies. + push(IdentityKind::Handle, profile.username.as_deref()); + } + } + + rows +} + +fn json_str<'a>(v: &'a Value, key: &str) -> Option<&'a str> { + v.get(key).and_then(|x| x.as_str()) +} + +// ──────────────────────────────────────────────────────────────────────── +// Read paths +// ──────────────────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ConnectedIdentity { + pub source: String, + pub identifier: String, + pub display_name: Option, + pub email: Option, + pub handle: Option, + pub phone: Option, + pub user_id: Option, + pub avatar_url: Option, + pub profile_url: Option, +} + +/// Load all provider-sourced identities, grouped by `(source, conn_id)`. +/// Rows whose last segment is not a known [`IdentityKind`] are silently +/// skipped — that includes legacy `username` rows from before the rewrite. +pub fn load_connected_identities() -> Vec { + let Some(client) = crate::global::client_if_ready() else { + tracing::debug!("[composio:profile] load_connected_identities: memory client not ready"); + return Vec::new(); + }; + let facets = match client.profile_store().facets_by_type(&FacetType::Workflow) { + Ok(f) => f, + Err(error) => { + tracing::warn!( + error = %error, + "[composio:profile] load_connected_identities: profile_facets_by_type failed" + ); + return Vec::new(); + } + }; + + let mut grouped: BTreeMap<(String, String), ConnectedIdentity> = BTreeMap::new(); + for facet in facets { + let Some((source, identifier, kind_str)) = parse_skill_identity_key(&facet.key) else { + continue; + }; + let Some(kind) = IdentityKind::parse(&kind_str) else { + continue; + }; + let entry = grouped + .entry((source.clone(), identifier.clone())) + .or_insert_with(|| ConnectedIdentity { + source, + identifier, + ..Default::default() + }); + match kind { + IdentityKind::DisplayName => entry.display_name = Some(facet.value), + IdentityKind::Email => entry.email = Some(facet.value), + IdentityKind::Handle => entry.handle = Some(facet.value), + IdentityKind::Phone => entry.phone = Some(facet.value), + IdentityKind::UserId => entry.user_id = Some(facet.value), + IdentityKind::AvatarUrl => entry.avatar_url = Some(facet.value), + IdentityKind::ProfileUrl => entry.profile_url = Some(facet.value), + } + } + grouped.into_values().collect() +} + +/// Direct self-check for the entity matcher and the chunk-build hook. +/// Returns true if any connection of `toolkit` has a row with this +/// `(kind, value)` after canonicalization. Non-matchable kinds +/// (avatar_url, profile_url) always return false. +pub fn is_self_identity(toolkit: &str, kind: IdentityKind, raw_value: &str) -> bool { + if !kind.is_matchable() { + return false; + } + let Some(canonical) = canonicalize(kind, raw_value) else { + return false; + }; + let Some(client) = crate::global::client_if_ready() else { + return false; + }; + let key_pattern = format!("skill:{}:%:{}", normalize_token(toolkit), kind.as_str()); + client + .profile_store() + .skill_identity_matches(&key_pattern, &canonical) +} + +/// Cross-toolkit variant — matches against every connected provider's +/// rows of this kind. Used for marking memory-tree entity rows: an email +/// in a Slack message that matches the user's Gmail address is still +/// "me," regardless of which source produced the chunk. +pub fn is_self_identity_any_toolkit(kind: IdentityKind, raw_value: &str) -> bool { + if !kind.is_matchable() { + return false; + } + let Some(canonical) = canonicalize(kind, raw_value) else { + return false; + }; + let Some(client) = crate::global::client_if_ready() else { + return false; + }; + let key_pattern = format!("skill:%:%:{}", kind.as_str()); + client + .profile_store() + .skill_identity_matches(&key_pattern, &canonical) +} + +/// Render a compact section for prompt injection. Skips `user_id` (not +/// human-readable), prefixes `handle` with `@`. +pub fn render_connected_identities_section(identities: &[ConnectedIdentity]) -> String { + if identities.is_empty() { + return String::new(); + } + let mut out = String::from("## Connected Identities\n\n"); + for id in identities { + let mut fields = Vec::::new(); + if let Some(v) = id.display_name.as_deref() { + let v = sanitize_prompt_value(v); + if !v.is_empty() { + fields.push(v); + } + } + if let Some(v) = id.email.as_deref() { + let v = sanitize_prompt_value(v); + if !v.is_empty() { + fields.push(v); + } + } + if let Some(v) = id.handle.as_deref() { + let v = sanitize_prompt_value(v); + if !v.is_empty() { + fields.push(format!("@{v}")); + } + } + if let Some(v) = id.profile_url.as_deref() { + let v = sanitize_prompt_value(v); + if !v.is_empty() { + fields.push(v); + } + } + if fields.is_empty() { + continue; + } + let identifier = sanitize_prompt_value(&id.identifier); + out.push_str(&format!( + "- {} ({}): {}\n", + title_case(&id.source), + identifier, + fields.join(" | ") + )); + } + if out.trim() == "## Connected Identities" { + return String::new(); + } + out +} + +/// Delete every row for a `(source, conn_id)` pair — used on disconnect. +pub fn delete_connected_identity_facets(source: &str, identifier: &str) -> usize { + // `persist_provider_profile` writes keys with `normalize_token`-applied + // segments; compare against the same normalized form here so a caller + // passing the raw toolkit/connection_id still matches stored rows + // (otherwise rows would survive disconnect and the user-tagger would + // keep treating the removed account as the user — #1381 review). + let source = normalize_token(source); + let identifier = normalize_token(identifier); + let Some(client) = crate::global::client_if_ready() else { + tracing::debug!( + source = %source, + identifier = %identifier, + "[composio:profile] delete_connected_identity_facets: memory client not ready" + ); + return 0; + }; + let store = client.profile_store(); + let Ok(facets) = store.facets_by_type(&FacetType::Workflow) else { + return 0; + }; + let mut deleted = 0usize; + for facet in facets { + let Some((s, i, _kind)) = parse_skill_identity_key(&facet.key) else { + continue; + }; + if s == source && i == identifier { + // Same swallow as before: a disconnect must not fail because one + // row was already gone. + if store.delete_by_facet_id(&facet.facet_id).unwrap_or(false) { + deleted += 1; + } + } + } + deleted +} + +// ──────────────────────────────────────────────────────────────────────── +// Helpers +// ──────────────────────────────────────────────────────────────────────── + +fn parse_skill_identity_key(key: &str) -> Option<(String, String, String)> { + let mut parts = key.split(':'); + let prefix = parts.next()?; + let source = parts.next()?; + let identifier = parts.next()?; + let kind = parts.next()?; + if prefix != "skill" || parts.next().is_some() { + return None; + } + Some((source.to_string(), identifier.to_string(), kind.to_string())) +} + +fn normalize_token(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for ch in raw.chars() { + let lower = ch.to_ascii_lowercase(); + if lower.is_ascii_alphanumeric() || lower == '-' || lower == '_' { + out.push(lower); + } else { + out.push('_'); + } + } + out.trim_matches('_').to_string() +} + +pub fn normalize_connection_identifier(raw: &str) -> String { + normalize_token(raw) +} + +fn title_case(raw: &str) -> String { + let mut chars = raw.chars(); + match chars.next() { + Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(), + None => String::new(), + } +} + +fn sanitize_prompt_value(raw: &str) -> String { + let replaced = raw.replace(['\n', '\r', '\t'], " ").replace('|', "/"); + replaced.split_whitespace().collect::>().join(" ") +} + +fn now_secs() -> f64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0) +} + +// ──────────────────────────────────────────────────────────────────────── +// Tests +// ──────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::profile::{self, profile_load_all, PROFILE_INIT_SQL}; + use parking_lot::Mutex; + use rusqlite::Connection; + use serde_json::json; + use std::sync::Arc; + + fn setup_db() -> Arc> { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(PROFILE_INIT_SQL).unwrap(); + Arc::new(Mutex::new(conn)) + } + + // ── IdentityKind ─────────────────────────────────────────────── + + #[test] + fn identity_kind_round_trips_through_str() { + for kind in [ + IdentityKind::UserId, + IdentityKind::Email, + IdentityKind::Handle, + IdentityKind::Phone, + IdentityKind::DisplayName, + IdentityKind::AvatarUrl, + IdentityKind::ProfileUrl, + ] { + assert_eq!(IdentityKind::parse(kind.as_str()), Some(kind)); + } + } + + #[test] + fn identity_kind_parse_rejects_unknown() { + assert_eq!(IdentityKind::parse("username"), None); + assert_eq!(IdentityKind::parse(""), None); + assert_eq!(IdentityKind::parse("UserId"), None); + } + + #[test] + fn matchable_kinds_exclude_url_fields() { + assert!(IdentityKind::UserId.is_matchable()); + assert!(IdentityKind::Email.is_matchable()); + assert!(IdentityKind::Handle.is_matchable()); + assert!(IdentityKind::Phone.is_matchable()); + assert!(IdentityKind::DisplayName.is_matchable()); + assert!(!IdentityKind::AvatarUrl.is_matchable()); + assert!(!IdentityKind::ProfileUrl.is_matchable()); + } + + #[test] + fn confidence_orders_hard_above_weak() { + assert!(IdentityKind::UserId.confidence() > IdentityKind::Email.confidence()); + assert!(IdentityKind::Email.confidence() > IdentityKind::Handle.confidence()); + assert!(IdentityKind::Handle.confidence() > IdentityKind::DisplayName.confidence()); + } + + // ── canonicalize ────────────────────────────────────────────── + + #[test] + fn canonicalize_email_lowercases_and_trims() { + assert_eq!( + canonicalize(IdentityKind::Email, " Cyrus@Example.COM "), + Some("cyrus@example.com".to_string()) + ); + } + + #[test] + fn canonicalize_handle_strips_at_and_lowercases() { + assert_eq!( + canonicalize(IdentityKind::Handle, "@Cyrus"), + Some("cyrus".to_string()) + ); + assert_eq!( + canonicalize(IdentityKind::Handle, "cyrus"), + Some("cyrus".to_string()) + ); + } + + #[test] + fn canonicalize_phone_keeps_only_digits_and_plus() { + assert_eq!( + canonicalize(IdentityKind::Phone, "+1 (555) 123-4567"), + Some("+15551234567".to_string()) + ); + } + + #[test] + fn canonicalize_display_name_collapses_whitespace() { + assert_eq!( + canonicalize(IdentityKind::DisplayName, " Cyrus Smith "), + Some("Cyrus Smith".to_string()) + ); + } + + #[test] + fn canonicalize_user_id_preserved_as_is() { + // Slack user_ids are case-sensitive; do not lowercase. + assert_eq!( + canonicalize(IdentityKind::UserId, "U123ABC"), + Some("U123ABC".to_string()) + ); + } + + #[test] + fn canonicalize_empty_returns_none() { + assert_eq!(canonicalize(IdentityKind::Email, ""), None); + assert_eq!(canonicalize(IdentityKind::Email, " "), None); + } + + // ── expand_identity_rows ────────────────────────────────────── + + fn fixture_profile( + toolkit: &str, + username: Option<&str>, + extras: Value, + ) -> ProviderUserProfile { + ProviderUserProfile { + toolkit: toolkit.into(), + connection_id: Some("conn-1".into()), + display_name: Some("Cyrus Smith".into()), + email: Some("cyrus@example.com".into()), + username: username.map(str::to_string), + avatar_url: None, + profile_url: Some("https://example.com/cyrus".into()), + extras, + } + } + + #[test] + fn expand_slack_promotes_username_to_user_id_and_extras_handle() { + let p = fixture_profile("slack", Some("U123ABC"), json!({ "handle": "cyrus" })); + let rows = expand_identity_rows("slack", &p); + + assert!(rows.contains(&(IdentityKind::UserId, "U123ABC".to_string()))); + assert!(rows.contains(&(IdentityKind::Handle, "cyrus".to_string()))); + assert!(rows.contains(&(IdentityKind::Email, "cyrus@example.com".to_string()))); + assert!(rows.contains(&(IdentityKind::DisplayName, "Cyrus Smith".to_string()))); + assert!(rows.contains(&( + IdentityKind::ProfileUrl, + "https://example.com/cyrus".to_string() + ))); + } + + #[test] + fn expand_gmail_skips_username_with_no_user_id_concept() { + let p = fixture_profile("gmail", None, Value::Null); + let rows = expand_identity_rows("gmail", &p); + + assert!(rows + .iter() + .all(|(k, _)| !matches!(k, IdentityKind::UserId | IdentityKind::Handle))); + assert!(rows.contains(&(IdentityKind::Email, "cyrus@example.com".to_string()))); + } + + #[test] + fn expand_notion_treats_username_as_user_id() { + let p = fixture_profile( + "notion", + Some("f3c1a8e2-b9b7-4a8d-9d5b-31a2e9f44e2f"), + Value::Null, + ); + let rows = expand_identity_rows("notion", &p); + + assert!(rows.contains(&( + IdentityKind::UserId, + "f3c1a8e2-b9b7-4a8d-9d5b-31a2e9f44e2f".to_string() + ))); + } + + #[test] + fn expand_unknown_toolkit_falls_back_to_handle() { + let p = fixture_profile("hypothetical", Some("alice"), Value::Null); + let rows = expand_identity_rows("hypothetical", &p); + + assert!(rows.contains(&(IdentityKind::Handle, "alice".to_string()))); + } + + #[test] + fn expand_empty_profile_emits_nothing_matchable() { + let p = ProviderUserProfile { + toolkit: "gmail".into(), + connection_id: Some("c-1".into()), + display_name: None, + email: None, + username: None, + avatar_url: None, + profile_url: None, + extras: Value::Null, + }; + let rows = expand_identity_rows("gmail", &p); + assert!(rows.is_empty()); + } + + // ── upsert wiring (uses the underlying profile_upsert directly) ─ + + #[test] + fn upsert_writes_kind_tagged_key() { + let conn = setup_db(); + + profile::profile_upsert( + &conn, + "skill-slack-conn-1-user_id", + &FacetType::Workflow, + "skill:slack:conn-1:user_id", + "U123ABC", + IdentityKind::UserId.confidence(), + None, + 1000.0, + ) + .unwrap(); + + let facets = profile_load_all(&conn).unwrap(); + let row = facets + .iter() + .find(|f| f.key == "skill:slack:conn-1:user_id") + .expect("row exists"); + assert_eq!(row.value, "U123ABC"); + assert!((row.confidence - 1.00).abs() < f64::EPSILON); + } + + #[test] + fn upsert_repeated_increments_evidence() { + let conn = setup_db(); + + for now in [1000.0, 2000.0] { + profile::profile_upsert( + &conn, + "skill-notion-default-email", + &FacetType::Workflow, + "skill:notion:default:email", + "user@workspace.com", + IdentityKind::Email.confidence(), + None, + now, + ) + .unwrap(); + } + + let facets = profile_load_all(&conn).unwrap(); + assert_eq!(facets.len(), 1); + assert_eq!(facets[0].evidence_count, 2); + } + + // ── parse_skill_identity_key ────────────────────────────────── + + #[test] + fn parse_key_round_trip() { + let parsed = parse_skill_identity_key("skill:slack:conn_1:user_id"); + assert_eq!( + parsed, + Some(( + "slack".to_string(), + "conn_1".to_string(), + "user_id".to_string() + )) + ); + } + + #[test] + fn parse_key_rejects_wrong_prefix() { + assert!(parse_skill_identity_key("preference:slack:c:email").is_none()); + } + + #[test] + fn parse_key_rejects_extra_segments() { + assert!(parse_skill_identity_key("skill:slack:c:email:extra").is_none()); + } + + // ── render ──────────────────────────────────────────────────── + + #[test] + fn render_includes_handle_with_at_and_omits_user_id() { + let rendered = render_connected_identities_section(&[ConnectedIdentity { + source: "slack".into(), + identifier: "T01ABC".into(), + display_name: Some("Cyrus Smith".into()), + email: Some("cyrus@example.com".into()), + handle: Some("cyrus".into()), + phone: None, + user_id: Some("U123ABC".into()), + avatar_url: None, + profile_url: None, + }]); + assert!(rendered.contains("## Connected Identities")); + assert!(rendered.contains("- Slack (T01ABC): Cyrus Smith | cyrus@example.com | @cyrus")); + assert!( + !rendered.contains("U123ABC"), + "user_id should not appear in prompt" + ); + } + + #[test] + fn render_empty_list_returns_empty_string() { + assert_eq!(render_connected_identities_section(&[]), ""); + } + + // ── now_secs sanity ─────────────────────────────────────────── + + #[test] + fn now_secs_returns_recent_unix_seconds() { + let t = now_secs(); + assert!(t > 1_000_000_000.0); + } + + #[test] + fn persist_returns_zero_when_memory_client_not_ready() { + // Exercise the early-return branch. Global client may or may + // not be initialised in the test binary depending on ordering. + let p = fixture_profile("gmail", None, Value::Null); + let _ = persist_provider_profile(&p); + } +} diff --git a/core/src/sync/composio/providers/profile_md.rs b/core/src/sync/composio/providers/profile_md.rs new file mode 100644 index 0000000..f2c471d --- /dev/null +++ b/core/src/sync/composio/providers/profile_md.rs @@ -0,0 +1,718 @@ +//! `PROFILE.md` markdown bridge — mirrors managed facet blocks into +//! `{workspace_dir}/PROFILE.md` so the agent prompt loader +//! (`agent/prompts/mod.rs::UserFilesSection`) picks them up on the next +//! turn. +//! +//! ## Block convention +//! +//! Each managed section lives between a pair of HTML comment markers: +//! +//! ```md +//! +//! ##
+//! +//! +//! +//! +//! ``` +//! +//! Anything outside the markers is left untouched, so user-authored prose +//! or hand-edited bullets are preserved across provider reconnects or +//! cache rebuilds. +//! +//! All operations are best-effort — errors are logged rather than +//! propagated, matching the PII-discipline pattern used in +//! `on_connection_created`. + +use super::ProviderUserProfile; +use std::fs; +use std::io; +use std::path::Path; + +// ── Legacy connected-accounts constants (kept for internal helpers) ─────────── + +const CA_BLOCK: &str = "connected-accounts"; +const CA_HEADING: &str = "## Connected Accounts"; +const FILE_HEADER: &str = "# User Profile\n"; + +/// All managed block names, in the order they are appended when a new +/// `PROFILE.md` is created. +pub const BLOCKS: &[&str] = &[ + "connected-accounts", // written by provider path (merge_provider_into_profile_md) + "style", + "identity", + "tooling", + "vetoes", + "goals", +]; + +// ── Public API ──────────────────────────────────────────────────────────────── + +/// Upsert the per-toolkit bullet for `profile` inside the managed +/// `connected-accounts` block of `{workspace_dir}/PROFILE.md`. +/// +/// Creates the file with a `# User Profile` header if it does not exist. +/// Idempotent — re-connecting the same toolkit replaces the existing +/// bullet rather than duplicating it. +pub fn merge_provider_into_profile_md( + workspace_dir: &Path, + profile: &ProviderUserProfile, +) -> io::Result<()> { + let toolkit = normalize_token(&profile.toolkit); + if toolkit.is_empty() { + return Ok(()); + } + // Require a real connection_id so the bullet keys match what the + // disconnect path (`composio_delete_connection`) will look up. + let identifier = profile + .connection_id + .as_deref() + .map(normalize_token) + .filter(|v| !v.is_empty()); + let identifier = match identifier { + Some(id) => id, + None => { + tracing::debug!( + toolkit = %toolkit, + "[composio:profile_md] skipping merge — connection_id missing or empty" + ); + return Ok(()); + } + }; + + let bullet = match render_provider_bullet(&toolkit, &identifier, profile) { + Some(b) => b, + None => return Ok(()), + }; + + let path = workspace_dir.join("PROFILE.md"); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let existing = match fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == io::ErrorKind::NotFound => String::new(), + Err(e) => return Err(e), + }; + + let updated = upsert_provider_bullet(&existing, &toolkit, &identifier, &bullet); + fs::write(&path, updated)?; + tracing::debug!( + target_file = "PROFILE.md", + toolkit = %toolkit, + identifier = %identifier, + "[composio:profile_md] merged provider profile into PROFILE.md" + ); + Ok(()) +} + +/// Remove the per-toolkit bullet for `(source, identifier)` from the +/// managed Connected Accounts block. If the block becomes empty the whole +/// block is dropped. Missing file or missing block are no-ops. +pub fn remove_provider_from_profile_md( + workspace_dir: &Path, + source: &str, + identifier: &str, +) -> io::Result<()> { + let path = workspace_dir.join("PROFILE.md"); + let existing = match fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(e), + }; + let toolkit = normalize_token(source); + let identifier = normalize_token(identifier); + if toolkit.is_empty() || identifier.is_empty() { + return Ok(()); + } + let updated = remove_provider_bullet(&existing, &toolkit, &identifier); + if updated != existing { + fs::write(&path, updated)?; + tracing::debug!( + target_file = "PROFILE.md", + toolkit = %toolkit, + identifier = %identifier, + "[composio:profile_md] removed provider bullet from PROFILE.md" + ); + } + Ok(()) +} + +/// Upsert a generic managed block. +/// +/// * `block_name` — one of [`BLOCKS`] (e.g. `"style"`, `"identity"`). +/// * `section_heading` — heading rendered inside the block (e.g. `"## Style"`). +/// * `body_markdown` — pre-rendered content (bullets, prose). Must not +/// contain the block markers themselves. +/// +/// Creates `PROFILE.md` if it does not exist. If the block is absent it is +/// appended at the end of the file. If the block exists its body is replaced +/// in-place — content outside the markers is left byte-for-byte untouched. +/// +/// An empty `body_markdown` renders a `*(no entries yet)*` placeholder +/// instead of deleting the block; this preserves the block's position for the +/// next write. +/// +/// Idempotent: calling with the same inputs twice produces the same file. +pub fn replace_managed_block( + workspace_dir: &Path, + block_name: &str, + section_heading: &str, + body_markdown: String, +) -> io::Result<()> { + let path = workspace_dir.join("PROFILE.md"); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + let existing = match fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == io::ErrorKind::NotFound => String::new(), + Err(e) => return Err(e), + }; + + let updated = upsert_block(&existing, block_name, section_heading, &body_markdown); + fs::write(&path, updated)?; + tracing::debug!( + block_name = %block_name, + "[composio:profile_md] replaced managed block '{}' in PROFILE.md", + block_name + ); + Ok(()) +} + +// ── Connected-accounts internals ────────────────────────────────────────────── + +fn render_provider_bullet( + toolkit: &str, + identifier: &str, + profile: &ProviderUserProfile, +) -> Option { + let mut fields: Vec = Vec::new(); + if let Some(v) = profile.display_name.as_deref().map(sanitize) { + if !v.is_empty() { + fields.push(v); + } + } + if let Some(v) = profile.email.as_deref().map(sanitize) { + if !v.is_empty() { + fields.push(v); + } + } + if let Some(v) = profile.username.as_deref().map(sanitize) { + if !v.is_empty() { + fields.push(format!("@{v}")); + } + } + if let Some(v) = profile.profile_url.as_deref().map(sanitize) { + if !v.is_empty() { + fields.push(v); + } + } + if fields.is_empty() { + return None; + } + let marker = bullet_marker(toolkit, identifier); + Some(format!( + "- {marker} **{title}** ({identifier}): {fields}", + title = title_case(toolkit), + identifier = identifier, + fields = fields.join(" | ") + )) +} + +fn bullet_marker(toolkit: &str, identifier: &str) -> String { + format!("") +} + +/// Insert or replace `bullet` inside the connected-accounts managed block. +fn upsert_provider_bullet(existing: &str, toolkit: &str, identifier: &str, bullet: &str) -> String { + let marker = bullet_marker(toolkit, identifier); + let start_tag = block_start(CA_BLOCK); + let end_tag = block_end(CA_BLOCK); + let (prefix, block_body, suffix) = split_any_block(existing, &start_tag, &end_tag); + + let mut lines: Vec = block_body + .lines() + .filter(|l| !l.contains(&marker)) + .map(|l| l.to_string()) + .collect(); + lines.push(bullet.to_string()); + + let mut bullets = lines + .into_iter() + .filter(|l| l.trim_start().starts_with("- ") +} + +/// Build the end marker for `block_name`. +pub fn block_end(block_name: &str) -> String { + format!("") +} + +/// Insert or replace a generic managed block in `existing`. +/// +/// If the block is absent it is appended. If it exists its body (between the +/// markers) is replaced. Content outside the markers is returned unchanged. +fn upsert_block( + existing: &str, + block_name: &str, + section_heading: &str, + body_markdown: &str, +) -> String { + let start_tag = block_start(block_name); + let end_tag = block_end(block_name); + + let body = if body_markdown.trim().is_empty() { + "*(no entries yet)*".to_string() + } else { + body_markdown.to_string() + }; + + let block = format!("{start_tag}\n{section_heading}\n\n{body}\n\n{end_tag}"); + + let (prefix, _old_body, suffix) = split_any_block(existing, &start_tag, &end_tag); + + if prefix == existing { + // Block was absent — append. + assemble(existing, &block, "") + } else { + assemble(&prefix, &block, &suffix) + } +} + +/// Split `existing` around the markers `[start_tag, end_tag]`. +/// +/// Returns `(prefix, block_body, suffix)`. If no block is present, +/// `prefix` is the full string and `block_body` / `suffix` are empty. +/// `block_body` is the content *between* the markers (excluding the +/// markers themselves). +fn split_any_block(existing: &str, start_tag: &str, end_tag: &str) -> (String, String, String) { + if let (Some(start), Some(end)) = (existing.find(start_tag), existing.find(end_tag)) { + if end > start { + let prefix = existing[..start].to_string(); + let body = existing[start + start_tag.len()..end].to_string(); + let suffix_start = end + end_tag.len(); + let suffix = existing[suffix_start..].to_string(); + return (prefix, body, suffix); + } + } + (existing.to_string(), String::new(), String::new()) +} + +/// Assemble `prefix + block + suffix`, normalising the newlines immediately +/// adjacent to the managed block while leaving the user's bytes elsewhere +/// untouched. +fn assemble(prefix: &str, block: &str, suffix: &str) -> String { + if block.is_empty() { + // Removing the block entirely. + let p = prefix.trim_end_matches('\n'); + let s = suffix.trim_start_matches('\n'); + let mut out = String::with_capacity(p.len() + s.len() + 2); + out.push_str(p); + if !p.is_empty() { + out.push('\n'); + if !s.is_empty() { + out.push('\n'); + } + } + out.push_str(s); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + return out; + } + + let mut out = String::new(); + if prefix.trim().is_empty() { + // Seed with a header on first creation. + out.push_str(FILE_HEADER); + out.push('\n'); + } else { + let p = prefix.trim_end_matches('\n'); + out.push_str(p); + out.push_str("\n\n"); + } + out.push_str(block); + if suffix.is_empty() { + out.push('\n'); + } else { + let s = suffix.trim_start_matches('\n'); + if s.is_empty() { + // Suffix was only whitespace — end with single newline, no blank line. + out.push('\n'); + } else { + out.push_str("\n\n"); + out.push_str(s); + if !out.ends_with('\n') { + out.push('\n'); + } + } + } + out +} + +// ── Token / string helpers ──────────────────────────────────────────────────── + +fn normalize_token(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for ch in raw.chars() { + let lower = ch.to_ascii_lowercase(); + if lower.is_ascii_alphanumeric() || lower == '-' || lower == '_' { + out.push(lower); + } else { + out.push('_'); + } + } + out.trim_matches('_').to_string() +} + +fn title_case(raw: &str) -> String { + let mut chars = raw.chars(); + match chars.next() { + Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(), + None => String::new(), + } +} + +fn sanitize(raw: &str) -> String { + let replaced = raw.replace(['\n', '\r', '\t'], " ").replace('|', "/"); + replaced.split_whitespace().collect::>().join(" ") +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + // ── merge_provider_into_profile_md (legacy API, unchanged) ─────────────── + + fn sample(toolkit: &str, conn: &str) -> ProviderUserProfile { + ProviderUserProfile { + toolkit: toolkit.into(), + connection_id: Some(conn.into()), + display_name: Some("Jane Doe".into()), + email: Some("jane@example.com".into()), + username: Some("janedoe".into()), + avatar_url: None, + profile_url: Some("https://example.com/jane".into()), + extras: serde_json::Value::Null, + } + } + + #[test] + fn creates_file_when_missing() { + let tmp = TempDir::new().unwrap(); + merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(body.starts_with("# User Profile"), "body was:\n{body}"); + let start = block_start(CA_BLOCK); + let end = block_end(CA_BLOCK); + assert!(body.contains(&start)); + assert!(body.contains(CA_HEADING)); + assert!(body.contains("**Gmail** (c-1):")); + assert!(body.contains("jane@example.com")); + assert!(body.contains("@janedoe")); + assert!(body.contains(&end)); + } + + #[test] + fn upsert_is_idempotent_for_same_toolkit_connection() { + let tmp = TempDir::new().unwrap(); + let mut p = sample("gmail", "c-1"); + merge_provider_into_profile_md(tmp.path(), &p).unwrap(); + p.display_name = Some("Jane D.".into()); + merge_provider_into_profile_md(tmp.path(), &p).unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + let occurrences = body.matches("acct:gmail:c-1").count(); + assert_eq!(occurrences, 1, "duplicate bullet:\n{body}"); + assert!(body.contains("Jane D.")); + assert!(!body.contains("Jane Doe")); + } + + #[test] + fn multiple_toolkits_render_separate_bullets() { + let tmp = TempDir::new().unwrap(); + merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); + merge_provider_into_profile_md(tmp.path(), &sample("twitter", "c-2")).unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(body.contains("acct:gmail:c-1")); + assert!(body.contains("acct:twitter:c-2")); + let start = block_start(CA_BLOCK); + let end = block_end(CA_BLOCK); + assert_eq!(body.matches(&start).count(), 1); + assert_eq!(body.matches(&end).count(), 1); + } + + #[test] + fn preserves_user_authored_content_outside_block() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("PROFILE.md"); + fs::write( + &path, + "# User Profile\n\nSome bio paragraph from LinkedIn.\n\n## Key facts\n- a\n- b\n", + ) + .unwrap(); + merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); + let body = fs::read_to_string(&path).unwrap(); + assert!(body.contains("Some bio paragraph from LinkedIn.")); + assert!(body.contains("## Key facts")); + assert!(body.contains("- a")); + assert!(body.contains("acct:gmail:c-1")); + } + + #[test] + fn skips_when_no_useful_fields() { + let tmp = TempDir::new().unwrap(); + let p = ProviderUserProfile { + toolkit: "gmail".into(), + connection_id: Some("c-1".into()), + display_name: Some(" ".into()), + email: None, + username: Some("".into()), + avatar_url: None, + profile_url: None, + extras: serde_json::Value::Null, + }; + merge_provider_into_profile_md(tmp.path(), &p).unwrap(); + assert!(!tmp.path().join("PROFILE.md").exists()); + } + + #[test] + fn remove_drops_specific_bullet() { + let tmp = TempDir::new().unwrap(); + merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); + merge_provider_into_profile_md(tmp.path(), &sample("twitter", "c-2")).unwrap(); + remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(!body.contains("acct:gmail:c-1")); + assert!(body.contains("acct:twitter:c-2")); + } + + #[test] + fn remove_drops_block_when_empty() { + let tmp = TempDir::new().unwrap(); + merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); + remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + let start = block_start(CA_BLOCK); + let end = block_end(CA_BLOCK); + assert!(!body.contains(&start), "block remained:\n{body}"); + assert!(!body.contains(&end)); + assert!(body.starts_with("# User Profile")); + } + + #[test] + fn remove_is_noop_when_file_missing() { + let tmp = TempDir::new().unwrap(); + remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); + assert!(!tmp.path().join("PROFILE.md").exists()); + } + + #[test] + fn skips_when_connection_id_missing() { + let tmp = TempDir::new().unwrap(); + let p = ProviderUserProfile { + toolkit: "gmail".into(), + connection_id: None, + display_name: Some("Jane".into()), + email: Some("jane@example.com".into()), + username: None, + avatar_url: None, + profile_url: None, + extras: serde_json::Value::Null, + }; + merge_provider_into_profile_md(tmp.path(), &p).unwrap(); + assert!(!tmp.path().join("PROFILE.md").exists()); + } + + #[test] + fn preserves_indentation_and_blank_lines_around_block() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("PROFILE.md"); + let original = "# User Profile\n\n indented bio line\n\n## Notes\n- alpha\n- beta\n\n"; + fs::write(&path, original).unwrap(); + merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); + let body = fs::read_to_string(&path).unwrap(); + assert!(body.contains(" indented bio line")); + assert!(body.contains("## Notes\n- alpha\n- beta")); + let start = block_start(CA_BLOCK); + let end = block_end(CA_BLOCK); + assert!(body.contains(&start) && body.contains(&end)); + remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); + let after = fs::read_to_string(&path).unwrap(); + assert!(after.contains(" indented bio line")); + assert!(after.contains("## Notes\n- alpha\n- beta")); + assert!(!after.contains(&start)); + } + + #[test] + fn sanitize_strips_pipes_and_newlines() { + assert_eq!(sanitize("foo\nbar"), "foo bar"); + assert_eq!(sanitize("a | b"), "a / b"); + assert_eq!(sanitize(" multi space "), "multi space"); + } + + // ── replace_managed_block ───────────────────────────────────────────────── + + #[test] + fn replace_managed_block_creates_file_if_missing() { + let tmp = TempDir::new().unwrap(); + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: terse".into(), + ) + .unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(body.contains("# User Profile"), "missing header:\n{body}"); + assert!(body.contains(&block_start("style"))); + assert!(body.contains("## Style")); + assert!(body.contains("- **verbosity**: terse")); + assert!(body.contains(&block_end("style"))); + } + + #[test] + fn replace_managed_block_appends_block_when_absent() { + let tmp = TempDir::new().unwrap(); + let path = tmp.path().join("PROFILE.md"); + fs::write(&path, "# User Profile\n\nSome existing text.\n").unwrap(); + replace_managed_block( + tmp.path(), + "identity", + "## Identity", + "- **name**: Alice".into(), + ) + .unwrap(); + let body = fs::read_to_string(&path).unwrap(); + // Existing content preserved. + assert!(body.contains("Some existing text.")); + // New block appended. + assert!(body.contains(&block_start("identity"))); + assert!(body.contains("## Identity")); + assert!(body.contains("- **name**: Alice")); + } + + #[test] + fn replace_managed_block_replaces_body_in_place() { + let tmp = TempDir::new().unwrap(); + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: verbose".into(), + ) + .unwrap(); + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: terse".into(), + ) + .unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(body.contains("terse")); + assert!(!body.contains("verbose")); + // Only one start marker. + assert_eq!(body.matches(&block_start("style")).count(), 1); + } + + #[test] + fn replace_managed_block_preserves_other_blocks_and_user_text() { + let tmp = TempDir::new().unwrap(); + // Write two blocks. + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: terse".into(), + ) + .unwrap(); + replace_managed_block( + tmp.path(), + "identity", + "## Identity", + "- **name**: Bob".into(), + ) + .unwrap(); + // Update only style. + replace_managed_block( + tmp.path(), + "style", + "## Style", + "- **verbosity**: verbose".into(), + ) + .unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + // Identity block untouched. + assert!(body.contains("- **name**: Bob")); + // Style updated. + assert!(body.contains("verbose")); + assert!(!body.contains("terse")); + } + + #[test] + fn replace_managed_block_empty_body_renders_placeholder() { + let tmp = TempDir::new().unwrap(); + replace_managed_block(tmp.path(), "goals", "## Goals", String::new()).unwrap(); + let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert!(body.contains("*(no entries yet)*")); + // Block markers still present. + assert!(body.contains(&block_start("goals"))); + assert!(body.contains(&block_end("goals"))); + } + + #[test] + fn replace_managed_block_idempotent_on_repeat_invocation() { + let tmp = TempDir::new().unwrap(); + let content = "- **verbosity**: terse".to_string(); + replace_managed_block(tmp.path(), "style", "## Style", content.clone()).unwrap(); + let body1 = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + replace_managed_block(tmp.path(), "style", "## Style", content).unwrap(); + let body2 = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); + assert_eq!(body1, body2, "second write should be idempotent"); + } +} diff --git a/core/src/sync/composio/providers/registry.rs b/core/src/sync/composio/providers/registry.rs new file mode 100644 index 0000000..c08e79b --- /dev/null +++ b/core/src/sync/composio/providers/registry.rs @@ -0,0 +1,151 @@ +//! Process-global registry of [`ComposioProvider`] implementations. +//! +//! There is exactly one provider per toolkit slug — the trait is not +//! a fan-out fan-in dispatch, it is a 1:1 mapping. This keeps trigger +//! routing simple (`HashMap::get(toolkit)` → call) and avoids the +//! "which subscriber wins" ambiguity that would come with multiple +//! providers per toolkit. +//! +//! The registry is initialised once at startup via +//! [`init_default_providers`] and is intentionally write-rare: tests +//! can register additional providers ad-hoc, but the production path +//! only writes during the startup hook. + +use std::collections::HashMap; +use std::sync::{Arc, OnceLock, RwLock}; + +use super::ComposioProvider; + +/// Reference-counted handle to a registered provider. +pub type ProviderArc = Arc; + +/// Backing storage for the global registry. +/// +/// `RwLock>` is fine here — registration happens at +/// startup and lookups are very fast (no contention in steady state). +type Registry = RwLock>; + +static REGISTRY: OnceLock = OnceLock::new(); + +fn registry() -> &'static Registry { + REGISTRY.get_or_init(|| RwLock::new(HashMap::new())) +} + +/// Register or replace a provider for its toolkit slug. +/// +/// Idempotent — re-registering the same toolkit overwrites the +/// previous entry, which is what tests rely on for setup/teardown. +pub fn register_provider(provider: ProviderArc) { + let slug = provider.toolkit_slug().to_string(); + if slug.is_empty() { + tracing::warn!("[composio:registry] refusing to register provider with empty slug"); + return; + } + let mut guard = registry() + .write() + .expect("composio provider registry poisoned"); + let was_present = guard.insert(slug.clone(), provider).is_some(); + if was_present { + tracing::debug!(toolkit = %slug, "[composio:registry] replaced existing provider"); + } else { + tracing::info!(toolkit = %slug, "[composio:registry] provider registered"); + } +} + +/// Look up the provider for a toolkit slug, if one is registered. +pub fn get_provider(toolkit: &str) -> Option { + let key = toolkit.trim(); + if key.is_empty() { + return None; + } + let guard = registry() + .read() + .expect("composio provider registry poisoned"); + guard.get(key).cloned() +} + +/// Snapshot of every registered provider, in unspecified order. Used +/// by the periodic sync scheduler to walk every toolkit. +pub fn all_providers() -> Vec { + let guard = registry() + .read() + .expect("composio provider registry poisoned"); + guard.values().cloned().collect() +} + +/// Register the built-in providers shipped with the core. Called once +/// from `start_channels` / `bootstrap_core_runtime` startup paths. +/// +/// Idempotent: re-running just re-registers (no-op in practice). +pub fn init_default_providers() { + register_provider(Arc::new(super::clickup::ClickUpProvider::new())); + register_provider(Arc::new(super::github::GitHubProvider::new())); + register_provider(Arc::new(super::gmail::GmailProvider::new())); + register_provider(Arc::new(super::linear::LinearProvider::new())); + register_provider(Arc::new(super::notion::NotionProvider::new())); + register_provider(Arc::new(super::slack::SlackProvider::new())); + tracing::info!( + count = all_providers().len(), + "[composio:registry] default providers initialised" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sync::composio::providers::{ProviderContext, ProviderUserProfile}; + use async_trait::async_trait; + + struct DummyProvider { + slug: &'static str, + } + + #[async_trait] + impl ComposioProvider for DummyProvider { + fn toolkit_slug(&self) -> &'static str { + self.slug + } + async fn fetch_user_profile( + &self, + _ctx: &ProviderContext, + ) -> Result { + Ok(ProviderUserProfile::default()) + } + } + + #[test] + fn register_and_lookup_roundtrip() { + register_provider(Arc::new(DummyProvider { + slug: "test_dummy_a", + })); + let p = get_provider("test_dummy_a").expect("provider should be registered"); + assert_eq!(p.toolkit_slug(), "test_dummy_a"); + } + + #[test] + fn lookup_unknown_returns_none() { + assert!(get_provider("__definitely_not_a_real_toolkit__").is_none()); + } + + #[test] + fn register_replaces_existing() { + register_provider(Arc::new(DummyProvider { + slug: "test_dummy_b", + })); + register_provider(Arc::new(DummyProvider { + slug: "test_dummy_b", + })); + // Still exactly one entry under that slug. + let count_with_b = all_providers() + .iter() + .filter(|p| p.toolkit_slug() == "test_dummy_b") + .count(); + assert_eq!(count_with_b, 1); + } + + #[test] + fn empty_slug_is_rejected() { + register_provider(Arc::new(DummyProvider { slug: "" })); + assert!(get_provider("").is_none()); + } +} diff --git a/core/src/sync/composio/providers/scope_lookup.rs b/core/src/sync/composio/providers/scope_lookup.rs new file mode 100644 index 0000000..e3bae02 --- /dev/null +++ b/core/src/sync/composio/providers/scope_lookup.rs @@ -0,0 +1,78 @@ +//! Scope-lookup operational helpers for the curated tool catalogs. +//! +//! Lives in a sibling module (extracted from the formerly-thick +//! `providers/mod.rs`) to keep the module entrypoint export-focused — +//! matches the project rule "keep mod.rs light; operational logic in +//! ops.rs / store.rs / types.rs" from CLAUDE.md. +//! +//! - [`curated_scope_for`] answers "what scope does this action slug +//! require?" — used by `composio::ops` to render `gated_tools` +//! unlock hints. +//! - [`toolkit_has_scope`] answers "does this toolkit have any +//! actions at the given scope?" — currently used by tests; intended +//! for future UI hints (grey-out a toggle that unlocks nothing). +//! +//! Both walk the native provider catalog first, then fall back to the +//! static `catalog_for_toolkit` map — so the answers match what +//! [`super::is_action_visible_with_pref`] would gate against. + +use super::tool_scope::{find_curated, toolkit_from_slug, ToolScope}; +use super::{catalog_for_toolkit, get_provider}; + +/// Look up the curated scope for `slug` if it appears in any registered +/// catalog (native provider's `curated_tools()` first, then the fallback +/// catalog from [`super::catalog_for_toolkit`]). Returns `None` for +/// genuinely uncurated slugs — callers that want a defensible heuristic +/// for those should fall back to [`super::classify_unknown`] explicitly. +/// +/// Sibling of [`super::is_action_visible_with_pref`]: that one decides +/// "visible?", this one returns "what scope is required?" so callers +/// (e.g. the `gated_tools` partition in +/// `composio::ops::fetch_connected_integrations`) can render a useful +/// unlock hint to the agent without re-doing the catalog walk. +pub fn curated_scope_for(slug: &str) -> Option { + let toolkit = toolkit_from_slug(slug)?; + let catalog = get_provider(&toolkit) + .and_then(|p| p.curated_tools()) + .or_else(|| catalog_for_toolkit(&toolkit))?; + find_curated(catalog, slug).map(|c| c.scope) +} + +/// Does any curated action for `toolkit` require `scope`? +/// +/// Currently used by this module's tests only (added when the +/// now-removed `composio_enable_scope` meta-tool needed a no-op +/// short-circuit). Kept because the same probe is useful any time we +/// ask "would flipping the {scope} bit unlock anything in this +/// toolkit?" — e.g. a UI hint that greys out a toggle with no effect. +/// +/// Walks both the native provider catalog and the fallback +/// [`super::catalog_for_toolkit`] so the answer matches what +/// [`super::is_action_visible_with_pref`] would gate against. +pub fn toolkit_has_scope(toolkit: &str, scope: ToolScope) -> bool { + let catalog = get_provider(toolkit) + .and_then(|p| p.curated_tools()) + .or_else(|| catalog_for_toolkit(toolkit)); + match catalog { + Some(cat) => cat.iter().any(|t| t.scope == scope), + None => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn toolkit_has_scope_distinguishes_gated_from_ungated_scopes() { + // gmail catalog includes destructive verbs (delete / trash / + // batch_delete), so admin-gating actually unlocks something. + assert!(toolkit_has_scope("gmail", ToolScope::Admin)); + assert!(toolkit_has_scope("gmail", ToolScope::Read)); + assert!(toolkit_has_scope("gmail", ToolScope::Write)); + // Case-insensitive toolkit slug → still routes to the catalog. + assert!(toolkit_has_scope("GMAIL", ToolScope::Admin)); + // Unknown toolkit → no catalog → no scope is "gating" anything. + assert!(!toolkit_has_scope("nonexistent-toolkit", ToolScope::Admin)); + } +} diff --git a/core/src/sync/composio/providers/slack/mod.rs b/core/src/sync/composio/providers/slack/mod.rs new file mode 100644 index 0000000..034492e --- /dev/null +++ b/core/src/sync/composio/providers/slack/mod.rs @@ -0,0 +1,19 @@ +//! Composio-backed Slack provider. +//! +//! The provider is wired into the periodic-sync scheduler (see +//! [`super::registry::init_default_providers`]) and fires +//! `SLACK_LIST_CONVERSATIONS` + `SLACK_FETCH_CONVERSATION_HISTORY` +//! against the user's Composio-authorized Slack connection. The reusable +//! synchronization and ingestion engine is owned by tinycortex. + +// The Slack post-processor moved to tinycortex (a pure Value transform, i.e. +// driver-side). Re-exported under the old module name — `pub`, not a plain +// `use`, because `tests/raw_coverage/memory_threads_raw_coverage_e2e.rs` +// imports this path directly. +pub use tinycortex::memory::sync::composio::providers::normalize::slack_post_process as post_process; +pub mod types; + +mod provider; + +pub use provider::{run_backfill_via_search, SlackProvider, BACKFILL_DAYS}; +pub use types::{SlackChannel, SlackMessage}; diff --git a/core/src/sync/composio/providers/slack/provider.rs b/core/src/sync/composio/providers/slack/provider.rs new file mode 100644 index 0000000..5d22208 --- /dev/null +++ b/core/src/sync/composio/providers/slack/provider.rs @@ -0,0 +1,335 @@ +//! Composio-backed Slack provider. +//! +//! Drives Slack history ingestion **without** a user-managed bot token +//! — authorization lives in the user's Composio Slack connection, and +//! the actual API calls fan out through [`ComposioClient::execute_tool`] +//! against Composio's action catalog (`SLACK_LIST_CONVERSATIONS`, +//! `SLACK_FETCH_CONVERSATION_HISTORY`, `SLACK_FETCH_TEAM_INFO`, …). +//! +//! The product provider retains profile lookup, trigger handling, and response +//! normalization. Channel enumeration, cursors, history paging, and memory +//! ingestion execute in tinycortex's Slack sync pipeline. +//! +//! ## Idempotency +//! +//! Source id is `slack:{connection_id}` — stable per workspace. Chunk +//! IDs are stable, so repeated synchronization updates the same documents. + +use crate::sync::composio::providers::{ + pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, ProviderContext, + ProviderUserProfile, SyncOutcome, +}; +use async_trait::async_trait; +use serde_json::{json, Value}; + +/// Composio action slug for team/workspace profile fetch. +const ACTION_FETCH_TEAM_INFO: &str = "SLACK_FETCH_TEAM_INFO"; +/// Composio action slug for Slack `auth.test` — returns the authed +/// user's id, handle, and team. Required for self-identity capture. +const ACTION_AUTH_TEST: &str = "SLACK_TEST_AUTH"; +/// Composio action slug for Slack `users.info` — returns the user's +/// profile (email, real_name, avatar). Optional; needs `users:read.email` +/// scope for the email field. +const ACTION_USERS_INFO: &str = "SLACK_RETRIEVE_DETAILED_USER_INFORMATION"; + +/// Default backfill window (days) applied when a channel has no +/// cursor yet. +pub const BACKFILL_DAYS: i64 = 6; + +/// Sync cadence for provider catalog scheduling. +const SYNC_INTERVAL_SECS: u64 = 15 * 60; + +pub struct SlackProvider; + +impl SlackProvider { + pub fn new() -> Self { + Self + } +} + +impl Default for SlackProvider { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ComposioProvider for SlackProvider { + fn toolkit_slug(&self) -> &'static str { + "slack" + } + + fn curated_tools(&self) -> Option<&'static [CuratedTool]> { + Some(crate::sync::composio::providers::catalogs::SLACK_CURATED) + } + + fn sync_interval_secs(&self) -> Option { + Some(resolve_sync_interval_secs("slack", SYNC_INTERVAL_SECS)) + } + + fn post_process_action_result( + &self, + slug: &str, + arguments: Option<&serde_json::Value>, + data: &mut serde_json::Value, + ) { + super::post_process::post_process(slug, arguments, data); + } + + async fn fetch_user_profile( + &self, + ctx: &ProviderContext, + ) -> Result { + tracing::debug!( + connection_id = ?ctx.connection_id, + "[composio:slack] fetch_user_profile via {ACTION_AUTH_TEST}" + ); + + // Step 1 — auth.test: required. Returns user_id (canonical sender + // id on Slack messages), the user's handle, and the team. + let auth_resp = ctx + .execute(ACTION_AUTH_TEST, Some(json!({}))) + .await + .map_err(|e| format!("[composio:slack] {ACTION_AUTH_TEST} failed: {e:#}"))?; + + if !auth_resp.successful { + let err = auth_resp + .error + .clone() + .unwrap_or_else(|| "provider reported failure".to_string()); + return Err(format!("[composio:slack] {ACTION_AUTH_TEST}: {err}")); + } + + // `auth_data` is the inner Composio payload — paths are relative + // to it. Slack's auth.test returns user_id/user/team/team_id at + // the top of `data`. + let auth_data = &auth_resp.data; + let user_id = pick_str(auth_data, &["user_id"]); + let handle = pick_str(auth_data, &["user"]); + let team_id = pick_str(auth_data, &["team_id"]); + let team_name = pick_str(auth_data, &["team"]); + + // Step 2 — users.info: optional. Needs `users:read.email` scope + // for `email`; falls back to `auth.test` data on missing-scope or + // any other failure so the profile still carries user_id+handle. + let mut display_name: Option = None; + let mut email: Option = None; + let mut avatar_url: Option = None; + + if let Some(uid) = user_id.as_deref() { + match ctx + .execute(ACTION_USERS_INFO, Some(json!({ "user": uid }))) + .await + { + Ok(info) if info.successful => { + let d = &info.data; + email = pick_str(d, &["user.profile.email", "profile.email"]); + display_name = pick_str( + d, + &[ + "user.profile.real_name", + "user.real_name", + "user.profile.display_name", + ], + ); + avatar_url = pick_str(d, &["user.profile.image_192", "user.profile.image_72"]); + } + Ok(info) => { + tracing::info!( + connection_id = ?ctx.connection_id, + error = ?info.error, + "[composio:slack] {ACTION_USERS_INFO} returned non-success — \ + falling back to auth.test data only (likely missing users:read scope)" + ); + } + Err(e) => { + tracing::info!( + connection_id = ?ctx.connection_id, + error = %e, + "[composio:slack] {ACTION_USERS_INFO} call failed — \ + falling back to auth.test data only" + ); + } + } + } + + // Step 3 — team_info: optional. Adds workspace context to `extras` + // (email_domain, icon) so the prompt section / UI can show it. + let (team_email_domain, team_icon) = + match ctx.execute(ACTION_FETCH_TEAM_INFO, Some(json!({}))).await { + Ok(resp) if resp.successful => { + let d = &resp.data; + let domain = pick_str(d, &["team.email_domain", "email_domain"]); + let icon = pick_str(d, &["team.icon.image_132", "team.icon.image_68"]); + (domain, icon) + } + _ => (None, None), + }; + + // Display name preference: users.info real_name > auth.test handle + // > team_name (last-resort so the prompt isn't empty). + let final_display_name = display_name + .clone() + .or_else(|| handle.clone()) + .or_else(|| team_name.clone()); + + // Profile URL: users.info doesn't return one for the user + // directly; the workspace URL is acceptable as a navigational + // fallback. (Slack user profile pages are workspace-scoped and + // not stably linkable from auth.test alone.) + let profile_url = pick_str(auth_data, &["url"]); + + let avatar_url = avatar_url.or(team_icon); + + let profile = ProviderUserProfile { + toolkit: "slack".to_string(), + connection_id: ctx.connection_id.clone(), + display_name: final_display_name, + email, + // username carries the platform-canonical sender id so the + // self-identity matcher can compare against Slack message + // sender_user_id directly. Handle moves into `extras` — + // `expand_identity_rows` lifts it back out as IdentityKind::Handle. + username: user_id, + avatar_url, + profile_url, + extras: json!({ + "handle": handle, + "team_id": team_id, + "team_name": team_name, + "team_email_domain": team_email_domain, + }), + }; + + let has_email = profile.email.is_some(); + let email_domain = profile + .email + .as_deref() + .and_then(|e| e.split('@').nth(1)) + .map(|d| d.to_string()); + tracing::info!( + connection_id = ?profile.connection_id, + has_email, + email_domain = ?email_domain, + has_user_id = profile.username.is_some(), + "[composio:slack] fetched user profile" + ); + Ok(profile) + } + + /// Slack rides the generic orchestrator. Channel enumeration + the user + /// directory backfill happen in `super::source::SlackSource::preamble`; + /// per-channel `conversations.history` pagination, the per-channel `oldest` + /// watermark, dedup, the `max_items` cap, and per-channel error tolerance + /// all live in `run_sync`. The Slack-specific primitives live in + /// `super::source`. + async fn on_trigger( + &self, + ctx: &ProviderContext, + trigger: &str, + _payload: &Value, + ) -> Result<(), String> { + if trigger.to_ascii_uppercase().contains("MESSAGE") { + let Some(connection_id) = ctx.connection_id.as_deref() else { + return Err("[composio:slack] trigger missing connection_id".to_string()); + }; + if let Err(e) = crate::tinycortex::run_composio_connection( + "slack", + connection_id, + ctx.config.as_ref(), + ) + .await + { + tracing::warn!( + error = %e, + "[composio:slack] trigger-driven sync failed (non-fatal)" + ); + } + } + Ok(()) + } +} + +// ── Search-based backfill (one-shot) ──────────────────────────────── + +/// Compatibility wrapper for the tinycortex workspace-wide search pipeline. +pub async fn run_backfill_via_search( + ctx: &ProviderContext, + backfill_days: i64, +) -> Result { + let connection_id = ctx + .connection_id + .as_deref() + .ok_or_else(|| "[composio:slack] search backfill missing connection_id".to_string())?; + let started_at_ms = now_ms(); + let outcome = crate::tinycortex::run_slack_search_backfill( + connection_id, + backfill_days, + ctx.config.as_ref(), + ) + .await + .map_err(|error| error.to_string())?; + Ok(SyncOutcome { + toolkit: "slack".into(), + connection_id: Some(connection_id.into()), + reason: "manual".into(), + items_ingested: outcome.records_ingested as usize, + started_at_ms, + finished_at_ms: now_ms(), + summary: outcome + .note + .unwrap_or_else(|| "slack search-backfill complete".into()), + details: serde_json::json!({ + "more_pending": outcome.more_pending, + "actions_called": outcome.actions_called, + "provider_cost_usd": outcome.provider_cost_usd, + }), + }) +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn toolkit_slug_is_stable() { + assert_eq!(SlackProvider::new().toolkit_slug(), "slack"); + } + + #[test] + fn sync_interval_matches_constant() { + assert_eq!( + SlackProvider::new().sync_interval_secs(), + Some(SYNC_INTERVAL_SECS) + ); + } + + #[test] + fn curated_tools_returns_slack_catalog() { + let tools = SlackProvider::new().curated_tools().unwrap(); + assert!(tools + .iter() + .any(|t| t.slug == "SLACK_FETCH_CONVERSATION_HISTORY")); + assert!(tools.iter().any(|t| t.slug == "SLACK_LIST_CONVERSATIONS")); + } + + #[test] + fn post_process_action_result_delegates_to_post_process_module() { + let provider = SlackProvider::new(); + let mut data = serde_json::json!({ + "channels": [{"id": "C1", "name": "eng", "is_private": false}] + }); + // Calling with an unknown slug should be a no-op. + provider.post_process_action_result("SLACK_UNKNOWN_ACTION", None, &mut data); + assert!( + data.get("channels").is_some(), + "no-op slug must not mutate data" + ); + } +} diff --git a/core/src/sync/composio/providers/slack/types.rs b/core/src/sync/composio/providers/slack/types.rs new file mode 100644 index 0000000..4271442 --- /dev/null +++ b/core/src/sync/composio/providers/slack/types.rs @@ -0,0 +1,68 @@ +//! Canonical types for the Composio-backed Slack provider. +//! +//! These types are independent of the Composio/Slack API payload shape. +//! They remain as compatibility wire types for product RPC responses and +//! backfill reporting; tinycortex owns runtime parsing and ingestion. +//! +//! The old `Bucket` struct (6-hour UTC window) has been removed — the +//! memory tree's L0 seal cascade handles batching after PR #1348, so +//! tinycortex owns batching and incremental persistence. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// A single message fetched from Slack's `conversations.history` or +/// `search.messages`. +/// +/// The Slack API represents `ts` as a decimal string like +/// `"1714003200.123456"` where the integer part is Unix seconds and the +/// fractional part is a per-workspace message sequence. We retain the +/// original string in `ts_raw` so it can round-trip back to the API +/// (e.g. as the `oldest` cursor on the next poll, and as the permalink +/// suffix for provenance). +/// +/// `channel_name`, `is_private`, `author_id`, and `permalink` are added +/// vs the old `memory::slack_ingestion::types::SlackMessage` because we no +/// longer carry a separate `SlackChannel` through the ingest path — +/// per-message context is self-contained. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SlackMessage { + /// Channel ID this message belongs to (e.g. `"C0123456"`). + pub channel_id: String, + /// Human-readable channel name (e.g. `"eng"`). Injected by the enricher + /// from the channel directory; may be empty for search results whose + /// channel was not listed. + pub channel_name: String, + /// `true` if this is a private channel the bot has been invited to. + pub is_private: bool, + /// Resolved display name of the author. Falls back to the raw user id + /// when the user directory doesn't have an entry for this id. + pub author: String, + /// Raw Slack user id (e.g. `"U01234"`). Retained alongside the resolved + /// `author` so downstream code can still look up or log the stable id. + pub author_id: String, + /// Message body (plain text; may contain Slack-flavoured markdown). + pub text: String, + /// Canonical timestamp derived from `ts_raw`. + pub timestamp: DateTime, + /// Raw Slack `ts` string (used for API cursors + archive URLs). + pub ts_raw: String, + /// Root thread `ts` if this message is a reply; `None` for top-level + /// messages. Retained for future thread-aware ingestion. + pub thread_ts: Option, + /// Resolved HTTPS permalink, if Composio includes it in the response. + /// Falls back to the `slack://archives/…` scheme in ingest. + pub permalink: Option, +} + +/// A Slack channel visible to the bot, as returned by `conversations.list`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SlackChannel { + /// Channel ID (stable across renames). + pub id: String, + /// Human-readable name (e.g. `"eng"` → rendered as `"#eng"` in headers). + /// May change if admins rename the channel. + pub name: String, + /// `true` if this is a private channel the bot has been invited to. + pub is_private: bool, +} diff --git a/core/src/sync/composio/providers/sync_state.rs b/core/src/sync/composio/providers/sync_state.rs new file mode 100644 index 0000000..da33c44 --- /dev/null +++ b/core/src/sync/composio/providers/sync_state.rs @@ -0,0 +1,19 @@ +//! Compatibility exports for sync state now owned by tinycortex. + +pub use tinycortex::memory::sync::state::DEFAULT_DAILY_REQUEST_LIMIT; +pub use tinycortex::memory::sync::{DailyBudget, SyncState}; + +pub const KV_NAMESPACE: &str = crate::tinycortex::HOST_SYNC_STATE_NAMESPACE; + +pub fn extract_item_id(item: &serde_json::Value, paths: &[&str]) -> Option { + paths.iter().find_map(|path| { + let value = path + .split('.') + .try_fold(item, |current, segment| current.get(segment))?; + value + .as_str() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + }) +} diff --git a/core/src/sync/composio/providers/tool_scope.rs b/core/src/sync/composio/providers/tool_scope.rs new file mode 100644 index 0000000..9923e37 --- /dev/null +++ b/core/src/sync/composio/providers/tool_scope.rs @@ -0,0 +1,200 @@ +//! Per-action scope classification (read / write / admin) plus the +//! [`CuratedTool`] catalog type that providers use to whitelist the +//! actions they want surfaced to the agent. +//! +//! Composio publishes 60+ actions per toolkit; most are noise for the +//! agent's planning loop. Each provider exports a hand-curated +//! [`CuratedTool`] slice via [`super::ComposioProvider::curated_tools`] +//! that pares the surface down to a useful subset and tags every action +//! with a [`ToolScope`] so per-user scope preferences can gate execution. + +use serde::{Deserialize, Serialize}; + +/// Classification of how invasive an action is. +/// +/// Used both to filter the agent's visible tool list and to enforce +/// per-user scope preferences at execution time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ToolScope { + /// Pure reads — `GET` / `FETCH` / `LIST` / `SEARCH` / `GET_PROFILE`. + Read, + /// Side-effectful actions that create or mutate user data — + /// `SEND` / `CREATE` / `UPDATE` / `REPLY` / `APPEND`. + Write, + /// Destructive or permission-changing actions — `DELETE` / `TRASH` / + /// `REMOVE` / `MODIFY_LABELS` / `SHARE`. + Admin, +} + +impl ToolScope { + pub fn as_str(self) -> &'static str { + match self { + ToolScope::Read => "read", + ToolScope::Write => "write", + ToolScope::Admin => "admin", + } + } +} + +/// One curated entry in a provider's tool catalog. +/// +/// `slug` is the Composio action slug as returned by `composio_list_tools` +/// (e.g. `"GMAIL_SEND_EMAIL"`). `scope` controls whether the action is +/// gated by the user's read / write / admin preference. +#[derive(Debug, Clone, Copy)] +pub struct CuratedTool { + pub slug: &'static str, + pub scope: ToolScope, +} + +/// Heuristic fallback when we need to gate a tool that isn't in any +/// provider's curated list. Prefer the curated classification when +/// available; only call this when [`super::ComposioProvider::curated_tools`] +/// returned `None` or didn't include the slug. +pub fn classify_unknown(slug: &str) -> ToolScope { + let upper = slug.to_ascii_uppercase(); + // Admin verbs are checked first so e.g. `MODIFY_LABELS` doesn't slip + // into the Write bucket on the `UPDATE`-substring rule. + const ADMIN: &[&str] = &[ + "DELETE", + "TRASH", + "REMOVE", + "MODIFY_LABELS", + "SHARE", + "REVOKE", + "DESTROY", + ]; + const WRITE: &[&str] = &[ + "SEND", "CREATE", "UPDATE", "REPLY", "APPEND", "INSERT", "ADD", "POST", "PATCH", "WRITE", + "DRAFT", + ]; + if ADMIN.iter().any(|kw| upper.contains(kw)) { + return ToolScope::Admin; + } + if WRITE.iter().any(|kw| upper.contains(kw)) { + return ToolScope::Write; + } + ToolScope::Read +} + +/// Look up a slug inside a curated catalog. +pub fn find_curated<'a>(catalog: &'a [CuratedTool], slug: &str) -> Option<&'a CuratedTool> { + catalog.iter().find(|t| t.slug.eq_ignore_ascii_case(slug)) +} + +/// Extract the toolkit slug from a Composio action slug. +/// +/// Most Composio action slugs follow `__…` +/// (e.g. `GMAIL_SEND_EMAIL` → `gmail`). A few toolkit identifiers contain +/// underscores themselves; those need known-prefix handling so connected +/// toolkit checks do not drop actions such as `ZOHO_MAIL_*`. +pub fn toolkit_from_slug(slug: &str) -> Option { + let trimmed = slug.trim(); + if trimmed.is_empty() { + return None; + } + const MULTI_SEGMENT_TOOLKIT_PREFIXES: &[(&str, &str)] = &[ + ("MICROSOFT_TEAMS_", "microsoft_teams"), + ("ONE_DRIVE_", "one_drive"), + ("ZOHO_MAIL_", "zoho_mail"), + ]; + let upper = trimmed.to_ascii_uppercase(); + for (prefix, toolkit) in MULTI_SEGMENT_TOOLKIT_PREFIXES { + if upper.starts_with(prefix) { + return Some((*toolkit).to_string()); + } + } + let prefix = trimmed.split('_').next()?; + if prefix.is_empty() { + None + } else { + Some(prefix.to_ascii_lowercase()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classify_unknown_picks_admin_for_destructive_verbs() { + assert_eq!(classify_unknown("GMAIL_DELETE_EMAIL"), ToolScope::Admin); + assert_eq!(classify_unknown("GMAIL_TRASH_EMAIL"), ToolScope::Admin); + assert_eq!(classify_unknown("GMAIL_MODIFY_LABELS"), ToolScope::Admin); + } + + #[test] + fn classify_unknown_picks_write_for_mutating_verbs() { + assert_eq!(classify_unknown("GMAIL_SEND_EMAIL"), ToolScope::Write); + assert_eq!(classify_unknown("NOTION_CREATE_PAGE"), ToolScope::Write); + assert_eq!(classify_unknown("NOTION_UPDATE_PAGE"), ToolScope::Write); + } + + #[test] + fn classify_unknown_defaults_to_read() { + assert_eq!(classify_unknown("GMAIL_FETCH_EMAILS"), ToolScope::Read); + assert_eq!(classify_unknown("NOTION_SEARCH"), ToolScope::Read); + assert_eq!(classify_unknown("GMAIL_GET_PROFILE"), ToolScope::Read); + } + + #[test] + fn classify_unknown_admin_takes_precedence_over_write() { + // MODIFY_LABELS contains no write verb but DELETE_DRAFT does — make + // sure the admin check wins. + assert_eq!(classify_unknown("GMAIL_DELETE_DRAFT"), ToolScope::Admin); + } + + #[test] + fn toolkit_from_slug_extracts_lowercase_prefix() { + assert_eq!( + toolkit_from_slug("GMAIL_SEND_EMAIL"), + Some("gmail".to_string()) + ); + assert_eq!( + toolkit_from_slug("NOTION_FETCH_DATA"), + Some("notion".to_string()) + ); + assert_eq!(toolkit_from_slug(""), None); + assert_eq!( + toolkit_from_slug("noUnderscore"), + Some("nounderscore".into()) + ); + } + + #[test] + fn toolkit_from_slug_handles_known_multi_segment_toolkits() { + assert_eq!( + toolkit_from_slug("ZOHO_MAIL_SEND_EMAIL"), + Some("zoho_mail".to_string()) + ); + assert_eq!( + toolkit_from_slug("ONE_DRIVE_GET_FILE"), + Some("one_drive".to_string()) + ); + assert_eq!( + toolkit_from_slug("MICROSOFT_TEAMS_SEND_MESSAGE"), + Some("microsoft_teams".to_string()) + ); + } + + #[test] + fn find_curated_is_case_insensitive() { + let catalog = &[CuratedTool { + slug: "GMAIL_SEND_EMAIL", + scope: ToolScope::Write, + }]; + assert!(find_curated(catalog, "gmail_send_email").is_some()); + assert!(find_curated(catalog, "GMAIL_SEND_EMAIL").is_some()); + assert!(find_curated(catalog, "GMAIL_DELETE_EMAIL").is_none()); + } + + #[test] + fn tool_scope_serializes_lowercase() { + assert_eq!(serde_json::to_string(&ToolScope::Read).unwrap(), "\"read\""); + assert_eq!( + serde_json::to_string(&ToolScope::Admin).unwrap(), + "\"admin\"" + ); + } +} diff --git a/core/src/sync/composio/providers/traits.rs b/core/src/sync/composio/providers/traits.rs new file mode 100644 index 0000000..8585eb2 --- /dev/null +++ b/core/src/sync/composio/providers/traits.rs @@ -0,0 +1,405 @@ +//! The core provider trait for Composio toolkit implementations. + +use async_trait::async_trait; + +use super::tool_scope::CuratedTool; +use super::types::{ + NormalizedTask, ProviderContext, ProviderUserProfile, SyncOutcome, SyncReason, TaskContainer, + TaskFetchFilter, +}; + +/// Native provider implementation for a specific Composio toolkit. +/// +/// All methods are async and return `Result<_, String>` so the bus +/// subscriber + RPC layer can forward errors as user-visible strings +/// without `anyhow` round-tripping. +#[async_trait] +pub trait ComposioProvider: Send + Sync { + /// Toolkit slug (e.g. `"gmail"`). Must match the slug Composio / + /// the backend allowlist uses — the registry keys on this. + fn toolkit_slug(&self) -> &'static str; + + /// Suggested periodic sync interval in seconds. Return `None` to + /// opt out of the periodic scheduler entirely (e.g. for write-only + /// providers like Slack send-message). + fn sync_interval_secs(&self) -> Option { + Some(15 * 60) + } + + /// Curated whitelist of Composio actions this provider considers + /// useful for the agent, classified by [`super::tool_scope::ToolScope`]. + /// + /// When `Some(&[...])`, the meta-tool layer hides every action not + /// in this list from `composio_list_tools` and rejects execution of + /// any slug not in this list (or whose scope is disabled in the + /// user's pref). + /// + /// Default: `None` — toolkits without a curated catalog (e.g. + /// integrations not yet hand-tuned) pass through all actions and + /// rely on the [`super::tool_scope::classify_unknown`] heuristic for + /// scope gating. + fn curated_tools(&self) -> Option<&'static [CuratedTool]> { + None + } + + /// Fetch a normalized user profile for the current connection in + /// `ctx`. Most providers implement this by calling a provider + /// "get profile / about me" action via `super::super::ops::composio_execute`. + async fn fetch_user_profile( + &self, + ctx: &ProviderContext, + ) -> Result; + + /// Compatibility entry point backed exclusively by tinycortex. + async fn sync(&self, ctx: &ProviderContext, reason: SyncReason) -> Result { + let connection_id = ctx.connection_id.as_deref().ok_or_else(|| { + format!( + "[composio:{}] sync missing connection_id", + self.toolkit_slug() + ) + })?; + let started_at_ms = now_ms(); + let outcome = crate::tinycortex::run_composio_connection_with_budgets( + self.toolkit_slug(), + connection_id, + ctx.config.as_ref(), + ctx.max_items, + ctx.sync_depth_days, + ) + .await + .map_err(|error| error.to_string())?; + Ok(SyncOutcome { + toolkit: self.toolkit_slug().into(), + connection_id: Some(connection_id.into()), + reason: reason.as_str().into(), + items_ingested: outcome.records_ingested as usize, + started_at_ms, + finished_at_ms: now_ms(), + summary: outcome.note.unwrap_or_else(|| "sync completed".into()), + details: serde_json::json!({ + "more_pending": outcome.more_pending, + "actions_called": outcome.actions_called, + "provider_cost_usd": outcome.provider_cost_usd, + }), + }) + } + + /// Fetch a filtered set of work items as structured + /// [`NormalizedTask`]s — the read path that powers the + /// `task_sources` domain. + /// + /// Unlike [`Self::sync`], this does **not** persist anything into + /// the memory store; it *returns* normalized tasks so the caller can + /// enrich them and route them onto the agent's todo board. `filter` + /// is provider-agnostic — implementations read only the fields that + /// apply to their toolkit and translate them into their own action + /// slug + arguments, then map the upstream payload back into + /// `NormalizedTask`. Implementations must honour + /// [`TaskFetchFilter::effective_max`] as an upper bound on the + /// number of tasks returned. + /// + /// Default impl: `Err` — providers without a task surface (e.g. + /// gmail, slack) opt out, exactly as + /// [`Self::sync_interval_secs`] returning `None` opts out of the + /// periodic scheduler. + async fn fetch_tasks( + &self, + ctx: &ProviderContext, + filter: &TaskFetchFilter, + ) -> Result, String> { + let _ = (ctx, filter); + Err(format!( + "[composio:{}] provider has no task-fetch surface", + self.toolkit_slug() + )) + } + + /// List the selectable containers the connected account exposes — + /// today Notion databases — so the task-source UI can offer a picker + /// instead of a raw-id text field. + /// + /// Default impl: `Err` — providers without a container surface opt out, + /// mirroring [`Self::fetch_tasks`]. + async fn list_databases(&self, ctx: &ProviderContext) -> Result, String> { + let _ = ctx; + Err(format!( + "[composio:{}] provider has no database/container surface", + self.toolkit_slug() + )) + } + + /// Standardized identity callback for provider implementations. + /// + /// Providers can override this to customize how identity fragments + /// are persisted. Default behavior stores a normalized identity + /// fragment in profile facets via `skill:{source}:{identifier}:{field}` + /// keys and returns the number of facets written. + fn identity_set(&self, profile: &ProviderUserProfile) -> usize { + super::profile::persist_provider_profile(profile) + } + + /// Hook fired when an OAuth handoff completes + /// (the host's `DomainEvent::ComposioConnectionCreated`). + /// + /// Default impl: fetch and persist the user profile. Initial memory + /// ingestion is dispatched separately through tinycortex by the bus. + /// Providers can override to add provider-specific bootstrapping + /// (e.g. registering Composio triggers, seeding labels, …). + async fn on_connection_created(&self, ctx: &ProviderContext) -> Result<(), String> { + let toolkit = self.toolkit_slug(); + tracing::info!( + toolkit = %toolkit, + connection_id = ?ctx.connection_id, + "[composio:provider] on_connection_created: fetching user profile" + ); + match self.fetch_user_profile(ctx).await { + Ok(profile) => { + // PII discipline: do not log raw display_name or email. + // We log only presence indicators and the email domain + // (non-PII) so the trace is debuggable without leaking + // the user's identity. Provider-specific impls follow + // the same convention. + let has_display_name = profile.display_name.is_some(); + let has_email = profile.email.is_some(); + let email_domain = profile + .email + .as_deref() + .and_then(|e| e.split('@').nth(1)) + .map(|d| d.to_string()); + tracing::info!( + toolkit = %toolkit, + has_display_name, + has_email, + email_domain = ?email_domain, + "[composio:provider] user profile fetched" + ); + + // Persist profile fields into the local user_profile + // facet table so display_name / email / avatar are + // available to the agent context and UI without a + // round-trip to the upstream provider. + let facets = self.identity_set(&profile); + tracing::debug!( + toolkit = %toolkit, + facets_written = facets, + "[composio:provider] identity_set persisted profile facets" + ); + + // Mirror the same identity fragment into PROFILE.md so + // it lands in the agent's prompt context on the next + // turn (the facets table feeds queries; PROFILE.md + // feeds the system prompt). + if let Err(e) = super::profile_md::merge_provider_into_profile_md( + ctx.config.workspace_dir(), + &profile, + ) { + tracing::warn!( + toolkit = %toolkit, + error = %e, + "[composio:provider] PROFILE.md merge failed (non-fatal)" + ); + } + } + Err(e) => { + tracing::warn!( + toolkit = %toolkit, + error = %e, + "[composio:provider] user profile fetch failed (continuing to sync)" + ); + } + } + Ok(()) + } + + /// Hook fired immediately after a Composio action executed against + /// this toolkit returns a **successful** response. The provider may + /// mutate `data` in place to reshape the upstream payload before it + /// is handed back to the agent / RPC caller (e.g. convert Gmail's + /// HTML message bodies to markdown to save context tokens). + /// + /// `slug` is the full action slug (e.g. `"GMAIL_FETCH_EMAILS"`) so + /// providers can dispatch per action. `arguments` is the caller's + /// original argument object — providers can read opt-out flags from + /// it (e.g. `raw_html: true` to preserve raw HTML). + /// + /// Errors from upstream are not routed here; only `successful` + /// responses. Default impl is a no-op so providers that have nothing + /// to rewrite don't need to override. + fn post_process_action_result( + &self, + slug: &str, + arguments: Option<&serde_json::Value>, + data: &mut serde_json::Value, + ) { + let _ = (slug, arguments, data); + } + + /// Hook fired when a Composio trigger webhook arrives for this + /// toolkit. `payload` is the raw provider payload as forwarded by + /// the backend. Implementations should be defensive — payload + /// shapes vary across triggers. + /// + /// Default impl: log and no-op. Most providers will want to + /// override this to react to specific triggers. + async fn on_trigger( + &self, + ctx: &ProviderContext, + trigger: &str, + payload: &serde_json::Value, + ) -> Result<(), String> { + tracing::debug!( + toolkit = %self.toolkit_slug(), + trigger = %trigger, + connection_id = ?ctx.connection_id, + payload_bytes = payload.to_string().len(), + "[composio:provider] on_trigger (default no-op)" + ); + Ok(()) + } +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +/// Build the env var name read by [`resolve_sync_interval_secs`] for a +/// given toolkit slug. Exposed so tests (and `.env.example`) can stay in +/// lockstep with the runtime lookup without re-implementing the casing. +pub fn sync_interval_env_var(toolkit: &str) -> String { + format!( + "OPENHUMAN_COMPOSIO_{}_SYNC_INTERVAL_SECS", + toolkit.to_ascii_uppercase() + ) +} + +/// Resolve the effective periodic sync interval (seconds) for a provider. +/// Reads `OPENHUMAN_COMPOSIO__SYNC_INTERVAL_SECS` if set; +/// otherwise returns `default_secs`. A non-positive or unparseable value +/// is rejected with a `warn` and the default is used — `0` would burn the +/// scheduler in a tight loop, so it is never honoured. +/// +/// Each provider's `sync_interval_secs()` impl calls this with its own +/// compile-time default so operators can independently slow down a +/// chatty toolkit (e.g. Slack) without rebuilding. +pub fn resolve_sync_interval_secs(toolkit: &str, default_secs: u64) -> u64 { + let key = sync_interval_env_var(toolkit); + match std::env::var(&key) { + Ok(s) => match s.trim().parse::() { + Ok(n) if n >= 1 => n, + _ => { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + tracing::warn!( + env = %key, + value = %s, + default = default_secs, + "[composio:provider] sync-interval env override not a positive u64; using default" + ); + }); + default_secs + } + }, + Err(_) => default_secs, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sync_interval_env_var_uppercases_slug() { + assert_eq!( + sync_interval_env_var("slack"), + "OPENHUMAN_COMPOSIO_SLACK_SYNC_INTERVAL_SECS" + ); + assert_eq!( + sync_interval_env_var("GitHub"), + "OPENHUMAN_COMPOSIO_GITHUB_SYNC_INTERVAL_SECS" + ); + } + + /// RAII guard for env var save/restore so the test does not leak + /// state to siblings within the same process. + struct EnvGuard { + key: String, + previous: Option, + } + + impl EnvGuard { + fn set(key: &str, value: &str) -> Self { + let previous = std::env::var(key).ok(); + std::env::set_var(key, value); + Self { + key: key.to_string(), + previous, + } + } + fn unset(key: &str) -> Self { + let previous = std::env::var(key).ok(); + std::env::remove_var(key); + Self { + key: key.to_string(), + previous, + } + } + } + + impl Drop for EnvGuard { + fn drop(&mut self) { + match self.previous.take() { + Some(v) => std::env::set_var(&self.key, v), + None => std::env::remove_var(&self.key), + } + } + } + + // Bundled into a single `#[test]` so cargo's per-test parallelism + // does not race on the shared env var. Each scenario explicitly + // drops its guard before the next so the env is in a known state. + #[test] + fn resolve_sync_interval_honors_per_toolkit_env() { + let _lock = crate::test_env_lock::TEST_ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + let key = sync_interval_env_var("slack"); + let default = 15 * 60; + + // Unset → default. + let _g = EnvGuard::unset(&key); + assert_eq!(resolve_sync_interval_secs("slack", default), default); + drop(_g); + + // Valid override slows the cadence. + let _g = EnvGuard::set(&key, "3600"); + assert_eq!(resolve_sync_interval_secs("slack", default), 3600); + drop(_g); + + // Whitespace tolerated. + let _g = EnvGuard::set(&key, " 1800 "); + assert_eq!(resolve_sync_interval_secs("slack", default), 1800); + drop(_g); + + // Zero rejected (would spin the scheduler). + let _g = EnvGuard::set(&key, "0"); + assert_eq!(resolve_sync_interval_secs("slack", default), default); + drop(_g); + + // Garbage rejected. + let _g = EnvGuard::set(&key, "soon"); + assert_eq!(resolve_sync_interval_secs("slack", default), default); + drop(_g); + + // Per-toolkit scoping: a different toolkit's var does not bleed + // into slack's lookup. + let gmail_key = sync_interval_env_var("gmail"); + let _slack_unset = EnvGuard::unset(&key); + let _gmail_set = EnvGuard::set(&gmail_key, "120"); + assert_eq!(resolve_sync_interval_secs("slack", default), default); + assert_eq!(resolve_sync_interval_secs("gmail", default), 120); + } +} diff --git a/core/src/sync/composio/providers/types.rs b/core/src/sync/composio/providers/types.rs new file mode 100644 index 0000000..3825c4a --- /dev/null +++ b/core/src/sync/composio/providers/types.rs @@ -0,0 +1,523 @@ +//! Shared types for Composio provider implementations. + +use serde::{Deserialize, Serialize}; +use std::sync::{Arc, Mutex}; + +// Test-only: the tests below build a `TestHostConfig` and call +// `MemoryHostConfig` methods on it directly. Production code in this module +// goes through `crate::Config`, so neither name is needed in a non-test build. +#[cfg(test)] +use tinymemory_api::host::{test_support::TestHostConfig, MemoryHostConfig}; + +use crate::composio_host::{self, ComposioExecuteResponse}; +use crate::config_loader as config_rpc; +use crate::Config; + +/// Reason a sync was triggered. Providers can use this to decide +/// whether to do a full backfill or an incremental pull. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SyncReason { + /// First sync immediately after an OAuth handoff completes. + ConnectionCreated, + /// Periodic background sync from the scheduler. + Periodic, + /// Explicit user-driven sync from RPC / UI. + Manual, +} + +impl SyncReason { + pub fn as_str(&self) -> &'static str { + match self { + SyncReason::ConnectionCreated => "connection_created", + SyncReason::Periodic => "periodic", + SyncReason::Manual => "manual", + } + } +} + +/// What kind of work an ingested task implies. GitHub's issues-and-PRs +/// search returns both shapes, and the job differs fundamentally — +/// *resolve* an issue vs *review* a pull request — so providers tag each +/// task and the `task_sources` enrichment phrases the objective / agent +/// prompt accordingly (the triage LLM then knows what to do). Providers +/// that don't distinguish (notion, linear, clickup) leave this `Generic`. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum TaskKind { + /// No issue/PR distinction — the default for non-code providers. + #[default] + Generic, + /// A tracker issue: the job is to resolve / implement it. + Issue, + /// A pull request: the job is to review it (read the diff, give feedback). + PullRequest, +} + +impl TaskKind { + /// Stable lowercase tag, mirrored into the card's `source_metadata`. + pub fn as_str(&self) -> &'static str { + match self { + TaskKind::Generic => "generic", + TaskKind::Issue => "issue", + TaskKind::PullRequest => "pull_request", + } + } +} + +/// Normalized user profile shape returned by every provider. +/// +/// The shared fields (`display_name`, `email`, `username`, `avatar_url`, +/// `profile_url`) +/// cover what the desktop UI actually needs to render a connected +/// account card. Anything provider-specific (Gmail's `messagesTotal`, +/// Notion's workspace ids, …) goes into [`extras`](Self::extras) so +/// callers don't have to widen the shape every time a new toolkit +/// lands. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ProviderUserProfile { + pub toolkit: String, + pub connection_id: Option, + pub display_name: Option, + pub email: Option, + pub username: Option, + pub avatar_url: Option, + pub profile_url: Option, + /// Provider-specific extras (raw JSON object). + #[serde(default)] + pub extras: serde_json::Value, +} + +/// Result of a provider sync run. Mostly used for logging + UI status. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SyncOutcome { + pub toolkit: String, + pub connection_id: Option, + pub reason: String, + pub items_ingested: usize, + pub started_at_ms: u64, + pub finished_at_ms: u64, + pub summary: String, + /// Provider-specific extras (raw JSON object). + #[serde(default)] + pub details: serde_json::Value, +} + +impl SyncOutcome { + pub fn elapsed_ms(&self) -> u64 { + self.finished_at_ms.saturating_sub(self.started_at_ms) + } +} + +/// A provider-agnostic, structured work item produced by +/// [`super::ComposioProvider::fetch_tasks`]. +/// +/// Unlike the `sync()` path — which persists upstream items into the +/// memory store as passive context — `fetch_tasks` *returns* normalized +/// tasks so the `task_sources` domain can enrich them and route them +/// onto the agent's todo board. Every native task provider (github, +/// notion, linear, clickup) maps its upstream payload shape into this +/// common envelope. +/// +/// `source_id` is left empty by providers and stamped by the +/// `task_sources` pipeline with the originating `TaskSource.id` — a +/// provider has no knowledge of which configured source asked for the +/// fetch. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct NormalizedTask { + /// Upstream provider's stable id for the item (issue/task/page id). + pub external_id: String, + /// The `TaskSource.id` that produced this task. Empty until the + /// pipeline stamps it. + #[serde(default)] + pub source_id: String, + /// Toolkit slug, e.g. `"github"`. + pub provider: String, + /// Whether this task is an issue, a pull request, or undifferentiated. + /// Drives intent-aware objective / prompt phrasing in enrichment. + #[serde(default)] + pub kind: TaskKind, + pub title: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub body: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assignee: Option, + /// Due date as an ISO-8601 string, when the provider exposes one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub due: Option, + #[serde(default)] + pub labels: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub priority: Option, + /// Last-updated ISO-8601 timestamp — used for cursor advancement and + /// edit-aware dedup (`{external_id}@{updated_at}`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub updated_at: Option, + /// The raw upstream payload, retained for enrichment / debugging. + #[serde(default)] + pub raw: serde_json::Value, +} + +/// A selectable upstream task container (board / database / list) used to +/// populate a picker so the user chooses from a list instead of pasting a +/// raw id. Today this is a Notion database, later a Linear team or ClickUp +/// list. Surfaced to the task-source UI as `{ id, title }`. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TaskContainer { + /// Provider-native id (e.g. a Notion database id) used as the filter id. + pub id: String, + /// Human-readable label for the picker. + pub title: String, +} + +/// Provider-agnostic filter passed into +/// [`super::ComposioProvider::fetch_tasks`]. +/// +/// The `task_sources` domain builds this from a user-configured, +/// per-provider `FilterSpec`. Each provider reads only the fields that +/// apply to it (github reads `repo`/`labels`; notion reads +/// `database_id`; linear/clickup read `team_id`; …) and ignores the +/// rest. `extra` is a free-form escape hatch surfaced in the UI for +/// advanced provider-native query fragments. +/// How the GitHub task-source fetch reaches GitHub. Shipped desktop users +/// connect GitHub via Composio OAuth (no `gh` on PATH, no `GITHUB_TOKEN`), +/// while local dev / self-host setups often have the reverse. `Auto` does the +/// right thing for both; `Composio` / `Local` force a path when the user wants. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum GithubFetchMode { + /// Try the connected Composio account first; fall back to local `gh`/REST + /// only when Composio is unavailable. The safe default — no regression for + /// shipped users, still a true fallback for local/dev. + #[default] + Auto, + /// Force the connected Composio account (classic shipped-app behaviour). + Composio, + /// Force local `gh` CLI / REST with a `GH_TOKEN`/`GITHUB_TOKEN` env token. + Local, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TaskFetchFilter { + /// Scope to items assigned to (or involving) the authenticated user. + #[serde(default)] + pub assignee_is_me: bool, + /// GitHub fetch path selector (Composio vs local `gh`/REST). Default `Auto`. + #[serde(default)] + pub github_fetch_mode: GithubFetchMode, + /// GitHub `owner/name` repository scope. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repo: Option, + /// GitHub label filter. + #[serde(default)] + pub labels: Vec, + /// Issue/task state filter (e.g. `"open"`, `"todo"`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state: Option, + /// Notion database (board) id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database_id: Option, + /// Notion status property filter. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status: Option, + /// Linear / ClickUp team (workspace) id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub team_id: Option, + /// ClickUp list id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub list_id: Option, + /// Free-form provider-native filter fragment (advanced). + #[serde(default)] + pub extra: serde_json::Value, + /// Hard cap on how many tasks a single fetch returns. + #[serde(default)] + pub max: u32, +} + +impl TaskFetchFilter { + /// Effective per-fetch item cap, defaulting to a safe bound when the + /// caller leaves `max` unset (0). + pub fn effective_max(&self) -> usize { + if self.max == 0 { + 25 + } else { + self.max as usize + } + } +} + +/// Per-call context handed to provider methods. +/// +/// `connection_id` is `None` when a method runs in a "no specific +/// connection" mode (e.g. an across-the-board periodic sync that +/// already iterated). For per-connection paths it is always populated. +/// +/// **Mode-aware dispatch (#1710)**: pre-fix, `ProviderContext` cached a +/// pre-baked `ComposioClient` built once at construction time. Toggling +/// `composio.mode = "direct"` mid-session left provider syncs still +/// routing through the backend tinyhumans tenant. The current shape +/// keeps an [`Arc`] and resolves the underlying client per call +/// through [`ProviderContext::execute`], mirroring the agent-tool +/// migration in the host's `integrations::composio::tools::ComposioExecuteTool`. +/// Per-sync accumulator for Composio billable-action usage. +/// +/// Lives behind a shared handle on [`ProviderContext`] so the single +/// `execute` chokepoint can tally every action a provider fires during one +/// sync run, regardless of which provider (gmail / slack / github / notion / +/// linear / clickup) or how many pages it paginates. +/// [`crate::sync::composio::run_connection_sync`] returns +/// the final tally alongside the [`SyncOutcome`] for the sync audit log +/// (#3111). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ComposioUsage { + /// Count of `execute` calls that returned a response this run. + pub actions_called: u32, + /// Sum of each response's backend-reported `cost_usd`. + pub cost_usd: f64, +} + +/// Shared, interior-mutable handle to a [`ComposioUsage`] tally. Cloning a +/// [`ProviderContext`] shares the same underlying counter, so the count is +/// stable no matter how the context is passed around within a sync. +pub type ComposioUsageHandle = Arc>; + +#[derive(Clone)] +pub struct ProviderContext { + pub config: Arc, + pub toolkit: String, + pub connection_id: Option, + /// Accumulates Composio billable-action usage across this context's + /// lifetime. Defaulted at every construction site; only the sync path + /// (`run_connection_sync`) reads it back. Non-sync callers (agent tools, + /// task-source fetches) leave it at zero — harmless. + pub usage: ComposioUsageHandle, + /// Maximum items to fetch in a single sync pass. + /// + /// Set from the corresponding `MemorySourceEntry.max_items` field at + /// sync-dispatch time. `None` means no cap beyond the provider's own + /// internal upper bounds. + pub max_items: Option, + /// Maximum sync depth window in days. + /// + /// Set from `MemorySourceEntry.sync_depth_days`. When `Some(n)`, the + /// provider only fetches items from the last `n` days. `None` means + /// no additional depth restriction beyond the provider's cursor. + pub sync_depth_days: Option, +} + +impl ProviderContext { + /// Build a context from the current config + a toolkit slug. + /// + /// Returns `None` only when we want to short-circuit early on the + /// "user clearly not signed in" path. In the post-#1710 shape this + /// is determined by attempting a factory resolve via + /// [`composio_host::is_available`] and treating a `false` there as + /// "skip silently" — the same UX as the pre-fix + /// `build_composio_client(...).is_some()` probe, but routed + /// through the mode-aware factory so direct-mode users (no backend + /// session token, BYO key in keychain) aren't falsely treated as + /// signed-out. + pub fn from_config( + config: Arc, + toolkit: impl Into, + connection_id: Option, + ) -> Option { + // Probe the factory: any successful resolve (Backend OR Direct) + // means the user has *some* viable Composio client. Direct-mode + // users typically have no backend session token, which would + // make a `build_composio_client` probe return None and falsely + // skip them. + if composio_host::is_available(&*config) { + Some(Self { + config, + toolkit: toolkit.into(), + connection_id, + usage: ComposioUsageHandle::default(), + max_items: None, + sync_depth_days: None, + }) + } else { + tracing::debug!( + "[composio:provider_context] from_config: no viable Composio client; \ + treating as not-signed-in" + ); + None + } + } + + /// Resolve the underlying composio client via the mode-aware + /// factory and dispatch a single action. This is the canonical + /// way for provider implementations to execute a Composio action + /// — going through here ensures the live `composio.mode` toggle is + /// honoured on every call (#1710). + /// + /// Returns the same [`ComposioExecuteResponse`] shape that + /// `ComposioClient::execute_tool` used to return so existing + /// provider call-sites can swap `ctx.client.execute_tool(...)` for + /// `ctx.execute(...)` with no other changes. + pub async fn execute( + &self, + action: &str, + arguments: Option, + ) -> anyhow::Result { + // [#1710 Wave 4] Reload config fresh per execute so a mid-session + // `composio.mode` toggle takes effect at the very next call. The + // Arc snapshot held by `self` was taken at agent-init time + // and is otherwise stale relative to subsequent set_api_key / + // clear_api_key RPCs. + // + // Use `reload_config_snapshot_with_timeout` (anchored to the snapshot's + // `config_path`) rather than `load_config_with_timeout` (which + // re-resolves `OPENHUMAN_WORKSPACE` from the process env). The config + // path is stable for the lifetime of a `ProviderContext` — it is set + // at context creation from the agent's scoped config — so reading from + // it always reaches the correct user workspace and avoids a data-race + // in tests that share the process env. + let live_config = config_rpc::reload_config_snapshot_with_timeout(&*self.config) + .await + .map_err(|e| { + tracing::warn!( + action = %action, + toolkit = %self.toolkit, + error = %e, + "[composio:provider_context] execute: reload_config failed" + ); + anyhow::anyhow!("composio provider_context: failed to reload live config: {e}") + })?; + // Mode dispatch (backend tenant vs the user's own direct v3 tenant) + // lives in the host's `ComposioHost` impl — this side just asks. + let result = composio_host::execute( + &*live_config, + action, + arguments, + &live_config.composio().entity_id, + self.connection_id.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!(e)); + + // Tally billable-action usage at the single chokepoint every provider + // routes through (#3111). We count any *completed* round-trip — even a + // provider-reported failure (`successful == false`) is a billable call + // — and sum the backend-reported `cost_usd`. Transport errors (the + // `Err` arm) never reached Composio, so they don't count. The lock is + // held only for the increment, never across an `.await`. + if let Ok(ref resp) = result { + if let Ok(mut usage) = self.usage.lock() { + usage.actions_called = usage.actions_called.saturating_add(1); + usage.cost_usd += resp.cost_usd; + } + } + result + } + + /// Memory client handle if the global memory singleton is ready. + /// Used by providers that want to persist sync snapshots. + /// + /// Under `cfg(test)` the global singleton is not booted, so build a + /// workspace-scoped client directly instead. + #[cfg(test)] + pub fn memory_client(&self) -> Option { + crate::store::MemoryClient::from_workspace_dir(self.config.workspace_dir().clone()) + .ok() + .map(std::sync::Arc::new) + } + + /// Memory client handle if the global memory singleton is ready. + /// Used by providers that want to persist sync snapshots. + #[cfg(not(test))] + pub fn memory_client(&self) -> Option { + crate::global::client_if_ready() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The whole #3111 tally relies on the `usage` handle being *shared* + /// across `ProviderContext` clones: a provider's `sync` runs against a + /// clone (or the same ctx passed by `&`), accumulates via `execute`, and + /// `run_connection_sync` reads the count back from its own handle. Pin + /// that the `Arc>` is genuinely shared so a clone's increments + /// are visible from the original — if this regressed to a per-clone + /// counter, the audit cost would silently always read zero. + #[test] + fn usage_handle_is_shared_across_context_clones() { + let ctx = ProviderContext { + config: Arc::new(TestHostConfig::default()) as Arc, + toolkit: "gmail".to_string(), + connection_id: None, + usage: ComposioUsageHandle::default(), + max_items: None, + sync_depth_days: None, + }; + let cloned = ctx.clone(); + + // Simulate two `execute` round-trips accumulating on the clone. + { + let mut usage = cloned.usage.lock().expect("lock usage"); + usage.actions_called = usage.actions_called.saturating_add(2); + usage.cost_usd += 0.015; + } + + // The original handle must observe the clone's tally. + let observed = ctx.usage.lock().expect("lock usage"); + assert_eq!(observed.actions_called, 2); + assert!((observed.cost_usd - 0.015).abs() < 1e-9); + } + + /// `ComposioUsage` defaults to a zero tally — the value + /// `run_connection_sync` returns for a sync that fired no Composio + /// actions, and what non-sync `ProviderContext` callers carry. + #[test] + fn composio_usage_defaults_to_zero() { + let usage = ComposioUsage::default(); + assert_eq!(usage.actions_called, 0); + assert_eq!(usage.cost_usd, 0.0); + } + + // `ProviderContext::execute` and `ProviderContext::backend_client` reload + // config from `ctx.config.config_path()` (via `reload_config_snapshot_with_timeout`) + // rather than from the process-global `OPENHUMAN_WORKSPACE`. Tests + // therefore only need to persist the config to `config_path` — no env var + // manipulation required. + + #[tokio::test] + async fn provider_context_execute_backend_branch_without_session_errors_cleanly() { + // Default `Config` (mode = "backend") with no stored session + // token: the factory should return a backend-session error from + // `ctx.execute`. Verifies the backend branch is reachable and + // the error surface is sensible. + let tmp = tempfile::tempdir().expect("tempdir"); + + let mut config = TestHostConfig::default(); + config.config_path = tmp.path().join("config.toml"); + config.workspace_dir = tmp.path().join("workspace"); + config.secrets_encrypt = false; + config.save().await.expect("save fake config to disk"); + + let ctx = ProviderContext { + config: Arc::new(config) as Arc, + toolkit: "gmail".to_string(), + connection_id: None, + usage: ComposioUsageHandle::default(), + max_items: None, + sync_depth_days: None, + }; + let res = ctx.execute("GMAIL_FETCH_EMAILS", None).await; + let err = res.expect_err("no backend session must error"); + let msg = err.to_string(); + assert!( + msg.contains("backend") || msg.contains("session"), + "expected backend-session error, got: {msg}" + ); + } +} diff --git a/core/src/sync/composio/providers/user_scopes.rs b/core/src/sync/composio/providers/user_scopes.rs new file mode 100644 index 0000000..af7beec --- /dev/null +++ b/core/src/sync/composio/providers/user_scopes.rs @@ -0,0 +1,159 @@ +//! Per-user, per-toolkit scope preferences. +//! +//! For each Composio toolkit a user has connected (or could connect), +//! we store a [`UserScopePref`] that records whether the agent is +//! allowed to call **read**, **write**, and / or **admin**-classified +//! actions for that toolkit. Defaults are `read=true, write=true, +//! admin=false` — the agent can use the integration productively out of +//! the box, but destructive / permission-changing actions require +//! explicit opt-in. +//! +//! Storage uses the same KV surface as [`super::sync_state`] +//! (`MemoryClient::kv_get` / `kv_set`) under a dedicated namespace so +//! prefs survive process restarts without any extra file management. + +use serde::{Deserialize, Serialize}; + +use crate::store::MemoryClientRef; + +use super::tool_scope::ToolScope; + +/// KV namespace for scope prefs. Separate from `composio-sync-state` so +/// the two never collide. +const KV_NAMESPACE: &str = "composio-user-scopes"; + +/// Per-toolkit scope preference. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct UserScopePref { + #[serde(default = "default_true")] + pub read: bool, + #[serde(default = "default_true")] + pub write: bool, + #[serde(default)] + pub admin: bool, +} + +fn default_true() -> bool { + true +} + +impl Default for UserScopePref { + fn default() -> Self { + Self { + read: true, + write: true, + admin: false, + } + } +} + +impl UserScopePref { + /// Returns `true` if the given scope is enabled in this preference. + pub fn allows(&self, scope: ToolScope) -> bool { + match scope { + ToolScope::Read => self.read, + ToolScope::Write => self.write, + ToolScope::Admin => self.admin, + } + } +} + +fn kv_key(toolkit: &str) -> String { + toolkit.trim().to_ascii_lowercase() +} + +/// Load the scope pref for `toolkit`. Returns the default +/// (`read+write`, no `admin`) when nothing is stored or when the KV +/// store can't be reached — the agent should always be able to use +/// connected integrations productively, even if pref storage is +/// temporarily unavailable. +pub async fn load(memory: &MemoryClientRef, toolkit: &str) -> UserScopePref { + let key = kv_key(toolkit); + if key.is_empty() { + return UserScopePref::default(); + } + match memory.kv_get(Some(KV_NAMESPACE), &key).await { + Ok(Some(value)) => match serde_json::from_value::(value) { + Ok(pref) => { + tracing::debug!( + toolkit = %key, + read = pref.read, + write = pref.write, + admin = pref.admin, + "[composio][scopes] pref loaded" + ); + pref + } + Err(e) => { + tracing::warn!( + toolkit = %key, + error = %e, + "[composio][scopes] pref deserialize failed, falling back to default" + ); + UserScopePref::default() + } + }, + Ok(None) => { + tracing::debug!( + toolkit = %key, + "[composio][scopes] no pref stored, using default (read+write)" + ); + UserScopePref::default() + } + Err(e) => { + tracing::warn!( + toolkit = %key, + error = %e, + "[composio][scopes] kv_get failed, falling back to default" + ); + UserScopePref::default() + } + } +} + +/// Persist a scope pref for `toolkit`. +pub async fn save( + memory: &MemoryClientRef, + toolkit: &str, + pref: UserScopePref, +) -> Result<(), String> { + let key = kv_key(toolkit); + if key.is_empty() { + return Err("user_scopes: toolkit must not be empty".to_string()); + } + let value = serde_json::to_value(pref) + .map_err(|e| format!("[composio][scopes] serialize failed: {e}"))?; + memory.kv_set(Some(KV_NAMESPACE), &key, &value).await?; + tracing::info!( + toolkit = %key, + read = pref.read, + write = pref.write, + admin = pref.admin, + "[composio][scopes] pref saved" + ); + Ok(()) +} + +/// Best-effort load that resolves the active memory client itself. Used +/// from the meta-tool layer where we don't have a `MemoryClientRef` in +/// scope. Falls back to the default pref when memory isn't initialised. +pub async fn load_or_default(toolkit: &str) -> UserScopePref { + match crate::global::client_if_ready() { + Some(client) => load(&client, toolkit).await, + None => { + // Match the normalized key form `load()` logs so traces + // grouped by `key` correlate across both code paths. + let key = kv_key(toolkit); + tracing::debug!( + toolkit = %toolkit, + key = %key, + "[composio][scopes] memory not ready, using default pref" + ); + UserScopePref::default() + } + } +} + +#[cfg(test)] +#[path = "user_scopes_tests.rs"] +mod tests; diff --git a/core/src/sync/composio/providers/user_scopes_tests.rs b/core/src/sync/composio/providers/user_scopes_tests.rs new file mode 100644 index 0000000..b6c9e0e --- /dev/null +++ b/core/src/sync/composio/providers/user_scopes_tests.rs @@ -0,0 +1,103 @@ +use super::*; +use crate::store::MemoryClient; +use std::sync::Arc; +use tempfile::TempDir; + +fn make_client() -> (TempDir, Arc) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let client = Arc::new( + MemoryClient::from_workspace_dir(tmp.path().join("workspace")) + .expect("memory client should initialize for user-scope tests"), + ); + (tmp, client) +} + +#[test] +fn default_is_read_write_no_admin() { + let p = UserScopePref::default(); + assert!(p.read); + assert!(p.write); + assert!(!p.admin); +} + +#[test] +fn allows_matches_scope() { + let p = UserScopePref { + read: true, + write: false, + admin: false, + }; + assert!(p.allows(ToolScope::Read)); + assert!(!p.allows(ToolScope::Write)); + assert!(!p.allows(ToolScope::Admin)); +} + +#[test] +fn round_trip_serde() { + let p = UserScopePref { + read: true, + write: true, + admin: true, + }; + let v = serde_json::to_value(p).unwrap(); + let back: UserScopePref = serde_json::from_value(v).unwrap(); + assert_eq!(p, back); +} + +#[test] +fn missing_fields_default_to_true_for_read_write() { + // Forward-compat: if we ever drop a field, existing stored + // documents still deserialize sensibly. + let v = serde_json::json!({}); + let p: UserScopePref = serde_json::from_value(v).unwrap(); + assert_eq!(p, UserScopePref::default()); +} + +#[tokio::test] +async fn save_and_load_round_trip_uses_normalized_toolkit_key() { + let (_tmp, client) = make_client(); + let pref = UserScopePref { + read: true, + write: false, + admin: true, + }; + + save(&client, " GMail ", pref).await.unwrap(); + + let loaded = load(&client, "gmail").await; + assert_eq!(loaded, pref); + + let raw = client + .kv_get(Some(KV_NAMESPACE), "gmail") + .await + .unwrap() + .expect("normalized toolkit key should be used"); + assert_eq!(raw.get("write").and_then(|v| v.as_bool()), Some(false)); + assert_eq!(raw.get("admin").and_then(|v| v.as_bool()), Some(true)); +} + +#[tokio::test] +async fn load_falls_back_to_default_when_stored_payload_is_invalid() { + let (_tmp, client) = make_client(); + client + .kv_set( + Some(KV_NAMESPACE), + "gmail", + &serde_json::json!("not-an-object"), + ) + .await + .unwrap(); + + let loaded = load(&client, "gmail").await; + assert_eq!(loaded, UserScopePref::default()); +} + +#[tokio::test] +async fn save_rejects_blank_toolkit() { + let (_tmp, client) = make_client(); + let err = save(&client, " ", UserScopePref::default()) + .await + .unwrap_err(); + assert!(err.contains("toolkit must not be empty")); +} diff --git a/core/src/sync/mcp/mod.rs b/core/src/sync/mcp/mod.rs new file mode 100644 index 0000000..d75e39b --- /dev/null +++ b/core/src/sync/mcp/mod.rs @@ -0,0 +1,18 @@ +//! Third-party MCP-server sync pipelines. +//! +//! Pipelines that pull from MCP (Model Context Protocol) servers the user +//! has connected. One pipeline per server. +//! +//! ## Layer rules +//! +//! - Transport (stdio / SSE / websocket) is owned by `mcp_clients/`; sync +//! here calls into that surface, never re-implements it. +//! - Data shapes are MCP-generic — the pipeline normalises into raw md +//! per record so the rest of memory_store doesn't have to know about +//! MCP at all. +//! +//! ## Status +//! +//! Scaffold only. The existing `mcp_clients/` module already knows how +//! to talk to a server; what's missing is the "drain new records since +//! last cursor and ingest" loop on top. diff --git a/core/src/sync/mod.rs b/core/src/sync/mod.rs new file mode 100644 index 0000000..33eb319 --- /dev/null +++ b/core/src/sync/mod.rs @@ -0,0 +1,32 @@ +//! Memory sync pipelines. +//! +//! One top-level module hosting every "pull data from upstream → land it +//! in memory_store" pipeline, organised by the kind of upstream it talks +//! to. Three kinds today: +//! +//! - [`composio`] — Composio managed connectors (Gmail, Slack, GitHub, +//! Notion, Linear, ClickUp, …). Pulls via the Composio Edge API. +//! - [`workspace`] — Local workspace connectors (filesystem vault sync, +//! local-only ingest, agent-experience capture from the harness). +//! - [`mcp`] — Third-party MCP servers. Pulls via the MCP protocol over +//! stdio/SSE. +//! +//! All three implement the `SyncPipeline` trait so the orchestrator +//! (`memory::jobs`) can drive them uniformly: `init` → `tick` → repeat. +//! +//! ## Layer rules +//! +//! - Sync writes into `memory_store` only — never directly into trees, +//! never directly into unified. The ingest pipeline in +//! `memory::ingest_pipeline` is the seam. +//! - One pipeline per upstream service. Composio's GitHub and MCP's +//! GitHub are distinct pipelines because they hit different surfaces +//! with different cadence and auth. +//! - Pipeline modules own their own types, their own state, and their +//! own retry/backoff policy. The trait gives the orchestrator a +//! single shape to call; everything else stays local. + +pub mod composio; +pub mod mcp; +pub mod sync_status; +pub mod workspace; diff --git a/core/src/sync/sync_status/mod.rs b/core/src/sync/sync_status/mod.rs new file mode 100644 index 0000000..019bbd6 --- /dev/null +++ b/core/src/sync/sync_status/mod.rs @@ -0,0 +1,16 @@ +//! Memory sync status surface (#1136 — simplified rewrite). +//! +//! The earlier push-based design (phase events from each provider's +//! sync loop, persisted KV store, subscriber that mirrored events +//! into storage) was replaced because it drifted from reality — +//! "downloading 0/0" was a common lie while the chunks table told +//! the truth. The pull-based replacement is one SQL query against +//! `mem_tree_chunks` GROUPED BY `source_kind` on each RPC call. +//! +//! Public surface: +//! +//! * [`MemorySyncStatus`] / [`FreshnessLabel`] — what the RPC returns +//! * `openhuman.memory_sync_status_list` — handler in `rpc` +//! * Controller registration via `schemas::all_registered_controllers` + +pub use tinycortex::memory::sync::{FreshnessLabel, MemorySyncStatus}; diff --git a/core/src/sync/workspace/mod.rs b/core/src/sync/workspace/mod.rs new file mode 100644 index 0000000..a21d477 --- /dev/null +++ b/core/src/sync/workspace/mod.rs @@ -0,0 +1,25 @@ +//! Workspace-scoped sync pipelines. +//! +//! Pipelines that pull from sources local to the user's workspace rather +//! than third-party services. Three flavors expected: +//! +//! | Submodule | Source | Notes | +//! | --- | --- | --- | +//! | `folder` | Files under a user-added folder memory source | Watch + diff | +//! | `harness` | Agent harness turns (TinyCortex archivist caller side) | Push-based | +//! | `dictation` | Local audio capture transcripts | Push-based | +//! +//! ## Status +//! +//! Mostly scaffold. Today folder ingestion lives in +//! `memory_sources/readers/folder.rs`, harness capture in +//! `agent_experience/`, and dictation in `dictation_hotkeys/`. Each will +//! land here as a `SyncPipeline` impl in a follow-up. +//! +//! [`periodic`] is live: the background cadence driver that keeps +//! workspace-kind memory sources (GitHub repos, folders, RSS, web pages) +//! syncing without manual "Sync now" clicks. + +pub mod periodic; + +pub use periodic::start_workspace_periodic_sync; diff --git a/core/src/sync/workspace/periodic.rs b/core/src/sync/workspace/periodic.rs new file mode 100644 index 0000000..a1d145e --- /dev/null +++ b/core/src/sync/workspace/periodic.rs @@ -0,0 +1,414 @@ +//! Periodic sync scheduler for workspace (non-Composio) memory sources. +//! +//! The Composio scheduler (`memory_sync::composio::periodic`) walks +//! Composio *connections* exclusively — GitHub repos, folders, RSS feeds +//! and web pages registered in `config.memory_sources` were only ever +//! synced when the user pressed "Sync now" (the `memory_sources.sync` +//! RPC). A GitHub source would sync once at setup and then silently go +//! stale forever. This loop closes that gap: it walks the registry on a +//! fixed tick and fires the existing [`sync_source`] dispatcher for every +//! enabled workspace-kind source whose cadence has elapsed. +//! +//! Cadence semantics mirror the Composio loop (#3302): +//! - `config.memory_sync_interval_secs() == Some(0)` → "Manual only", the +//! loop skips every source. +//! - `Some(n)` → sync every `max(n, 24h-default)` seconds. +//! - `None` → the 24h default. +//! +//! Due-check sources, in priority order: +//! 1. the in-memory fired-at map (most accurate within this process), and +//! 2. the persisted sync-audit log — keyed by `source_id` with the +//! source's own `source_kind` — so a configured cadence survives app +//! restarts instead of re-firing on every cold start. +//! +//! `sync_source` itself owns overlap protection (per-source `ACTIVE_SYNCS` +//! mutex), audit writes, and post-sync raw-coverage reconcile +//! (`check_and_rebuild_tree`), so this loop stays a thin cadence driver. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +use chrono::{DateTime, Utc}; +use tokio::time::interval; + +use crate::config_loader as config_rpc; +use crate::scheduler_gate::resume_notify; +use crate::sources::sync::sync_source; +use crate::sources::types::{MemorySourceEntry, SourceKind}; +use crate::sync::composio::periodic::{ + connection_is_due, effective_interval_secs, periodic_pause_reason, +}; +use crate::tinycortex::{try_read_audit_log, SyncAuditEntry}; +use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; + +/// How often the scheduler wakes up to look for due syncs. Matches the +/// Composio loop's cadence — per-source intervals (24h default) bound the +/// actual sync frequency; this only bounds how far past due we can drift. +const TICK_SECONDS: u64 = 1200; + +/// Process-wide guard: only the first call spawns the loop. +static SCHEDULER_STARTED: OnceLock<()> = OnceLock::new(); + +/// `source_id → last fired-at instant` for this process lifetime. Recorded +/// at *fire* time (the sync runs detached in `sync_source`'s spawned task), +/// so a failing source retries on the next due boundary, not every tick. +type FiredAtMap = Arc>>; + +static LAST_FIRED_AT: OnceLock = OnceLock::new(); + +fn fired_map() -> FiredAtMap { + LAST_FIRED_AT + .get_or_init(|| Arc::new(Mutex::new(HashMap::new()))) + .clone() +} + +/// Source kinds this loop schedules. Composio is owned by the Composio +/// scheduler; Conversation/Twitter have no periodic pull semantics today. +fn is_workspace_synced_kind(kind: &SourceKind) -> bool { + matches!( + kind, + SourceKind::GithubRepo | SourceKind::Folder | SourceKind::RssFeed | SourceKind::WebPage + ) +} + +/// Index `source_id → most recent successful sync timestamp` from the +/// persisted audit log, restricted to workspace source kinds. Failed runs +/// are skipped (matching the in-memory semantics — a failure retries at +/// the next tick after the cadence elapses). +fn index_last_success_by_source_id(entries: &[SyncAuditEntry]) -> HashMap> { + let mut idx: HashMap> = HashMap::new(); + for e in entries { + if !e.success { + continue; + } + let is_workspace_kind = matches!( + e.source_kind.as_str(), + "github_repo" | "folder" | "rss_feed" | "web_page" + ); + if !is_workspace_kind { + continue; + } + idx.entry(e.source_id.clone()) + .and_modify(|t| { + if e.timestamp > *t { + *t = e.timestamp; + } + }) + .or_insert(e.timestamp); + } + idx +} + +/// Wall-clock elapsed since the persisted last success, saturating at zero +/// for clock skew. `None` when the source has never successfully synced. +fn persisted_since_last_sync( + idx: &HashMap>, + source_id: &str, + now: DateTime, +) -> Option { + idx.get(source_id).map(|ts| { + let secs = (now - *ts).num_seconds().max(0) as u64; + Duration::from_secs(secs) + }) +} + +/// Spawn the workspace-source periodic sync task. Idempotent. +pub fn start_workspace_periodic_sync() { + if SCHEDULER_STARTED.set(()).is_err() { + tracing::debug!("[memory_sync:workspace:periodic] scheduler already running"); + return; + } + tokio::spawn(async move { + tracing::info!( + tick_seconds = TICK_SECONDS, + "[memory_sync:workspace:periodic] scheduler starting" + ); + run_loop().await; + tracing::error!("[memory_sync:workspace:periodic] scheduler loop exited"); + }); +} + +/// Tick loop: wakes on the steady cadence or a scheduler-gate resume +/// (Memory Tree toggled back on / sign-in), same shape as the Composio +/// loop — resume runs a tick immediately and re-bases the ticker. +async fn run_loop() { + let mut ticker = interval(Duration::from_secs(TICK_SECONDS)); + let resume = resume_notify(); + // Skip the immediate-fire tick so startup isn't slammed before sign-in. + ticker.tick().await; + + loop { + tokio::select! { + _ = ticker.tick() => {} + _ = resume.notified() => { + ticker.reset(); + } + } + if let Err(e) = run_one_tick().await { + tracing::warn!( + error = %e, + "[memory_sync:workspace:periodic] tick failed (continuing)" + ); + } + } +} + +/// Run a single scheduler tick. `pub(crate)` so tests can drive ticks +/// without the real interval. +pub(crate) async fn run_one_tick() -> Result<(), String> { + // Honour the same pause reasons as the Composio loop: user toggled + // Memory Tree off, or signed out. + if let Some(reason) = periodic_pause_reason() { + tracing::debug!( + reason = reason.as_str(), + "[memory_sync:workspace:periodic] scheduler-gate paused — skipping tick" + ); + return Ok(()); + } + + let config = config_rpc::load_config_with_timeout() + .await + .map_err(|e| format!("load_config: {e}"))?; + + let global_interval = config.memory_sync_interval_secs(); + let Some(interval_secs) = + effective_interval_secs(DEFAULT_MEMORY_SYNC_INTERVAL_SECS, global_interval) + else { + tracing::debug!( + "[memory_sync:workspace:periodic] manual-only mode — skipping all workspace sources" + ); + return Ok(()); + }; + + let (audit_index, audit_available) = workspace_audit_state(try_read_audit_log(&*config)); + if !audit_available { + tracing::warn!( + "[memory_sync:workspace:periodic] audit unavailable; sources without in-memory cadence will be skipped" + ); + } + let now = Utc::now(); + let map = fired_map(); + + let due_sources: Vec = crate::sources::decode_memory_sources(&*config) + .iter() + .filter(|s| s.enabled && is_workspace_synced_kind(&s.kind)) + .filter(|s| { + let in_memory_since = { + let guard = map.lock().unwrap_or_else(|e| e.into_inner()); + guard.get(&s.id).map(|when| when.elapsed()) + }; + let Some(since) = cadence_from_audit( + in_memory_since, + audit_available, + persisted_since_last_sync(&audit_index, &s.id, now), + ) else { + tracing::debug!( + source_kind = %s.kind.as_str(), + "[memory_sync:workspace:periodic] source has unknown cadence while audit is unavailable; skipping" + ); + return false; + }; + connection_is_due(interval_secs, since) + }) + .cloned() + .collect(); + + if due_sources.is_empty() { + tracing::debug!("[memory_sync:workspace:periodic] tick complete — nothing due"); + return Ok(()); + } + + let mut fired = 0usize; + for source in due_sources { + let source_id = source.id.clone(); + let kind = source.kind.as_str(); + tracing::info!( + source_id = %source_id, + kind = %kind, + interval_secs, + "[memory_sync:workspace:periodic] firing sync" + ); + // sync_source spawns the actual work and returns immediately; it + // rejects overlapping syncs of the same source internally. + match sync_source(source, config.to_arc()).await { + Ok(()) => { + if let Ok(mut guard) = map.lock() { + guard.insert(source_id, Instant::now()); + } + fired += 1; + } + Err(e) => { + tracing::warn!( + source_id = %source_id, + kind = %kind, + error = %e, + "[memory_sync:workspace:periodic] sync dispatch failed (will retry next tick)" + ); + } + } + } + + tracing::debug!(fired, "[memory_sync:workspace:periodic] tick complete"); + Ok(()) +} + +fn workspace_audit_state( + read: anyhow::Result>, +) -> (HashMap>, bool) { + match read { + Ok(entries) => (index_last_success_by_source_id(&entries), true), + Err(error) => { + tracing::warn!(%error, "[memory_sync:workspace:periodic] audit read failed"); + (HashMap::new(), false) + } + } +} + +fn cadence_from_audit( + in_memory_since: Option, + audit_available: bool, + persisted_since: Option, +) -> Option> { + match in_memory_since { + Some(since) => Some(Some(since)), + None if audit_available => Some(persisted_since), + None => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(source_id: &str, kind: &str, success: bool, ts: DateTime) -> SyncAuditEntry { + SyncAuditEntry { + timestamp: ts, + source_id: source_id.to_string(), + source_kind: kind.to_string(), + scope: format!("{kind}:{source_id}"), + items_fetched: 1, + batches: 0, + input_tokens: 0, + output_tokens: 0, + estimated_cost_usd: 0.0, + composio_actions_called: 0, + composio_cost_usd: 0.0, + actual_charged_usd: None, + duration_ms: 10, + success, + error: None, + } + } + + #[test] + fn workspace_kinds_are_scheduled_composio_is_not() { + assert!(is_workspace_synced_kind(&SourceKind::GithubRepo)); + assert!(is_workspace_synced_kind(&SourceKind::Folder)); + assert!(is_workspace_synced_kind(&SourceKind::RssFeed)); + assert!(is_workspace_synced_kind(&SourceKind::WebPage)); + assert!(!is_workspace_synced_kind(&SourceKind::Composio)); + assert!(!is_workspace_synced_kind(&SourceKind::Conversation)); + assert!(!is_workspace_synced_kind(&SourceKind::TwitterQuery)); + } + + #[test] + fn audit_index_keeps_latest_workspace_success_and_skips_others() { + let now = Utc::now(); + let older = now - chrono::Duration::hours(30); + let newer = now - chrono::Duration::hours(2); + let entries = vec![ + entry("src_gh", "github_repo", true, older), + entry("src_gh", "github_repo", true, newer), // newest success wins + entry("src_gh", "github_repo", false, now), // failure ignored + entry("conn_1", "composio", true, now), // composio kind ignored + ]; + let idx = index_last_success_by_source_id(&entries); + assert_eq!(idx.get("src_gh"), Some(&newer)); + assert!(!idx.contains_key("conn_1")); + } + + /// The headline regression: a GitHub source that synced once long ago + /// must read as DUE under the default 24h cadence — before this loop + /// existed, nothing ever consulted that staleness, so the source went + /// permanently dark after its first manual sync. + #[test] + fn stale_github_source_is_due_fresh_one_is_not() { + let now = Utc::now(); + let mut idx = HashMap::new(); + idx.insert("src_stale".to_string(), now - chrono::Duration::days(5)); + idx.insert("src_fresh".to_string(), now - chrono::Duration::hours(1)); + + let interval = + effective_interval_secs(DEFAULT_MEMORY_SYNC_INTERVAL_SECS, None).expect("interval"); + + let stale = persisted_since_last_sync(&idx, "src_stale", now); + assert!(connection_is_due(interval, stale), "5-day-old sync is due"); + + let fresh = persisted_since_last_sync(&idx, "src_fresh", now); + assert!( + !connection_is_due(interval, fresh), + "1h-old sync is not due" + ); + + // Never-synced source fires immediately. + let never = persisted_since_last_sync(&idx, "src_new", now); + assert!(connection_is_due(interval, never)); + } + + #[test] + fn manual_only_global_setting_disables_the_loop() { + assert_eq!( + effective_interval_secs(DEFAULT_MEMORY_SYNC_INTERVAL_SECS, Some(0)), + None + ); + } + + #[test] + fn persisted_since_last_sync_saturates_clock_skew() { + let now = Utc::now(); + let mut idx = HashMap::new(); + idx.insert("future".to_string(), now + chrono::Duration::hours(2)); + assert_eq!( + persisted_since_last_sync(&idx, "future", now), + Some(Duration::ZERO) + ); + assert_eq!(persisted_since_last_sync(&idx, "missing", now), None); + } + + #[test] + fn audit_failure_is_unavailable_and_unknown_cadence_is_excluded() { + let (index, available) = + workspace_audit_state(Err(anyhow::anyhow!("simulated audit I/O failure"))); + assert!(index.is_empty()); + assert!(!available); + assert_eq!(cadence_from_audit(None, available, None), None); + + let known = Duration::from_secs(60); + assert_eq!( + cadence_from_audit(Some(known), available, None), + Some(Some(known)) + ); + } + + #[test] + fn readable_empty_audit_keeps_never_synced_workspace_source_due() { + let (index, available) = workspace_audit_state(Ok(Vec::new())); + assert!(index.is_empty()); + assert!(available); + + let cadence = cadence_from_audit(None, available, None) + .expect("readable empty audit keeps the source eligible"); + assert!(connection_is_due( + DEFAULT_MEMORY_SYNC_INTERVAL_SECS, + cadence + )); + } + + #[tokio::test] + async fn start_workspace_periodic_sync_is_idempotent() { + start_workspace_periodic_sync(); + start_workspace_periodic_sync(); + assert!(SCHEDULER_STARTED.get().is_some()); + } +} diff --git a/core/src/sync/workspace/watcher.rs b/core/src/sync/workspace/watcher.rs new file mode 100644 index 0000000..27bdaf8 --- /dev/null +++ b/core/src/sync/workspace/watcher.rs @@ -0,0 +1,717 @@ +//! Vault file-system watcher. +//! +//! Watches a configurable directory (default: the Obsidian vault's +//! `wiki/notes/` folder) for `Create`, `Modify`, and `Remove` events +//! and ingests changes into the memory tree in near-real time. +//! +//! ## Design +//! +//! ```text +//! ┌─────────────────────────────────────────────────────────┐ +//! │ notify (OS-native watcher) │ +//! │ FSEvents / inotify / ReadDirectoryChanges │ +//! └────────────────────┬────────────────────────────────────┘ +//! │ raw events (debounced, 500 ms) +//! ▼ +//! ┌─────────────────────────────────────────────────────────┐ +//! │ run_loop() ← tokio task (singleton via OnceLock) │ +//! │ │ +//! │ ① scheduler-gate check (UserDisabled / SignedOut) │ +//! │ ② mtime guard (SQLite WatcherStateStore) │ +//! │ ③ path→source_id build (stable base + mtime suffix) │ +//! │ ④ ingest_document_with_scope() or mark_deleted() │ +//! └─────────────────────────────────────────────────────────┘ +//! ``` +//! +//! ## Dedup strategy (mtime-based source_id) +//! +//! `ingest_document_with_scope` deduplicates on `source_id`. A plain +//! `path`-based ID means edits are silently ignored (already-ingested +//! guard fires). We therefore build: +//! +//! ```text +//! source_id = "vault_watcher:@" +//! ``` +//! +//! Every modification creates a new `source_id`, bypassing the dedup +//! gate and letting the pipeline store a fresh version. The previous +//! version remains in the store but becomes unreachable via normal +//! queries (the tree rebuild naturally supersedes it). +//! +//! For `Remove` events we call `mark_document_deleted(source_id)` so +//! the entry is tombstoned rather than left as orphan data. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use notify::{ + event::{CreateKind, ModifyKind, RemoveKind}, + Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher, +}; +use notify_debouncer_mini::{new_debouncer, DebouncedEvent, Debouncer}; +use tokio::sync::mpsc; + +use crate::Config; +use crate::config_loader as config_rpc; +use crate::ingest_pipeline::ingest_document_with_scope; +use tinycortex::memory::ingest::canonicalize::document::DocumentInput; +use crate::sync::workspace::watcher::state::WatcherStateStore; +use crate::scheduler_gate::current_policy; +use crate::scheduler_gate::PauseReason; + +pub mod state; + +// ───────────────────────────────────────────────────────────────────────────── +// Constants +// ───────────────────────────────────────────────────────────────────────────── + +/// Debounce window: coalesce bursts of rapid saves into one event. +const DEBOUNCE_MS: u64 = 500; + +/// Only ingest files matching these extensions. +const WATCHED_EXTENSIONS: &[&str] = &["md", "txt"]; + +/// State DB filename inside the workspace directory. +const STATE_DB_FILENAME: &str = "vault_watcher_state.db"; + +// ───────────────────────────────────────────────────────────────────────────── +// Singleton guard — mirrors composio/periodic.rs pattern exactly +// ───────────────────────────────────────────────────────────────────────────── + +static WATCHER_STARTED: OnceLock<()> = OnceLock::new(); + +/// Spawn the vault watcher background task. Idempotent: only the first +/// call actually spawns; subsequent calls are cheap no-ops. +pub fn start_vault_watcher() { + if WATCHER_STARTED.get().is_some() { + tracing::debug!("[vault_watcher] already running, skipping start"); + return; + } + if WATCHER_STARTED.set(()).is_err() { + tracing::debug!("[vault_watcher] already running (race), skipping start"); + return; + } + + tokio::spawn(async move { + tracing::info!("[vault_watcher] starting"); + if let Err(e) = run_loop().await { + tracing::error!(error = %e, "[vault_watcher] loop exited with error"); + } + }); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Scheduler-gate check — same allow-list as composio/periodic.rs +// ───────────────────────────────────────────────────────────────────────────── + +fn watcher_pause_reason() -> Option { + let reason = current_policy().pause_reason()?; + matches!(reason, PauseReason::UserDisabled | PauseReason::SignedOut).then_some(reason) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Main loop +// ───────────────────────────────────────────────────────────────────────────── + +async fn run_loop() -> Result<(), String> { + let config = config_rpc::load_config_with_timeout() + .await + .map_err(|e| format!("[vault_watcher] load_config: {e}"))?; + + let watch_path = resolve_watch_path(&config)?; + + tracing::info!( + path = %watch_path.display(), + "[vault_watcher] watching vault directory" + ); + + // Open (or create) the SQLite state store. + let db_path = config + .workspace_dir() + .join(STATE_DB_FILENAME); + let state_store = Arc::new(Mutex::new( + WatcherStateStore::open(&db_path) + .map_err(|e| format!("[vault_watcher] state db open failed: {e}"))?, + )); + + // Seed in-memory mtime map from SQLite so a restart doesn't re-ingest + // everything from scratch. + let mtime_cache: Arc>> = { + let store = state_store.lock().unwrap_or_else(|e| e.into_inner()); + let all = store.load_all().map_err(|e| format!("[vault_watcher] load_all: {e}"))?; + let map: HashMap = all + .into_iter() + .filter(|s| !s.deleted) + .map(|s| (s.path, s.mtime_secs)) + .collect(); + Arc::new(Mutex::new(map)) + }; + + // Channel between the notify callback (sync) and our async handler. + let (tx, mut rx) = mpsc::unbounded_channel::(); + + // Build the debounced watcher. `new_debouncer` returns a + // `Debouncer` which we must keep alive. + let tx_clone = tx.clone(); + let mut debouncer: Debouncer = new_debouncer( + Duration::from_millis(DEBOUNCE_MS), + move |res: Result, _>| { + if let Ok(events) = res { + for ev in events { + let _ = tx_clone.send(ev); + } + } + }, + ) + .map_err(|e| format!("[vault_watcher] debouncer init: {e}"))?; + + debouncer + .watcher() + .watch(&watch_path, RecursiveMode::Recursive) + .map_err(|e| format!("[vault_watcher] watch failed: {e}"))?; + + tracing::info!("[vault_watcher] fs watch active, entering event loop"); + + while let Some(event) = rx.recv().await { + // ── scheduler-gate check ───────────────────────────────────────── + if let Some(reason) = watcher_pause_reason() { + tracing::debug!( + reason = reason.as_str(), + "[vault_watcher] paused — dropping event" + ); + continue; + } + + let path = event.path.clone(); + + // Only care about watched extensions. + if !is_watched_extension(&path) { + continue; + } + + match classify_event(&event) { + VaultEvent::CreateOrModify => { + handle_upsert( + &path, + &watch_path, + &config, + Arc::clone(&state_store), + Arc::clone(&mtime_cache), + ) + .await; + } + VaultEvent::Remove => { + handle_remove( + &path, + &watch_path, + &config, + Arc::clone(&state_store), + Arc::clone(&mtime_cache), + ) + .await; + } + VaultEvent::Ignore => {} + } + } + + // Channel closed — the debouncer was dropped (shouldn't happen in + // normal operation). + tracing::warn!("[vault_watcher] event channel closed, loop exiting"); + Ok(()) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Event classification +// ───────────────────────────────────────────────────────────────────────────── + +#[derive(Debug)] +enum VaultEvent { + CreateOrModify, + Remove, + Ignore, +} + +fn classify_event(ev: &DebouncedEvent) -> VaultEvent { + // notify-debouncer-mini exposes the underlying notify EventKind. + match &ev.kind { + EventKind::Create(_) | EventKind::Modify(ModifyKind::Data(_)) => { + VaultEvent::CreateOrModify + } + // ModifyKind::Name covers renames — treat the new path as a create. + EventKind::Modify(ModifyKind::Name(_)) => VaultEvent::CreateOrModify, + EventKind::Remove(_) => VaultEvent::Remove, + _ => VaultEvent::Ignore, + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Upsert handler (Create + Modify) +// ───────────────────────────────────────────────────────────────────────────── + +async fn handle_upsert( + path: &Path, + vault_root: &Path, + config: &Config, + state_store: Arc>, + mtime_cache: Arc>>, +) { + // ── mtime guard: skip if file unchanged since last ingest ──────────── + let mtime = match file_mtime(path) { + Some(m) => m, + None => { + tracing::debug!( + path = %path.display(), + "[vault_watcher] cannot read mtime, skipping" + ); + return; + } + }; + + { + let cache = mtime_cache.lock().unwrap_or_else(|e| e.into_inner()); + if cache.get(path) == Some(&mtime) { + tracing::debug!( + path = %path.display(), + "[vault_watcher] mtime unchanged, skipping" + ); + return; + } + } + + // ── read file content ──────────────────────────────────────────────── + let body = match tokio::fs::read_to_string(path).await { + Ok(b) => b, + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "[vault_watcher] read failed, skipping" + ); + return; + } + }; + + let rel = path + .strip_prefix(vault_root) + .unwrap_or(path) + .to_string_lossy() + .to_string(); + + // ── build mtime-scoped source_id ───────────────────────────────────── + // Format: "vault_watcher:@" + // Each edit produces a distinct ID, bypassing the dedup gate so the + // updated content actually reaches the pipeline. + let source_id = format!("vault_watcher:{rel}@{mtime}"); + + let doc = DocumentInput { + provider: "vault_watcher".to_string(), + title: rel.clone(), + body, + modified_at: chrono::Utc::now(), + source_ref: Some(format!("vault:{rel}")), + }; + + let tags = vec!["vault_watcher".to_string(), "obsidian".to_string()]; + + match ingest_document_with_scope(config, &source_id, "user", tags, doc, None).await { + Ok(result) => { + tracing::debug!( + path = %rel, + source_id = %source_id, + already_ingested = result.already_ingested, + "[vault_watcher] upsert ok" + ); + + // Update mtime cache + SQLite state. + { + let mut cache = mtime_cache.lock().unwrap_or_else(|e| e.into_inner()); + cache.insert(path.to_path_buf(), mtime); + } + if let Ok(mut store) = state_store.lock() { + if let Err(e) = store.record_seen(path, mtime) { + tracing::warn!(error = %e, "[vault_watcher] state db write failed"); + } + } + } + Err(e) => { + tracing::warn!( + path = %rel, + error = %e, + "[vault_watcher] ingest failed" + ); + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Remove handler +// ───────────────────────────────────────────────────────────────────────────── + +async fn handle_remove( + path: &Path, + vault_root: &Path, + config: &Config, + state_store: Arc>, + mtime_cache: Arc>>, +) { + let rel = path + .strip_prefix(vault_root) + .unwrap_or(path) + .to_string_lossy() + .to_string(); + + // We need the last-known mtime to reconstruct the source_id and + // tombstone the correct document. + let last_mtime = { + let cache = mtime_cache.lock().unwrap_or_else(|e| e.into_inner()); + cache.get(path).copied() + }; + + if let Some(mtime) = last_mtime { + let source_id = format!("vault_watcher:{rel}@{mtime}"); + if let Err(e) = + crate::ingest_pipeline::mark_document_deleted(config, &source_id) + .await + { + tracing::warn!( + path = %rel, + source_id = %source_id, + error = %e, + "[vault_watcher] mark_deleted failed" + ); + } else { + tracing::debug!( + path = %rel, + source_id = %source_id, + "[vault_watcher] marked deleted" + ); + } + } else { + tracing::debug!( + path = %rel, + "[vault_watcher] remove event but no prior ingest found, nothing to tombstone" + ); + } + + // Evict from cache + mark deleted in SQLite regardless. + { + let mut cache = mtime_cache.lock().unwrap_or_else(|e| e.into_inner()); + cache.remove(path); + } + if let Ok(mut store) = state_store.lock() { + if let Err(e) = store.record_deleted(path) { + tracing::warn!(error = %e, "[vault_watcher] state db delete failed"); + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────────────────────── + +/// Resolve the vault watch path from config, falling back to +/// `/obsidian_vault/wiki/notes/`. +fn resolve_watch_path(config: &Config) -> Result { + // Prefer an explicit setting; fall back to the conventional location. + let path = config + .vault_watch_path() + .unwrap_or_else(|| config.workspace_dir().join("obsidian_vault/wiki/notes")); + + if !path.exists() { + // Create the directory so the watcher can start; Obsidian will + // populate it when the vault is opened. + std::fs::create_dir_all(&path) + .map_err(|e| format!("[vault_watcher] cannot create watch dir: {e}"))?; + tracing::info!( + path = %path.display(), + "[vault_watcher] created watch directory (vault not yet populated)" + ); + } + + Ok(path) +} + +fn is_watched_extension(path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .map(|e| WATCHED_EXTENSIONS.contains(&e)) + .unwrap_or(false) +} + +fn file_mtime(path: &Path) -> Option { + std::fs::metadata(path) + .ok()? + .modified() + .ok()? + .duration_since(UNIX_EPOCH) + .ok() + .map(|d| d.as_secs()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + #[test] + fn is_watched_extension_md_and_txt() { + assert!(is_watched_extension(Path::new("note.md"))); + assert!(is_watched_extension(Path::new("note.txt"))); + assert!(!is_watched_extension(Path::new("image.png"))); + assert!(!is_watched_extension(Path::new("data.json"))); + } + + #[test] + fn file_mtime_returns_some_for_existing_file() { + let tmp = TempDir::new().unwrap(); + let p = tmp.path().join("test.md"); + fs::write(&p, "hello").unwrap(); + assert!(file_mtime(&p).is_some()); + } + + #[test] + fn file_mtime_returns_none_for_missing_file() { + assert!(file_mtime(Path::new("/nonexistent/file.md")).is_none()); + } + + #[test] + fn source_id_format_includes_mtime() { + let rel = "journal/2024-01-01.md"; + let mtime: u64 = 1_700_000_000; + let id = format!("vault_watcher:{rel}@{mtime}"); + assert_eq!(id, "vault_watcher:journal/2024-01-01.md@1700000000"); + } + + #[test] + fn start_vault_watcher_is_idempotent() { + // Two calls must not panic; the OnceLock ensures only one spawns. + // We can't assert much more without a live tokio runtime here, but + // this pins the guard logic doesn't regress. + // + // NOTE: deliberately does NOT use #[tokio::test] — calling + // start_vault_watcher() outside an async context exercises the + // OnceLock-already-set branch, which is the important regression + // target. The actual `tokio::spawn` inside will no-op gracefully. + // WATCHER_STARTED may already be set by a prior test in this + // process; that's fine — the second-call path is what we're testing. + start_vault_watcher(); + start_vault_watcher(); + assert!(WATCHER_STARTED.get().is_some()); + } +} + +//! Integration tests for the vault watcher. +//! +//! These tests exercise the watcher end-to-end against a real temp +//! directory and a real SQLite state store, without starting the +//! background tokio task (which needs a live config + ingest pipeline). +//! +//! What is tested here: +//! - WatcherStateStore round-trips +//! - mtime-guard logic (skip unchanged file, process changed file) +//! - source_id format expected by the ingest pipeline +//! - Extension filter +//! +//! The actual `ingest_document_with_scope` call is covered by the +//! ingest pipeline's own test suite; we don't re-test it here. + +#[cfg(test)] +mod vault_watcher_integration { + use std::fs; + use std::path::Path; + use std::time::Duration; + use tempfile::TempDir; + + use crate::sync::workspace::watcher::state::WatcherStateStore; + + // ── helpers ─────────────────────────────────────────────────────────── + + fn mtime_secs(path: &Path) -> u64 { + std::fs::metadata(path) + .unwrap() + .modified() + .unwrap() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + } + + // ── state store ─────────────────────────────────────────────────────── + + #[test] + fn state_store_persists_across_reopen() { + let tmp = TempDir::new().unwrap(); + let db = tmp.path().join("state.db"); + let note = Path::new("/vault/note.md"); + + { + let mut store = WatcherStateStore::open(&db).unwrap(); + store.record_seen(note, 1_700_000_000).unwrap(); + } + // Re-open simulates a process restart. + { + let store = WatcherStateStore::open(&db).unwrap(); + assert_eq!(store.last_mtime(note).unwrap(), Some(1_700_000_000)); + } + } + + #[test] + fn state_store_deleted_survives_reopen() { + let tmp = TempDir::new().unwrap(); + let db = tmp.path().join("state.db"); + let note = Path::new("/vault/deleted.md"); + + { + let mut store = WatcherStateStore::open(&db).unwrap(); + store.record_seen(note, 1_000).unwrap(); + store.record_deleted(note).unwrap(); + } + { + let store = WatcherStateStore::open(&db).unwrap(); + // `last_mtime` returns None for deleted entries. + assert_eq!(store.last_mtime(note).unwrap(), None); + // But the row is still there (deleted=1). + let rows = store.load_all().unwrap(); + assert!(rows.iter().any(|r| r.path == note && r.deleted)); + } + } + + // ── mtime-guard: skip unchanged file ───────────────────────────────── + + #[test] + fn mtime_guard_skips_file_with_same_mtime() { + let tmp = TempDir::new().unwrap(); + let db = tmp.path().join("state.db"); + let note = tmp.path().join("note.md"); + fs::write(¬e, "initial").unwrap(); + + let mtime = mtime_secs(¬e); + + let mut store = WatcherStateStore::open(&db).unwrap(); + store.record_seen(¬e, mtime).unwrap(); + + // Simulate the in-memory cache check: stored mtime == current mtime + // → the watcher should skip this file. + let cached = store.last_mtime(¬e).unwrap(); + assert_eq!( + cached, + Some(mtime), + "cache should report the file as already seen at this mtime" + ); + } + + // ── mtime-guard: process changed file ──────────────────────────────── + + #[test] + fn mtime_guard_processes_file_with_new_mtime() { + let tmp = TempDir::new().unwrap(); + let db = tmp.path().join("state.db"); + let note = tmp.path().join("note.md"); + + fs::write(¬e, "version 1").unwrap(); + let mtime_v1 = mtime_secs(¬e); + + let mut store = WatcherStateStore::open(&db).unwrap(); + store.record_seen(¬e, mtime_v1).unwrap(); + + // Simulate a file edit: write new content and sleep 1s to bump mtime. + // On most filesystems 1-second granularity is the minimum resolution. + std::thread::sleep(Duration::from_secs(1)); + fs::write(¬e, "version 2").unwrap(); + let mtime_v2 = mtime_secs(¬e); + + assert_ne!( + mtime_v1, mtime_v2, + "mtime must advance when file is modified" + ); + + // The cache holds mtime_v1; current file has mtime_v2 → watcher proceeds. + let cached = store.last_mtime(¬e).unwrap().unwrap(); + assert!( + mtime_v2 > cached, + "new mtime should be greater than cached mtime" + ); + } + + // ── source_id format ───────────────────────────────────────────────── + + #[test] + fn source_id_is_stable_and_version_scoped() { + let rel = "journal/2024-01-01.md"; + let mtime: u64 = 1_700_000_000; + + let id_v1 = format!("vault_watcher:{rel}@{mtime}"); + let id_v2 = format!("vault_watcher:{rel}@{}", mtime + 1); + + // Same path, different mtime → different source_id → bypasses dedup. + assert_ne!(id_v1, id_v2); + + // Stable format — the ingest pipeline stores this as the document key. + assert_eq!( + id_v1, + "vault_watcher:journal/2024-01-01.md@1700000000" + ); + } + + // ── extension filter ───────────────────────────────────────────────── + + #[test] + fn extension_filter_accepts_md_and_txt_only() { + let accepted = ["note.md", "draft.txt"]; + let rejected = ["image.png", "data.json", "script.js", "Makefile"]; + + let is_watched = |name: &str| { + std::path::Path::new(name) + .extension() + .and_then(|e| e.to_str()) + .map(|e| ["md", "txt"].contains(&e)) + .unwrap_or(false) + }; + + for name in accepted { + assert!(is_watched(name), "{name} should be watched"); + } + for name in rejected { + assert!(!is_watched(name), "{name} should not be watched"); + } + } + + // ── multiple files: only changed one gets ingested ─────────────────── + + #[test] + fn only_changed_file_gets_new_source_id() { + let tmp = TempDir::new().unwrap(); + let db = tmp.path().join("state.db"); + + let file_a = tmp.path().join("a.md"); + let file_b = tmp.path().join("b.md"); + fs::write(&file_a, "content a").unwrap(); + fs::write(&file_b, "content b").unwrap(); + + let mtime_a = mtime_secs(&file_a); + let mtime_b = mtime_secs(&file_b); + + let mut store = WatcherStateStore::open(&db).unwrap(); + store.record_seen(&file_a, mtime_a).unwrap(); + store.record_seen(&file_b, mtime_b).unwrap(); + + // Modify only file_b. + std::thread::sleep(Duration::from_secs(1)); + fs::write(&file_b, "content b v2").unwrap(); + let new_mtime_b = mtime_secs(&file_b); + + // file_a: cached == current → skip. + let cached_a = store.last_mtime(&file_a).unwrap().unwrap(); + assert_eq!(cached_a, mtime_a, "file_a should still be cached at v1"); + + // file_b: cached < current → process. + let cached_b = store.last_mtime(&file_b).unwrap().unwrap(); + assert!( + new_mtime_b > cached_b, + "file_b new mtime should exceed cached mtime" + ); + } +} diff --git a/core/src/sync_events.rs b/core/src/sync_events.rs new file mode 100644 index 0000000..36e96db --- /dev/null +++ b/core/src/sync_events.rs @@ -0,0 +1,117 @@ +//! High-level memory sync orchestration. +//! +//! This module owns the user-facing "sync my memory" workflow: +//! +//! 1. accept a manual or scheduled sync request +//! 2. emit coarse lifecycle events for UI visibility +//! 3. dispatch into [`crate::sync`] backends +//! 4. rely on `memory_store` + `memory_queue` + `memory_tree` backends to +//! persist, enqueue, ingest, and seal the resulting data +//! +//! The low-level provider implementations live in `memory_sync/*`; this module +//! is the orchestration seam the `memory` domain presents to RPC/tools/UI. + +use serde::{Deserialize, Serialize}; + +/// Why a sync run was requested. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MemorySyncTrigger { + Manual, + Cron, +} + +impl MemorySyncTrigger { + pub fn as_str(self) -> &'static str { + match self { + Self::Manual => "manual", + Self::Cron => "cron", + } + } +} + +/// Coarse orchestration stages surfaced to the frontend. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MemorySyncStage { + Requested, + Fetching, + Stored, + Queued, + Ingesting, + Completed, + Failed, +} + +impl MemorySyncStage { + pub fn as_str(self) -> &'static str { + match self { + Self::Requested => "requested", + Self::Fetching => "fetching", + Self::Stored => "stored", + Self::Queued => "queued", + Self::Ingesting => "ingesting", + Self::Completed => "completed", + Self::Failed => "failed", + } + } +} + +/// Publish a coarse sync lifecycle event for UI subscribers. +/// +/// `source_id` is the originating `MemorySourceEntry.id` when this event +/// can be attributed to a specific memory-source row. Pass `None` for +/// non-memory-source sync paths (channel-provider syncs, etc.) to avoid +/// corrupting the per-row indicator on the frontend. +pub fn emit_sync_stage( + trigger: MemorySyncTrigger, + stage: MemorySyncStage, + provider: Option<&str>, + connection_id: Option<&str>, + detail: Option, + source_id: Option<&str>, +) { + log::debug!( + "[memory-sync] emit stage={} trigger={} provider={:?} connection_id={:?} source_id={:?}", + stage.as_str(), + trigger.as_str(), + provider, + connection_id, + source_id + ); + crate::events::publish(crate::events::MemoryEvent::SyncStageChanged { + trigger: trigger.as_str().to_string(), + stage: stage.as_str().to_string(), + provider: provider.map(str::to_string), + connection_id: connection_id.map(str::to_string), + detail, + source_id: source_id.map(str::to_string), + }); +} + +/// Extract the originating memory-source id from a composite `source_id` of +/// the form `"mem_src::"` used by the reader-based ingest +/// path (folder, RSS, web-page sources). +/// +/// The encoding is: `mem_src:` prefix, followed by the memory-source id (a +/// short alphanumeric slug, no colons), then `:`, then the item id (which +/// may contain colons, e.g. RSS GUIDs that are URLs like +/// `https://example.com/feed/1`). +/// +/// Because the **source_id** is always the first colon-delimited segment after +/// `"mem_src:"`, we find the **first** colon — not the last — to extract it. +/// +/// Returns `None` when the source_id is not in this format (e.g. channel- +/// provider syncs such as `"slack:workspace-1"`). +pub fn extract_mem_src_id(composite_source_id: &str) -> Option<&str> { + let rest = composite_source_id.strip_prefix("mem_src:")?; + // format: mem_src:: + // source_id is a plain slug (no colons). item_id follows after the first colon. + let colon_pos = rest.find(':')?; + let source_id = &rest[..colon_pos]; + // Ensure there's something after the colon (item_id is non-empty). + if colon_pos + 1 >= rest.len() { + return None; + } + Some(source_id) +} diff --git a/core/src/test_env_lock.rs b/core/src/test_env_lock.rs new file mode 100644 index 0000000..61c83c5 --- /dev/null +++ b/core/src/test_env_lock.rs @@ -0,0 +1,15 @@ +//! Serialises tests that mutate process environment. +//! +//! The host has a lock of the same name (`config::TEST_ENV_LOCK`) over the same +//! variables. They are deliberately *different* locks: each crate's tests link +//! into their own binary and therefore their own process, so a shared lock +//! would buy nothing and would mean this crate owning a mutex for the host's +//! benefit. Same reasoning as +//! [`crate::embedding_host::embedding_test_guard`]. + +use std::sync::Mutex; + +/// Held for the duration of any test that sets or clears an env var the config +/// loader reads. Poison is deliberately ignored — a panicking test must not +/// cascade into every later one. +pub static TEST_ENV_LOCK: Mutex<()> = Mutex::new(()); diff --git a/core/src/test_seams.rs b/core/src/test_seams.rs new file mode 100644 index 0000000..17ca16b --- /dev/null +++ b/core/src/test_seams.rs @@ -0,0 +1,133 @@ +//! One-shot installation of stub host seams for this crate's own tests. +//! +//! The seams fail loudly when unwired — see [`crate::embedding_host`] for why +//! that is deliberate — so a test that reaches any of them needs a host +//! installed. These stubs are the smallest thing that makes the *core's* +//! behaviour observable: a noop embedder, known cloud defaults. +//! +//! # The chat stub answers availability, not routing +//! +//! [`TestChatHost`] reports that a summariser *is* available, so the doctor's +//! aggregation — which is core logic — is reachable. It refuses to build a +//! model. Which provider answers a role and what model id that resolves to is +//! host routing policy that a stub could only assert against itself; those +//! tests moved to the host, where the real implementation is. + +use std::sync::{Arc, Once}; + +use async_trait::async_trait; + +use crate::config_loader::ConfigLoader; +use crate::Config; + +/// A [`ConfigLoader`] that hands back a default test config. +/// +/// The background loops reload config on every tick by design; without a loader +/// they fail with the unwired-seam error before reaching the behaviour under +/// test. +#[derive(Debug)] +struct TestConfigLoader; + +#[async_trait] +impl ConfigLoader for TestConfigLoader { + async fn load(&self) -> Result, String> { + Ok(Box::new( + tinymemory_api::host::test_support::TestHostConfig::default(), + )) + } + + async fn reload_snapshot(&self, _snapshot: &Config) -> Result, String> { + Ok(Arc::new( + tinymemory_api::host::test_support::TestHostConfig::default(), + )) + } +} + +static INIT: Once = Once::new(); + +/// Install the stub seams. Idempotent; safe to call from every test. +pub(crate) fn init() { + INIT.call_once(|| { + crate::embedding_host::TestEmbeddingHost::install(); + crate::config_loader::set_config_loader(Arc::new(TestConfigLoader)); + crate::chat_host::set_chat_host(Arc::new(TestChatHost)); + crate::composio_host::set_composio_host(Arc::new(TestComposioHost)); + }); +} + +/// A [`ChatHost`] that reports an available summariser and nothing else. +/// +/// The doctor aggregates summariser availability into its report, and that +/// aggregation is core behaviour worth testing. Actually *building* a model is +/// host routing, so this refuses — no test in this crate should need one. +#[derive(Debug)] +struct TestChatHost; + +impl crate::chat_host::ChatHost for TestChatHost { + fn provider_for_role(&self, _role: &str, _config: &Config) -> String { + "test".to_string() + } + + fn create_chat_model_with_model_id( + &self, + _role: &str, + _config: &Config, + _temperature: f64, + ) -> Result<(Arc>, String), String> { + Err("TestChatHost does not build models — model routing is host behaviour".to_string()) + } + + fn usage_from_response( + &self, + _response: &tinyagents::harness::model::ModelResponse, + ) -> Option { + None + } + + fn summarizer_available(&self, _config: &Config) -> (bool, &'static str) { + (true, "test chat host reports a summariser") + } +} + +/// A [`ComposioHost`] that behaves like a signed-out user. +/// +/// Every method reports the no-backend-session state, which is the branch the +/// core's own tests exercise; a stub that succeeded would need to fake Composio +/// itself. +#[derive(Debug)] +struct TestComposioHost; + +#[async_trait] +impl crate::composio_host::ComposioHost for TestComposioHost { + async fn list_connections( + &self, + _config: &Config, + ) -> Result, String> { + Err(NO_SESSION.to_string()) + } + + async fn execute( + &self, + _config: &Config, + _tool: &str, + _arguments: Option, + _entity_id: &str, + _connection_id: Option<&str>, + ) -> Result { + Err(NO_SESSION.to_string()) + } + + fn api_key(&self, _config: &Config) -> Option { + None + } + + fn is_available(&self, _config: &Config) -> bool { + false + } +} + +/// The message [`TestComposioHost`] reports. Matches the shape the real backend +/// client produces when no session token is stored, which is what the tests +/// assert on. +const NO_SESSION: &str = "composio backend mode unavailable: no backend session token. \ + Sign in first (auth_store_session)."; diff --git a/core/src/thread_context.rs b/core/src/thread_context.rs new file mode 100644 index 0000000..1e41363 --- /dev/null +++ b/core/src/thread_context.rs @@ -0,0 +1,120 @@ +//! Ambient `thread_id` propagation across an agent turn. +//! +//! The web channel keys runtime sessions by `(client_id, thread_id)` and the +//! backend's `/openai/v1/chat/completions` endpoint accepts an optional +//! `thread_id` field so it can group inference logs and align KV-cache keys +//! with the same logical chat the user sees on screen. +//! +//! Threading the identifier through every layer (`Agent` → tool loop → +//! sub-agent runner → `Provider` impl) would touch dozens of call sites +//! and tests. Instead, the channel sets a [`tokio::task_local`] before +//! invoking the agent loop, and the OpenAI-compatible provider reads it +//! when serializing the request body. Other call paths see `None` and +//! omit the field — backward-compatible with backends that don't accept +//! it. +//! +//! ```ignore +//! use crate::thread_context::{with_thread_id, current_thread_id}; +//! +//! with_thread_id("abc123", async { +//! // any provider.chat() call inside this future sees thread_id=Some("abc123") +//! assert_eq!(current_thread_id().as_deref(), Some("abc123")); +//! }).await; +//! ``` + +use std::future::Future; + +tokio::task_local! { + static THREAD_ID: Option; +} + +/// Run `fut` with the given `thread_id` available to any descendant task +/// that calls [`current_thread_id`]. Empty / whitespace-only ids are +/// normalized to `None` so callers can pass through user input without +/// guarding for it. +pub async fn with_thread_id(thread_id: impl Into, fut: F) -> T +where + F: Future, +{ + let id = thread_id.into(); + let trimmed = id.trim(); + let value = if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + }; + log::debug!( + "[thread-context] entering scope thread_id={}", + value.as_deref().unwrap_or("") + ); + THREAD_ID.scope(value, fut).await +} + +/// Return the ambient `thread_id` set by an enclosing [`with_thread_id`] +/// scope, or `None` when called outside one (tests, CLI, sub-systems +/// that don't participate in chat sessions). +pub fn current_thread_id() -> Option { + THREAD_ID + .try_with(|v| v.clone()) + .ok() + .flatten() + .filter(|s| !s.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn scope_sets_and_clears_thread_id() { + assert!(current_thread_id().is_none(), "baseline outside scope"); + with_thread_id("thread-123", async { + assert_eq!(current_thread_id().as_deref(), Some("thread-123")); + }) + .await; + assert!( + current_thread_id().is_none(), + "thread_id must not leak past scope" + ); + } + + #[tokio::test] + async fn empty_or_whitespace_id_normalizes_to_none() { + with_thread_id(" ", async { + assert!(current_thread_id().is_none()); + }) + .await; + with_thread_id("", async { + assert!(current_thread_id().is_none()); + }) + .await; + } + + #[tokio::test] + async fn nested_scope_overrides_outer() { + with_thread_id("outer", async { + assert_eq!(current_thread_id().as_deref(), Some("outer")); + with_thread_id("inner", async { + assert_eq!(current_thread_id().as_deref(), Some("inner")); + }) + .await; + assert_eq!(current_thread_id().as_deref(), Some("outer")); + }) + .await; + } + + #[tokio::test] + async fn spawned_task_inherits_via_explicit_propagation() { + // tokio::task_local does not propagate across spawn by default. + // Document the expected pattern: capture before spawning. + with_thread_id("propagated", async { + let captured = current_thread_id(); + let handle = tokio::spawn(async move { + with_thread_id(captured.unwrap_or_default(), async { current_thread_id() }).await + }); + let observed = handle.await.unwrap(); + assert_eq!(observed.as_deref(), Some("propagated")); + }) + .await; + } +} diff --git a/core/src/tinycortex/chat.rs b/core/src/tinycortex/chat.rs new file mode 100644 index 0000000..f31e0c9 --- /dev/null +++ b/core/src/tinycortex/chat.rs @@ -0,0 +1,114 @@ +//! LLM chat seam — bridge OpenHuman's memory chat runtime onto the crate's +//! [`ChatProvider`] (W1). +//! +//! TinyCortex extracts entities/topics and summarises via an injected +//! `ChatProvider` (it never makes a network call). OpenHuman already owns a +//! memory LLM surface — `memory::chat::{ChatProvider, ChatPrompt}` with +//! `build_chat_provider(&Config)` routing through `openhuman::inference` +//! (provider selection, credit metering, usage accounting). This adapter wraps +//! that host provider and re-exposes it as the crate's `ChatProvider`, so the +//! engine's LLM entity extractor / summariser drive OpenHuman inference without +//! duplicating any routing. +//! +//! The two contracts are near-identical (`name` + async `chat_for_json`). The +//! only conversion is the prompt: the host `ChatPrompt.temperature` is `f64`, +//! the crate's is `f32`; every other field maps 1:1. + +use std::sync::Arc; + +use async_trait::async_trait; +use tinycortex::memory::score::extract::{ + ChatPrompt as CortexChatPrompt, ChatProvider as CortexChatProvider, +}; + +use crate::chat::{ + build_chat_provider as build_host_chat_provider, ChatPrompt as HostChatPrompt, + ChatProvider as HostChatProvider, +}; +use crate::Config; + +/// Wraps an OpenHuman [`HostChatProvider`] as the crate's [`CortexChatProvider`]. +pub struct SeamChatProvider { + inner: Arc, +} + +impl SeamChatProvider { + /// Build the adapter over a host chat provider (already routed through + /// `openhuman::inference`). + pub fn new(inner: Arc) -> Self { + tracing::debug!( + provider = inner.name(), + "[memory] constructing tinycortex chat seam over memory::chat::ChatProvider" + ); + Self { inner } + } +} + +#[async_trait] +impl CortexChatProvider for SeamChatProvider { + fn name(&self) -> &str { + self.inner.name() + } + + async fn chat_for_json(&self, prompt: &CortexChatPrompt) -> anyhow::Result { + let host = HostChatPrompt { + system: prompt.system.clone(), + user: prompt.user.clone(), + // Crate temperature is f32; host takes f64. + temperature: f64::from(prompt.temperature), + kind: prompt.kind, + max_tokens: prompt.max_tokens, + }; + self.inner.chat_for_json(&host).await + } +} + +/// Build a crate [`CortexChatProvider`] from the host [`Config`], routed through +/// `openhuman::inference` — the entry point the seam's LLM entity extractor and +/// summariser construct from. +pub fn build_chat_provider(config: &Config) -> anyhow::Result> { + let host = build_host_chat_provider(config)?; + Ok(Arc::new(SeamChatProvider::new(host))) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Echoes the fields it received back as a JSON body so the test can assert + /// the crate->host prompt conversion (incl. the f32->f64 temperature). + struct EchoHostProvider; + + #[async_trait] + impl HostChatProvider for EchoHostProvider { + fn name(&self) -> &str { + "echo" + } + async fn chat_for_json(&self, prompt: &HostChatPrompt) -> anyhow::Result { + Ok(format!( + "system={};user={};temp={};kind={};max={:?}", + prompt.system, prompt.user, prompt.temperature, prompt.kind, prompt.max_tokens + )) + } + } + + #[tokio::test] + async fn converts_prompt_and_delegates_to_host_provider() { + let seam = SeamChatProvider::new(Arc::new(EchoHostProvider)); + assert_eq!(CortexChatProvider::name(&seam), "echo"); + + let prompt = CortexChatPrompt { + system: "sys".to_string(), + user: "usr".to_string(), + temperature: 0.5, + kind: "extract", + max_tokens: Some(64), + }; + let out = seam.chat_for_json(&prompt).await.unwrap(); + // Every field maps 1:1; temperature widens f32 0.5 -> f64 0.5. + assert_eq!( + out, + "system=sys;user=usr;temp=0.5;kind=extract;max=Some(64)" + ); + } +} diff --git a/core/src/tinycortex/config.rs b/core/src/tinycortex/config.rs new file mode 100644 index 0000000..d06c0b4 --- /dev/null +++ b/core/src/tinycortex/config.rs @@ -0,0 +1,112 @@ +//! `Config` → [`tinycortex::memory::MemoryConfig`] mapping (W1). +//! +//! The crate's [`MemoryConfig`] is the single input every engine primitive +//! takes (`workspace`, embedding dims/model/strict, tree budgets, retrieval +//! weight profile, sync budget). This adapter derives it from OpenHuman's +//! host [`Config`] plus the resolved memory workspace root, so the rest of the +//! seam constructs engine calls from real product configuration. +//! +//! Field provenance: +//! - `workspace` ← the memory workspace root (same root `MemoryClient` opens). +//! - `embedding.dim` ← `config.memory().embedding_dimensions`. +//! - `embedding.model` ← `config.memory().embedding_model`. +//! - `embedding.strict` ← `config.memory_tree().embedding_strict` (when false the +//! engine tolerates an inert embedder and falls back to scope+recency rerank). +//! - `tree` / `retrieval` / `sync_budget` ← crate defaults, which already match +//! the host engine's constants (`INPUT_TOKEN_BUDGET = 50_000`, +//! `OUTPUT_TOKEN_BUDGET = 5_000`, `SUMMARY_FANOUT = 10`, +//! `DEFAULT_FLUSH_AGE_SECS = 604_800`). The `tree_policy.rs` flavour overlays +//! and per-source `WeightProfile` selection are layered on at call sites in +//! later workstreams; this base mapping is the W1 foundation. + +use std::path::PathBuf; + +use tinycortex::memory::config::EmbeddingConfig; +use tinycortex::memory::MemoryConfig; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::Config; + +/// Build a [`MemoryConfig`] from the host [`Config`] and the resolved memory +/// workspace root. +/// +/// `workspace` is the directory under which the engine stores `chunks.db`, the +/// content vault, and the tree DBs — it must be the same root the host +/// `MemoryClient` opens so an existing user workspace is read in place (parity +/// is gated by the W3 golden-workspace harness). +pub fn memory_config_from(config: &Config, workspace: PathBuf) -> MemoryConfig { + let mut mc = MemoryConfig::new(workspace); + mc.content_root = Some(config.memory_tree_content_root()); + mc.embedding = EmbeddingConfig { + dim: config.memory().embedding_dimensions, + model: config.memory().embedding_model.clone(), + strict: config.memory_tree().embedding_strict, + }; + mc +} + +/// Build a [`MemoryConfig`] rooted at the host's own `workspace_dir`. +/// +/// This is the shape ~15 `memory/**` adapter modules each used to re-declare as +/// a private `fn engine_config` / `fn memory_config` / `fn config`; they were +/// byte-identical, so they now all call this. Use [`memory_config_from`] +/// directly only when the workspace root is *not* `config.workspace_dir()` (the +/// sync/rebuild paths that address an alternate root). +pub fn engine_config(config: &Config) -> MemoryConfig { + memory_config_from(config, config.workspace_dir().clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_workspace_and_embedding_from_host_config() { + let mut config = TestHostConfig::default(); + config.memory.embedding_dimensions = 1024; + config.memory.embedding_model = "embedding-v1".to_string(); + config.memory_tree.embedding_strict = true; + + let workspace = PathBuf::from("/tmp/openhuman/ws"); + let mc = memory_config_from(&config, workspace.clone()); + + assert_eq!(mc.workspace, workspace); + assert_eq!(mc.embedding.dim, 1024); + assert_eq!(mc.embedding.model, "embedding-v1"); + assert!(mc.embedding.strict); + } + + #[test] + fn tree_defaults_match_engine_constants() { + // The base mapping leaves tree budgets at the crate defaults, which are + // the host engine's own constants — asserted here so a crate-side change + // to those defaults surfaces as a failing parity test rather than a + // silent behaviour drift. + let mc = memory_config_from(&TestHostConfig::default(), PathBuf::from("/tmp/ws")); + assert_eq!(mc.tree.input_token_budget, 50_000); + assert_eq!(mc.tree.output_token_budget, 5_000); + assert_eq!(mc.tree.summary_fanout, 10); + assert_eq!(mc.tree.flush_age_secs, 604_800); + } + + #[test] + fn engine_config_roots_at_host_workspace_dir() { + // Pins the wrapper's only behavioural claim: identical to + // `memory_config_from(config, config.workspace_dir().clone())`. + let mut config = TestHostConfig::default(); + config.memory.embedding_dimensions = 768; + config.memory_tree.embedding_strict = true; + + let via_wrapper = engine_config(&config); + let via_explicit = memory_config_from(&config, config.workspace_dir.clone()); + + assert_eq!(via_wrapper.workspace, config.workspace_dir); + assert_eq!(via_wrapper.workspace, via_explicit.workspace); + assert_eq!(via_wrapper.content_root, via_explicit.content_root); + assert_eq!(via_wrapper.embedding.dim, via_explicit.embedding.dim); + assert_eq!(via_wrapper.embedding.model, via_explicit.embedding.model); + assert_eq!(via_wrapper.embedding.strict, via_explicit.embedding.strict); + } +} diff --git a/core/src/tinycortex/embeddings.rs b/core/src/tinycortex/embeddings.rs new file mode 100644 index 0000000..15c5adb --- /dev/null +++ b/core/src/tinycortex/embeddings.rs @@ -0,0 +1,140 @@ +//! Embedding seam — bridge OpenHuman's [`EmbeddingProvider`] onto the crate's +//! two embedding traits (W1). +//! +//! OpenHuman owns the concrete providers (voyage / openai / cohere / ollama / +//! cloud / noop) and the `embeddings/factory.rs` construction policy (rate-limit +//! + retry). TinyCortex "never makes a network call" — it takes compute through +//! +//! [`EmbeddingBackend`] (the vector store) and [`Embedder`] (retrieval / seal +//! scoring). This adapter wraps one `Arc` and re-exposes +//! it as both, so the engine drives OpenHuman embeddings without cloning any +//! provider logic. +//! +//! The two host and crate contracts are shape-identical (`name` / `model_id` / +//! `dimensions` / `signature` / async `embed`), and both use `anyhow::Result`, +//! so this is a near-pure pass-through. Critically, `signature()` delegates to +//! the provider so the persisted embedding-space signature +//! (`provider=…;model=…;dims=…`) stays byte-identical whether the store keys off +//! the crate backend or the raw provider (#1574 fidelity). + +use std::sync::Arc; + +use async_trait::async_trait; +use tinycortex::memory::score::embed::Embedder; +use tinycortex::memory::store::vectors::EmbeddingBackend; + +use tinymemory_api::host::EmbeddingProvider; + +/// Wraps an OpenHuman [`EmbeddingProvider`] as the crate's [`EmbeddingBackend`] +/// (vector store) and [`Embedder`] (retrieval / seal scoring). +pub struct SeamEmbedder { + provider: Arc, +} + +impl SeamEmbedder { + /// Build the adapter over an OpenHuman embedding provider, preserving its + /// factory-configured rate-limit + retry policy. + pub fn new(provider: Arc) -> Self { + tracing::debug!( + provider = provider.name(), + model_id = provider.model_id(), + dimensions = provider.dimensions(), + signature = %provider.signature(), + "[memory] constructing tinycortex embedding seam over EmbeddingProvider" + ); + Self { provider } + } +} + +#[async_trait] +impl EmbeddingBackend for SeamEmbedder { + fn name(&self) -> &str { + self.provider.name() + } + + fn model_id(&self) -> &str { + self.provider.model_id() + } + + fn dimensions(&self) -> usize { + self.provider.dimensions() + } + + /// Delegate to the provider so the persisted signature is byte-identical to + /// the config-derived `active_embedding_signature` — a mismatch would split + /// one embedding space into two (#1574). + fn signature(&self) -> String { + self.provider.signature() + } + + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + self.provider.embed(texts).await + } +} + +#[async_trait] +impl Embedder for SeamEmbedder { + fn name(&self) -> &'static str { + // The crate's `Embedder` requires a `'static` name (debug/diagnostics + // only); the provider's own `name()` is borrowed, so report a stable + // seam label rather than leaking a lifetime. + "openhuman-seam" + } + + async fn embed(&self, text: &str) -> anyhow::Result> { + self.provider.embed_one(text).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct FakeProvider; + + #[async_trait] + impl EmbeddingProvider for FakeProvider { + fn name(&self) -> &str { + "fake" + } + fn model_id(&self) -> &str { + "fake-model" + } + fn dimensions(&self) -> usize { + 3 + } + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(texts + .iter() + .map(|t| vec![t.len() as f32, 0.0, 0.0]) + .collect()) + } + } + + #[tokio::test] + async fn backend_passes_through_metadata_and_signature() { + let seam = SeamEmbedder::new(Arc::new(FakeProvider)); + assert_eq!(EmbeddingBackend::name(&seam), "fake"); + assert_eq!(seam.model_id(), "fake-model"); + assert_eq!(seam.dimensions(), 3); + // Byte-identical to format_embedding_signature(name, model_id, dims). + assert_eq!( + EmbeddingBackend::signature(&seam), + "provider=fake;model=fake-model;dims=3" + ); + } + + #[tokio::test] + async fn backend_and_embedder_both_delegate_to_provider() { + let seam = SeamEmbedder::new(Arc::new(FakeProvider)); + + let batch = EmbeddingBackend::embed(&seam, &["ab", "cde"]) + .await + .unwrap(); + assert_eq!(batch, vec![vec![2.0, 0.0, 0.0], vec![3.0, 0.0, 0.0]]); + + let one = Embedder::embed(&seam, "abcd").await.unwrap(); + assert_eq!(one, vec![4.0, 0.0, 0.0]); + assert_eq!(Embedder::name(&seam), "openhuman-seam"); + } +} diff --git a/core/src/tinycortex/ingest.rs b/core/src/tinycortex/ingest.rs new file mode 100644 index 0000000..222a54f --- /dev/null +++ b/core/src/tinycortex/ingest.rs @@ -0,0 +1,79 @@ +//! Host adapters for tinycortex on-demand ingestion. + +use rusqlite::Transaction; +use tinycortex::memory::ingest::{QueueJobSink, TreeJobSink}; +use tinycortex::memory::score::extract::{LlmEntityExtractor, LlmExtractorConfig}; +use tinycortex::memory::score::ScoringConfig; + +use crate::Config; + +#[derive(Default)] +pub struct HostTreeJobSink; + +impl HostTreeJobSink { + pub fn new() -> Self { + Self + } +} + +impl TreeJobSink for HostTreeJobSink { + fn enqueue_extract_tx( + &self, + tx: &Transaction<'_>, + chunk_id: &str, + default_max_attempts: u32, + ) -> anyhow::Result { + tracing::trace!( + chunk_id, + default_max_attempts, + "[memory:ingest] enqueue extract job in chunk transaction" + ); + let enqueued = QueueJobSink + .enqueue_extract_tx(tx, chunk_id, default_max_attempts) + .inspect_err(|error| { + tracing::error!( + chunk_id, + error = %error, + "[memory:ingest] enqueue extract job failed" + ); + })?; + tracing::trace!( + chunk_id, + enqueued, + "[memory:ingest] enqueue extract job outcome (false = already queued)" + ); + Ok(enqueued) + } +} + +fn scoring_config(config: &Config) -> ScoringConfig { + match super::build_chat_provider(config) { + Ok(provider) => { + let extractor = LlmExtractorConfig { + output_language: config.output_language().map(str::to_string), + ..Default::default() + }; + ScoringConfig::with_llm_extractor(std::sync::Arc::new(LlmEntityExtractor::new( + extractor, provider, + ))) + } + Err(error) => { + tracing::warn!(%error, "[memory:ingest] chat provider unavailable; using regex scoring"); + ScoringConfig::default_regex_only() + } + } +} + +pub fn context( + config: &Config, +) -> ( + tinycortex::memory::MemoryConfig, + HostTreeJobSink, + ScoringConfig, +) { + ( + super::memory_config_from(config, config.workspace_dir().clone()), + HostTreeJobSink::new(), + scoring_config(config), + ) +} diff --git a/core/src/tinycortex/mod.rs b/core/src/tinycortex/mod.rs new file mode 100644 index 0000000..6c4f116 --- /dev/null +++ b/core/src/tinycortex/mod.rs @@ -0,0 +1,72 @@ +//! `tinycortex` integration — run OpenHuman's memory engine on the published +//! [`tinycortex`](https://crates.io/crates/tinycortex) crate. +//! +//! OpenHuman's memory subsystem migrates onto the `tinycortex` crate (store / +//! chunks / tree / retrieval / queue / ingest / score + the long tail). This +//! module is the **adapter seam**, mirroring `src/openhuman/agent/tinyagents/`: it +//! implements the crate's engine traits over OpenHuman services and derives the +//! engine's [`tinycortex::memory::MemoryConfig`] from the host `Config`. Nothing here contains +//! engine logic — that lives in the crate. +//! +//! ## Ownership boundary (the seam contract) +//! +//! **Engine (crate):** content store + YAML vault, SQLite vectors/kv/entity +//! index, chunk lifecycle, summary trees, hybrid retrieval, scoring, the async +//! job model, ingest canonicalize/extract, and the diff/entities/graph/goals/ +//! archivist/tool-memory/conversations long tail. +//! +//! **Product (host, stays in OpenHuman):** JSON-RPC schemas/ops/read_rpc, agent +//! tools + `SecurityPolicy` gating, sync scheduling/credentials/events, the +//! event bus, preferences, `source_scope` per-turn allowlist, redaction, the +//! global singleton + background queue worker, embedding/LLM **compute**, and the +//! host-retained `UnifiedMemory` namespace-document tier (episodic/event/ +//! segment/doc/graph/profile tables) plus the `wiki_git`/`obsidian` content +//! surfaces the crate deliberately excludes. +//! +//! Network-capable sync providers are feature-gated in the crate; their +//! credentials and product policy stay in the host. LLM/embedding compute is +//! injected through `EmbeddingBackend`, `ChatProvider`, `Summariser`, and +//! `EntityExtractor`; the job queue is driven by the host worker loop via +//! `queue::run_once` / `drain_until_idle`. Those adapters live beside this file +//! (`embeddings.rs`, `chat.rs`, `queue_driver.rs`, `ingest.rs`, `seal.rs`, and +//! `sync.rs`). +//! +//! See `docs/tinycortex-migration-spec.md` for the full ownership split, +//! drift/gap/parity ledgers, and the workstream order. + +mod chat; +mod config; +mod embeddings; +mod ingest; +#[cfg(test)] +mod parity; +mod persona; +mod queue_driver; +mod seal; +mod summariser; +mod sync; + +pub use chat::{build_chat_provider, SeamChatProvider}; +pub use config::{engine_config, memory_config_from}; +pub use embeddings::SeamEmbedder; +pub use ingest::{context as ingest_context, HostTreeJobSink}; +pub use persona::{ + coding_session_status, coding_session_status_for_roots, ingest_coding_sessions, + CodingSessionIngestRequest, CodingSessionIngestResponse, CodingSessionSourceStatus, +}; +pub use queue_driver::{ + classify_worker_error, HostQueueDelegates, WorkerErrorAction, WorkerReport, +}; +pub use seal::{ + cascade_tree, flush_stale_tree_buffers, seal_document_subtree, + seal_one_level as seal_tree_level, +}; +pub use summariser::HostSummariser; +pub use sync::{ + append_audit_entry, estimate_cost_usd, load_composio_sync_state, needs_rebuild, raw_coverage, + read_audit_log, rebuild_tree_from_raw, run_composio_connection, + run_composio_connection_with_budgets, run_github_sync, run_gmail_backfill, + run_slack_search_backfill, run_source_pipeline, sync_context, try_read_audit_log, + HostSyncAdapter, RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, + SourcePipelineFailure, SyncAuditEntry, HOST_SYNC_STATE_NAMESPACE, +}; diff --git a/core/src/tinycortex/parity.rs b/core/src/tinycortex/parity.rs new file mode 100644 index 0000000..7dea6a0 --- /dev/null +++ b/core/src/tinycortex/parity.rs @@ -0,0 +1,256 @@ +//! On-disk format parity — Layer-1 regression pins (migration W3 gate, spec §0.3). +//! +//! Existing user workspaces must open unchanged after the store cutover. These +//! are the cheap, fixture-free asserters from the parity checklist: they pin the +//! crate's deterministic **on-disk contracts** to the exact byte forms that +//! historical OpenHuman workspaces were written with, so any future crate change +//! that would silently reshape chunk IDs, vector encoding, or vault paths fails +//! here instead of corrupting a real workspace. +//! +//! The golden constants were computed from the format spec (SHA-256 first-32-hex +//! chunk IDs; little-endian packed f32 vectors) and cross-checked against the +//! crate at the W3 baseline. The Layer-2 golden-workspace differential harness +//! (a real `chunks.db` + vault opened and compared) is the merge gate for the +//! actual store flips; this layer runs on every PR. +//! +//! Test-only module — no runtime code. + +#[cfg(test)] +mod tests { + use tinycortex::memory::chunks::{chunk_id, SourceKind}; + use tinycortex::memory::store::content::chunk_rel_path; + use tinycortex::memory::store::vectors::{bytes_to_vec, vec_to_bytes}; + + /// P1 — the deterministic chunk ID is SHA-256 over + /// `source_kind \0 source_id \0 seq_be \0 content`, first 32 hex chars. + /// This golden is the value historical workspaces indexed by; a change to + /// the hash inputs / order / separators would strand every existing chunk. + #[test] + fn chunk_id_matches_historical_golden() { + let id = chunk_id(SourceKind::Document, "src-1", 5, "hello world"); + assert_eq!(id, "2be5fac18b12bfb417736b54deaf5f9d"); + assert_eq!(id.len(), 32); + assert!(id.chars().all(|c| c.is_ascii_hexdigit())); + } + + /// P1 — every field participates in the hash, and `seq` is order-sensitive. + /// Guards against an input being dropped or reordered (which a property-free + /// golden alone would miss on a symmetric swap). + #[test] + fn chunk_id_is_sensitive_to_every_field() { + let base = chunk_id(SourceKind::Document, "src-1", 5, "hello world"); + assert_ne!(base, chunk_id(SourceKind::Chat, "src-1", 5, "hello world")); + assert_ne!( + base, + chunk_id(SourceKind::Document, "src-2", 5, "hello world") + ); + assert_ne!( + base, + chunk_id(SourceKind::Document, "src-1", 6, "hello world") + ); + assert_ne!( + base, + chunk_id(SourceKind::Document, "src-1", 5, "hello worlds") + ); + // Determinism: same inputs, same id. + assert_eq!( + base, + chunk_id(SourceKind::Document, "src-1", 5, "hello world") + ); + } + + /// P2 — vectors persist as little-endian packed f32, 4 bytes/element, no + /// header. The golden byte string is what existing `vectors.embedding` + /// BLOBs and `mem_tree_*_embeddings` sidecars were written with. + #[test] + fn vector_encoding_is_le_packed_f32() { + let v = vec![1.0f32, -2.0, 0.5]; + let bytes = vec_to_bytes(&v); + assert_eq!(bytes.len(), v.len() * 4); + assert_eq!(hex(&bytes), "0000803f000000c00000003f"); + // Round-trips exactly. + assert_eq!(bytes_to_vec(&bytes).expect("valid packed f32 bytes"), v); + } + + /// P6 — vault paths sanitize IDs to cross-platform-safe filenames. Chunk IDs + /// contain colons (`chat:slack:#eng:0`) that are illegal on Windows NTFS; + /// the path must not leak them, and must be deterministic so an existing + /// vault file is found in place. + #[test] + fn content_paths_are_windows_safe_and_stable() { + let p1 = chunk_rel_path("chat", "slack:#eng", "chat:slack:#eng:0"); + let p2 = chunk_rel_path("chat", "slack:#eng", "chat:slack:#eng:0"); + assert_eq!(p1, p2, "path derivation must be deterministic"); + assert!( + !p1.contains(':'), + "path must not contain Windows-illegal ':' -> {p1}" + ); + assert!(p1.ends_with(".md"), "chunk files are markdown -> {p1}"); + } + + /// P6 (differential) — the host and crate `chunk_rel_path` must produce + /// **byte-identical** vault paths for every id shape a real workspace holds. + /// Both impls still exist (content is not flipped until W3), so a crate-side + /// change to `slugify_source_id` / `sanitize_filename` / the email special + /// case would silently strand every existing chunk file under a new path. + /// This pins them together over an adversarial corpus (colons, all + /// Windows-illegal chars, unicode, >255-char ids, gmail participant slugs, + /// malformed email source_ids) so any drift fails here, not on a user's disk. + #[test] + fn chunk_rel_path_host_crate_byte_parity() { + use crate::store::content::paths as host; + use tinycortex::memory::store::content as cortex; + + let long_id = "x".repeat(300); + let corpus: &[(&str, &str, &str)] = &[ + // (source_kind, source_id, chunk_id) + ("chat", "slack:#eng", "chat:slack:#eng:0"), + ("chat", "Slack:#Eng__Team", "chat:slack:#eng:0"), + ("document", "file:///Users/x/Notes.md", "doc:notes:3"), + ("document", "weird__source__id", "id-with-no-illegal-chars"), + ("chat", "src", "a\\b/c:d*e?f\"gi|j"), + ("chat", "东京:room", "chat:东京:0"), + ("chat", "src", &long_id), + // Email: well-formed gmail participants → one slugified folder. + ( + "email", + "gmail:notifications@github.com|sanil@x.com", + "email:msg:0", + ), + ("email", "gmail:Alice@X.com|bob@y.com", "email:msg:1"), + // Email: malformed / legacy source_id → flat fallback layout. + ("email", "legacyid", "email:legacy:0"), + ("email", "gmail:", "email:empty-participants:0"), + ]; + + for (kind, source_id, chunk_id) in corpus { + let h = host::chunk_rel_path(kind, source_id, chunk_id); + let c = cortex::chunk_rel_path(kind, source_id, chunk_id); + assert_eq!( + h, c, + "chunk_rel_path diverged for (kind={kind}, source_id={source_id}, chunk_id={chunk_id}): host={h} crate={c}" + ); + assert!(!h.contains(':'), "host path leaked ':' -> {h}"); + assert!(h.ends_with(".md"), "chunk files are markdown -> {h}"); + } + } + + /// P6 (differential) — the same byte-parity requirement for summary paths. + /// The summary basename (`summary_filename`) and the `wiki/summaries/...` + /// layout per `SummaryTreeKind` must match across host and crate, or a + /// re-open would not find an existing sealed summary in place. + #[test] + fn summary_rel_path_host_crate_byte_parity() { + use crate::store::content::paths as host; + use tinycortex::memory::store::content as cortex; + + // (host kind, crate kind, scope_slug) — variants are 1:1 across sides. + let kinds = [ + ( + host::SummaryTreeKind::Source, + cortex::SummaryTreeKind::Source, + "source-slug", + ), + ( + host::SummaryTreeKind::Global, + cortex::SummaryTreeKind::Global, + "ignored-for-global", + ), + ( + host::SummaryTreeKind::Topic, + cortex::SummaryTreeKind::Topic, + "phoenix-migration", + ), + ]; + // Canonical ms-first ids, legacy level-first ids, and malformed shapes + // that must fall back through `sanitize_filename` on both sides. + let summary_ids: &[&str] = &[ + "summary:1700000000000:L2-abc-uuid", + "summary:L3:legacy-uuid", + "summary:1700000000000:L2-a/b", // illegal tail → sanitized + "summary:notms:L1-tail", // non-13-digit ms → fallback + "raw-unknown-shape:with:colons", // unknown → sanitize_filename + "东京-summary", // unicode + ]; + + for (hk, ck, scope) in kinds { + for level in [0u32, 1, 4] { + for sid in summary_ids { + let h = host::summary_rel_path(hk, scope, level, sid); + let c = cortex::summary_rel_path(ck, scope, level, sid); + assert_eq!( + h, c, + "summary_rel_path diverged for (scope={scope}, level={level}, id={sid}): host={h} crate={c}" + ); + assert!(!h.contains(':'), "host summary path leaked ':' -> {h}"); + } + } + } + } + + /// P10 — the embedding-space **signature** string that keys every persisted + /// vector. Host (`embeddings::format_embedding_signature`) and crate + /// (`store::vectors::format_embedding_signature`) each own their **own** copy + /// of this formatter, so a change to either would silently split one + /// embedding space into two — every existing vector would look stale under + /// the new signature and trigger a full re-embed storm on the next open. + /// Pin both to the golden `provider={name};model={model};dims={dims}` form + /// over a corpus (real provider triples plus empties / special chars). + #[test] + fn embedding_signature_host_crate_byte_parity() { + use tinycortex::memory::store::vectors::format_embedding_signature as cortex_sig; + use tinymemory_api::host::format_embedding_signature as host_sig; + + // (name, model_id, dims, expected golden) + let corpus: &[(&str, &str, usize, &str)] = &[ + ( + "voyage", + "voyage-3", + 1024, + "provider=voyage;model=voyage-3;dims=1024", + ), + ( + "openai", + "text-embedding-3-small", + 1536, + "provider=openai;model=text-embedding-3-small;dims=1536", + ), + ( + "ollama", + "nomic-embed-text", + 768, + "provider=ollama;model=nomic-embed-text;dims=768", + ), + ( + "cohere", + "embed-english-v3.0", + 1024, + "provider=cohere;model=embed-english-v3.0;dims=1024", + ), + ("inert", "none", 0, "provider=inert;model=none;dims=0"), + // Edge shapes: empty model, punctuation in model id. + ("noop", "", 3, "provider=noop;model=;dims=3"), + ("x", "m-1_2.3", 42, "provider=x;model=m-1_2.3;dims=42"), + ]; + + for (name, model, dims, golden) in corpus { + let h = host_sig(name, model, *dims); + let c = cortex_sig(name, model, *dims); + assert_eq!( + h, c, + "signature diverged for (name={name}, model={model}, dims={dims}): host={h} crate={c}" + ); + assert_eq!(&h, golden, "signature format drifted from the golden form"); + } + } + + fn hex(bytes: &[u8]) -> String { + use std::fmt::Write; + bytes + .iter() + .fold(String::with_capacity(bytes.len() * 2), |mut acc, b| { + let _ = write!(acc, "{b:02x}"); + acc + }) + } +} diff --git a/core/src/tinycortex/persona.rs b/core/src/tinycortex/persona.rs new file mode 100644 index 0000000..4b9105e --- /dev/null +++ b/core/src/tinycortex/persona.rs @@ -0,0 +1,446 @@ +//! Host orchestration for TinyCortex coding-session persona ingestion. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use tinycortex::memory::persona::readers::{claude_code, codex, RawSession}; +use tinycortex::memory::persona::state::FileStateStore; +use tinycortex::memory::persona::{PersonaConfig, Pipeline, RunMode}; +use walkdir::WalkDir; + +use crate::Config; + +const DEFAULT_MAX_SESSIONS: usize = 100; +const MAX_MAX_SESSIONS: usize = 1_000; +const MAX_STATUS_SESSION_FILES: usize = 1_000; +const MAX_STATUS_SESSION_FILE_BYTES: u64 = 4 * 1024 * 1024; +const MAX_STATUS_TOTAL_BYTES: u64 = 16 * 1024 * 1024; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct CodingSessionSourceStatus { + pub kind: String, + pub available: bool, + pub session_files: usize, + pub evidence_units: usize, + pub invalid_files: usize, + pub scan_truncated: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct CodingSessionIngestRequest { + #[serde(default)] + pub backfill: bool, + #[serde(default = "default_max_sessions")] + pub max_sessions: usize, +} + +fn default_max_sessions() -> usize { + DEFAULT_MAX_SESSIONS +} + +#[derive(Debug, Clone, Serialize)] +pub struct CodingSessionIngestResponse { + pub mode: String, + pub files_seen: usize, + pub sessions_processed: usize, + pub sessions_skipped: usize, + pub sessions_failed: usize, + pub evidence_units: usize, + pub observations: usize, + pub budget_hit: bool, + pub pack_path: Option, +} + +fn roots_from_environment() -> (PathBuf, PathBuf) { + let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")); + let claude_home = std::env::var_os("CLAUDE_CONFIG_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| home.join(".claude")); + let codex_home = std::env::var_os("CODEX_HOME") + .map(PathBuf::from) + .unwrap_or_else(|| home.join(".codex")); + (claude_home.join("projects"), codex_home.join("sessions")) +} + +fn source_status( + kind: &str, + root: &Path, + max_files: usize, + discover: impl Fn(&Path, usize) -> (Vec, bool), + read: impl Fn(&Path) -> anyhow::Result, +) -> CodingSessionSourceStatus { + let (files, mut scan_truncated) = discover(root, max_files); + if scan_truncated { + tracing::debug!( + source = kind, + max_files, + "[memory_persona] coding session status scan capped" + ); + } + let mut evidence_units = 0; + let mut invalid_files = 0; + let mut bytes_scheduled = 0_u64; + for path in &files { + if let Ok(metadata) = path.metadata() { + let file_bytes = metadata.len(); + if file_bytes > MAX_STATUS_SESSION_FILE_BYTES + || bytes_scheduled.saturating_add(file_bytes) > MAX_STATUS_TOTAL_BYTES + { + scan_truncated = true; + tracing::debug!( + source = kind, + file_bytes, + bytes_scheduled, + max_file_bytes = MAX_STATUS_SESSION_FILE_BYTES, + max_total_bytes = MAX_STATUS_TOTAL_BYTES, + reason = "status-byte-budget", + "[memory_persona] skipped coding session during bounded status scan" + ); + continue; + } + bytes_scheduled += file_bytes; + } + match read(path) { + Ok(session) => evidence_units += session.evidence.len(), + Err(_error) => { + invalid_files += 1; + tracing::debug!( + source = kind, + reason = "read-or-parse-failed", + "[memory_persona] skipped unreadable coding session" + ); + } + } + } + CodingSessionSourceStatus { + kind: kind.to_string(), + available: root.is_dir(), + session_files: files.len(), + evidence_units, + invalid_files, + scan_truncated, + } +} + +fn discover_session_files( + root: &Path, + max_files: usize, + is_candidate: impl Fn(&Path) -> bool, +) -> (Vec, bool) { + let mut files = Vec::with_capacity(max_files.min(64)); + // Keep traversal unsorted: `sort_by_file_name` buffers and sorts every + // directory before yielding its first entry, which defeats `max_files` + // for users with very large Codex day or Claude project directories. + for entry in WalkDir::new(root) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + { + let path = entry.path(); + if !is_candidate(path) { + continue; + } + if files.len() == max_files { + return (files, true); + } + files.push(path.to_path_buf()); + } + (files, false) +} + +fn discover_claude_sessions(root: &Path, max_files: usize) -> (Vec, bool) { + discover_session_files(root, max_files, |path| { + path.extension() + .is_some_and(|extension| extension == "jsonl") + }) +} + +fn discover_codex_sessions(root: &Path, max_files: usize) -> (Vec, bool) { + discover_session_files(root, max_files, |path| { + path.extension() + .is_some_and(|extension| extension == "jsonl") + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("rollout-")) + }) +} + +pub fn coding_session_status_for_roots( + claude_root: &Path, + codex_root: &Path, +) -> Vec { + tracing::debug!("[memory_persona] coding session scan: entry"); + let statuses = vec![ + source_status( + "claude_code", + claude_root, + MAX_STATUS_SESSION_FILES, + discover_claude_sessions, + claude_code::read_session, + ), + source_status( + "codex", + codex_root, + MAX_STATUS_SESSION_FILES, + discover_codex_sessions, + codex::read_session, + ), + ]; + tracing::debug!( + files = statuses + .iter() + .map(|status| status.session_files) + .sum::(), + evidence = statuses + .iter() + .map(|status| status.evidence_units) + .sum::(), + invalid = statuses + .iter() + .map(|status| status.invalid_files) + .sum::(), + "[memory_persona] coding session scan: exit" + ); + statuses +} + +pub fn coding_session_status() -> Vec { + let (claude_root, codex_root) = roots_from_environment(); + coding_session_status_for_roots(&claude_root, &codex_root) +} + +pub async fn ingest_coding_sessions( + config: &Config, + request: CodingSessionIngestRequest, +) -> anyhow::Result { + let (claude_root, codex_root) = roots_from_environment(); + let max_sessions = request.max_sessions.clamp(1, MAX_MAX_SESSIONS); + let mode = if request.backfill { + RunMode::Backfill + } else { + RunMode::Incremental + }; + tracing::info!( + mode = if request.backfill { + "backfill" + } else { + "incremental" + }, + max_sessions, + "[memory_persona] coding session ingestion: entry" + ); + + let memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + let mut persona = PersonaConfig::with_home( + dirs::home_dir() + .as_deref() + .unwrap_or_else(|| Path::new(".")), + "OpenHuman user", + ); + persona.claude_code_root = Some(claude_root); + persona.codex_root = Some(codex_root); + // This product surface is deliberately scoped to coding-session history. + // Repository history and instruction files can be wired separately with + // their own disclosure and cost controls. + persona.project_roots.clear(); + persona.global_instruction_files.clear(); + persona.author_emails.clear(); + persona.run_budget.max_sessions = max_sessions; + persona.run_budget.max_llm_calls = max_sessions as u32; + + let provider = super::build_chat_provider(config).inspect_err(|error| { + tracing::error!( + error = %error, + "[memory_persona] coding session ingestion: build_chat_provider failed" + ); + })?; + let summariser = super::HostSummariser::new(config.to_arc()); + let store = FileStateStore::open_in_workspace(config.workspace_dir()).inspect_err(|error| { + tracing::error!( + error = %error, + "[memory_persona] coding session ingestion: open state store failed" + ); + })?; + let report = Pipeline { + config: &memory_config, + persona: &persona, + provider: provider.as_ref(), + summariser: &summariser, + store: &store, + } + .run(mode) + .await + .inspect_err(|error| { + tracing::error!( + error = %error, + "[memory_persona] coding session ingestion: pipeline run failed" + ); + })?; + + tracing::info!( + files_seen = report.files_seen, + sessions_processed = report.sessions_processed, + sessions_failed = report.sessions_failed, + evidence_units = report.evidence_units, + observations = report.observations, + budget_hit = report.budget_hit, + "[memory_persona] coding session ingestion: exit" + ); + Ok(CodingSessionIngestResponse { + mode: report.mode, + files_seen: report.files_seen, + sessions_processed: report.sessions_processed, + sessions_skipped: report.sessions_skipped, + sessions_failed: report.sessions_failed, + evidence_units: report.evidence_units, + observations: report.observations, + budget_hit: report.budget_hit, + pack_path: report.pack_path, + }) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::tempdir; + + use super::*; + + #[test] + fn scans_codex_and_claude_sessions_and_filters_machine_content() { + let temp = tempdir().unwrap(); + let claude = temp.path().join("claude"); + let codex = temp.path().join("codex/2026/07/14"); + fs::create_dir_all(&claude).unwrap(); + fs::create_dir_all(&codex).unwrap(); + fs::write( + claude.join("session.jsonl"), + concat!( + "{\"type\":\"assistant\",\"message\":{\"content\":[{\"type\":\"text\",\"text\":\"machine\"}]}}\n", + "{\"type\":\"user\",\"sessionId\":\"c1\",\"cwd\":\"/repo\",\"timestamp\":\"2026-07-14T00:00:00Z\",\"message\":{\"content\":\"Prefer small modules\"}}\n" + ), + ) + .unwrap(); + fs::write( + codex.join("rollout-test.jsonl"), + concat!( + "{\"type\":\"session_meta\",\"payload\":{\"id\":\"x1\",\"cwd\":\"/repo\"}}\n", + "{\"type\":\"response_item\",\"timestamp\":\"2026-07-14T00:00:00Z\",\"payload\":{\"type\":\"message\",\"role\":\"developer\",\"content\":[{\"type\":\"input_text\",\"text\":\"secret scaffolding\"}]}}\n", + "{\"type\":\"response_item\",\"timestamp\":\"2026-07-14T00:00:01Z\",\"payload\":{\"type\":\"message\",\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Run focused tests first\"}]}}\n" + ), + ) + .unwrap(); + + let statuses = coding_session_status_for_roots(&claude, &temp.path().join("codex")); + assert_eq!(statuses.len(), 2); + assert_eq!(statuses[0].session_files, 1); + assert_eq!(statuses[0].evidence_units, 1); + assert_eq!(statuses[1].session_files, 1); + assert_eq!(statuses[1].evidence_units, 1); + assert_eq!(statuses[0].invalid_files + statuses[1].invalid_files, 0); + } + + #[test] + fn status_scan_stops_parsing_at_the_configured_limit() { + let paths = [PathBuf::from("one"), PathBuf::from("two")]; + let reads = std::cell::Cell::new(0); + let status = source_status( + "fixture", + Path::new("."), + 1, + |_, max_files| (paths[..max_files].to_vec(), paths.len() > max_files), + |_| { + reads.set(reads.get() + 1); + Ok(RawSession::new( + tinycortex::memory::persona::types::EvidenceSource::new( + tinycortex::memory::persona::types::PersonaSourceKind::Codex, + ), + )) + }, + ); + + assert_eq!(reads.get(), 1); + assert_eq!(status.session_files, 1); + assert!(status.scan_truncated); + } + + #[test] + fn bounded_discovery_stops_after_finding_one_extra_candidate_without_ordering() { + let temp = tempdir().unwrap(); + fs::write(temp.path().join("a.jsonl"), "").unwrap(); + fs::write(temp.path().join("b.jsonl"), "").unwrap(); + fs::write(temp.path().join("ignored.txt"), "").unwrap(); + + let (files, truncated) = discover_claude_sessions(temp.path(), 1); + + assert_eq!(files.len(), 1); + assert_eq!(files[0].extension().unwrap(), "jsonl"); + assert!(truncated); + } + + #[test] + fn status_scan_skips_oversized_sessions_without_parsing_them() { + let temp = tempdir().unwrap(); + let oversized = temp.path().join("oversized.jsonl"); + let small = temp.path().join("small.jsonl"); + let file = fs::File::create(&oversized).unwrap(); + file.set_len(MAX_STATUS_SESSION_FILE_BYTES + 1).unwrap(); + fs::write(&small, "{}\n").unwrap(); + let reads = std::cell::Cell::new(0); + + let status = source_status( + "fixture", + temp.path(), + 2, + |_, _| (vec![oversized.clone(), small.clone()], false), + |_| { + reads.set(reads.get() + 1); + Ok(RawSession::new( + tinycortex::memory::persona::types::EvidenceSource::new( + tinycortex::memory::persona::types::PersonaSourceKind::Codex, + ), + )) + }, + ); + + assert_eq!(reads.get(), 1); + assert_eq!(status.session_files, 2); + assert_eq!(status.invalid_files, 0); + assert!(status.scan_truncated); + } + + #[test] + fn status_scan_enforces_the_aggregate_byte_budget() { + let temp = tempdir().unwrap(); + let paths = (0..5) + .map(|index| { + let path = temp.path().join(format!("session-{index}.jsonl")); + let file = fs::File::create(&path).unwrap(); + file.set_len(MAX_STATUS_SESSION_FILE_BYTES).unwrap(); + path + }) + .collect::>(); + let reads = std::cell::Cell::new(0); + + let status = source_status( + "fixture", + temp.path(), + paths.len(), + |_, _| (paths.clone(), false), + |_| { + reads.set(reads.get() + 1); + Ok(RawSession::new( + tinycortex::memory::persona::types::EvidenceSource::new( + tinycortex::memory::persona::types::PersonaSourceKind::Codex, + ), + )) + }, + ); + + assert_eq!(reads.get(), 4); + assert_eq!(status.session_files, 5); + assert!(status.scan_truncated); + } +} diff --git a/core/src/tinycortex/queue_driver.rs b/core/src/tinycortex/queue_driver.rs new file mode 100644 index 0000000..87f16ff --- /dev/null +++ b/core/src/tinycortex/queue_driver.rs @@ -0,0 +1,1020 @@ +//! Queue worker-loop driver seam (migration W4). +//! +//! TinyCortex owns the job *store* and the single-step engine +//! (`queue::run_once` claims one `mem_tree_jobs` row, dispatches it through +//! [`QueueDelegates`], and settles it) but deliberately drops the tokio worker +//! pool, the wall-clock scheduler, Sentry reporting, and the storage-degraded +//! state machine — those are host concerns (plan §1, deletion ledger: "host +//! worker loop + Sentry/degraded wiring kept host"). This seam is where the +//! host drives the crate queue. +//! +//! **This module is the first W4 brick: the host-retained error policy.** When +//! `run_once` returns an error, the legacy `memory_queue::worker` loop applied a +//! carefully-tuned "back off, don't page" policy per failure class — the product +//! of several Sentry floods (OPENHUMAN-TAURI-BP, #2206, TAURI-RUST-4R8/E93, +//! CORE-RUST-19J). [`classify_worker_error`] ports that decision table verbatim +//! on top of the crate's now-merged classifiers +//! ([`is_host_io_error`] etc., tinycortex#63), so the crate-driven loop +//! reproduces it exactly. It is a pure function so the policy is unit-tested +//! without spinning a live loop. +//! +//! The host worker is flipped to `tinycortex::memory::queue::run_once` through +//! `HostQueueDelegates`. The adapter preserves host-owned scheduling, health +//! reporting, event-bus publishing, and product-policy hooks while the crate +//! owns claim/dispatch/settle. + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Context; +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use tinycortex::memory::queue::worker::{ + is_host_io_error, is_sqlite_busy, is_sqlite_corrupt, is_sqlite_disk_full, + is_sqlite_io_transient, +}; +use tinycortex::memory::queue::{ + AppendDecision, AppendTarget, ExtractDecision, NodeRef, QueueDelegates, ReembedProgress, + SealDocumentPayload, SealPayload, StaleBuffer, +}; +use tinycortex::memory::MemoryConfig; + +use crate::store::chunks::store as chunk_store; +use crate::store::chunks::types::{truncate_to_conservative_tokens, Chunk, Metadata}; +use crate::store::content as content_store; +use crate::store::content::read as content_read; +use crate::store::content::tags as content_tags; +use crate::store::trees::store as trees_store; +use crate::tree::health; +use crate::tree::score; +use crate::tree::score::embed::{build_write_embedder, pack_checked, Embedder}; +use crate::tree::score::store as score_store; +use crate::tree::tree::TreeFactory; +use crate::tree_source::get_or_create_source_tree; +use crate::Config; + +// ── Pure scope helpers (ported verbatim from `memory_queue::handlers`) ──────── +// These pin the SAME source→tree mapping the append-buffer path uses, so reads +// look up the tree the seal worker wrote to. Copied (not imported) because +// `memory_queue` is deleted at the W4 flip and these belong with the seam. + +/// Derive the tree scope from a source_id. GitHub per-item ids like +/// `github:owner/repo:commit:sha` collapse to `github:owner/repo` so a repo's +/// items share one tree; other ids pass through. +fn derive_tree_scope(source_id: &str) -> String { + if let Some(rest) = source_id.strip_prefix("github:") { + if let Some(idx) = rest.find(':') { + return format!("github:{}", &rest[..idx]); + } + } + source_id.to_string() +} + +/// The source-tree scope a chunk appends under: its `path_scope` when set +/// (shared-directory sources like Notion), else the GitHub-aware scope. +/// +/// `pub(crate)` and re-exported from [`crate::tinycortex`] so read +/// paths (e.g. `memory_tree::retrieval::cover`) look up the tree the seal +/// worker actually wrote to. This is the single canonical host copy — the +/// legacy `memory_queue::handlers` copy was deleted at the W4 flip. +pub(crate) fn chunk_tree_scope(metadata: &Metadata) -> String { + metadata + .path_scope + .clone() + .unwrap_or_else(|| derive_tree_scope(&metadata.source_id)) +} + +/// Whether a chunk's source uses the per-document rollup/versioning path +/// (Notion) — those skip the flat L0 buffer; their tree is built by SealDocument. +fn uses_document_subtree(chunk: &Chunk) -> bool { + const DOC_SUBTREE_PREFIX: &str = "notion:"; + chunk.metadata.source_id.starts_with(DOC_SUBTREE_PREFIX) + || chunk + .metadata + .path_scope + .as_deref() + .is_some_and(|s| s.starts_with(DOC_SUBTREE_PREFIX)) +} + +// ── Re-embed backfill helpers (ported from `memory_queue::handlers`) ────────── + +/// Texts per re-embed batch — sized to the batch API (Voyage: 1000/req). +const REEMBED_BACKFILL_BATCH: usize = 1000; +/// Conservative per-text embed token budget; caps any body that reaches an embed +/// call so no single input overflows the embedder's context and fails the batch. +const EMBED_SAFE_TOKENS: u32 = 7500; + +fn cap_embed_text(text: &str) -> &str { + truncate_to_conservative_tokens(text, EMBED_SAFE_TOKENS) +} + +fn try_mark_chunk_reembed_skipped(config: &Config, chunk_id: &str, sig: &str, reason: &str) { + if let Err(e) = chunk_store::mark_chunk_reembed_skipped(config, chunk_id, sig, reason) { + log::warn!( + "[tinycortex::queue_driver] reembed: failed to persist chunk tombstone chunk_id={chunk_id} sig={sig}: {e}" + ); + } +} + +fn try_mark_summary_reembed_skipped(config: &Config, summary_id: &str, sig: &str, reason: &str) { + if let Err(e) = trees_store::mark_summary_reembed_skipped(config, summary_id, sig, reason) { + log::warn!( + "[tinycortex::queue_driver] reembed: failed to persist summary tombstone summary_id={summary_id} sig={sig}: {e}" + ); + } +} + +/// Read each row's source text, embed the readable bodies in one batched call, +/// and classify per position (ported verbatim from `handlers::reembed_collect`, +/// preserving the #1574 §6 failure semantics: body-read/wrong-dim/unrecoverable +/// → persistent tombstone; cloud `AuthMissing` → fail without tombstone so rows +/// stay re-embeddable after login; other transient → propagate). +async fn reembed_collect( + config: &Config, + embedder: &dyn Embedder, + active_sig: &str, + ids: &[String], + label: &str, + read_body: impl Fn(&Config, &str) -> anyhow::Result, + mark_skipped: impl Fn(&Config, &str, &str, &str), +) -> anyhow::Result)>> { + let mut readable: Vec<(&String, String)> = Vec::with_capacity(ids.len()); + for id in ids { + match read_body(config, id) { + Ok(body) => readable.push((id, body)), + Err(e) => { + log::warn!( + "[tinycortex::queue_driver] reembed: {label} {id} body read failed: {e}; skipping (sig={active_sig})" + ); + mark_skipped(config, id, active_sig, &format!("body read failed: {e}")); + } + } + } + if readable.is_empty() { + return Ok(Vec::new()); + } + + let results = { + let texts: Vec<&str> = readable + .iter() + .map(|(_, body)| cap_embed_text(body)) + .collect(); + embedder.embed_batch(&texts).await + }; + if results.len() != readable.len() { + anyhow::bail!( + "reembed: {label} embed_batch returned {} results for {} texts (sig={active_sig})", + results.len(), + readable.len() + ); + } + + let mut out: Vec<(String, Vec)> = Vec::with_capacity(readable.len()); + for ((id, _body), result) in readable.into_iter().zip(results) { + match result { + Ok(v) if pack_checked(&v).is_ok() => out.push((id.clone(), v)), + Ok(_) => { + log::warn!( + "[tinycortex::queue_driver] reembed: {label} {id} embed wrong dim, skipping (sig={active_sig})" + ); + mark_skipped(config, id, active_sig, "embed wrong dim"); + } + Err(e) => { + let failure = health::classify_embed_error(&e); + // Correlation is the re-embed operation identity + typed + // outcome only — never the raw provider error or row content. + log::debug!( + "[tinycortex::queue_driver] action=classify_embed_failure op=reembed \ + label={label} id={id} sig={active_sig} code={} class={}", + failure.code.as_str(), + failure.class.as_str() + ); + // #5354: name the local-runtime fix on the status panel now + // rather than after the retry budget drains. + health::mark_local_model_unavailable_if_applicable(&failure); + if matches!(failure.code, health::FailureCode::AuthMissing) { + return Err(anyhow::Error::new(failure).context(format!( + "reembed: {label} {id} cloud auth missing (sig={active_sig}): {e:#}" + ))); + } + if !failure.is_unrecoverable() { + return Err(anyhow::Error::new(failure).context(format!( + "reembed: {label} {id} transient embed failed (sig={active_sig}): {e:#}" + ))); + } + log::warn!( + "[tinycortex::queue_driver] reembed: {label} {id} embed failed unrecoverably: {e}; skipping (sig={active_sig})" + ); + mark_skipped(config, id, active_sig, &format!("embed failed: {e}")); + } + } + } + Ok(out) +} + +/// How the host worker loop should report an errored `run_once` poll to Sentry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkerReport { + /// Do not page — the condition is transient, or persistent-but-user-only- + /// fixable and flood-prone (re-polling every second would bury the + /// dashboard). The `log::warn!` breadcrumb is enough. + Silent, + /// Report exactly once via a process-wide latch keyed by this reason tag, + /// then stay silent until the condition clears (so a genuinely-new later + /// failure can still page once). + Once(&'static str), + /// Report every occurrence — a genuinely unexpected error that should keep + /// surfacing. + Always(&'static str), +} + +/// The host-retained decision for an errored `run_once` poll: how long to back +/// off, whether/how to page, whether to flip the storage-degraded flag, and +/// whether to drive corrupt-DB quarantine+rebuild recovery. +/// +/// Ported verbatim from the `memory_queue::worker` error arms. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WorkerErrorAction { + /// How long the worker sleeps before the next poll. + pub backoff: Duration, + /// Sentry reporting policy for this failure class. + pub report: WorkerReport, + /// Mark the memory_tree storage-degraded (`StorageUnavailable`) so the status + /// panel shows the user an actionable "check your disk" banner — a persistent + /// host-FS failure only the user can clear. + pub mark_degraded: bool, + /// Drive the corrupt-DB quarantine+rebuild recovery path (which owns its own + /// report-once latch), rather than paging directly. + pub recover_corrupt: bool, +} + +/// Classify a `run_once` error into the host's back-off/report/degrade policy. +/// +/// Mirrors the legacy `memory_queue::worker` arms exactly, on the crate's +/// classifiers: +/// - **busy/locked** (`SQLITE_BUSY`/`LOCKED`): 1s, silent — transient write-lock +/// contention that `busy_timeout` + the next poll almost always clears. +/// - **transient I/O** (`-shm` family, `CANTOPEN`, `IOERR_TRUNCATE`, breaker): +/// 30s, silent (#2206 flooded ~19k events/4d). +/// - **disk full** (`SQLITE_FULL`): 300s, silent — persistent, user-only-fixable +/// (TAURI-RUST-4R8: ~95k events). +/// - **corrupt** (`SQLITE_CORRUPT`/`NOTADB`): 300s + quarantine/rebuild recovery +/// (which reports once) — never clears on its own (TAURI-RUST-E93). +/// - **host-FS** (EIO/ENOSPC/EROFS): 300s + storage-degraded + report-once — +/// failing/read-only storage (CORE-RUST-19J: ~10k events/50min). +/// - **anything else**: 1s + report-always — a genuine, unexpected error. +pub fn classify_worker_error(err: &anyhow::Error) -> WorkerErrorAction { + if is_sqlite_busy(err) { + WorkerErrorAction { + backoff: Duration::from_secs(1), + report: WorkerReport::Silent, + mark_degraded: false, + recover_corrupt: false, + } + } else if is_sqlite_io_transient(err) { + WorkerErrorAction { + backoff: Duration::from_secs(30), + report: WorkerReport::Silent, + mark_degraded: false, + recover_corrupt: false, + } + } else if is_sqlite_disk_full(err) { + WorkerErrorAction { + backoff: Duration::from_secs(300), + report: WorkerReport::Silent, + mark_degraded: false, + recover_corrupt: false, + } + } else if is_sqlite_corrupt(err) { + WorkerErrorAction { + backoff: Duration::from_secs(300), + // The recovery path owns the report-once latch, so the classifier + // itself stays silent and just requests recovery. + report: WorkerReport::Silent, + mark_degraded: false, + recover_corrupt: true, + } + } else if is_host_io_error(err) { + WorkerErrorAction { + backoff: Duration::from_secs(300), + report: WorkerReport::Once("tree_jobs_worker_host_io"), + mark_degraded: true, + recover_corrupt: false, + } + } else { + WorkerErrorAction { + backoff: Duration::from_secs(1), + report: WorkerReport::Always("tree_jobs_worker"), + mark_degraded: false, + recover_corrupt: false, + } + } +} + +/// Host implementation of the crate's [`QueueDelegates`] — the engine seam the +/// crate queue pushes its heavy per-job work through. +/// +/// TinyCortex owns the job store + dispatch (`handle_job` parses payloads, +/// enqueues follow-ups, decides `Done`/`Defer`) but delegates the parts it +/// cannot do itself — scoring/admission, buffer pushes, sealing, embedding — +/// because they need `memory_tree` / `memory_store` internals that are host +/// (and, for tree/score, host until W5). This bridges each delegate method to +/// the existing host engine, holding the host [`Config`] the calls need (the +/// `&MemoryConfig` the crate passes is derived from this same workspace). +/// +/// **Brick 2 status (additive — the driver is not flipped to this yet):** all 8 +/// delegate methods are wired to the real host engine (`memory_tree` / score / +/// embed / `memory_store`), porting the `memory_queue::handlers` bodies into the +/// crate's decision-returning shape — the delegate does only the heavy engine +/// work and returns the outcome; the crate's `handle_job` owns payload parsing, +/// follow-up enqueues, and `Done`/`Defer`, so the delegate must never enqueue. +/// Nothing is flipped: the live queue still runs on `memory_queue` until brick 3 +/// re-points `global.rs`/enqueue onto the crate store and deletes the legacy +/// engine. +pub struct HostQueueDelegates { + config: Arc, +} + +impl HostQueueDelegates { + /// Build the delegates over the host [`Config`] whose workspace the crate + /// queue is driving. + pub fn new(config: Arc) -> Self { + Self { config } + } +} + +#[async_trait] +impl QueueDelegates for HostQueueDelegates { + /// Ported from `prepare_extract` + `finalize_extract`: score + admit one + /// chunk and persist its score/lifecycle. Returns the admission decision; + /// the crate's `handle_extract` enqueues the append-buffer follow-up and arms + /// the re-embed backfill from it (so this must NOT enqueue — that would + /// double-enqueue). `Ok(None)` when the chunk row vanished. + async fn extract_chunk( + &self, + _config: &MemoryConfig, + chunk_id: &str, + ) -> anyhow::Result> { + let config = &*self.config; + let Some(mut chunk) = chunk_store::get_chunk(config, chunk_id)? else { + return Ok(None); + }; + + // The `content` column is a ≤500-char preview after the MD-on-disk + // migration; the scorer needs the full body. Swap it in for scoring, + // then restore the preview (avoids retaining the full body afterward). + let body = content_read::read_chunk_body(config, &chunk.id) + .with_context(|| format!("read full body for extract chunk_id={}", chunk.id))?; + let preview = std::mem::replace(&mut chunk.content, body); + let scoring_cfg = score::scoring_config_from(config); + let result = score::score_chunk(&chunk, &scoring_cfg).await?; + chunk.content = preview; + + let kept = result.kept; + let uses_doc = uses_document_subtree(&chunk); + let tree_scope = chunk_tree_scope(&chunk.metadata); + let timestamp_ms = chunk.metadata.timestamp.timestamp_millis(); + + // Persist score + lifecycle atomically. No follow-up enqueue here. + chunk_store::with_connection(config, |conn| { + let tx = conn.unchecked_transaction()?; + score::persist_score_tx(&tx, &result, timestamp_ms, None)?; + let status = if kept { + chunk_store::CHUNK_STATUS_ADMITTED + } else { + chunk_store::CHUNK_STATUS_DROPPED + }; + tx.execute( + "UPDATE mem_tree_chunks SET lifecycle_status = ?1 WHERE id = ?2", + rusqlite::params![status, chunk.id], + )?; + tx.commit()?; + Ok(()) + })?; + + // Best-effort: rewrite the on-disk chunk file's obsidian tags from the + // extracted entities (visible after the tx commits). Non-fatal. + if kept { + if let Some(content_path) = chunk_store::get_chunk_content_path(config, &chunk.id)? { + let content_root = config.memory_tree_content_root(); + let entity_ids = score_store::list_entity_ids_for_node(config, &chunk.id)?; + let obsidian_tags: Vec = entity_ids + .iter() + .filter_map(|eid| { + let (kind, surface) = eid.split_once(':')?; + Some(content_tags::entity_tag(kind, surface)) + }) + .collect(); + let mut abs_path = content_root; + for component in content_path.split('/') { + abs_path.push(component); + } + if let Err(e) = content_tags::update_chunk_tags(&abs_path, &obsidian_tags) { + log::warn!( + "[tinycortex::queue_driver] update_chunk_tags failed chunk_id={}: {e}", + chunk.id + ); + } + } + } + + Ok(Some(ExtractDecision { + kept, + uses_document_subtree: uses_doc, + tree_scope, + })) + } + + /// Ported from `handle_append_buffer`: push a leaf/summary node into its + /// target tree's L0 buffer and report whether the buffer crossed its seal + /// gate. The crate's `handle_append_buffer` enqueues the seal from the + /// returned `should_seal` (so this must NOT enqueue). `Ok(None)` when the + /// node or target tree is missing. + async fn append_node( + &self, + _config: &MemoryConfig, + node: &NodeRef, + target: &AppendTarget, + ) -> anyhow::Result> { + let config = &*self.config; + + // Buffer accounting needs only (item_id, token_count, timestamp); the + // full body/entities are re-read from disk at seal time, so — unlike the + // legacy handler's `LeafRef` — we don't read them here. + let (item_id, token_count, timestamp, lifecycle_chunk_id): ( + String, + i64, + DateTime, + Option, + ) = match node { + NodeRef::Leaf { chunk_id } => { + let Some(chunk) = chunk_store::get_chunk(config, chunk_id)? else { + return Ok(None); + }; + let id = chunk.id.clone(); + ( + id.clone(), + chunk.token_count as i64, + chunk.metadata.timestamp, + Some(id), + ) + } + NodeRef::Summary { summary_id } => { + let Some(summary) = trees_store::get_summary(config, summary_id)? else { + return Ok(None); + }; + // Summaries carry no chunk lifecycle to update. + ( + summary.id, + summary.token_count as i64, + summary.time_range_start, + None, + ) + } + }; + + let tree = match target { + AppendTarget::Source { source_id } => { + Some(get_or_create_source_tree(config, source_id)?) + } + AppendTarget::Topic { tree_id } => trees_store::get_tree(config, tree_id)?, + }; + let Some(tree) = tree else { + // Target topic tree archived between route and append — drop. + return Ok(None); + }; + let is_source_target = matches!(target, AppendTarget::Source { .. }); + let tree_id = tree.id.clone(); + + // ATOMIC: buffer push + lifecycle update. (The seal enqueue that the + // legacy handler did in this same tx is now the crate's job, driven by + // the returned `should_seal`.) + let should_seal = chunk_store::with_connection(config, move |conn| { + let tx = conn.unchecked_transaction()?; + let mut buf = trees_store::get_buffer_conn(&tx, &tree.id, 0)?; + if !buf.item_ids.iter().any(|x| x == &item_id) { + buf.item_ids.push(item_id.clone()); + buf.token_sum = buf.token_sum.saturating_add(token_count); + buf.oldest_at = match buf.oldest_at { + Some(existing) => Some(existing.min(timestamp)), + None => Some(timestamp), + }; + trees_store::upsert_buffer_tx(&tx, &buf)?; + } + let memory_config = + super::memory_config_from(&*self.config, self.config.workspace_dir().clone()); + let should_seal = tinycortex::memory::tree::should_seal(&memory_config, &buf); + if is_source_target { + if let Some(cid) = lifecycle_chunk_id.as_deref() { + chunk_store::set_chunk_lifecycle_status_tx( + &tx, + cid, + chunk_store::CHUNK_STATUS_BUFFERED, + )?; + } + } + tx.commit()?; + Ok(should_seal) + })?; + + Ok(Some(AppendDecision { + tree_id, + should_seal, + })) + } + + /// Ported from `handle_seal`: seal exactly one buffer level. Returns `None` + /// for the crate to enqueue as the parent — the host `seal_one_level` + /// (called with `enqueue_follow_ups = true`) drives the cascade itself by + /// enqueuing the summary's append + parent seal into the shared + /// `mem_tree_jobs` table, which the crate's `run_once` then claims (identical + /// schema, parity P4). A no-op (missing tree, empty buffer, gate not met) + /// also returns `None`. *(Transitional: once the crate tree cascade is + /// adopted in W5 this should switch to `enqueue_follow_ups = false` and + /// return the parent `SealPayload` for the crate to enqueue.)* + async fn seal_level( + &self, + _config: &MemoryConfig, + payload: &SealPayload, + ) -> anyhow::Result> { + let Some(tree) = trees_store::get_tree(&*self.config, &payload.tree_id)? else { + return Ok(None); + }; + let buf = trees_store::get_buffer(&*self.config, &tree.id, payload.level)?; + let forced = payload.force_now_ms.is_some(); + let memory_config = + super::memory_config_from(&*self.config, self.config.workspace_dir().clone()); + if buf.is_empty() + || (!forced && !tinycortex::memory::tree::should_seal(&memory_config, &buf)) + { + return Ok(None); + } + let strategy = TreeFactory::from_tree(&tree).label_strategy(&*self.config); + let summary_id = + super::seal_tree_level(&*self.config, &tree, &buf, &strategy, true).await?; + // Best-effort: rewrite the sealed summary's on-disk obsidian tags. Entity + // rows were committed inside seal_one_level, so they are visible here. + if let Err(e) = content_store::update_summary_tags(&*self.config, &summary_id) { + log::warn!( + "[tinycortex::queue_driver] update_summary_tags failed for summary_id={summary_id}: {e:#}" + ); + } + Ok(None) + } + + /// Ported from `handle_flush_stale`: list L0/summary buffers older than + /// `max_age_secs` that the crate should force-seal. + async fn list_stale_buffers( + &self, + _config: &MemoryConfig, + max_age_secs: i64, + ) -> anyhow::Result> { + let cutoff = chrono::Utc::now() - chrono::Duration::seconds(max_age_secs); + let buffers = trees_store::list_stale_buffers(&*self.config, cutoff)?; + Ok(buffers + .into_iter() + .map(|b| StaleBuffer { + tree_id: b.tree_id, + level: b.level, + }) + .collect()) + } + + /// Ported from `handle_seal_document`: build/rebuild one document version's + /// per-doc subtree and merge its doc-root into the connection tree. + async fn seal_document( + &self, + _config: &MemoryConfig, + payload: &SealDocumentPayload, + ) -> anyhow::Result<()> { + if payload.chunk_ids.is_empty() { + return Ok(()); + } + // One physical tree per connection scope (e.g. notion:{connection_id}). + let tree = get_or_create_source_tree(&*self.config, &payload.tree_scope)?; + let strategy = TreeFactory::from_tree(&tree).label_strategy(&*self.config); + super::seal_document_subtree( + &*self.config, + &tree, + &payload.doc_id, + payload.version_ms, + &payload.chunk_ids, + &strategy, + ) + .await?; + Ok(()) + } + + /// Ported from `handle_reembed_backfill`: embed one bounded batch of + /// chunks/summaries lacking a vector at `signature`. Maps the host handler's + /// control flow onto [`ReembedProgress`] (the crate's `handle_reembed_backfill` + /// turns `Wrote{more_pending:true}` into `Defer` and the terminal variants + /// into `Done`). + async fn reembed_batch( + &self, + _config: &MemoryConfig, + signature: &str, + ) -> anyhow::Result { + let config = &*self.config; + let active_sig = chunk_store::tree_active_signature(config); + if active_sig != signature { + // The embedder changed since this chain started — a fresh chain for + // the new signature supersedes it. + return Ok(ReembedProgress::StaleSignature); + } + + // Phase 1: up to BATCH ids lacking a sidecar vector at the active + // signature (excluding persistently-tombstoned rows) — chunks first, + // then summaries to fill the batch. + let (chunk_ids, summary_ids): (Vec, Vec) = + chunk_store::with_connection(config, |conn| { + let chunks: Vec = { + let mut stmt = conn.prepare( + "SELECT id FROM mem_tree_chunks c + WHERE NOT EXISTS ( + SELECT 1 FROM mem_tree_chunk_embeddings e + WHERE e.chunk_id = c.id AND e.model_signature = ?1) + AND NOT EXISTS ( + SELECT 1 FROM mem_tree_chunk_reembed_skipped s + WHERE s.chunk_id = c.id AND s.model_signature = ?1) + LIMIT ?2", + )?; + let ids = stmt + .query_map( + rusqlite::params![active_sig, REEMBED_BACKFILL_BATCH as i64], + |r| r.get::<_, String>(0), + )? + .collect::>>()?; + ids + }; + let remaining = REEMBED_BACKFILL_BATCH.saturating_sub(chunks.len()); + let summaries: Vec = if remaining == 0 { + Vec::new() + } else { + let mut stmt = conn.prepare( + "SELECT id FROM mem_tree_summaries s + WHERE s.deleted = 0 + AND NOT EXISTS ( + SELECT 1 FROM mem_tree_summary_embeddings e + WHERE e.summary_id = s.id AND e.model_signature = ?1) + AND NOT EXISTS ( + SELECT 1 FROM mem_tree_summary_reembed_skipped sk + WHERE sk.summary_id = s.id AND sk.model_signature = ?1) + LIMIT ?2", + )?; + let ids = stmt + .query_map(rusqlite::params![active_sig, remaining as i64], |r| { + r.get::<_, String>(0) + })? + .collect::>>()?; + ids + }; + Ok((chunks, summaries)) + })?; + + if chunk_ids.is_empty() && summary_ids.is_empty() { + return Ok(ReembedProgress::Covered); + } + + // Phase 2: WRITE-path embedder. A missing/unusable provider skips (rows + // stay re-embeddable) rather than poisoning recall with inert vectors. + let embedder = match build_write_embedder(config).context("build embedder in reembed")? { + Some(e) => e, + None => return Ok(ReembedProgress::NoProvider), + }; + let chunk_vecs = reembed_collect( + config, + embedder.as_ref(), + &active_sig, + &chunk_ids, + "chunk", + content_read::read_chunk_body, + try_mark_chunk_reembed_skipped, + ) + .await?; + let summary_vecs = reembed_collect( + config, + embedder.as_ref(), + &active_sig, + &summary_ids, + "summary", + content_read::read_summary_body, + try_mark_summary_reembed_skipped, + ) + .await?; + + // Phase 3: persist all collected vectors to the sidecars in one tx. + chunk_store::with_connection(config, |conn| { + let tx = conn.unchecked_transaction()?; + for (id, v) in &chunk_vecs { + chunk_store::set_chunk_embedding_for_signature_tx(&tx, id, &active_sig, v)?; + } + for (id, v) in &summary_vecs { + trees_store::set_summary_embedding_for_signature_tx(&tx, id, &active_sig, v)?; + } + tx.commit()?; + Ok(()) + })?; + + // This batch was bounded — more rows may remain; revisit. + Ok(ReembedProgress::Wrote { more_pending: true }) + } + + /// The active embedding-space signature the queue re-embed switch-path keys + /// on — the config-derived `provider={};model={};dims={}` string (P10). + fn active_signature(&self, _config: &MemoryConfig) -> String { + chunk_store::tree_active_signature(&*self.config) + } + + /// Whether any chunk/summary still lacks a vector at `signature` — the + /// coverage probe the re-embed backfill trigger uses (ported from + /// `memory_queue::ops::ensure_reembed_backfill`). + fn has_uncovered_reembed_work( + &self, + _config: &MemoryConfig, + signature: &str, + ) -> anyhow::Result { + chunk_store::with_connection(&*self.config, |conn| { + Ok(chunk_store::has_uncovered_reembed_work(conn, signature)?) + }) + } +} + +#[cfg(test)] +mod tests { + // Engine/queue types (`QueueDelegates`, `MemoryConfig`, the payload types, + // `async_trait`) come through `super::*` from the module-level imports. + use super::*; + + fn sqlite_failure(code: rusqlite::ErrorCode, extended: i32, msg: &str) -> anyhow::Error { + anyhow::Error::from(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error { + code, + extended_code: extended, + }, + Some(msg.into()), + )) + } + + #[test] + fn busy_backs_off_one_second_silently() { + let a = classify_worker_error(&sqlite_failure( + rusqlite::ErrorCode::DatabaseBusy, + 5, + "database is locked", + )); + assert_eq!(a.backoff, Duration::from_secs(1)); + assert_eq!(a.report, WorkerReport::Silent); + assert!(!a.mark_degraded && !a.recover_corrupt); + } + + #[test] + fn transient_io_backs_off_thirty_seconds_silently() { + let a = classify_worker_error(&sqlite_failure( + rusqlite::ErrorCode::SystemIoFailure, + 1546, + "disk I/O error", + )); + assert_eq!(a.backoff, Duration::from_secs(30)); + assert_eq!(a.report, WorkerReport::Silent); + } + + #[test] + fn disk_full_backs_off_long_and_silent() { + let a = classify_worker_error(&sqlite_failure( + rusqlite::ErrorCode::DiskFull, + 13, + "database or disk is full", + )); + assert_eq!(a.backoff, Duration::from_secs(300)); + assert_eq!(a.report, WorkerReport::Silent); + assert!(!a.mark_degraded && !a.recover_corrupt); + } + + #[test] + fn corrupt_drives_recovery_not_a_direct_page() { + let a = classify_worker_error(&sqlite_failure( + rusqlite::ErrorCode::DatabaseCorrupt, + 11, + "database disk image is malformed", + )); + assert_eq!(a.backoff, Duration::from_secs(300)); + assert!(a.recover_corrupt, "corrupt must drive quarantine+rebuild"); + assert_eq!( + a.report, + WorkerReport::Silent, + "recovery owns the report-once latch" + ); + assert!(!a.mark_degraded); + } + + #[test] + fn host_io_marks_degraded_and_reports_once() { + let a = classify_worker_error(&anyhow::Error::from(std::io::Error::from_raw_os_error(5))); + assert_eq!(a.backoff, Duration::from_secs(300)); + assert!( + a.mark_degraded, + "host-FS failure must flip storage-degraded" + ); + assert_eq!(a.report, WorkerReport::Once("tree_jobs_worker_host_io")); + assert!(!a.recover_corrupt); + } + + #[test] + fn unknown_error_reports_every_time_short_backoff() { + let a = classify_worker_error(&anyhow::anyhow!("upstream returned 500")); + assert_eq!(a.backoff, Duration::from_secs(1)); + assert_eq!(a.report, WorkerReport::Always("tree_jobs_worker")); + assert!(!a.mark_degraded && !a.recover_corrupt); + } + + /// A minimal host-side [`QueueDelegates`] — proves the host can satisfy the + /// crate trait (all delegate arg/return types resolve) and that the host can + /// drive `queue::run_once` end-to-end. The real engine bridge lands with the + /// W4 delegates brick; this no-op stands in so the driver integration is + /// exercised now. + struct NoopDelegates; + + #[async_trait] + impl QueueDelegates for NoopDelegates { + async fn extract_chunk( + &self, + _config: &MemoryConfig, + _chunk_id: &str, + ) -> anyhow::Result> { + Ok(None) + } + async fn append_node( + &self, + _config: &MemoryConfig, + _node: &NodeRef, + _target: &AppendTarget, + ) -> anyhow::Result> { + Ok(None) + } + async fn seal_level( + &self, + _config: &MemoryConfig, + _payload: &SealPayload, + ) -> anyhow::Result> { + Ok(None) + } + async fn list_stale_buffers( + &self, + _config: &MemoryConfig, + _max_age_secs: i64, + ) -> anyhow::Result> { + Ok(Vec::new()) + } + async fn seal_document( + &self, + _config: &MemoryConfig, + _payload: &SealDocumentPayload, + ) -> anyhow::Result<()> { + Ok(()) + } + async fn reembed_batch( + &self, + _config: &MemoryConfig, + _signature: &str, + ) -> anyhow::Result { + Ok(ReembedProgress::Covered) + } + fn active_signature(&self, _config: &MemoryConfig) -> String { + "provider=inert;model=none;dims=0".to_string() + } + fn has_uncovered_reembed_work( + &self, + _config: &MemoryConfig, + _signature: &str, + ) -> anyhow::Result { + Ok(false) + } + } + + /// End-to-end smoke: the host can drive the crate queue. An empty workspace + /// queue → `run_once` claims nothing → `Ok(false)`, and initialising the + /// chunk DB along the way does not error. + #[tokio::test] + async fn host_drives_run_once_on_empty_queue() { + let tmp = tempfile::tempdir().expect("tempdir"); + let mc = MemoryConfig::new(tmp.path()); + let processed = tinycortex::memory::queue::run_once(&mc, &NoopDelegates) + .await + .expect("run_once on empty queue"); + assert!(!processed, "empty queue processes nothing"); + } + + fn host_delegates_on_tempdir() -> (tempfile::TempDir, HostQueueDelegates) { + crate::test_seams::init(); + let tmp = tempfile::tempdir().expect("tempdir"); + let mut config = tinymemory_api::host::test_support::TestHostConfig::default(); + config.workspace_dir = tmp.path().to_path_buf(); + ( + tmp, + HostQueueDelegates::new(std::sync::Arc::new(config) as std::sync::Arc), + ) + } + + /// The self-contained `HostQueueDelegates` methods bind to the real host + /// engine and run on a fresh workspace: the signature is non-empty, and an + /// empty workspace reports no uncovered re-embed work and no stale buffers. + #[tokio::test] + async fn host_delegates_selfcontained_methods_bind_and_run() { + let (tmp, d) = host_delegates_on_tempdir(); + let mc = MemoryConfig::new(tmp.path()); + + let sig = d.active_signature(&mc); + assert!(!sig.is_empty(), "active signature should be non-empty"); + + assert!( + !d.has_uncovered_reembed_work(&mc, &sig) + .expect("coverage probe"), + "a fresh workspace has no uncovered re-embed work" + ); + + assert!( + d.list_stale_buffers(&mc, 3600) + .await + .expect("list stale buffers") + .is_empty(), + "a fresh workspace has no stale buffers" + ); + } + + /// `extract_chunk` / `append_node` are ported: on a missing chunk row they + /// are a no-op (`Ok(None)`), matching the legacy handlers' "row vanished + /// between enqueue and claim" path. + #[tokio::test] + async fn host_delegates_extract_and_append_missing_chunk_are_noop() { + let (tmp, d) = host_delegates_on_tempdir(); + let mc = MemoryConfig::new(tmp.path()); + assert!(d + .extract_chunk(&mc, "nonexistent") + .await + .expect("extract_chunk") + .is_none()); + assert!(d + .append_node( + &mc, + &NodeRef::Leaf { + chunk_id: "nonexistent".into() + }, + &AppendTarget::Source { + source_id: "s".into() + }, + ) + .await + .expect("append_node") + .is_none()); + } + + /// `reembed_batch` is ported: a job signature that differs from the config's + /// active embedding signature is superseded (`StaleSignature`), exactly as + /// the legacy `handle_reembed_backfill` finished a stale chain — and this + /// path returns before touching the worklist SQL. + #[tokio::test] + async fn host_delegates_reembed_batch_supersedes_stale_signature() { + let (tmp, d) = host_delegates_on_tempdir(); + let mc = MemoryConfig::new(tmp.path()); + let progress = d + .reembed_batch(&mc, "provider=stale-does-not-match;model=old;dims=1") + .await + .expect("reembed_batch stale path"); + assert!(matches!(progress, ReembedProgress::StaleSignature)); + } + + /// The ported seal methods handle empty/missing state without error: an + /// empty document version is a no-op, and sealing a level of a tree that + /// doesn't exist yields no parent to cascade. + #[tokio::test] + async fn host_delegates_seal_methods_handle_empty_state() { + let (tmp, d) = host_delegates_on_tempdir(); + let mc = MemoryConfig::new(tmp.path()); + + d.seal_document( + &mc, + &SealDocumentPayload { + tree_scope: "notion:conn".into(), + doc_id: "notion:conn:page".into(), + version_ms: Some(1), + chunk_ids: vec![], + }, + ) + .await + .expect("seal_document on an empty version is a no-op"); + + let parent = d + .seal_level( + &mc, + &SealPayload { + tree_id: "nonexistent-tree".into(), + level: 0, + force_now_ms: None, + }, + ) + .await + .expect("seal_level on a missing tree"); + assert!(parent.is_none(), "missing tree has no parent to cascade"); + } +} diff --git a/core/src/tinycortex/seal.rs b/core/src/tinycortex/seal.rs new file mode 100644 index 0000000..d614da4 --- /dev/null +++ b/core/src/tinycortex/seal.rs @@ -0,0 +1,266 @@ +//! Product compute and notification adapters for tinycortex sealing. + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use chrono::Duration; + +#[cfg(feature = "memory-git")] +use crate::store::content::wiki_git::{SummaryCommitBatch, SummaryCommitEntry}; +use crate::store::trees::types::{Buffer, SummaryNode, Tree}; +use crate::tree::score::embed::{build_write_embedder, Embedder as HostEmbedder}; +use crate::tree::tree::bucket_seal::LabelStrategy; +use crate::Config; + +use super::{memory_config_from, HostSummariser}; + +struct EmbedderBridge<'a>(&'a dyn HostEmbedder); + +#[async_trait] +impl tinycortex::memory::score::embed::Embedder for EmbedderBridge<'_> { + fn name(&self) -> &'static str { + self.0.name() + } + + async fn embed(&self, text: &str) -> Result> { + let vector = self.0.embed(text).await.map_err(|error| { + let failure = crate::tree::health::classify_embed_error(&error); + // Correlation is the embedder identity + typed outcome only — never + // the raw provider error, endpoint, or the text being embedded. + log::debug!( + "[memory_tree::seal] action=classify_embed_failure embedder={} code={} class={}", + self.0.name(), + failure.code.as_str(), + failure.class.as_str() + ); + // #5354: name the local-runtime fix on the status panel now rather + // than after the retry budget drains. + crate::tree::health::mark_local_model_unavailable_if_applicable(&failure); + anyhow::Error::new(failure).context(format!("seal embedding failed: {error:#}")) + })?; + crate::tree::score::embed::pack_checked(&vector) + .context("seal embedding dimension check")?; + crate::tree::health::clear_semantic_recall_degraded(); + Ok(vector) + } +} + +struct Observer<'a> { + // Only read by the `memory-git` `summary_committed` impl below; with the + // feature off that impl is a no-op and never touches `config`. + #[cfg_attr(not(feature = "memory-git"), allow(dead_code))] + config: &'a Config, +} + +impl tinycortex::memory::tree::SealObserver for Observer<'_> { + fn progress(&self, tree: &Tree, step: &str, level: u32, item_count: Option) { + crate::events::publish(crate::events::MemoryEvent::TreeBuildProgress { + phase: "seal".to_string(), + step: step.to_string(), + tree_scope: Some(tree.scope.clone()), + level: Some(level), + item_count, + detail: None, + }); + } + + /// Record a sealed summary in the git wiki mirror. + /// + /// A no-op without `memory-git`: the summary's own content file is written + /// by the caller either way, and this only mirrors it into the git ledger. + /// Returning `Ok(())` is therefore accurate rather than lenient — nothing + /// the caller depends on failed to happen. + #[cfg(not(feature = "memory-git"))] + fn summary_committed( + &self, + _tree: &Tree, + _node: &SummaryNode, + _content_path: &str, + _reason: &str, + ) -> Result<()> { + Ok(()) + } + + #[cfg(feature = "memory-git")] + fn summary_committed( + &self, + tree: &Tree, + node: &SummaryNode, + content_path: &str, + reason: &str, + ) -> Result<()> { + crate::store::content::wiki_git::commit_summaries( + &self.config.memory_tree_content_root(), + &SummaryCommitBatch { + reason: reason.to_string(), + tree_id: tree.id.clone(), + tree_scope: tree.scope.clone(), + entries: vec![SummaryCommitEntry { + summary_id: node.id.clone(), + content_path: content_path.to_string(), + level: node.level, + child_count: node.child_ids.len(), + token_count: node.token_count, + time_range_start: node.time_range_start, + time_range_end: node.time_range_end, + }], + }, + ) + } +} + +pub async fn seal_one_level( + config: &Config, + tree: &Tree, + buffer: &Buffer, + strategy: &LabelStrategy, + enqueue_follow_ups: bool, +) -> Result { + if let Err(error) = crate::store::content::obsidian::ensure_obsidian_defaults( + &config.memory_tree_content_root(), + ) { + log::warn!("[tree::bucket_seal] obsidian defaults failed: {error:#}"); + } + let host_embedder = build_write_embedder(config)?; + let embedder_bridge = host_embedder.as_deref().map(EmbedderBridge); + let summariser = HostSummariser::new(config.to_arc()); + let observer = Observer { config }; + let strategy = match strategy { + LabelStrategy::ExtractFromContent(extractor) => { + tinycortex::memory::tree::LabelStrategy::ExtractFromContent(extractor.clone()) + } + LabelStrategy::UnionFromChildren => { + tinycortex::memory::tree::LabelStrategy::UnionFromChildren + } + LabelStrategy::Empty => tinycortex::memory::tree::LabelStrategy::Empty, + }; + tinycortex::memory::tree::seal_one_level_with_services( + &memory_config_from(config, config.workspace_dir().clone()), + tree, + buffer, + &tinycortex::memory::tree::SealServices { + summariser: &summariser, + embedder: embedder_bridge + .as_ref() + .map(|bridge| bridge as &dyn tinycortex::memory::score::embed::Embedder), + observer: &observer, + }, + &strategy, + enqueue_follow_ups, + ) + .await +} + +pub async fn seal_document_subtree( + config: &Config, + tree: &Tree, + doc_id: &str, + version_ms: Option, + chunk_ids: &[String], + strategy: &LabelStrategy, +) -> Result { + if let Err(error) = crate::store::content::obsidian::ensure_obsidian_defaults( + &config.memory_tree_content_root(), + ) { + log::warn!("[tree::bucket_seal] obsidian defaults failed: {error:#}"); + } + let host_embedder = build_write_embedder(config)?; + let embedder_bridge = host_embedder.as_deref().map(EmbedderBridge); + let summariser = HostSummariser::new(config.to_arc()); + let observer = Observer { config }; + let strategy = match strategy { + LabelStrategy::ExtractFromContent(extractor) => { + tinycortex::memory::tree::LabelStrategy::ExtractFromContent(extractor.clone()) + } + LabelStrategy::UnionFromChildren => { + tinycortex::memory::tree::LabelStrategy::UnionFromChildren + } + LabelStrategy::Empty => tinycortex::memory::tree::LabelStrategy::Empty, + }; + tinycortex::memory::tree::seal_document_subtree_with_services( + &memory_config_from(config, config.workspace_dir().clone()), + tree, + doc_id, + version_ms, + chunk_ids, + &tinycortex::memory::tree::SealServices { + summariser: &summariser, + embedder: embedder_bridge + .as_ref() + .map(|bridge| bridge as &dyn tinycortex::memory::score::embed::Embedder), + observer: &observer, + }, + &strategy, + ) + .await +} + +pub async fn cascade_tree( + config: &Config, + tree: &Tree, + start_level: u32, + force: bool, + strategy: &LabelStrategy, +) -> Result> { + let host_embedder = build_write_embedder(config)?; + let embedder_bridge = host_embedder.as_deref().map(EmbedderBridge); + let summariser = HostSummariser::new(config.to_arc()); + let observer = Observer { config }; + let strategy = match strategy { + LabelStrategy::ExtractFromContent(extractor) => { + tinycortex::memory::tree::LabelStrategy::ExtractFromContent(extractor.clone()) + } + LabelStrategy::UnionFromChildren => { + tinycortex::memory::tree::LabelStrategy::UnionFromChildren + } + LabelStrategy::Empty => tinycortex::memory::tree::LabelStrategy::Empty, + }; + tinycortex::memory::tree::cascade_all_from_with_services( + &memory_config_from(config, config.workspace_dir().clone()), + tree, + start_level, + force, + &tinycortex::memory::tree::SealServices { + summariser: &summariser, + embedder: embedder_bridge + .as_ref() + .map(|bridge| bridge as &dyn tinycortex::memory::score::embed::Embedder), + observer: &observer, + }, + &strategy, + false, + ) + .await +} + +pub async fn flush_stale_tree_buffers( + config: &Config, + max_age: Duration, + strategy: &LabelStrategy, +) -> Result { + let host_embedder = build_write_embedder(config)?; + let embedder_bridge = host_embedder.as_deref().map(EmbedderBridge); + let summariser = HostSummariser::new(config.to_arc()); + let observer = Observer { config }; + let strategy = match strategy { + LabelStrategy::ExtractFromContent(extractor) => { + tinycortex::memory::tree::LabelStrategy::ExtractFromContent(extractor.clone()) + } + LabelStrategy::UnionFromChildren => { + tinycortex::memory::tree::LabelStrategy::UnionFromChildren + } + LabelStrategy::Empty => tinycortex::memory::tree::LabelStrategy::Empty, + }; + tinycortex::memory::tree::flush_stale_buffers_with_services( + &memory_config_from(config, config.workspace_dir().clone()), + max_age, + &tinycortex::memory::tree::SealServices { + summariser: &summariser, + embedder: embedder_bridge + .as_ref() + .map(|bridge| bridge as &dyn tinycortex::memory::score::embed::Embedder), + observer: &observer, + }, + &strategy, + ) + .await +} diff --git a/core/src/tinycortex/summariser.rs b/core/src/tinycortex/summariser.rs new file mode 100644 index 0000000..c975c82 --- /dev/null +++ b/core/src/tinycortex/summariser.rs @@ -0,0 +1,62 @@ +//! OpenHuman LLM adapter for tinycortex tree summarization. + +use async_trait::async_trait; +use std::sync::Arc; +use tinycortex::memory::tree::{ + Summariser, SummaryCall, SummaryContext, SummaryInput, SummaryOutput, +}; + +use crate::Config; + +#[derive(Clone)] +pub struct HostSummariser { + config: Arc, +} + +impl HostSummariser { + pub fn new(config: Arc) -> Self { + Self { config } + } + + async fn call( + &self, + inputs: &[SummaryInput], + context: &SummaryContext<'_>, + ) -> anyhow::Result { + let output = crate::tree::summarise::summarise(&*self.config, inputs, context).await?; + Ok(SummaryCall { + output: SummaryOutput { + content: output.content, + token_count: output.token_count, + entities: output.entities, + topics: output.topics, + }, + input_tokens: output.input_tokens, + output_tokens: output.output_tokens, + charged_amount_usd: output.charged_amount_usd, + }) + } +} + +#[async_trait] +impl Summariser for HostSummariser { + fn name(&self) -> &str { + "openhuman" + } + + async fn summarise( + &self, + inputs: &[SummaryInput], + context: &SummaryContext<'_>, + ) -> anyhow::Result { + Ok(self.call(inputs, context).await?.output) + } + + async fn summarise_with_usage( + &self, + inputs: &[SummaryInput], + context: &SummaryContext<'_>, + ) -> anyhow::Result { + self.call(inputs, context).await + } +} diff --git a/core/src/tinycortex/sync.rs b/core/src/tinycortex/sync.rs new file mode 100644 index 0000000..1531740 --- /dev/null +++ b/core/src/tinycortex/sync.rs @@ -0,0 +1,837 @@ +//! OpenHuman service adapters for tinycortex live synchronization. + +use async_trait::async_trait; +use std::sync::Arc; +use tinycortex::memory::sync::{ + ClickUpSyncPipeline, ComposioClient, ExternalSourceReader, GitHubSyncPipeline, + GithubRepoSyncPipeline, GmailSyncPipeline, LinearSyncPipeline, LocalDocument, + LocalDocumentSink, NotionSyncPipeline, SkillDocSink, SkillDocument, + SlackSearchBackfillPipeline, SlackSyncPipeline, SyncContext, SyncDispatcher, SyncEvent, + SyncEventSink, SyncOutcome, SyncPipeline, SyncStage, SyncStateStore, WorkspaceSourcePipeline, +}; + +use crate::sources::{MemorySourceEntry, SourceKind}; +use crate::store::MemoryClientRef; +use crate::Config; + +pub const HOST_SYNC_STATE_NAMESPACE: &str = "composio-sync-state"; +pub use tinycortex::memory::sync::{ + RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, SyncAuditEntry, +}; + +pub struct HostSyncAdapter { + memory: MemoryClientRef, + config: Option>, +} + +#[derive(Debug)] +pub struct SourcePipelineFailure { + pub message: String, + pub actions_called: u32, + pub provider_cost_usd: f64, +} + +impl std::fmt::Display for SourcePipelineFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl SourcePipelineFailure { + fn without_usage(message: impl Into) -> Self { + Self { + message: message.into(), + actions_called: 0, + provider_cost_usd: 0.0, + } + } +} + +impl HostSyncAdapter { + pub fn new(memory: MemoryClientRef) -> Self { + Self { + memory, + config: None, + } + } + + fn with_config(memory: MemoryClientRef, config: Arc) -> Self { + Self { + memory, + config: Some(config), + } + } +} + +/// Append one host sync audit record, logging failures without exposing source identifiers. +pub fn append_audit_entry(config: &Config, entry: &SyncAuditEntry) { + tracing::debug!( + source_kind = %entry.source_kind, + success = entry.success, + items_fetched = entry.items_fetched, + "[tinycortex:sync] audit append starting" + ); + let memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + match tinycortex::memory::sync::append_audit_entry(&memory_config, entry) { + Ok(()) => tracing::debug!( + source_kind = %entry.source_kind, + success = entry.success, + "[tinycortex:sync] audit append completed" + ), + Err(error) => { + tracing::warn!(%error, source_kind = %entry.source_kind, "[tinycortex:sync] audit append failed"); + } + } +} + +/// Read persisted sync audit records while preserving storage failures for fail-closed callers. +pub fn try_read_audit_log(config: &Config) -> anyhow::Result> { + tracing::debug!("[tinycortex:sync] audit read starting"); + let memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + let entries = tinycortex::memory::sync::read_audit_log(&memory_config).map_err(|error| { + tracing::warn!(%error, "[tinycortex:sync] audit read failed"); + error + })?; + tracing::debug!( + entries = entries.len(), + "[tinycortex:sync] audit read completed" + ); + Ok(entries) +} + +/// Read persisted sync audit records for best-effort RPC and reporting surfaces. +pub fn read_audit_log(config: &Config) -> Vec { + try_read_audit_log(config).unwrap_or_default() +} + +/// Estimate sync inference cost using TinyCortex's canonical pricing model. +pub fn estimate_cost_usd(input_tokens: u64, output_tokens: u64) -> f64 { + tinycortex::memory::sync::estimate_cost_usd(input_tokens, output_tokens) +} + +/// Measure coverage of a raw archive by its TinyCortex memory tree. +pub fn raw_coverage( + config: &Config, + tree_scope: &str, + archive_source_id: &str, +) -> anyhow::Result { + tracing::debug!("[tinycortex:sync] raw coverage scan starting"); + let memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + let coverage = + tinycortex::memory::sync::raw_coverage(&memory_config, tree_scope, archive_source_id) + .map_err(|error| { + tracing::warn!(%error, "[tinycortex:sync] raw coverage scan failed"); + error + })?; + tracing::debug!( + total = coverage.total, + covered = coverage.covered, + pending = coverage.pending.len(), + "[tinycortex:sync] raw coverage scan completed" + ); + Ok(coverage) +} + +/// Return whether a raw archive contains records absent from its memory tree. +pub fn needs_rebuild(config: &Config, tree_scope: &str, archive_source_id: &str) -> bool { + let memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + let required = + tinycortex::memory::sync::needs_rebuild(&memory_config, tree_scope, archive_source_id); + tracing::debug!( + required, + "[tinycortex:sync] raw rebuild requirement evaluated" + ); + required +} + +/// Rebuild a memory tree from its raw archive through the host summarizer. +pub async fn rebuild_tree_from_raw( + config: &Config, + tree_scope: &str, + archive_source_id: &str, +) -> anyhow::Result { + tracing::info!("[tinycortex:sync] raw rebuild starting"); + let memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + let summariser = super::HostSummariser::new(config.to_arc()); + let outcome = tinycortex::memory::sync::rebuild_tree_from_raw( + &memory_config, + tree_scope, + archive_source_id, + &summariser, + ) + .await + .map_err(|error| { + tracing::warn!(%error, "[tinycortex:sync] raw rebuild failed"); + error + })?; + tracing::info!( + files_read = outcome.files_read, + batches = outcome.batches, + "[tinycortex:sync] raw rebuild completed" + ); + Ok(outcome) +} + +/// Run a registered GitHub repository source through TinyCortex synchronization. +pub async fn run_github_sync( + source: &MemorySourceEntry, + config: &Config, +) -> anyhow::Result { + tracing::info!("[tinycortex:sync] GitHub repository sync starting"); + if crate::global::client_if_ready().is_none() { + tracing::debug!("[tinycortex:sync] GitHub sync initializing memory client"); + crate::global::init(config.workspace_dir().clone()) + .map_err(anyhow::Error::msg) + .map_err(|error| { + tracing::warn!(%error, "[tinycortex:sync] GitHub sync memory initialization failed"); + error + })?; + } + let outcome = run_source_pipeline(source, config) + .await + .map_err(|error| anyhow::anyhow!(error.to_string())) + .map_err(|error| { + tracing::warn!(%error, "[tinycortex:sync] GitHub repository sync failed"); + error + })?; + tracing::info!( + records_ingested = outcome.records_ingested, + more_pending = outcome.more_pending, + actions_called = outcome.actions_called, + "[tinycortex:sync] GitHub repository sync completed" + ); + Ok(outcome) +} + +#[async_trait] +impl ExternalSourceReader for HostSyncAdapter { + async fn list_items( + &self, + source: &tinycortex::memory::sources::MemorySourceEntry, + ) -> anyhow::Result> { + let config = self + .config + .as_ref() + .ok_or_else(|| anyhow::anyhow!("external source reader requires host config"))?; + let host_source: MemorySourceEntry = serde_json::from_value(serde_json::to_value(source)?)?; + let reader = crate::sources::readers::reader_for(&host_source.kind); + let items = reader + .list_items(&host_source, &**config) + .await + .map_err(anyhow::Error::msg)?; + serde_json::from_value(serde_json::to_value(items)?).map_err(Into::into) + } + + async fn read_item( + &self, + source: &tinycortex::memory::sources::MemorySourceEntry, + item_id: &str, + ) -> anyhow::Result { + let config = self + .config + .as_ref() + .ok_or_else(|| anyhow::anyhow!("external source reader requires host config"))?; + let host_source: MemorySourceEntry = serde_json::from_value(serde_json::to_value(source)?)?; + let reader = crate::sources::readers::reader_for(&host_source.kind); + let content = reader + .read_item(&host_source, item_id, &**config) + .await + .map_err(anyhow::Error::msg)?; + serde_json::from_value(serde_json::to_value(content)?).map_err(Into::into) + } +} + +pub fn sync_context(memory: MemoryClientRef) -> SyncContext { + let adapter = std::sync::Arc::new(HostSyncAdapter::new(memory)); + SyncContext { + events: adapter.clone(), + documents: adapter.clone(), + state: adapter, + local_documents: None, + external_sources: None, + summariser: None, + } +} + +fn source_sync_context(memory: MemoryClientRef, config: &Config, local: bool) -> SyncContext { + let adapter = std::sync::Arc::new(HostSyncAdapter::with_config(memory, config.to_arc())); + SyncContext { + events: adapter.clone(), + documents: adapter.clone(), + state: adapter.clone(), + local_documents: local.then(|| adapter.clone() as std::sync::Arc), + external_sources: local.then_some(adapter as std::sync::Arc), + summariser: local.then(|| { + std::sync::Arc::new(super::HostSummariser::new(config.to_arc())) + as std::sync::Arc + }), + } +} + +pub async fn run_source_pipeline( + source: &MemorySourceEntry, + config: &Config, +) -> Result { + let memory = crate::global::client_if_ready() + .ok_or_else(|| SourcePipelineFailure::without_usage("memory client is not ready"))?; + let mut memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + memory_config.sync.interval_secs = config.memory_sync_interval_secs(); + memory_config.sync.budget.max_items = source.max_items; + memory_config.sync.budget.max_tokens_per_sync = source.max_tokens_per_sync; + memory_config.sync.budget.max_cost_per_sync_usd = source.max_cost_per_sync_usd; + memory_config.sync.budget.sync_depth_days = source.sync_depth_days; + + let pipeline = build_pipeline(source, config, &mut memory_config) + .map_err(SourcePipelineFailure::without_usage)?; + let pipeline_id = pipeline.id().to_owned(); + let mut dispatcher = SyncDispatcher::new(); + dispatcher + .register(pipeline) + .map_err(|error| SourcePipelineFailure::without_usage(error.to_string()))?; + dispatcher + .tick( + &pipeline_id, + &memory_config, + &source_sync_context(memory, config, source.kind != SourceKind::Composio), + ) + .await + .map_err(|error| { + let usage = error.downcast_ref::(); + SourcePipelineFailure { + message: error.to_string(), + actions_called: usage.map_or(0, |error| error.actions_called), + provider_cost_usd: usage.map_or(0.0, |error| error.provider_cost_usd), + } + }) +} + +/// Run a Composio connection through tinycortex, preserving any source-level +/// budgets already configured in OpenHuman's registry. +pub async fn run_composio_connection( + toolkit: &str, + connection_id: &str, + config: &Config, +) -> Result { + run_composio_connection_with_budgets(toolkit, connection_id, config, None, None).await +} + +/// Run a Composio connection with request-scoped budget overrides. +/// +/// Provider RPCs carry these values in `ProviderContext`, before a source has +/// necessarily been persisted in the registry. Explicit values therefore take +/// precedence, while `None` preserves the registered/default source budget. +pub async fn run_composio_connection_with_budgets( + toolkit: &str, + connection_id: &str, + config: &Config, + max_items: Option, + sync_depth_days: Option, +) -> Result { + let mut source = crate::sources::decode_memory_sources(config) + .iter() + .find(|source| { + source.kind == SourceKind::Composio + && source.connection_id.as_deref() == Some(connection_id) + }) + .cloned() + .unwrap_or_else(|| { + let (max_items, sync_depth_days) = + crate::sources::memory_sync_defaults_for_toolkit(toolkit); + MemorySourceEntry { + id: format!("composio:{toolkit}:{connection_id}"), + kind: SourceKind::Composio, + label: format!("{toolkit} connection"), + enabled: true, + toolkit: Some(toolkit.to_ascii_lowercase()), + connection_id: Some(connection_id.to_string()), + path: None, + glob: None, + url: None, + branch: None, + paths: Vec::new(), + max_commits: None, + max_issues: None, + max_prs: None, + query: None, + since_days: None, + max_items, + selector: None, + max_tokens_per_sync: None, + max_cost_per_sync_usd: None, + sync_depth_days, + } + }); + + source.max_items = max_items; + source.sync_depth_days = sync_depth_days; + + tracing::debug!( + toolkit, + connection_id, + source_id = %source.id, + max_items = ?source.max_items, + sync_depth_days = ?source.sync_depth_days, + "[tinycortex:sync] dispatching Composio connection" + ); + run_source_pipeline(&source, config).await +} + +pub async fn load_composio_sync_state( + toolkit: &str, + connection_id: &str, +) -> anyhow::Result { + let memory = crate::global::client_if_ready() + .ok_or_else(|| anyhow::anyhow!("memory client is not ready"))?; + let adapter = HostSyncAdapter::new(memory); + tinycortex::memory::sync::SyncState::load(&adapter, toolkit, connection_id).await +} + +pub async fn run_slack_search_backfill( + connection_id: &str, + backfill_days: i64, + config: &Config, +) -> Result { + let memory = crate::global::client_if_ready() + .ok_or_else(|| SourcePipelineFailure::without_usage("memory client is not ready"))?; + let mut memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + let composio = composio_config(config).map_err(SourcePipelineFailure::without_usage)?; + memory_config.sync.composio = Some(composio.clone()); + let pipeline = std::sync::Arc::new(SlackSearchBackfillPipeline::new( + ComposioClient::new(composio), + connection_id, + backfill_days, + )); + let pipeline_id = pipeline.id().to_owned(); + let mut dispatcher = SyncDispatcher::new(); + dispatcher + .register(pipeline) + .map_err(|error| SourcePipelineFailure::without_usage(error.to_string()))?; + dispatcher + .tick( + &pipeline_id, + &memory_config, + &source_sync_context(memory, config, false), + ) + .await + .map_err(|error| SourcePipelineFailure::without_usage(error.to_string())) +} + +pub async fn run_gmail_backfill( + connection_id: &str, + query: &str, + max_pages: usize, + page_size: usize, + config: &Config, +) -> Result { + let memory = crate::global::client_if_ready() + .ok_or_else(|| SourcePipelineFailure::without_usage("memory client is not ready"))?; + let mut memory_config = super::memory_config_from(config, config.workspace_dir().clone()); + let composio = composio_config(config).map_err(SourcePipelineFailure::without_usage)?; + memory_config.sync.composio = Some(composio.clone()); + let pipeline = std::sync::Arc::new( + GmailSyncPipeline::new(ComposioClient::new(composio), connection_id) + .with_limits(max_pages, page_size) + .with_query(query), + ); + let pipeline_id = pipeline.id().to_owned(); + let mut dispatcher = SyncDispatcher::new(); + dispatcher + .register(pipeline) + .map_err(|error| SourcePipelineFailure::without_usage(error.to_string()))?; + dispatcher + .tick( + &pipeline_id, + &memory_config, + &source_sync_context(memory, config, false), + ) + .await + .map_err(|error| SourcePipelineFailure::without_usage(error.to_string())) +} + +/// Composio toolkit slugs that have a native memory-sync pipeline in +/// [`build_pipeline`] — the authoritative "can actually ingest into memory" set. +/// +/// This MUST stay in lockstep with the registered memory-sync providers +/// (`memory_sync::composio::all_composio_sync_providers`): the +/// `memory_sources.supported_toolkits` RPC advertises the provider registry, and +/// the `connection_created` auto-register gates on it, so any divergence +/// reintroduces the "connection reports ACTIVE but silently never ingests" +/// failure this guards against (#4957). The registry↔pipeline equality is pinned +/// by `composio_syncable_set_matches_provider_registry` in the tests below, and +/// the arms of the `match` in [`build_pipeline`] map 1:1 to these slugs. +pub fn syncable_composio_toolkits() -> &'static [&'static str] { + &["clickup", "github", "gmail", "linear", "notion", "slack"] +} + +/// Whether `toolkit` has a native memory-sync pipeline (case-insensitive). +/// Callers deciding *whether to offer/register* a Composio source should prefer +/// the provider registry (`get_composio_sync_provider`) so there is a single +/// advertised source of truth; this mirror exists for the sync layer itself. +pub fn is_composio_toolkit_syncable(toolkit: &str) -> bool { + let slug = toolkit.trim().to_ascii_lowercase(); + syncable_composio_toolkits().contains(&slug.as_str()) +} + +fn build_pipeline( + source: &MemorySourceEntry, + config: &Config, + memory_config: &mut tinycortex::memory::config::MemoryConfig, +) -> Result, String> { + if source.kind != SourceKind::Composio { + let crate_source: tinycortex::memory::sources::MemorySourceEntry = serde_json::from_value( + serde_json::to_value(source).map_err(|error| error.to_string())?, + ) + .map_err(|error| error.to_string())?; + if source.kind == SourceKind::GithubRepo { + return GithubRepoSyncPipeline::new(crate_source) + .map(|pipeline| std::sync::Arc::new(pipeline) as std::sync::Arc) + .map_err(|error| error.to_string()); + } + return WorkspaceSourcePipeline::new(crate_source) + .map(|pipeline| std::sync::Arc::new(pipeline) as std::sync::Arc) + .map_err(|error| error.to_string()); + } + + let toolkit = source + .toolkit + .as_deref() + .map(str::trim) + .filter(|toolkit| !toolkit.is_empty()) + .ok_or_else(|| "composio source missing toolkit".to_string())? + .to_ascii_lowercase(); + let connection_id = source + .connection_id + .as_deref() + .map(str::trim) + .filter(|connection_id| !connection_id.is_empty()) + .ok_or_else(|| "composio source missing connection_id".to_string())?; + // Fail closed *before* resolving credentials/client for any toolkit without a + // native pipeline. This keeps the unsupported-toolkit error identical to the + // match's fallback while making the syncable set a single, testable gate that + // stays pinned to the provider registry (#4957). + if !is_composio_toolkit_syncable(&toolkit) { + return Err(format!( + "tinycortex sync does not support toolkit '{toolkit}'" + )); + } + let composio = composio_config(config)?; + memory_config.sync.composio = Some(composio.clone()); + let client = ComposioClient::new(composio); + let pipeline: std::sync::Arc = match toolkit.as_str() { + "gmail" => std::sync::Arc::new(GmailSyncPipeline::new(client, connection_id)), + "github" => std::sync::Arc::new(GitHubSyncPipeline::new(client, connection_id)), + "notion" => std::sync::Arc::new(NotionSyncPipeline::new(client, connection_id)), + "linear" => std::sync::Arc::new(LinearSyncPipeline::new(client, connection_id)), + "clickup" => std::sync::Arc::new(ClickUpSyncPipeline::new(client, connection_id)), + "slack" => std::sync::Arc::new(SlackSyncPipeline::new(client, connection_id)), + _ => { + return Err(format!( + "tinycortex sync does not support toolkit '{toolkit}'" + )) + } + }; + Ok(pipeline) +} + +fn composio_config( + config: &Config, +) -> Result { + use tinycortex::memory::config::{ComposioMode, ComposioSyncConfig, SecretString}; + + if config.composio().mode.eq_ignore_ascii_case("direct") { + let api_key = crate::composio_host::api_key(config) + .or_else(|| config.composio().api_key.clone()) + .ok_or_else(|| "Composio direct API key is not configured".to_string())?; + Ok(ComposioSyncConfig { + mode: ComposioMode::Direct, + base_url: "https://backend.composio.dev/api/v3".into(), + api_key: Some(SecretString::new(api_key)), + bearer_token: None, + entity_id: Some(config.composio().entity_id.clone()), + }) + } else { + let bearer = config + .session_token()? + .ok_or_else(|| "OpenHuman backend bearer token is not configured".to_string())?; + Ok(ComposioSyncConfig { + mode: ComposioMode::Proxied, + base_url: config.effective_backend_api_url(), + api_key: None, + bearer_token: Some(SecretString::new(bearer)), + entity_id: Some(config.composio().entity_id.clone()), + }) + } +} + +#[async_trait] +impl SkillDocSink for HostSyncAdapter { + async fn store(&self, document: SkillDocument) -> anyhow::Result<()> { + tracing::debug!( + toolkit = %document.toolkit, + connection_id = %document.connection_id, + document_id = %document.document_id, + "[tinycortex:sync] storing synchronized document" + ); + self.memory + .store_skill_sync( + &document.namespace_skill_id, + &document.connection_id, + &document.title, + &document.content, + Some("tinycortex-sync".into()), + Some(document.metadata), + Some("medium".into()), + None, + None, + Some(document.document_id), + ) + .await + .map_err(anyhow::Error::msg) + } + + async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()> { + let namespace = format!("skill-{}", namespace_skill_id.trim()); + tracing::debug!( + namespace, + document_id, + "[tinycortex:sync] deleting synchronized document" + ); + self.memory + .delete_document(&namespace, document_id) + .await + .map(|_| ()) + .map_err(anyhow::Error::msg) + } +} + +#[async_trait] +impl LocalDocumentSink for HostSyncAdapter { + async fn upsert(&self, document: LocalDocument) -> anyhow::Result<()> { + let config = self + .config + .as_ref() + .ok_or_else(|| anyhow::anyhow!("local document sink missing host config"))?; + let input = tinycortex::memory::ingest::canonicalize::document::DocumentInput { + provider: "memory_sources:local".into(), + title: document.title, + body: document.body, + modified_at: document.modified_at, + source_ref: document.source_ref, + }; + crate::ingest_pipeline::ingest_document_with_scope( + &**config, + &document.source_id, + &document.owner, + document.tags, + input, + document.path_scope, + ) + .await + .map(|_| ()) + .map_err(anyhow::Error::msg) + } + + async fn delete(&self, source_id: &str) -> anyhow::Result<()> { + let config = self + .config + .clone() + .ok_or_else(|| anyhow::anyhow!("local document sink missing host config"))?; + let source_id = source_id.to_owned(); + tokio::task::spawn_blocking(move || { + crate::store::chunks::store::delete_chunks_by_source( + &*config, + crate::store::chunks::types::SourceKind::Document, + &source_id, + ) + }) + .await + .map_err(|error| anyhow::anyhow!("local delete task failed: {error}"))??; + Ok(()) + } +} + +#[async_trait] +impl SyncStateStore for HostSyncAdapter { + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { + self.memory + .kv_get(Some(namespace), key) + .await + .map_err(anyhow::Error::msg) + } + + async fn set( + &self, + namespace: &str, + key: &str, + value: &serde_json::Value, + ) -> anyhow::Result<()> { + self.memory + .kv_set(Some(namespace), key, value) + .await + .map_err(anyhow::Error::msg) + } +} + +#[async_trait] +impl SyncEventSink for HostSyncAdapter { + async fn emit(&self, event: SyncEvent) -> anyhow::Result<()> { + crate::events::publish(crate::events::MemoryEvent::SyncStageChanged { + trigger: "tinycortex".into(), + stage: stage_name(event.stage).into(), + provider: Some(event.toolkit), + connection_id: event.connection_id, + detail: event.message, + source_id: Some(event.source_id), + }); + Ok(()) + } +} + +fn stage_name(stage: SyncStage) -> &'static str { + match stage { + SyncStage::Requested => "requested", + SyncStage::Fetching => "fetching", + SyncStage::Stored => "stored", + SyncStage::Ingesting => "ingesting", + SyncStage::Completed => "completed", + SyncStage::Failed => "failed", + } +} + +#[cfg(test)] +mod tests { + use super::{ + build_pipeline, is_composio_toolkit_syncable, syncable_composio_toolkits, + try_read_audit_log, + }; + use crate::sources::MemorySourceEntry; + use crate::sync::composio::{get_composio_sync_provider, init_default_composio_sync_providers}; + + /// The advertised set (`memory_sources.supported_toolkits`, sourced from the + /// provider registry) and the syncable set (`build_pipeline`) must not + /// diverge: a toolkit that is advertised but has no pipeline reports ACTIVE + /// and then silently never ingests — the exact defect of #4957. + /// + /// Both directions are asserted against an explicit built-in slug set. The + /// provider registry is process-global and sibling tests register throwaway + /// providers into it without unregistering, so walking it directly would be + /// order-flaky; pinning the built-in set keeps this deterministic. + #[test] + fn advertised_and_syncable_toolkit_sets_cannot_diverge() { + init_default_composio_sync_providers(); + + // Every syncable toolkit must have a registered provider — otherwise it + // could never be advertised or auto-registered in the first place. + for &slug in syncable_composio_toolkits() { + assert!( + get_composio_sync_provider(slug).is_some(), + "syncable toolkit `{slug}` has no registered memory-sync provider" + ); + } + + // Every built-in provider shipped by `init_default_composio_sync_providers` + // must be syncable. This is the #4957 direction: advertising a provider + // that `build_pipeline` rejects is the silent failure we guard against. + // + // We pin the built-in slug set explicitly rather than walking + // `all_composio_sync_providers()`: that registry is process-global and + // sibling tests register throwaway providers into it that they never + // unregister (e.g. `provideronly` in composio/tools_tests.rs, `stub-no-active` + // in composio/identity.rs), so a raw registry walk fails nondeterministically + // depending on test execution order. A new built-in toolkit must be added to + // this list, to `syncable_composio_toolkits`, and to `build_pipeline` together + // — the assert_eq below fails loudly if the first two ever drift apart. + const BUILTIN_SYNC_PROVIDERS: &[&str] = + &["clickup", "github", "gmail", "linear", "notion", "slack"]; + + let mut builtin = BUILTIN_SYNC_PROVIDERS.to_vec(); + builtin.sort_unstable(); + let mut syncable = syncable_composio_toolkits().to_vec(); + syncable.sort_unstable(); + assert_eq!( + builtin, syncable, + "the built-in provider set and syncable set diverged — a provider is \ + advertised without a matching `build_pipeline` arm, or vice versa (#4957)" + ); + + for &slug in BUILTIN_SYNC_PROVIDERS { + assert!( + get_composio_sync_provider(slug).is_some(), + "built-in provider `{slug}` is not registered by \ + init_default_composio_sync_providers" + ); + assert!( + is_composio_toolkit_syncable(slug), + "built-in provider `{slug}` is advertised but has no build_pipeline arm — \ + it would report ACTIVE and silently fail to sync (#4957)" + ); + } + } + + /// Behavioural regression for #4957: an unsupported Composio toolkit is + /// rejected by `build_pipeline` *before* any credential/client resolution. + /// + /// We hand it a default `Config` (no Composio auth configured). If the gate + /// ran AFTER config resolution we would get a config error ("backend bearer + /// token is not configured" / "direct API key is not configured"); instead + /// we must get the unsupported-toolkit error, proving the fail-closed + /// ordering that stops an unsyncable toolkit from ever reaching a pipeline. + #[test] + fn build_pipeline_rejects_unsupported_toolkit_before_resolving_config() { + // `googlecalendar` is a real Composio toolkit with no native pipeline — + // exactly the prod case from #4957. + let source: MemorySourceEntry = serde_json::from_value(serde_json::json!({ + "id": "composio:googlecalendar:conn-1", + "kind": "composio", + "label": "googlecalendar connection", + "toolkit": "googlecalendar", + "connection_id": "conn-1", + })) + .expect("construct composio source"); + + let config = tinymemory_api::host::test_support::TestHostConfig::default(); + let mut memory_config = + tinycortex::memory::config::MemoryConfig::new("/tmp/openhuman-test-ws"); + + // `build_pipeline` returns `Result, String>`; the + // Ok arm is not `Debug`, so match rather than `expect_err`. + let err = match build_pipeline(&source, &config, &mut memory_config) { + Ok(_) => panic!("unsupported toolkit must be rejected before config resolution"), + Err(e) => e, + }; + assert!( + err.contains("does not support toolkit 'googlecalendar'"), + "expected the unsupported-toolkit error (proving rejection precedes \ + config resolution), got: {err}" + ); + } + + /// Locks the reported prod failures (googlecalendar / googlesheets) as + /// non-syncable, and pins case-insensitive/trimming behaviour. + #[test] + fn is_composio_toolkit_syncable_classifies_known_slugs() { + assert!(!is_composio_toolkit_syncable("googlecalendar")); + assert!(!is_composio_toolkit_syncable("googlesheets")); + assert!(!is_composio_toolkit_syncable("discord")); + assert!(!is_composio_toolkit_syncable("")); + assert!(is_composio_toolkit_syncable("gmail")); + assert!(is_composio_toolkit_syncable("Gmail")); + assert!(is_composio_toolkit_syncable(" slack ")); + } + + #[test] + fn fallible_audit_read_distinguishes_io_failure_from_empty_log() { + let workspace = tempfile::tempdir().expect("workspace"); + let audit_path = workspace.path().join("memory_tree/sync_audit.jsonl"); + std::fs::create_dir_all(&audit_path).expect("create directory at audit file path"); + + let mut config = tinymemory_api::host::test_support::TestHostConfig::default(); + config.workspace_dir = workspace.path().to_path_buf(); + + let error = try_read_audit_log(&config).expect_err("directory read must fail"); + assert!( + error.downcast_ref::().is_some(), + "expected the audit I/O error to remain distinguishable: {error:#}" + ); + } +} diff --git a/core/src/tool_memory/README.md b/core/src/tool_memory/README.md new file mode 100644 index 0000000..cab74b5 --- /dev/null +++ b/core/src/tool_memory/README.md @@ -0,0 +1,38 @@ +# memory_tools + +Tool-scoped memory: durable rules / learnings keyed per tool name. Distinct +from generic namespace memory and from `learning::tool_tracker` statistics. + +## Namespace convention + +Each tool gets its own namespace `tool-{tool_name}`. Build the string via the +`tool_memory_namespace` re-export — never hard-code it. + +## Layout + +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Module root + public re-exports. | +| [`mod.rs`](mod.rs) | Re-exports crate types/store and defines `tool_memory_store(Arc)`. | +| [`capture.rs`](capture.rs) | `ToolMemoryCaptureHook` — `PostTurnHook` impl that captures user edicts and repeated tool failures into the store (host-retained). | +| [`prompt.rs`](prompt.rs) | **Shim** — re-exports the crate `ToolMemoryRulesSection` + `render_tool_memory_rules` + `TOOL_MEMORY_HEADING`, and keeps the host `PromptSection` impl that plugs the section into the system-prompt builder. | +| [`tools/`](tools/) | Agent-facing read/write tools: `MemoryToolsListTool` (list rules for a tool), `MemoryToolsPutTool` (upsert a rule). | +| [`test_helpers.rs`](test_helpers.rs) | `#[cfg(test)]` `MockMemory` used by `capture::tests` (the store engine's own coverage lives in the crate). | + +## How it fits + +The agent harness: +1. **Reads** at session build — `ToolMemoryRulesSection::render` walks every + `tool-*` namespace and pins Critical/High rules into the system prompt. +2. **Writes** at turn end — `ToolMemoryCaptureHook` parses the user message + for edicts (`"never do X"`, `"always Y"`, …) and inserts rules. +3. **Direct read/write** — `tools::MemoryTools{List,Put}Tool` let the agent + itself inspect / record rules mid-session. + +## Layer rules + +- No upward dependencies — only `memory::Memory` trait (via `Arc`) + and project-wide primitives (`tools::traits::Tool`, `serde_json`). +- `MockMemory` is `#[cfg(test)]`-only — never available outside test builds. +- Re-exports in `mod.rs` are the public surface. Crate-owned type and store + forwarding files were removed; consumers should use the domain root. diff --git a/core/src/tool_memory/mod.rs b/core/src/tool_memory/mod.rs new file mode 100644 index 0000000..0bbf8fa --- /dev/null +++ b/core/src/tool_memory/mod.rs @@ -0,0 +1,44 @@ +//! Tool-scoped memory layer for durable learnings and high-priority rules. +//! +//! Implements the dedicated memory namespace requested in +//! [issue #1400](https://github.com/tinyhumansai/openhuman/issues/1400): +//! a first-class storage and retrieval surface for **actionable** +//! tool-specific guidance, distinct from the +//! `tool_effectiveness` +//! statistics namespace and from the generic `global` / `skill-*` +//! namespaces. +//! +//! ## Namespace convention +//! +//! Each tool gets its own namespace `tool-{tool_name}`. The prefix is +//! distinct from `global`, `skill-{id}`, `tool_effectiveness`, and the +//! learning namespaces so list/clear operations can reason about it +//! without ambiguity. Build the namespace string via +//! [`tool_memory_namespace`] — never hard-code the format. +//! +//! ## Components +//! +//! - [`tinycortex::memory::tool_memory::types`] owns [`ToolMemoryRule`], +//! [`ToolMemoryPriority`], and [`ToolMemorySource`]. +//! - [`tinycortex::memory::tool_memory::store`] owns [`ToolMemoryStore`], the +//! put/list/delete/prompt API built on top of an `Arc`. +//! - `capture` — `ToolMemoryCaptureHook`, the post-turn +//! `PostTurnHook` that records user edicts and repeated tool +//! failures. +//! - `prompt` — `ToolMemoryRulesSection`, the prompt section that +//! pins Critical / High rules into the system prompt so they survive +//! mid-session compression. +//! - `tools` — agent-facing read/write tools: +//! `tools::MemoryToolsListTool`, `tools::MemoryToolsPutTool`. +//! +//! `PostTurnHook`: the host's `agent::hooks::PostTurnHook` + +mod store; +#[cfg(any(test, feature = "test-support"))] +pub mod test_helpers; + +pub use store::tool_memory_store; +pub use tinycortex::memory::tool_memory::{ + store::{ToolMemoryStore, TOOL_MEMORY_PROMPT_CAP}, + types::{tool_memory_namespace, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource}, +}; diff --git a/core/src/tool_memory/store.rs b/core/src/tool_memory/store.rs new file mode 100644 index 0000000..3446fba --- /dev/null +++ b/core/src/tool_memory/store.rs @@ -0,0 +1,10 @@ +use std::sync::Arc; + +use crate::Memory; + +use tinycortex::memory::tool_memory::store::ToolMemoryStore; + +/// Build the crate-owned store over OpenHuman's shared memory object. +pub fn tool_memory_store(memory: Arc) -> ToolMemoryStore { + ToolMemoryStore::new(memory) +} diff --git a/core/src/tool_memory/test_helpers.rs b/core/src/tool_memory/test_helpers.rs new file mode 100644 index 0000000..c2b0e87 --- /dev/null +++ b/core/src/tool_memory/test_helpers.rs @@ -0,0 +1,228 @@ +//! Shared test infrastructure for the tool-scoped memory layer. +//! +//! Only compiled under `#[cfg(any(test, feature = "test-support"))]`. + +use std::collections::HashMap; + +use async_trait::async_trait; +use parking_lot::Mutex; + +use crate::{Memory, MemoryCategory, MemoryEntry, NamespaceSummary, RecallOpts}; + +/// Minimal in-memory [`Memory`] backend for unit tests. +/// +/// Stores entries in a `HashMap` keyed by `(namespace, key)`. All methods +/// that are not needed by the store/capture tests are no-ops. +#[derive(Default)] +pub struct MockMemory { + pub entries: Mutex>, +} + +#[async_trait] +impl Memory for MockMemory { + fn name(&self) -> &str { + "mock" + } + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + ) -> anyhow::Result<()> { + self.entries.lock().insert( + (namespace.to_string(), key.to_string()), + MemoryEntry { + id: format!("{namespace}/{key}"), + key: key.to_string(), + content: content.to_string(), + namespace: Some(namespace.to_string()), + category, + timestamp: "now".into(), + session_id: session_id.map(str::to_string), + score: None, + taint: Default::default(), + }, + ); + Ok(()) + } + async fn recall( + &self, + _query: &str, + _limit: usize, + _opts: RecallOpts<'_>, + ) -> anyhow::Result> { + Ok(Vec::new()) + } + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { + Ok(self + .entries + .lock() + .get(&(namespace.to_string(), key.to_string())) + .cloned()) + } + async fn list( + &self, + namespace: Option<&str>, + _category: Option<&MemoryCategory>, + _session_id: Option<&str>, + ) -> anyhow::Result> { + let lock = self.entries.lock(); + Ok(match namespace { + Some(ns) => lock + .iter() + .filter(|((n, _), _)| n == ns) + .map(|(_, v)| v.clone()) + .collect(), + None => lock.values().cloned().collect(), + }) + } + async fn forget(&self, namespace: &str, key: &str) -> anyhow::Result { + Ok(self + .entries + .lock() + .remove(&(namespace.to_string(), key.to_string())) + .is_some()) + } + async fn namespace_summaries(&self) -> anyhow::Result> { + let mut counts: HashMap = HashMap::new(); + for (ns, _) in self.entries.lock().keys() { + *counts.entry(ns.clone()).or_default() += 1; + } + Ok(counts + .into_iter() + .map(|(namespace, count)| NamespaceSummary { + namespace, + count, + last_updated: None, + }) + .collect()) + } + async fn count(&self) -> anyhow::Result { + Ok(self.entries.lock().len()) + } + async fn health_check(&self) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn mock_memory_store_get_list_and_count_roundtrip() { + let memory = MockMemory::default(); + memory + .store( + "tool-bash", + "rule/1", + "always dry run first", + MemoryCategory::Custom("tool_memory".into()), + Some("session-1"), + ) + .await + .unwrap(); + memory + .store( + "tool-web", + "rule/2", + "cite sources", + MemoryCategory::Conversation, + None, + ) + .await + .unwrap(); + + let got = memory.get("tool-bash", "rule/1").await.unwrap().unwrap(); + assert_eq!(got.id, "tool-bash/rule/1"); + assert_eq!(got.content, "always dry run first"); + assert_eq!(got.namespace.as_deref(), Some("tool-bash")); + assert_eq!(got.session_id.as_deref(), Some("session-1")); + + let scoped = memory.list(Some("tool-bash"), None, None).await.unwrap(); + assert_eq!(scoped.len(), 1); + assert_eq!(scoped[0].key, "rule/1"); + + let all = memory.list(None, None, None).await.unwrap(); + assert_eq!(all.len(), 2); + assert_eq!(memory.count().await.unwrap(), 2); + assert!(memory.health_check().await); + assert_eq!(memory.name(), "mock"); + + // The mock intentionally ignores category/session filters so tool + // tests can focus on caller behavior instead of backend indexing. + let filtered = memory + .list( + Some("tool-bash"), + Some(&MemoryCategory::Core), + Some("different-session"), + ) + .await + .unwrap(); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].key, "rule/1"); + } + + #[tokio::test] + async fn mock_memory_forget_and_namespace_summaries_track_entries() { + let memory = MockMemory::default(); + memory + .store("tool-bash", "rule/1", "first", MemoryCategory::Core, None) + .await + .unwrap(); + memory + .store("tool-bash", "rule/2", "second", MemoryCategory::Daily, None) + .await + .unwrap(); + memory + .store( + "tool-web", + "rule/3", + "third", + MemoryCategory::Conversation, + None, + ) + .await + .unwrap(); + + let mut summaries = memory.namespace_summaries().await.unwrap(); + summaries.sort_by(|a, b| a.namespace.cmp(&b.namespace)); + assert_eq!(summaries.len(), 2); + assert_eq!(summaries[0].namespace, "tool-bash"); + assert_eq!(summaries[0].count, 2); + assert_eq!(summaries[1].namespace, "tool-web"); + assert_eq!(summaries[1].count, 1); + + assert!(memory.forget("tool-bash", "rule/1").await.unwrap()); + assert!(!memory.forget("tool-bash", "missing").await.unwrap()); + + let remaining = memory.list(Some("tool-bash"), None, None).await.unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].key, "rule/2"); + } + + #[tokio::test] + async fn mock_memory_recall_is_empty_noop() { + let memory = MockMemory::default(); + let recalled = memory + .recall("anything", 5, RecallOpts::default()) + .await + .unwrap(); + assert!(recalled.is_empty()); + } + + #[tokio::test] + async fn mock_memory_empty_state_helpers_return_empty_values() { + let memory = MockMemory::default(); + assert!(memory.get("missing", "rule").await.unwrap().is_none()); + assert!(memory + .list(Some("missing"), None, None) + .await + .unwrap() + .is_empty()); + assert!(memory.namespace_summaries().await.unwrap().is_empty()); + assert_eq!(memory.count().await.unwrap(), 0); + } +} diff --git a/core/src/traits.rs b/core/src/traits.rs new file mode 100644 index 0000000..022bfb0 --- /dev/null +++ b/core/src/traits.rs @@ -0,0 +1,169 @@ +//! Core traits and data structures for the OpenHuman memory system. +//! +//! This module defines the foundational `Memory` trait that all storage backends +//! must implement. The standard memory value types (`MemoryEntry`, +//! `MemoryCategory`, `MemoryTaint`, `RecallOpts`, `NamespaceSummary`) are +//! **re-exported from the `tinycortex` crate** (migration W2, spec §0.5): the +//! crate is the single source of truth for these wire-compatible types, and the +//! 30+ host consumers keep their `memory::traits::…` import paths unchanged. +//! +//! `MemoryTaint` is security-critical provenance — it fails closed to +//! `ExternalSync` for unknown/corrupt values so the subconscious gate refuses +//! external-effect tools on chunks of unknown origin. Its semantics were proven +//! byte-identical to the former host definition before re-exporting; the tests +//! below are the host-side seam that pins that contract on the crate type. +//! +//! The `Memory` trait is also re-exported from `tinycortex`; backend-specific +//! resources such as SQLite connections are carried explicitly by factories +//! instead of being exposed through the storage abstraction. + +// ── Value types: re-exported from the crate (W2 type-unification, spec §0.5) ── +// +// These were formerly defined here. They are now the crate's types verbatim +// (identical fields, derives, serde attrs, and — for `MemoryTaint` — the same +// fail-closed `from_db_str`). Re-exporting keeps one source of truth while every +// `use crate::traits::{MemoryEntry, …}` site compiles unchanged. +pub use tinycortex::memory::{ + Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memory_category_display_outputs_expected_values() { + assert_eq!(MemoryCategory::Core.to_string(), "core"); + assert_eq!(MemoryCategory::Daily.to_string(), "daily"); + assert_eq!(MemoryCategory::Conversation.to_string(), "conversation"); + // TinyCortex renders `Custom(name)` with a `custom:` prefix so it stays + // distinct from the built-in variants and `Display`/`FromStr` are true + // inverses (see `memory_category_from_stored`). + assert_eq!( + MemoryCategory::Custom("project_notes".into()).to_string(), + "custom:project_notes" + ); + } + + #[test] + fn memory_category_custom_wire_values_round_trip_and_accept_legacy_bare_values() { + let current: MemoryCategory = "custom:project_notes".parse().unwrap(); + let legacy: MemoryCategory = "project_notes".parse().unwrap(); + + assert_eq!(current, MemoryCategory::Custom("project_notes".into())); + assert_eq!(legacy, MemoryCategory::Custom("project_notes".into())); + assert_eq!( + serde_json::to_string(¤t).unwrap(), + "\"custom:project_notes\"" + ); + } + + #[test] + fn memory_category_serde_uses_snake_case() { + let core = serde_json::to_string(&MemoryCategory::Core).unwrap(); + let daily = serde_json::to_string(&MemoryCategory::Daily).unwrap(); + let conversation = serde_json::to_string(&MemoryCategory::Conversation).unwrap(); + + assert_eq!(core, "\"core\""); + assert_eq!(daily, "\"daily\""); + assert_eq!(conversation, "\"conversation\""); + } + + #[test] + fn memory_entry_roundtrip_preserves_optional_fields() { + let entry = MemoryEntry { + id: "id-1".into(), + key: "favorite_language".into(), + content: "Rust".into(), + namespace: Some("global".into()), + category: MemoryCategory::Core, + timestamp: "2026-02-16T00:00:00Z".into(), + session_id: Some("session-abc".into()), + score: Some(0.98), + taint: MemoryTaint::Internal, + }; + + let json = serde_json::to_string(&entry).unwrap(); + let parsed: MemoryEntry = serde_json::from_str(&json).unwrap(); + + assert_eq!(parsed.id, "id-1"); + assert_eq!(parsed.key, "favorite_language"); + assert_eq!(parsed.content, "Rust"); + assert_eq!(parsed.namespace.as_deref(), Some("global")); + assert_eq!(parsed.category, MemoryCategory::Core); + assert_eq!(parsed.session_id.as_deref(), Some("session-abc")); + assert_eq!(parsed.score, Some(0.98)); + assert_eq!(parsed.taint, MemoryTaint::Internal); + } + + #[test] + fn memory_taint_defaults_to_internal_for_legacy_rows() { + // Legacy rows persisted before the taint column existed deserialize + // to MemoryTaint::Internal, so the gate's tainted-subconscious + // escalation never fires for entries we cannot classify. + let legacy = r#"{ + "id":"x", + "key":"k", + "content":"c", + "namespace":null, + "category":"core", + "timestamp":"2026-01-01T00:00:00Z", + "session_id":null, + "score":null + }"#; + let parsed: MemoryEntry = serde_json::from_str(legacy).unwrap(); + assert_eq!(parsed.taint, MemoryTaint::Internal); + } + + #[test] + fn memory_taint_as_db_str_uses_snake_case_form() { + assert_eq!(MemoryTaint::Internal.as_db_str(), "internal"); + assert_eq!(MemoryTaint::ExternalSync.as_db_str(), "external_sync"); + } + + #[test] + fn memory_taint_from_db_str_known_values_roundtrip_unknown_fails_closed() { + // Round-trip both known values. + assert_eq!( + MemoryTaint::from_db_str(MemoryTaint::Internal.as_db_str()), + MemoryTaint::Internal + ); + assert_eq!( + MemoryTaint::from_db_str(MemoryTaint::ExternalSync.as_db_str()), + MemoryTaint::ExternalSync + ); + // Unknown / corrupted column values fail closed to the more + // restrictive `ExternalSync` so the subconscious gate refuses + // external_effect tools on chunks of unknown provenance rather + // than silently treating them as user-authored. This is the W2 + // security seam test on the re-exported crate type. + assert_eq!(MemoryTaint::from_db_str(""), MemoryTaint::ExternalSync); + assert_eq!( + MemoryTaint::from_db_str("EXTERNAL_SYNC"), + MemoryTaint::ExternalSync + ); + assert_eq!( + MemoryTaint::from_db_str("future"), + MemoryTaint::ExternalSync + ); + } + + #[test] + fn memory_taint_roundtrips_external_sync() { + let entry = MemoryEntry { + id: "x".into(), + key: "k".into(), + content: "c".into(), + namespace: None, + category: MemoryCategory::Conversation, + timestamp: "2026-01-01T00:00:00Z".into(), + session_id: None, + score: None, + taint: MemoryTaint::ExternalSync, + }; + let json = serde_json::to_string(&entry).unwrap(); + assert!(json.contains("\"taint\":\"external_sync\"")); + let parsed: MemoryEntry = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.taint, MemoryTaint::ExternalSync); + } +} diff --git a/core/src/tree/README.md b/core/src/tree/README.md new file mode 100644 index 0000000..22bbd00 --- /dev/null +++ b/core/src/tree/README.md @@ -0,0 +1,42 @@ +# memory_tree + +Generic tree mechanics on top of `memory_store::trees`. Kind-agnostic: a +`Source`, `Global`, or `Topic` tree all flow through the same code here. +Kind-specific policy (when to spawn a topic tree, what scope a global tree +covers, how digests are written) lives in `memory::tree_global` and +`memory::tree_topic`; this module is unaware of it. + +```text +memory (orchestrator) ──┐ + │ writes leaves via TreeWriteRequest + ▼ +memory_tree (this module — generic mechanics) + ├── tree/ append + cascade seal + flush + ├── summarise.rs L_n -> L_{n+1} text via the chat model + ├── retrieval/ agent-facing read tools (walk, drill, fetch) + ├── score/ scoring, embedding, entity extraction + ├── tools.rs re-exports from memory::query + └── mod.rs re-exports the canonical Tree{Write,Read}{Request,Outcome,Result} + │ contract types from tinycortex::memory::tree + ▼ +memory_store::trees (persistence: one Tree table, one schema) +``` + +## Layout + +| Path | Role | +| --- | --- | +| [`mod.rs`](mod.rs) | Re-exports the canonical contract types from `tinycortex::memory::tree` (`TreeWriteRequest`/`TreeWriteOutcome`, `TreeReadRequest`/`TreeReadHit`/`TreeReadResult`, `TreeLeafPayload`, `TreeLabelStrategy` — pure types, no IO) and the controller-schema registries hosted in `memory`. Re-exports `memory::tree_global` + `memory::tree_topic` under the legacy `memory_tree::tree_{global,topic}` paths. | +| [`tree/`](tree/) | `bucket_seal` (append leaf + cascade seal), `flush` (time-based partial seal), `registry` (kind-parameterized `get_or_create_tree` with UNIQUE-race recovery), `mod.rs` (re-exports + `memory_store::trees` shims for legacy paths). | +| [`summarise.rs`](summarise.rs) | One function: produce the next-level summary text for a bucket. Wraps the chat model with a fixed prompt and token budget. | +| [`retrieval/`](retrieval/) | Agent-facing tools. Read: `walk` (agentic), `drill_down`, `fetch_leaves`, `query_{source,global,topic}`, `search_entities`. Write: `ingest_document` (orchestrator-facing). | +| [`score/`](score/) | Product adapters over TinyCortex scoring and TinyAgents embedding models, plus entity extraction and the entity index store. | + +## Layer rules + +- **No tree-kind branching here.** `bucket_seal`, `flush`, `registry`, + `summarise` all take `TreeKind` as a parameter or treat it as opaque. +- **No persistence here.** Reads and writes go through + `memory_store::trees::{store, registry, hotness}`. +- **No policy here.** Curator gates (hotness thresholds), digest cadence, + global scope sentinels — all live in `memory::tree_{global,topic}`. diff --git a/core/src/tree/graph/bfs.rs b/core/src/tree/graph/bfs.rs new file mode 100644 index 0000000..444d068 --- /dev/null +++ b/core/src/tree/graph/bfs.rs @@ -0,0 +1,19 @@ +//! `Config` adapter for tinycortex-owned bounded graph traversal. + +use anyhow::Result; + +use crate::Config; + +pub use tinycortex::memory::graph::PairDistance; + +pub fn pair_distances( + config: &Config, + entity_ids: &[String], + max_h: u32, +) -> Result> { + tinycortex::memory::graph::pair_distances( + &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + entity_ids, + max_h, + ) +} diff --git a/core/src/tree/graph/mod.rs b/core/src/tree/graph/mod.rs new file mode 100644 index 0000000..37fc92e --- /dev/null +++ b/core/src/tree/graph/mod.rs @@ -0,0 +1,19 @@ +//! Entity co-occurrence graph for E2GraphRAG-style deterministic retrieval. +//! +//! Two pieces: +//! - [`store`] — persistence for `mem_tree_entity_edges`, the undirected +//! weighted co-occurrence graph built incrementally at ingest time. +//! - [`bfs`] — bounded shortest-path (hop distance) over that graph, used as +//! the query-time "graph filter" that routes retrieval between the local +//! (entity-index intersection) and global (dense summary-tree) branches. +//! +//! The graph bridges the entity index and the summary tree without any LLM in +//! the loop: query entities → hop-distance filter → candidate chunk lookup. + +pub mod bfs; +pub mod store; + +pub use bfs::{pair_distances, PairDistance}; +pub use store::{ + clear_edges_for_entities_tx, neighbors, pairs_from_entities, upsert_edges, upsert_edges_tx, +}; diff --git a/core/src/tree/graph/store.rs b/core/src/tree/graph/store.rs new file mode 100644 index 0000000..2408300 --- /dev/null +++ b/core/src/tree/graph/store.rs @@ -0,0 +1,40 @@ +//! `Config` adapters for tinycortex-owned persisted graph edges. + +use anyhow::Result; +use rusqlite::Transaction; + +use crate::tinycortex::engine_config; +use crate::Config; + +pub use tinycortex::memory::graph::pairs_from_entities; + +pub fn upsert_edges_tx( + transaction: &Transaction<'_>, + pairs: &[(String, String)], + timestamp_ms: i64, +) -> Result { + tinycortex::memory::graph::upsert_edges_tx(transaction, pairs, timestamp_ms) +} + +pub fn upsert_edges( + config: &Config, + pairs: &[(String, String)], + timestamp_ms: i64, +) -> Result { + tinycortex::memory::graph::upsert_edges(&engine_config(config), pairs, timestamp_ms) +} + +pub fn neighbors(config: &Config, entity_id: &str) -> Result> { + tinycortex::memory::graph::edge_neighbors(&engine_config(config), entity_id) +} + +pub fn clear_edges_for_entities_tx( + transaction: &Transaction<'_>, + entity_ids: &[String], +) -> Result { + tinycortex::memory::graph::clear_edges_for_entities_tx(transaction, entity_ids) +} + +pub fn count_edges(config: &Config) -> Result { + tinycortex::memory::graph::count_edges(&engine_config(config)) +} diff --git a/core/src/tree/health/doctor.rs b/core/src/tree/health/doctor.rs new file mode 100644 index 0000000..515f3ee --- /dev/null +++ b/core/src/tree/health/doctor.rs @@ -0,0 +1,438 @@ +//! One-shot memory-pipeline diagnostic (#002 FR-009). +//! +//! `run_doctor` walks each stage of the chunk→wiki + summary-tree pipeline and +//! returns a [`DoctorReport`]: per-stage health, the single first blocking +//! cause (so the agent / CLI gets one actionable answer instead of a wall of +//! counters), and the current counters. It is exposed as an agent tool and a +//! CLI/RPC method — there is no UI surface this round (the status panel +//! already renders `first_blocking_cause`). +//! +//! Design: this is a **config + persisted-state** diagnosis — it reads the +//! routing config, the scheduler-gate mode, the process-global degraded flags +//! (set by the embed/extract stages), the job-queue counters, and the chunk +//! count. It intentionally does **not** fire a live embed/extract probe in this +//! cut: a network call would make the doctor slow, flaky, and order-dependent, +//! and the degraded flags already capture "did the last real run fail and how". +//! A time-boxed live probe is a clean follow-up if we want pre-run validation. + +use serde::{Deserialize, Serialize}; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use super::{current_degraded_state, DegradedState, FailureCode, PipelineFailure}; +use crate::Config; +use tinymemory_api::host::SchedulerGateMode; + +/// Health of one named pipeline stage. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct StageHealth { + /// Stable stage id: `routing`, `scheduler_gate`, `embeddings`, + /// `extraction`, `queue`, `summary_tree`. + pub stage: String, + /// True when this stage is healthy / not blocking. + pub ok: bool, + /// Typed failure when `ok == false`; `None` when healthy. Carries the + /// i18n remediation key the surfaces render. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure: Option, + /// Short non-localized human note for logs / CLI (never a secret). + pub note: String, +} + +impl StageHealth { + fn ok(stage: &str, note: impl Into) -> Self { + Self { + stage: stage.to_string(), + ok: true, + failure: None, + note: note.into(), + } + } + + fn bad(stage: &str, failure: PipelineFailure, note: impl Into) -> Self { + Self { + stage: stage.to_string(), + ok: false, + failure: Some(failure), + note: note.into(), + } + } +} + +/// Current pipeline counters, mirrored from the status surface so the doctor +/// is a one-call snapshot. +// No `Eq`: `extraction_coverage` is `Option` — `f32` never implements `Eq`. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct DoctorCounters { + pub total_chunks: u64, + pub jobs_ready: u64, + pub jobs_running: u64, + pub jobs_failed: u64, + /// #002 (FR-010 / US5): fraction of chunks with ≥1 indexed entity, in + /// `[0.0, 1.0]`. Near 0 with `total_chunks > 0` means extraction is + /// producing no structure. `None` when the metric could not be measured + /// (DB read error) — deliberately distinct from a genuine `0.0` so a + /// broken measurement is never misreported as a structure failure. + #[serde(default)] + pub extraction_coverage: Option, +} + +/// The full diagnostic. `first_blocking_cause` is the failure of the first +/// non-ok stage in pipeline order (`stages` is already ordered), so a caller +/// can act on one thing; `healthy` is the convenience roll-up. +// No `Eq`: transitively contains `DoctorCounters` (Option — f32: !Eq). +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct DoctorReport { + pub healthy: bool, + pub stages: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub first_blocking_cause: Option, + pub degraded: DegradedState, + pub counters: DoctorCounters, +} + +/// Run the diagnostic against `config` + persisted/queue/degraded state. +/// +/// Best-effort: counter reads that error degrade to 0 (the doctor is a +/// convenience, not an audit) and never fail the whole call. Stage order is +/// the pipeline order so the first non-ok stage is the first blocking cause. +pub fn run_doctor(config: &Config) -> DoctorReport { + use crate::queue::store as queue; + use crate::queue::types::JobStatus; + use crate::store::chunks::store as chunks; + + let degraded = current_degraded_state(); + let counters = DoctorCounters { + total_chunks: chunks::count_chunks(config).unwrap_or(0), + jobs_ready: queue::count_by_status(config, JobStatus::Ready).unwrap_or(0), + jobs_running: queue::count_by_status(config, JobStatus::Running).unwrap_or(0), + jobs_failed: queue::count_by_status(config, JobStatus::Failed).unwrap_or(0), + extraction_coverage: chunks::extraction_coverage(config).ok(), + }; + + let mut stages = Vec::new(); + + // 0. Storage health — the foundational layer. If the host filesystem can't + // service the memory_tree path (EIO/ENOSPC/EROFS on dir-create / DB + // open, flagged by the queue worker's host-I/O arm), nothing downstream + // can run. Pushed FIRST so it becomes `first_blocking_cause` and the + // status panel leads the user to the disk fix instead of a misleading + // "configure embeddings" message. Only the user owns this lever. + if degraded.storage { + let cause = degraded + .cause + .clone() + .filter(|c| c.code == FailureCode::StorageUnavailable) + .unwrap_or_else(|| PipelineFailure::new(FailureCode::StorageUnavailable)); + stages.push(StageHealth::bad( + "storage", + cause, + "memory storage path is unavailable — the host filesystem returned a \ + persistent I/O error (failing/read-only disk or SD card)", + )); + } else { + stages.push(StageHealth::ok( + "storage", + "memory storage path is writable", + )); + } + + // 1. Routing/config sanity — is *any* embeddings provider configured? + // (`build_write_embedder` skips embedding when none is, so this is the + // most common "empty wiki" root cause.) + let embeddings_provider = config + .memory_tree() + .embedding_endpoint + .as_deref() + .filter(|s| !s.trim().is_empty()) + .map(|_| "ollama-override".to_string()) + .or_else(|| config.embeddings_provider().map(str::to_string)) + .filter(|s| !s.trim().is_empty()); + stages.push(match embeddings_provider.as_deref() { + // Explicit `none` opt-out: semantic recall is off by the user's choice, + // not a fault. Reported `ok` (consistent with a `scheduler_gate=off` + // pause and the write-path opt-out treatment) but with an honest note, + // so the prior "provider configured: none" can't read as a working + // embeddings provider. (CodeRabbit on doctor.rs) + Some("none") => StageHealth::ok( + "embeddings", + "embeddings disabled by you (provider = none) — semantic recall is intentionally off", + ), + Some(p) => StageHealth::ok("embeddings", format!("provider configured: {p}")), + None => StageHealth::bad( + "embeddings", + PipelineFailure::new(FailureCode::EmbeddingsUnconfigured), + "no embeddings provider configured — semantic recall is off", + ), + }); + + // 2. Scheduler gate — `off` means the user paused background work. Report + // it as a *user choice*, not a fault (ok == true), but note it so a + // confused "nothing is happening" reads clearly. + let gate_off = config.scheduler_gate().mode == SchedulerGateMode::Off; + stages.push(StageHealth::ok( + "scheduler_gate", + if gate_off { + "paused by you (scheduler gate = off) — background sync is intentionally stopped" + } else { + "auto — background sync runs" + }, + )); + + // 3. Queue health — failed jobs are a hard signal. The typed reason (when + // present on the most-recent failed row) is surfaced by the status RPC; + // here we just flag that failures exist and how many. + if counters.jobs_failed > 0 { + stages.push(StageHealth::bad( + "queue", + // The most-recent typed reason is surfaced by pipeline_status; + // doctor reports the count + a transient-by-default placeholder so + // the stage is non-ok and actionable. + PipelineFailure::new(FailureCode::Transient), + format!("{} failed job(s) in mem_tree_jobs", counters.jobs_failed), + )); + } else { + stages.push(StageHealth::ok("queue", "no failed jobs")); + } + + // 4. Degraded signals from the last real run. + if degraded.semantic_recall { + let cause = degraded + .cause + .clone() + .unwrap_or_else(|| PipelineFailure::new(FailureCode::EmbeddingsUnconfigured)); + stages.push(StageHealth::bad( + "extraction", + // semantic_recall degradation is an embeddings problem, but reuse + // the recorded cause which names the real reason. + cause, + "semantic recall degraded — embeddings were skipped on the last run", + )); + } else if degraded.structure { + let cause = degraded + .cause + .clone() + .unwrap_or_else(|| PipelineFailure::new(FailureCode::ExtractionTimeout)); + stages.push(StageHealth::bad( + "extraction", + cause, + "wiki structure degraded — extraction produced no entities on the last run", + )); + } else { + stages.push(StageHealth::ok("extraction", "no degradation recorded")); + } + + // 5. Summary-tree precondition. Reuse the runtime's own capability check + // (`tree_runtime::ops::summarizer_available`) so the doctor matches what + // "Build Summary Trees" will actually do — since #002 FR-007 it runs on + // the configured cloud provider when local AI is off, so local-AI-off is + // NOT a fault by itself. Only `bad` when no provider resolves at all. + let (summary_ok, summary_note) = crate::chat_host::summarizer_available(config); + stages.push(if summary_ok { + StageHealth::ok("summary_tree", summary_note) + } else { + StageHealth::bad( + "summary_tree", + PipelineFailure::new(FailureCode::SummarizerUnavailable), + summary_note, + ) + }); + + let first_blocking_cause = stages + .iter() + .find(|s| !s.ok) + .and_then(|s| s.failure.clone()); + let healthy = first_blocking_cause.is_none(); + + DoctorReport { + healthy, + stages, + first_blocking_cause, + degraded, + counters, + } +} + +/// Async wrapper around [`run_doctor`] for async call sites (the RPC + agent +/// tool). `run_doctor` does synchronous SQLite reads (chunk/job counts + +/// extraction coverage); a contended DB could pin a Tokio worker for the +/// busy-timeout window, so offload the whole diagnostic to a blocking thread. +pub async fn async_run_doctor(config: &Config) -> DoctorReport { + let cfg = config.to_arc(); + match tokio::task::spawn_blocking(move || run_doctor(&*cfg)).await { + Ok(report) => report, + Err(join_err) => { + // The blocking task panicked — surface a degraded-but-shaped report + // rather than propagating, since the doctor is a best-effort + // diagnostic and callers expect a report, not an error. + log::warn!("[memory_tree::health::doctor] run_doctor task failed: {join_err}"); + DoctorReport { + healthy: false, + stages: Vec::new(), + first_blocking_cause: Some(PipelineFailure::new(FailureCode::Transient)), + degraded: current_degraded_state(), + counters: DoctorCounters::default(), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + (tmp, cfg) + } + + #[test] + fn misconfigured_workspace_reports_embeddings_as_first_blocking_cause() { + let _g = super::super::test_guard(); + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = None; // no provider at all + cfg.local_ai.runtime_enabled = false; + + let report = run_doctor(&cfg); + assert!(!report.healthy); + // Embeddings is stage 1, so it is the first blocking cause. + let cause = report.first_blocking_cause.expect("should have a cause"); + assert_eq!(cause.code, FailureCode::EmbeddingsUnconfigured); + // The embeddings stage is non-ok with the same code. + let embed = report + .stages + .iter() + .find(|s| s.stage == "embeddings") + .unwrap(); + assert!(!embed.ok); + } + + #[test] + fn healthy_when_embeddings_and_local_ai_configured() { + let _g = super::super::test_guard(); + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = Some("none".into()); // a configured choice + cfg.local_ai.runtime_enabled = true; + + let report = run_doctor(&cfg); + assert!( + report.healthy, + "expected healthy, got {:?}", + report.first_blocking_cause + ); + assert!(report.first_blocking_cause.is_none()); + // Every stage ok. + assert!( + report.stages.iter().all(|s| s.ok), + "stages: {:?}", + report.stages + ); + } + + #[test] + fn embeddings_none_opt_out_is_ok_but_note_is_honest() { + // `embeddings_provider = "none"` is a deliberate opt-out: the stage stays + // ok (a configured choice, like a paused scheduler gate) but the note must + // not read as a working provider ("provider configured: none"). (CodeRabbit) + let _g = super::super::test_guard(); + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = Some("none".into()); + cfg.local_ai.runtime_enabled = true; + + let report = run_doctor(&cfg); + let embed = report + .stages + .iter() + .find(|s| s.stage == "embeddings") + .unwrap(); + assert!(embed.ok, "opt-out is a choice, not a fault"); + assert!( + embed.note.contains("disabled") && embed.note.contains("intentionally off"), + "note must name the intentional opt-out, got: {}", + embed.note + ); + assert!( + !embed.note.contains("provider configured"), + "must not read as a working provider, got: {}", + embed.note + ); + } + + #[test] + fn scheduler_gate_off_is_a_choice_not_a_fault() { + use tinymemory_api::host::SchedulerGateMode; + let _g = super::super::test_guard(); + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = Some("ollama:bge-m3".into()); + cfg.local_ai.runtime_enabled = true; + cfg.scheduler_gate.mode = SchedulerGateMode::Off; + + // Double-reset: guard resets on entry, but a concurrent non-guarded + // code path (e.g. a tokio task draining after its test dropped its + // guard) may have re-set the flags between guard acquisition and here. + super::super::clear_semantic_recall_degraded(); + super::super::clear_structure_degraded(); + + let report = run_doctor(&cfg); + // Paused is reported but does NOT make the pipeline unhealthy. + assert!( + report.healthy, + "expected healthy, failing stages: {:?}", + report.stages.iter().filter(|s| !s.ok).collect::>() + ); + let gate = report + .stages + .iter() + .find(|s| s.stage == "scheduler_gate") + .unwrap(); + assert!(gate.ok); + assert!(gate.note.contains("paused")); + } + + /// A host-FS storage failure must surface as the doctor's + /// `first_blocking_cause` (stage 0), outranking everything else — even a + /// fully-misconfigured embeddings setup — so the user is told to fix their + /// disk, not their provider config. + #[test] + fn storage_failure_is_first_blocking_cause() { + let _g = super::super::test_guard(); + let (_tmp, mut cfg) = test_config(); + // Deliberately also break embeddings so we prove storage wins. + cfg.embeddings_provider = None; + cfg.local_ai.runtime_enabled = false; + super::super::mark_storage_degraded(FailureCode::StorageUnavailable); + + let report = run_doctor(&cfg); + assert!(!report.healthy); + let cause = report.first_blocking_cause.expect("should have a cause"); + assert_eq!( + cause.code, + FailureCode::StorageUnavailable, + "storage must outrank the embeddings misconfig" + ); + let storage = report + .stages + .iter() + .find(|s| s.stage == "storage") + .expect("storage stage present"); + assert!(!storage.ok); + assert!(report.degraded.storage); + } + + #[test] + fn report_serde_roundtrips() { + let _g = super::super::test_guard(); + let (_tmp, cfg) = test_config(); + let report = run_doctor(&cfg); + let json = serde_json::to_string(&report).unwrap(); + let back: DoctorReport = serde_json::from_str(&json).unwrap(); + assert_eq!(report, back); + } +} diff --git a/core/src/tree/health/mod.rs b/core/src/tree/health/mod.rs new file mode 100644 index 0000000..87bbbc8 --- /dev/null +++ b/core/src/tree/health/mod.rs @@ -0,0 +1,542 @@ +//! Host-side surface of the memory pipeline's failure + degradation model. +//! +//! The **taxonomy itself** — [`FailureCode`], [`FailureClass`], +//! [`PipelineFailure`], [`DegradedState`], and the `classify_embed_error` +//! classifier — now lives in the engine crate at +//! `tinycortex::memory::health`, and is re-exported below so every existing +//! `memory::tree::health::…` path keeps resolving. It moved because a build +//! whose only driver was a third-party external backend would have no use for +//! the engine's private failure vocabulary. +//! +//! What stays here is everything that is *host* surface rather than engine +//! vocabulary: +//! +//! - the **process-visible degradation flags** (`mark_*` / `clear_*` / +//! [`current_degraded_state`]) — set by host plumbing deep in the job worker, +//! read by the `pipeline_status` RPC, and coupled to a socket broadcast; +//! - [`doctor`] — the health report, which reads the host's scheduler-gate +//! config; +//! - `user_error` — whose `kind` string is a pinned contract with the frontend. + +pub mod doctor; +pub use doctor::{async_run_doctor, run_doctor, DoctorCounters, DoctorReport, StageHealth}; + +pub(crate) mod user_error; +pub(crate) use user_error::publish_local_model_unavailable_user_error; + +/// The failure taxonomy proper. Re-exported (rather than re-declared) so the +/// ~30 `crate::tree::health::{…}` call sites across the host +/// are unaffected by the move, and so there is exactly one definition. +pub use tinycortex::memory::health::{ + classify_embed_error, classify_embed_error_str, DegradedState, FailureClass, FailureCode, + PipelineFailure, +}; + +// ── Process-visible degradation flags ──────────────────────────────────── +// +// The embed/extract stages run deep inside the job worker, far from the +// `pipeline_status` RPC. Rather than thread a `DegradedState` return up +// through every call site, the stages set these process-global atomics when +// they detect a degraded condition (no usable embedder → semantic recall +// disabled; extraction empty across the board → no structure). The status / +// doctor surface reads them via [`current_degraded_state`]. They reflect the +// most recent run, are cheap, and never block — a coarse "is recall/structure +// currently degraded?" signal, intentionally not per-namespace. + +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; + +static SEMANTIC_RECALL_DEGRADED: AtomicBool = AtomicBool::new(false); +/// Whether the clients have already been told about the *current* local-runtime +/// outage. Separate from [`SEMANTIC_RECALL_DEGRADED`] because "is recall +/// degraded" and "have we announced it" are different questions, and the +/// announcement must be claimed by exactly one caller: the embed path runs +/// concurrently across worker tasks, and a plain read-then-write of the +/// degraded flag lets two of them both decide they are the first +/// (CodeRabbit, #5398). Claimed with `compare_exchange`, released by +/// [`clear_semantic_recall_degraded`] so a later outage announces again. +static LOCAL_MODEL_USER_ERROR_SURFACED: AtomicBool = AtomicBool::new(false); +static STRUCTURE_DEGRADED: AtomicBool = AtomicBool::new(false); +/// The host filesystem can't service the memory_tree path (EIO/ENOSPC/EROFS). +/// Set by the queue worker's host-I/O arm; cleared on the next successful +/// claim (storage recovered). Most severe — outranks recall/structure. +static STORAGE_DEGRADED: AtomicBool = AtomicBool::new(false); +/// Per-flag degradation cause as a `FailureCode` discriminant (0 = none). +/// Tracked separately per flag so clearing one degradation can't leave the +/// other reporting a stale cause (e.g. mark recall, mark structure, clear +/// structure → recall must still report its OWN cause, not structure's). +static SEMANTIC_RECALL_CAUSE: AtomicU8 = AtomicU8::new(0); +static STRUCTURE_CAUSE: AtomicU8 = AtomicU8::new(0); +static STORAGE_CAUSE: AtomicU8 = AtomicU8::new(0); + +fn code_to_u8(code: FailureCode) -> u8 { + match code { + FailureCode::BudgetExhausted => 1, + FailureCode::AuthMissing => 2, + FailureCode::AuthInvalid => 3, + FailureCode::EmbeddingsUnconfigured => 4, + FailureCode::EmbeddingDimMismatch => 5, + FailureCode::LocalModelUnavailable => 6, + FailureCode::ExtractionTimeout => 7, + FailureCode::SummarizerUnavailable => 8, + FailureCode::Transient => 9, + FailureCode::EmptyInputRefused => 10, + FailureCode::StorageUnavailable => 11, + } +} + +fn u8_to_code(v: u8) -> Option { + Some(match v { + 1 => FailureCode::BudgetExhausted, + 2 => FailureCode::AuthMissing, + 3 => FailureCode::AuthInvalid, + 4 => FailureCode::EmbeddingsUnconfigured, + 5 => FailureCode::EmbeddingDimMismatch, + 6 => FailureCode::LocalModelUnavailable, + 7 => FailureCode::ExtractionTimeout, + 8 => FailureCode::SummarizerUnavailable, + 9 => FailureCode::Transient, + 10 => FailureCode::EmptyInputRefused, + 11 => FailureCode::StorageUnavailable, + _ => return None, + }) +} + +/// Record that semantic recall is degraded (embeddings were skipped because no +/// usable provider is available). `cause` names why so the status surface can +/// lead the user to the fix. Idempotent / cheap; safe to call per embed-stage. +/// +/// The cause is published **before** the flag, and the flag with `Release`, so +/// a concurrent [`current_degraded_state`] that observes the flag set cannot +/// still read the previous degradation's cause and render the wrong +/// remediation (CodeRabbit, #5398). Same ordering in every `mark_*` below. +pub fn mark_semantic_recall_degraded(cause: FailureCode) { + SEMANTIC_RECALL_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + SEMANTIC_RECALL_DEGRADED.store(true, Ordering::Release); +} + +/// Surface a local-runtime embed failure on the status panel immediately +/// (#5354). No-op for every other cause. +/// +/// The typed `failure_reason` a job persists is only read back once that job +/// settles *terminally* — for a transient class that means after the whole +/// retry budget has drained. The local-runtime causes (Ollama daemon stopped, +/// model never pulled) are user-fixable right now, so waiting out the backoff +/// before naming the fix is exactly the silent window this issue is about. +/// Setting the degraded flag at classification time puts the remediation on +/// the panel from the first failure; the flag self-clears on the next +/// successful embed, so a user who starts Ollama sees it disappear. +/// +/// This is also the **only** producer of the durable UserErrorCenter entry for +/// the "model was never pulled" half of the cause. The embedder health gate in +/// `memory::store::factories` probes `GET /api/tags`, which succeeds whenever +/// the daemon is up — so a running daemon with a missing model never trips that +/// gate and never publishes its `user_error` (codex, #5398). Publishing here +/// covers both halves from the one place that has actually classified the +/// failure. +/// +/// The broadcast fires only on the **transition** into the state, not on every +/// failed embed: the re-embed path calls this per row, and while the panel +/// store dedupes on the descriptor identity, emitting one socket event per +/// chunk would be pointless traffic. +/// +/// The announcement is claimed with a `compare_exchange` on a dedicated latch +/// rather than by reading the degraded flag, so exactly one of several +/// concurrent embed tasks publishes (CodeRabbit, #5398). +/// +/// The latch is released by [`clear_semantic_recall_degraded`], which every +/// write-embedder build calls once per seal / re-embed operation. That is +/// deliberate: it makes the announcement **re-emit once per failing operation +/// until recovery**, so a client that was not yet connected when the outage +/// began still receives it on the next operation. `publish_web_channel_event` +/// is an unbuffered broadcast with no replay, so bounded re-emission is what +/// stands in for one. +pub fn mark_local_model_unavailable_if_applicable(failure: &PipelineFailure) { + if failure.code != FailureCode::LocalModelUnavailable { + return; + } + // Claim the announcement before mutating anything else. `compare_exchange` + // makes the check-and-claim one indivisible step, so concurrent callers + // cannot all conclude they are first. + let claimed_announcement = LOCAL_MODEL_USER_ERROR_SURFACED + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok(); + + log::warn!( + "[memory_tree::health] action=mark_degraded surface=semantic_recall \ + cause=local_model_unavailable class={} announced={}", + failure.class.as_str(), + claimed_announcement + ); + mark_semantic_recall_degraded(FailureCode::LocalModelUnavailable); + + if claimed_announcement { + publish_local_model_unavailable_user_error("embed_classify"); + } +} + +/// Clear the semantic-recall degraded flag — call when an embed succeeds, so +/// the surface recovers once the user fixes the provider. Clears only this +/// flag's cause; a still-active structure degradation keeps its own. +pub fn clear_semantic_recall_degraded() { + SEMANTIC_RECALL_DEGRADED.store(false, Ordering::Relaxed); + SEMANTIC_RECALL_CAUSE.store(0, Ordering::Relaxed); + // Release the announcement claim so a later local-runtime failure tells the + // clients again. Called once per write-embedder build, which is what turns + // the single announcement into bounded re-emission until recovery — the + // reason a client connecting mid-outage still gets told. + LOCAL_MODEL_USER_ERROR_SURFACED.store(false, Ordering::Release); +} + +/// Record that wiki structure is degraded (extraction yielded nothing across +/// the board). `cause` is typically [`FailureCode::ExtractionTimeout`]. +pub fn mark_structure_degraded(cause: FailureCode) { + STRUCTURE_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + STRUCTURE_DEGRADED.store(true, Ordering::Release); +} + +/// Clear the structure degraded flag — call when extraction yields entities. +/// Clears only this flag's cause. +pub fn clear_structure_degraded() { + STRUCTURE_DEGRADED.store(false, Ordering::Relaxed); + STRUCTURE_CAUSE.store(0, Ordering::Relaxed); +} + +/// Record that the memory_tree storage path is unusable — the host filesystem +/// returned a persistent I/O error (EIO/ENOSPC/EROFS) on dir-create / DB open. +/// `cause` is typically [`FailureCode::StorageUnavailable`]. Set by the queue +/// worker's host-I/O arm so the status surface tells the user to check their +/// disk; idempotent / cheap. +pub fn mark_storage_degraded(cause: FailureCode) { + STORAGE_CAUSE.store(code_to_u8(cause), Ordering::Relaxed); + STORAGE_DEGRADED.store(true, Ordering::Release); +} + +/// Clear the storage degraded flag — call when a claim succeeds (the DB opened, +/// so the host filesystem recovered), so the surface self-heals. Clears only +/// this flag's cause. +pub fn clear_storage_degraded() { + STORAGE_DEGRADED.store(false, Ordering::Relaxed); + STORAGE_CAUSE.store(0, Ordering::Relaxed); +} + +/// Test-only serialization + reset for the process-global degraded flags. +/// +/// The flags are a single process-wide signal, so tests across *different* +/// modules (factory, extract::llm, tree::rpc) that set or read them race under +/// cargo's parallel runner. Any such test must `let _g = test_guard();` at the +/// top: it takes a shared mutex (serialising all flag-touching tests) and +/// resets both flags to a clean baseline so the test starts deterministic. +#[cfg(any(test, feature = "test-support"))] +pub fn test_guard() -> std::sync::MutexGuard<'static, ()> { + static LOCK: std::sync::OnceLock> = std::sync::OnceLock::new(); + let g = LOCK + .get_or_init(|| std::sync::Mutex::new(())) + .lock() + .unwrap_or_else(|p| p.into_inner()); + SEMANTIC_RECALL_DEGRADED.store(false, Ordering::Relaxed); + LOCAL_MODEL_USER_ERROR_SURFACED.store(false, Ordering::Relaxed); + STRUCTURE_DEGRADED.store(false, Ordering::Relaxed); + STORAGE_DEGRADED.store(false, Ordering::Relaxed); + SEMANTIC_RECALL_CAUSE.store(0, Ordering::Relaxed); + STRUCTURE_CAUSE.store(0, Ordering::Relaxed); + STORAGE_CAUSE.store(0, Ordering::Relaxed); + g +} + +/// Snapshot the current process-global [`DegradedState`] for the status / +/// doctor surface. The `cause` is populated from the last recorded +/// [`FailureCode`] when either flag is set. +pub fn current_degraded_state() -> DegradedState { + // Acquire pairs with the Release store on each flag in `mark_*_degraded`, + // which publishes the cause FIRST. A reader that observes a set flag is + // therefore guaranteed to observe the cause that was stored with it, never + // a stale one from a previous degradation (CodeRabbit, #5398). + let semantic_recall = SEMANTIC_RECALL_DEGRADED.load(Ordering::Acquire); + let structure = STRUCTURE_DEGRADED.load(Ordering::Acquire); + let storage = STORAGE_DEGRADED.load(Ordering::Acquire); + // Each flag carries its own cause; pick the most actionable one to surface. + // Storage degradation is reported first — the host FS can't open the DB, so + // it's the foundational failure beneath both recall and structure (no point + // telling the user "configure embeddings" when the disk is dying). Then + // structure (extraction failing → empty wiki), then recall. Either way the + // cause reflects a CURRENTLY-active flag. + let cause = if storage { + u8_to_code(STORAGE_CAUSE.load(Ordering::Relaxed)).map(PipelineFailure::new) + } else if structure { + u8_to_code(STRUCTURE_CAUSE.load(Ordering::Relaxed)).map(PipelineFailure::new) + } else if semantic_recall { + u8_to_code(SEMANTIC_RECALL_CAUSE.load(Ordering::Relaxed)).map(PipelineFailure::new) + } else { + None + }; + DegradedState { + semantic_recall, + structure, + storage, + cause, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// #5354 — a classified local-runtime failure flips the recall flag with + /// its own cause, so the panel names the Ollama fix from the first failed + /// embed instead of waiting out the retry budget. + #[test] + fn local_model_unavailable_marks_recall_degraded_with_its_cause() { + let _g = test_guard(); + + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + + let s = current_degraded_state(); + assert!(s.semantic_recall, "recall must be flagged degraded"); + assert_eq!( + s.cause.as_ref().map(|c| c.code), + Some(FailureCode::LocalModelUnavailable) + ); + assert_eq!( + s.cause.as_ref().map(|c| c.remediation_key.as_str()), + Some("memory.health.remediation.local_model_unavailable") + ); + } + + /// #5398 (codex) — the classifier is the ONLY producer of the durable + /// UserErrorCenter entry when Ollama is running but the model was never + /// pulled: the factory health gate probes `GET /api/tags`, which succeeds + /// in that case, so it never fires. It must broadcast on the transition + /// into the state, and must not re-broadcast per failed row afterwards. + #[test] + fn local_model_unavailable_broadcasts_once_per_transition() { + let _g = test_guard(); + let sink = crate::events::RecordingSink::install(); + + let failure = PipelineFailure::new(FailureCode::LocalModelUnavailable); + + // First failure of the outage → clients are told. + mark_local_model_unavailable_if_applicable(&failure); + let recorded = sink.drain(); + assert_eq!(recorded.len(), 1, "transition must broadcast"); + let event = &recorded[0]; + assert!( + matches!( + event, + crate::events::MemoryEvent::LocalModelUnavailable { .. } + ), + "the transition must publish the local-model-unavailable event, got {event:?}" + ); + + // Subsequent failures in the same outage must stay quiet — the re-embed + // path calls this per row. + mark_local_model_unavailable_if_applicable(&failure); + mark_local_model_unavailable_if_applicable(&failure); + assert!( + sink.drain().is_empty(), + "must not re-broadcast while already degraded for this cause" + ); + + // A successful embed clears the flag; the next outage is a new + // transition and must tell the clients again. + clear_semantic_recall_degraded(); + mark_local_model_unavailable_if_applicable(&failure); + assert!( + !sink.drain().is_empty(), + "a fresh outage after recovery must broadcast again" + ); + } + + /// #5398 (CodeRabbit) — concurrent embed tasks must not all decide they are + /// the first to announce. The claim is a `compare_exchange`, so exactly one + /// of N racing callers publishes. Deterministic: the assertion is on the + /// count of claims, which the atomic makes exact regardless of scheduling. + #[test] + fn concurrent_failures_announce_exactly_once() { + let _g = test_guard(); + let sink = crate::events::RecordingSink::install(); + + const THREADS: usize = 8; + std::thread::scope(|scope| { + for _ in 0..THREADS { + scope.spawn(|| { + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + }); + } + }); + + let published = sink.drain().len(); + assert_eq!( + published, 1, + "{THREADS} concurrent failures must yield exactly one announcement" + ); + } + + /// #5398 (CodeRabbit) — `publish_web_channel_event` is an unbuffered + /// broadcast: an announcement made before any client subscribed is dropped + /// with no replay. Bounded re-emission is what covers that, so a client + /// connecting mid-outage must still be told on the next failing operation. + #[test] + fn announcement_reaches_a_client_that_connects_mid_outage() { + let _g = test_guard(); + let failure = PipelineFailure::new(FailureCode::LocalModelUnavailable); + + // Outage starts with nobody listening — this send goes nowhere. + mark_local_model_unavailable_if_applicable(&failure); + + // The client connects now, after the first failure. + let sink = crate::events::RecordingSink::install(); + assert!( + sink.drain().is_empty(), + "the pre-subscription announcement is genuinely gone, not buffered" + ); + + // Next seal / re-embed operation builds its write embedder, which + // clears the degraded state, then fails again against the same dead + // runtime. The late subscriber must receive that one. + clear_semantic_recall_degraded(); + mark_local_model_unavailable_if_applicable(&failure); + + let recorded = sink.drain(); + assert_eq!( + recorded.len(), + 1, + "a client connecting mid-outage must still be told" + ); + assert!(matches!( + recorded[0], + crate::events::MemoryEvent::LocalModelUnavailable { .. } + )); + } + + /// A different active cause must not be mistaken for "already surfaced" — + /// recall degraded for an unrelated reason still needs the local-runtime + /// entry when Ollama then goes away. + #[test] + fn local_model_unavailable_broadcasts_over_a_different_active_cause() { + let _g = test_guard(); + let sink = crate::events::RecordingSink::install(); + + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + mark_local_model_unavailable_if_applicable(&PipelineFailure::new( + FailureCode::LocalModelUnavailable, + )); + + assert!( + !sink.drain().is_empty(), + "a cause change into local_model_unavailable is a transition" + ); + } + + /// The helper must stay a no-op for every other cause — a cloud budget or + /// transport failure has nothing to do with the local runtime, and marking + /// recall degraded there would show the wrong remediation. + #[test] + fn other_failure_codes_do_not_mark_recall_degraded() { + let _g = test_guard(); + + for code in [ + FailureCode::Transient, + FailureCode::BudgetExhausted, + FailureCode::AuthMissing, + ] { + mark_local_model_unavailable_if_applicable(&PipelineFailure::new(code)); + assert!( + !current_degraded_state().semantic_recall, + "{} must not flip the recall flag", + code.as_str() + ); + } + } + + /// Regression (CodeRabbit): per-flag causes. Mark recall, then structure, + /// then clear structure — recall must still report its OWN cause, not the + /// (now-cleared) structure cause. With the old single shared slot this + /// surfaced the wrong remediation. + #[test] + fn degraded_cause_is_per_flag_not_shared() { + let _g = test_guard(); // resets both flags + causes + + // Recall degraded for embeddings reason; structure degraded for extraction. + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + mark_structure_degraded(FailureCode::ExtractionTimeout); + + // Structure takes precedence while both are active. + let s = current_degraded_state(); + assert!(s.semantic_recall && s.structure); + assert_eq!( + s.cause.as_ref().map(|c| c.code), + Some(FailureCode::ExtractionTimeout) + ); + + // Clear structure — recall stays, and its cause must be the RECALL one, + // not the cleared structure cause. + clear_structure_degraded(); + let s = current_degraded_state(); + assert!(s.semantic_recall && !s.structure); + assert_eq!( + s.cause.as_ref().map(|c| c.code), + Some(FailureCode::EmbeddingsUnconfigured), + "recall must keep its own cause after structure clears" + ); + + // Clear recall too — fully healthy, no cause. + clear_semantic_recall_degraded(); + let s = current_degraded_state(); + assert!(!s.is_degraded()); + assert!(s.cause.is_none()); + } + + /// `StorageUnavailable` is the foundational host-FS failure: unrecoverable, + /// with its own remediation key. + #[test] + fn storage_unavailable_is_unrecoverable_with_key() { + let f = PipelineFailure::new(FailureCode::StorageUnavailable); + assert_eq!(f.class, FailureClass::Unrecoverable); + assert!(f.is_unrecoverable()); + assert_eq!( + f.remediation_key, + "memory.health.remediation.storage_unavailable" + ); + // discriminant round-trips through the per-flag u8 mapping. + assert_eq!( + u8_to_code(code_to_u8(FailureCode::StorageUnavailable)), + Some(FailureCode::StorageUnavailable) + ); + } + + /// Storage degradation outranks both structure and recall in + /// `current_degraded_state` — the host can't open the DB, so the disk fix + /// is the one actionable thing to surface. Clearing storage falls back to + /// the next-most-severe active cause (structure), each keeping its own. + #[test] + fn storage_degradation_outranks_structure_and_recall() { + let _g = test_guard(); // resets all flags + causes + + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + mark_structure_degraded(FailureCode::ExtractionTimeout); + mark_storage_degraded(FailureCode::StorageUnavailable); + + // All three active → storage wins. + let s = current_degraded_state(); + assert!(s.storage && s.structure && s.semantic_recall); + assert!(s.is_degraded()); + assert_eq!( + s.cause.as_ref().map(|c| c.code), + Some(FailureCode::StorageUnavailable) + ); + + // Clear storage → structure becomes the surfaced cause (its OWN, not + // storage's stale one). + clear_storage_degraded(); + let s = current_degraded_state(); + assert!(!s.storage && s.structure); + assert_eq!( + s.cause.as_ref().map(|c| c.code), + Some(FailureCode::ExtractionTimeout) + ); + } +} diff --git a/core/src/tree/health/user_error.rs b/core/src/tree/health/user_error.rs new file mode 100644 index 0000000..3aabcb5 --- /dev/null +++ b/core/src/tree/health/user_error.rs @@ -0,0 +1,37 @@ +//! Client-facing `user_error` surfacing for memory-pipeline health causes. +//! +//! The memory pipeline already records typed causes for the status panel, but +//! the panel only exists while the user is looking at it. A cause the user must +//! act on outside the app — the local Ollama runtime being unusable — also +//! belongs in the durable UserErrorCenter. +//! +//! # What is here, and what is in the host +//! +//! Getting that into the UserErrorCenter means a `user_error` web-channel +//! event, and web channels are host surface. So this module owns only the +//! *decision to report*, published as +//! [`MemoryEvent::LocalModelUnavailable`]; the host's sink builds the wire +//! payload and broadcasts it. The `error_type` token both sides key on lives in +//! the contract crate ([`LOCAL_MODEL_UNAVAILABLE_KIND`]) so it cannot drift. +//! +//! The two producers — the embedder health gate in +//! [`crate::store::factories`] and the failure classifier in the parent module +//! — both go through [`publish_local_model_unavailable_user_error`], so they +//! emit one identical shape. + +pub(crate) use tinymemory_api::host::LOCAL_MODEL_UNAVAILABLE_KIND; + +/// Report that the local embedding runtime is unusable. +/// +/// `origin` is a short, non-sensitive tag naming which producer fired +/// (`health_gate` / `embed_classify`) so the two paths stay distinguishable in +/// the log without threading a correlation id through the health API. +pub(crate) fn publish_local_model_unavailable_user_error(origin: &str) { + log::debug!( + "[memory_tree::health] action=surface_user_error kind={LOCAL_MODEL_UNAVAILABLE_KIND} \ + origin={origin}" + ); + crate::events::publish(crate::events::MemoryEvent::LocalModelUnavailable { + origin: origin.to_string(), + }); +} diff --git a/core/src/tree/ingest.rs b/core/src/tree/ingest.rs new file mode 100644 index 0000000..805c60e --- /dev/null +++ b/core/src/tree/ingest.rs @@ -0,0 +1,67 @@ +//! Product artifact hooks around tinycortex-owned direct summary ingestion. + +#[cfg(feature = "memory-git")] +use anyhow::Context; +use anyhow::Result; + +#[cfg(feature = "memory-git")] +use crate::store::content::wiki_git::{SummaryCommitBatch, SummaryCommitEntry}; +use crate::store::trees::types::Tree; +use crate::tinycortex::{memory_config_from, HostSummariser}; +use crate::Config; + +pub use tinycortex::memory::tree::{SummaryIngestInput, SummaryIngestOutcome}; + +pub async fn ingest_summary( + config: &Config, + tree: &Tree, + input: SummaryIngestInput, +) -> Result { + log::debug!( + "[memory_tree::ingest] tinycortex enter tree_kind={} children={}", + tree.kind.as_str(), + input.child_labels.len() + ); + let content_root = config.memory_tree_content_root(); + if let Err(error) = crate::store::content::obsidian::ensure_obsidian_defaults(&content_root) { + log::warn!("[memory_tree::ingest] obsidian defaults failed: {error:#}"); + } + + let outcome = tinycortex::memory::tree::ingest_summary( + &memory_config_from(config, config.workspace_dir().clone()), + tree, + input.clone(), + &HostSummariser::new(config.to_arc()), + ) + .await?; + + // The git wiki mirror is a DERIVED view: `ingest_summary` above has already + // written the summary to disk, and this only records it in the git-backed + // mirror. Skipping it when `memory-git` is off loses the mirror, not the + // summary — so the call site is gated rather than stubbed. + #[cfg(feature = "memory-git")] + crate::store::content::wiki_git::commit_summaries( + &content_root, + &SummaryCommitBatch { + reason: "summary_ingest".to_string(), + tree_id: tree.id.clone(), + tree_scope: tree.scope.clone(), + entries: vec![SummaryCommitEntry { + summary_id: outcome.summary_id.clone(), + content_path: outcome.content_path.clone(), + level: 1, + child_count: input.child_labels.len(), + token_count: input.token_count, + time_range_start: input.time_range_start, + time_range_end: input.time_range_end, + }], + }, + ) + .with_context(|| format!("commit ingested summary {}", outcome.summary_id))?; + + log::debug!( + "[memory_tree::ingest] tinycortex complete sealed={}", + outcome.sealed_ids.len() + ); + Ok(outcome) +} diff --git a/core/src/tree/mod.rs b/core/src/tree/mod.rs new file mode 100644 index 0000000..61e0ab4 --- /dev/null +++ b/core/src/tree/mod.rs @@ -0,0 +1,27 @@ +//! Memory tree — generic summary-tree engine. +//! +//! This module provides the core tree mechanics: bucket-seal cascades, +//! scoring, embedding, entity extraction, retrieval, and summarisation. +//! It is flavor-agnostic; the specific tree instances (global, topic, +//! source) and their policies live in [`crate`]. + +pub mod graph; +pub mod health; +pub mod ingest; +pub mod nlp; +pub mod retrieval; +pub mod score; +pub mod summarise; +// `module_inception` is a byproduct of the domain-family reorg: the parent was +// renamed from `memory_tree` to `memory/tree`, which shortened it to match this +// long-standing inner module. Renaming the inner module would be a real rename +// on top of a pure move, so it is allowed here and left as follow-up. +#[allow(clippy::module_inception)] +pub mod tree; +pub mod tree_runtime; + +// Tree I/O contracts are engine-owned. +pub use tinycortex::memory::tree::{ + TreeLabelStrategy, TreeLeafPayload, TreeReadHit, TreeReadRequest, TreeReadResult, + TreeWriteOutcome, TreeWriteRequest, +}; diff --git a/core/src/tree/nlp/mod.rs b/core/src/tree/nlp/mod.rs new file mode 100644 index 0000000..ed8e40f --- /dev/null +++ b/core/src/tree/nlp/mod.rs @@ -0,0 +1,187 @@ +//! Query-side NLP for the deterministic (E2GraphRAG) retriever. +//! +//! [`extract_query_entities`] turns a natural-language query into a set of +//! canonical entity ids that key into `mem_tree_entity_index` and the +//! co-occurrence graph. It prefers the runtime Python server's spaCy backend +//! (named entities + +//! salient nouns) and falls back to the in-Rust regex extractor whenever +//! spaCy is disabled or unavailable — so retrieval always works offline, just +//! with lower person/org recall. +//! +//! Output is intentionally `Vec`: it reuses +//! `score::resolver::canonicalise` so query entity ids land in the exact +//! same `:` namespace as the indexed chunk entities. No id +//! mismatch, no bespoke join. + +// Provisioning (`ensure_spacy`, `spacy_provisioned`, the model id) stayed in +// the host — it downloads a Python toolchain and supervises a server. Only the +// extraction call and its wire types cross the seam. +pub use crate::nlp_host::{SpacyEntity, SpacyResponse}; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::tree::score::extract::{EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic}; +use crate::tree::score::resolver::{canonicalise, CanonicalEntity}; +use crate::Config; + +/// Map a spaCy entity label to our [`EntityKind`]. Unknown labels collapse to +/// [`EntityKind::Misc`] so they still participate as graph anchors. +fn map_spacy_label(label: &str) -> EntityKind { + match label { + "PERSON" => EntityKind::Person, + "ORG" | "NORP" => EntityKind::Organization, + "GPE" | "LOC" | "FAC" => EntityKind::Location, + "PRODUCT" => EntityKind::Product, + "EVENT" => EntityKind::Event, + "DATE" | "TIME" => EntityKind::Datetime, + "MONEY" | "QUANTITY" | "PERCENT" | "CARDINAL" | "ORDINAL" => EntityKind::Quantity, + "LANGUAGE" => EntityKind::Technology, + _ => EntityKind::Misc, + } +} + +/// Extract canonical query entities, preferring spaCy and falling back to the +/// in-Rust regex extractor. Never fails: an unavailable sidecar degrades to +/// the fallback rather than erroring, because an empty/partial entity set just +/// routes retrieval toward the global (dense) branch. +pub async fn extract_query_entities(config: &Config, query: &str) -> Vec { + let trimmed = query.trim(); + if trimmed.is_empty() { + return Vec::new(); + } + + if config.memory_tree().spacy_enabled { + match crate::nlp_host::extract_spacy(config, trimmed).await { + Ok(resp) => { + let extracted = spacy_to_extracted(&resp); + let canon = canonicalise(&extracted); + log::debug!( + "[memory_tree::nlp] spaCy query extraction: entities={} nouns={} canonical={}", + resp.entities.len(), + resp.nouns.len(), + canon.len() + ); + return canon; + } + Err(e) => { + log::warn!("[memory_tree::nlp] spaCy extraction failed, falling back: {e:#}"); + } + } + } else { + log::debug!("[memory_tree::nlp] spaCy disabled by config — using regex fallback"); + } + + fallback_extract(trimmed).await +} + +/// Build [`ExtractedEntities`] from a spaCy response: named entities become +/// entity spans, salient nouns become topics. Topics are promoted to +/// `topic:` canonical ids by [`canonicalise`]. +fn spacy_to_extracted(resp: &SpacyResponse) -> ExtractedEntities { + let entities = resp + .entities + .iter() + .map(|e| ExtractedEntity { + kind: map_spacy_label(&e.label), + text: e.text.clone(), + span_start: e.start, + span_end: e.end, + score: 1.0, + }) + .collect(); + let topics = resp + .nouns + .iter() + .map(|n| ExtractedTopic { + label: n.clone(), + score: 1.0, + }) + .collect(); + ExtractedEntities { + entities, + topics, + llm_importance: None, + llm_importance_reason: None, + } +} + +/// Regex-only fallback. Deterministic, no network, no LLM — catches +/// emails/urls/handles/hashtags in the query. Person/org recall is lost +/// (spaCy's job), which simply biases retrieval toward the global branch. +async fn fallback_extract(query: &str) -> Vec { + use crate::tree::score::extract::{CompositeExtractor, EntityExtractor}; + let extractor = CompositeExtractor::regex_only(); + match extractor.extract(query).await { + Ok(extracted) => { + let canon = canonicalise(&extracted); + log::debug!( + "[memory_tree::nlp] regex fallback query extraction: canonical={}", + canon.len() + ); + canon + } + Err(e) => { + log::warn!("[memory_tree::nlp] regex fallback failed: {e:#}"); + Vec::new() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg_spacy_off() -> TestHostConfig { + crate::test_seams::init(); + let mut c = TestHostConfig::default(); + c.memory_tree.spacy_enabled = false; + c + } + + #[test] + fn label_mapping_covers_common_kinds() { + assert_eq!(map_spacy_label("PERSON"), EntityKind::Person); + assert_eq!(map_spacy_label("ORG"), EntityKind::Organization); + assert_eq!(map_spacy_label("GPE"), EntityKind::Location); + assert_eq!(map_spacy_label("WHATEVER"), EntityKind::Misc); + } + + #[tokio::test] + async fn fallback_used_when_spacy_disabled_extracts_mechanical_entities() { + let cfg = cfg_spacy_off(); + let ents = extract_query_entities(&cfg, "ping alice@example.com about #launch").await; + assert!( + ents.iter() + .any(|e| e.canonical_id == "email:alice@example.com"), + "regex fallback should find the email; got {ents:?}" + ); + assert!( + ents.iter().any(|e| e.kind == EntityKind::Hashtag), + "regex fallback should find the hashtag; got {ents:?}" + ); + } + + #[tokio::test] + async fn empty_query_yields_no_entities() { + let cfg = cfg_spacy_off(); + assert!(extract_query_entities(&cfg, " ").await.is_empty()); + } + + #[test] + fn spacy_response_maps_nouns_to_topics() { + let resp = SpacyResponse { + entities: vec![crate::nlp_host::SpacyEntity { + text: "Alice".into(), + label: "PERSON".into(), + start: 0, + end: 5, + }], + nouns: vec!["migration".into()], + }; + let extracted = spacy_to_extracted(&resp); + let canon = canonicalise(&extracted); + assert!(canon.iter().any(|c| c.canonical_id == "person:alice")); + assert!(canon.iter().any(|c| c.canonical_id == "topic:migration")); + } +} diff --git a/core/src/tree/retrieval/README.md b/core/src/tree/retrieval/README.md new file mode 100644 index 0000000..a580fbb --- /dev/null +++ b/core/src/tree/retrieval/README.md @@ -0,0 +1,30 @@ +# Retrieval + +Phase 4 (#710) — search-time pipeline for the hierarchical memory tree. Exposes six LLM-callable primitives that read across the source / topic / global trees built by Phase 3 and surface results in a uniform [`RetrievalHit`] shape. There is no classifier, gate, or composer in this phase — orchestration (which tool to call, how to combine) is left to the calling LLM. + +## Public surface + +- `pub fn query_source` / `pub struct QuerySourceRequest` — `source.rs`, `rpc.rs` — per-source summary retrieval, optional semantic rerank. +- `pub fn query_global` / `pub struct QueryGlobalRequest` — `global.rs`, `rpc.rs` — cross-source digest for a window in days. +- `pub fn query_topic` / `pub struct QueryTopicRequest` — `topic.rs`, `rpc.rs` — entity-scoped retrieval across every tree. +- `pub fn search_entities` / `pub struct SearchEntitiesRequest` — `search.rs`, `rpc.rs` — fuzzy LIKE lookup over the entity index. +- `pub fn drill_down` / `pub struct DrillDownRequest` — `drill_down.rs`, `rpc.rs` — walk `child_ids` from a summary one (or more) levels down. +- `pub fn fetch_leaves` / `pub struct FetchLeavesRequest` — `fetch.rs`, `rpc.rs` — batch-hydrate raw chunks by id (cap 20). +- `pub struct RetrievalHit` / `pub enum NodeKind` / `pub struct QueryResponse` / `pub struct EntityMatch` — `types.rs` — wire shapes shared by every tool. +- `pub fn all_retrieval_controller_schemas` / `pub fn all_retrieval_registered_controllers` — `schemas.rs` — registry exports wired into `core::all`. + +## Files + +- `mod.rs` — module surface; declares submodules and the `pub use` re-exports. +- `types.rs` — shared wire types and the `hit_from_summary` / `hit_from_chunk` helpers. +- `source.rs` / `global.rs` / `topic.rs` — query the corresponding tree level. +- `search.rs` — free-text LIKE search over `mem_tree_entity_index`. +- `drill_down.rs` — BFS walk of summary children with optional semantic rerank. +- `fetch.rs` — batch hydration of leaf chunks. +- `rpc.rs` — request / response structs and the JSON-RPC handler bodies. +- `schemas.rs` — `ControllerSchema` definitions and dispatch table for the controller registry. +- `integration_test.rs` — end-to-end test that drives the real ingest pipeline through every retrieval tool. + +## Tests + +Per-tool unit tests live in `mod tests` inside each file. The `integration_test.rs` module is private to this crate and exercises ingest → seal → retrieve in one workspace. diff --git a/core/src/tree/retrieval/benchmarks.rs b/core/src/tree/retrieval/benchmarks.rs new file mode 100644 index 0000000..f59338d --- /dev/null +++ b/core/src/tree/retrieval/benchmarks.rs @@ -0,0 +1,361 @@ +//! Memory retrieval benchmark fixtures — #1538 +//! +//! Deterministic test scenarios that verify retrieval quality and safety +//! for OpenHuman's memory tree. Each scenario exercises the full pipeline +//! (ingest → extract → score → seal → retrieve) using synthetic fixture data +//! so no real user data is required. +//! +//! ## Scenarios +//! +//! | # | Scenario | What it tests | +//! |---|----------|---------------| +//! | 2 | Citation bundle | Retrieval returns chunk/source IDs alongside content | +//! | 5 | Long-source compression | Large source retrieves exact relevant leaf chunk | +//! | 6 | Scale/soak | 20 sources stay correct under `query_source` + `search_entities` | +//! +//! The entity-/topic-retrieval scenarios (cross-chat recall, stale +//! preference, contradiction handling, drill-down isolation) were retired +//! with the topic tree — source trees + the entity index remain the substrate. +//! +//! Run with: `cargo test --package openhuman_core -- retrieval_benchmarks` + +use chrono::{TimeZone, Utc}; +use tempfile::TempDir; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::ingest_pipeline::ingest_chat; +use crate::queue::testing::drain_until_idle; +use crate::store::chunks::types::SourceKind; +use crate::tree::retrieval::{fetch_leaves, query_source, search_entities}; +use crate::Config; +use tinycortex::memory::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; + +/// Shared test config — disables embedding for deterministic inert behaviour. +fn bench_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + (tmp, cfg) +} + +/// Helper: ingest a chat batch with deterministic timestamps. +/// Each message is padded with entity-bearing text (email + hashtag) to ensure +/// the entity index gets populated reliably. This is required because: +/// 1. The regex extractor finds emails (alice@example.com) and hashtags (#phoenix) +/// 2. Without these, `search_entities` returns 0 hits and entity-based tests fail +/// 3. The sealing threshold also needs sufficient content per message +async fn ingest_chat_batch( + cfg: &Config, + scope: &str, + owner: &str, + messages: Vec<(String, String)>, + base_ts_millis: i64, +) -> Vec { + let batch = ChatBatch { + platform: "slack".into(), + channel_label: scope.into(), + messages: messages + .into_iter() + .enumerate() + .map(|(i, (author, text))| { + // Pad messages with entity-bearing content to ensure reliable extraction. + // The entity extractor needs: + // - Email pattern: test@entity.example (regex finds emails) + // - Hashtag pattern: #topic (regex finds hashtags + emits topic entities) + // - Minimum content for sealing: ~200+ chars total + let padded_text = format!("{} #benchmark test@entity.example", text); + ChatMessage { + author, + timestamp: Utc + .timestamp_millis_opt(base_ts_millis + (i as i64) * 60_000) + .unwrap(), + text: padded_text, + source_ref: None, + } + }) + .collect(), + }; + let result = ingest_chat(cfg, scope, owner, vec![], batch).await.unwrap(); + drain_until_idle(cfg).await.unwrap(); + result.chunk_ids +} + +/// Verify search_entities surfaces entities from both chats independently. +#[tokio::test] +async fn bench_cross_chat_entity_discoverable() { + let (_tmp, cfg) = bench_config(); + + ingest_chat_batch( + &cfg, + "slack:#eng", + "alice", + vec![( + "alice".into(), + "alice@example.com is leading the Phoenix migration.".into(), + )], + 1_700_000_000_000, + ) + .await; + + ingest_chat_batch( + &cfg, + "slack:#ops", + "carol", + vec![( + "carol".into(), + "alice@example.com confirmed the Friday timeline.".into(), + )], + 1_700_100_000_000, + ) + .await; + + let matches = search_entities(&cfg, "alice", None, 10).await.unwrap(); + + // alice should be discoverable via canonical email id + let alice = matches + .iter() + .find(|m| m.canonical_id.contains("alice@example.com")) + .expect("alice should be discoverable from both chats"); + + assert!( + alice.mention_count >= 2, + "alice should have >= 2 mentions across both chats, got {}", + alice.mention_count + ); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Scenario 2 — Citation bundle +// ───────────────────────────────────────────────────────────────────────────── + +/// Verify retrieval returns chunk IDs and source refs (provenance chain). +#[tokio::test] +async fn bench_citation_bundle_provenance() { + let (_tmp, cfg) = bench_config(); + + // Use a URL-bearing message to ensure entity indexing works + // and pad to trigger sealing (sealing needs sufficient content) + ingest_chat_batch( + &cfg, + "slack:#eng", + "alice", + vec![( + "alice".into(), + "RFC-42 v3 is approved. Link: https://example.com/rfc42 is ready for review.".into(), + )], + 1_700_000_000_000, + ) + .await; + + // query_source for Chat — should return hits with source_ref populated + let source_resp = query_source(&cfg, None, Some(SourceKind::Chat), None, None, 20) + .await + .unwrap(); + + // Guard: source trees only seal when summarization runs (depends on embedder config). + // Without a sealed tree query_source returns 0 hits — skip assertions in that case. + if source_resp.total == 0 { + return; + } + + // Find hits with provenance + let prov_hits: Vec<_> = source_resp + .hits + .iter() + .filter(|h| h.source_ref.is_some()) + .collect(); + + assert!( + !prov_hits.is_empty(), + "retrieval hits should include source_ref provenance (citation bundle)" + ); + + for hit in prov_hits { + assert!( + !hit.node_id.is_empty(), + "hit node_id must be populated for citation" + ); + assert!( + hit.tree_kind.as_str() == "source" || hit.tree_kind.as_str() == "chat", + "hit tree_kind should be source or chat, got {:?}", + hit.tree_kind + ); + } +} + +/// fetch_leaves should hydrate exact chunk IDs with full content. +#[tokio::test] +async fn bench_citation_fetch_leaves_hydrates() { + let (_tmp, cfg) = bench_config(); + + let chunk_ids = ingest_chat_batch( + &cfg, + "slack:#eng", + "alice", + vec![( + "alice".into(), + "Critical decision: all services must migrate to TLS 1.3 by Q4.".into(), + )], + 1_700_000_000_000, + ) + .await; + + drain_until_idle(&cfg).await.unwrap(); + + let leaves = fetch_leaves(&cfg, &chunk_ids).await.unwrap(); + + assert_eq!( + leaves.len(), + chunk_ids.len(), + "fetch_leaves must hydrate all requested chunk IDs" + ); + + for (leaf, expected_id) in leaves.iter().zip(chunk_ids.iter()) { + assert_eq!( + leaf.node_id, *expected_id, + "fetch_leaves response node_id should match requested chunk_id" + ); + assert!( + !leaf.content.is_empty(), + "fetch_leaves should return non-empty content" + ); + // source_ref is populated during summarization (sealed trees). If the + // embedder is disabled the tree won't seal and source_ref will be None + // — this is not a test failure, just an environment constraint. + if leaf.source_ref.is_none() { + continue; + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Scenario 3 — Stale preference +// ───────────────────────────────────────────────────────────────────────────── + +// ───────────────────────────────────────────────────────────────────────────── +// Scenario 5 — Long-source compression +// ───────────────────────────────────────────────────────────────────────────── + +/// A large source (> 10k tokens) should retrieve only the exact relevant leaf +/// chunk, not the entire source content. +#[tokio::test] +async fn bench_long_source_retrieves_exact_leaf() { + let (_tmp, cfg) = bench_config(); + + // Build a long conversation — 30 messages, each ~200 tokens + // Total far exceeds the chunk size, forcing multiple chunks + let messages: Vec<(String, String)> = (0..30) + .map(|i| { + ( + "alice".into(), + format!( + "Engineering log {}: Detailed technical note about system architecture \ + design decisions, database sharding strategy, and deployment \ + pipeline configuration for the Phoenix project. This entry contains \ + specific implementation details for iteration {}.", + i, i + ), + ) + }) + .collect(); + + ingest_chat_batch(&cfg, "slack:#eng", "alice", messages, 1_700_000_000_000).await; + + drain_until_idle(&cfg).await.unwrap(); + + // Query the long source — should return summaries, not raw chunks + let source_resp = query_source(&cfg, None, Some(SourceKind::Chat), None, None, 20) + .await + .unwrap(); + + // Guard: if nothing sealed (budget not crossed), skip assertions. + if source_resp.total == 0 { + return; + } + + // Total hits should be bounded (summaries, not all raw chunks) + assert!( + source_resp.total <= 10, + "long source should not dump all chunks; expected <= 10 summaries, got {}", + source_resp.total + ); + + // If we have summaries, they should be compact + for hit in &source_resp.hits { + assert!( + hit.content.len() <= 1000, + "summary hit should be compact (≤ 1000 chars), got {} for: {}", + hit.content.len(), + hit.content.chars().take(50).collect::() + ); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Scenario 6 — Scale/soak fixture (no real user data) +// ───────────────────────────────────────────────────────────────────────────── + +/// Ingest 20 sources across 5 platforms — verify retrieval remains correct +/// at scale without any real user data. +#[tokio::test] +async fn bench_scale_ingest_20_sources_no_real_data() { + let (_tmp, cfg) = bench_config(); + + let platforms = vec![ + ("slack:#eng", "alice"), + ("slack:#ops", "bob"), + ("slack:#product", "carol"), + ("email:team", "dave"), + ("email:security", "eve"), + ]; + + for (i, (scope, owner)) in platforms.iter().cycle().take(20).enumerate() { + let scope_str = scope.to_string(); + let owner_str = owner.to_string(); + ingest_chat_batch( + &cfg, + &scope_str, + &owner_str, + vec![( + owner_str.clone(), + format!( + "Scale test message {} from {} — verifying retrieval correctness \ + at volume with deterministic synthetic data. No PII present.", + i, owner_str + ), + )], + 1_700_000_000_000 + (i as i64) * 60_000, + ) + .await; + } + + drain_until_idle(&cfg).await.unwrap(); + + // query_source should show activity across the window + let source_resp = query_source(&cfg, None, None, None, None, 30) + .await + .unwrap(); + + // Guard: query_source returns hits only from sealed (summarized) source trees. + // Without an embedder configured the summarizer won't run, so trees will + // remain unsealed and query_source returns 0 hits — skip in that case. + if source_resp.total == 0 { + return; + } + + // search_entities for each owner should return results + for (_, owner) in &platforms { + let matches = search_entities(&cfg, owner, None, 5).await.unwrap(); + assert!( + !matches.is_empty(), + "search_entities should find owner '{}' after scale ingest", + owner + ); + } +} diff --git a/core/src/tree/retrieval/cover.rs b/core/src/tree/retrieval/cover.rs new file mode 100644 index 0000000..6f72f9a --- /dev/null +++ b/core/src/tree/retrieval/cover.rs @@ -0,0 +1,42 @@ +use anyhow::Result; + +use crate::source_scope::current_source_scope; +use crate::store::chunks::types::SourceKind; +use crate::tinycortex::engine_config; +use crate::tree::retrieval::types::QueryResponse; +use crate::Config; + +const DEFAULT_LIMIT: usize = 200; + +pub async fn cover_window( + config: &Config, + since_ms: i64, + until_ms: i64, + source_id: Option<&str>, + source_kind: Option, + limit: usize, +) -> Result { + let limit = if limit == 0 { DEFAULT_LIMIT } else { limit }; + let scope = current_source_scope(); + if source_id.is_some_and(|id| scope.as_ref().is_some_and(|set| !set.contains(id))) { + return Ok(QueryResponse::empty()); + } + log::debug!( + "[retrieval::cover] tinycortex has_source_id={} source_kind={:?} limit={}", + source_id.is_some(), + source_kind.map(|k| k.as_str()), + limit + ); + let mut response = tinycortex::memory::retrieval::cover_window_scoped( + &engine_config(config), + since_ms, + until_ms, + source_id, + source_kind, + scope, + usize::MAX, + )?; + let total = response.hits.len(); + response.hits.truncate(limit); + Ok(QueryResponse::new(response.hits, total)) +} diff --git a/core/src/tree/retrieval/drill_down.rs b/core/src/tree/retrieval/drill_down.rs new file mode 100644 index 0000000..d6fe480 --- /dev/null +++ b/core/src/tree/retrieval/drill_down.rs @@ -0,0 +1,50 @@ +use anyhow::Result; + +use crate::source_scope::current_source_scope; +use crate::tinycortex::engine_config; +use crate::tree::retrieval::engine::EmbedderBridge; +use crate::tree::retrieval::types::RetrievalHit; +use crate::tree::score::embed::{build_embedder_from_config, InertEmbedder}; +use crate::Config; + +pub async fn drill_down( + config: &Config, + node_id: &str, + max_depth: u32, + query: Option<&str>, + limit: Option, +) -> Result> { + log::debug!( + "[retrieval::drill_down] tinycortex max_depth={} has_query={} limit={:?}", + max_depth, + query.is_some(), + limit + ); + let embedder = if query.is_none() || max_depth == 0 { + log::debug!("[retrieval::drill_down] using inert embedder for non-semantic traversal"); + Box::new(InertEmbedder::new()) as Box + } else { + build_embedder_from_config(config)? + }; + let bridge = EmbedderBridge(embedder.as_ref()); + let engine_limit = current_source_scope() + .as_ref() + .map(|_| None) + .unwrap_or(limit); + let mut hits = tinycortex::memory::retrieval::drill_down( + &engine_config(config), + node_id, + max_depth, + query, + &bridge, + engine_limit, + ) + .await?; + if let Some(set) = current_source_scope() { + hits.retain(|hit| set.contains(&hit.tree_scope)); + } + if let Some(limit) = limit { + hits.truncate(limit); + } + Ok(hits) +} diff --git a/core/src/tree/retrieval/engine.rs b/core/src/tree/retrieval/engine.rs new file mode 100644 index 0000000..a23bcfe --- /dev/null +++ b/core/src/tree/retrieval/engine.rs @@ -0,0 +1,21 @@ +use anyhow::Result; +use async_trait::async_trait; + +use crate::tree::score::embed::Embedder as HostEmbedder; + +pub(super) struct EmbedderBridge<'a>(pub &'a dyn HostEmbedder); + +#[async_trait] +impl tinycortex::memory::score::embed::Embedder for EmbedderBridge<'_> { + fn name(&self) -> &'static str { + self.0.name() + } + + async fn embed(&self, text: &str) -> Result> { + self.0.embed(text).await + } + + async fn embed_batch(&self, texts: &[&str]) -> Vec>> { + self.0.embed_batch(texts).await + } +} diff --git a/core/src/tree/retrieval/fast.rs b/core/src/tree/retrieval/fast.rs new file mode 100644 index 0000000..5b219e1 --- /dev/null +++ b/core/src/tree/retrieval/fast.rs @@ -0,0 +1,42 @@ +//! Product adapters for tinycortex-owned deterministic fast retrieval. + +use anyhow::Result; + +use crate::source_scope::current_source_scope; +use crate::tinycortex::engine_config; +use crate::tree::nlp; +use crate::tree::retrieval::engine::EmbedderBridge; +use crate::tree::retrieval::types::QueryResponse; +use crate::tree::score::embed::build_embedder_from_config; +use crate::Config; + +pub use tinycortex::memory::retrieval::FastRetrieveOptions; + +pub async fn fast_retrieve( + config: &Config, + query: &str, + options: FastRetrieveOptions, +) -> Result { + let query_entities = nlp::extract_query_entities(config, query).await; + let entity_ids: Vec<_> = query_entities + .into_iter() + .map(|entity| entity.canonical_id) + .collect(); + log::debug!( + "[retrieval::fast] tinycortex query_len={} entities={} limit={} hops={}", + query.len(), + entity_ids.len(), + options.limit, + options.max_hops + ); + let embedder = build_embedder_from_config(config)?; + tinycortex::memory::retrieval::fast_retrieve( + &engine_config(config), + query, + &entity_ids, + &EmbedderBridge(embedder.as_ref()), + current_source_scope().as_ref(), + options, + ) + .await +} diff --git a/core/src/tree/retrieval/fetch.rs b/core/src/tree/retrieval/fetch.rs new file mode 100644 index 0000000..79fdaa4 --- /dev/null +++ b/core/src/tree/retrieval/fetch.rs @@ -0,0 +1,32 @@ +use anyhow::Result; + +use crate::source_scope::chunk_source_allowed_in; +use crate::source_scope::current_source_scope; +use crate::store::chunks::store::get_chunks_batch; +use crate::tinycortex::engine_config; +use crate::tree::retrieval::types::RetrievalHit; +use crate::Config; + +pub use tinycortex::memory::retrieval::MAX_BATCH; + +pub async fn fetch_leaves(config: &Config, chunk_ids: &[String]) -> Result> { + log::debug!( + "[retrieval::fetch] tinycortex requested={}", + chunk_ids.len() + ); + let permitted_ids = if let Some(set) = current_source_scope() { + let chunks = get_chunks_batch(config, chunk_ids)?; + chunk_ids + .iter() + .filter(|id| { + chunks.get(*id).is_some_and(|chunk| { + chunk_source_allowed_in(&set, &chunk.metadata.tags, &chunk.metadata.source_id) + }) + }) + .cloned() + .collect::>() + } else { + chunk_ids.to_vec() + }; + tinycortex::memory::retrieval::fetch_leaves(&engine_config(config), &permitted_ids) +} diff --git a/core/src/tree/retrieval/integration_tests.rs b/core/src/tree/retrieval/integration_tests.rs new file mode 100644 index 0000000..9e41c8f --- /dev/null +++ b/core/src/tree/retrieval/integration_tests.rs @@ -0,0 +1,292 @@ +//! End-to-end integration test for Phase 4 retrieval tools (#710). +//! +//! Wires the real ingest pipeline (`ingest_chat`) + the six retrieval +//! primitives together to catch drift between ingestion-side schema +//! writes (entity index, trees, summaries) and retrieval-side reads. +//! +//! This lives next to the per-tool unit tests rather than under `tests/` +//! because it needs access to private internals (`Config::default`, +//! `score::store::*`) without spinning the full RPC stack. + +#![cfg(test)] + +use chrono::{TimeZone, Utc}; +use tempfile::TempDir; + +use tinymemory_api::host::MemoryHostConfig; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::ingest_pipeline::ingest_chat; +use crate::store::chunks::types::SourceKind; +use crate::tree::retrieval::{drill_down, fetch_leaves, query_source, search_entities}; +use tinycortex::memory::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; + +fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + // Phase 4 (#710): ingest embeds chunks; tests use inert for determinism. + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + // #002 (FR-002): the write path now SKIPS embedding (returns None) when no + // provider is configured, instead of silently using a zero-vector inert + // embedder. These integration tests assert embeddings ARE populated + // end-to-end, so opt into the inert embedder explicitly — `provider=none` + // is the deterministic "vector search by choice" path that + // `build_write_embedder` returns as Some(inert). + cfg.embeddings_provider = Some("none".into()); + (tmp, cfg) +} + +fn chat_about_phoenix(seq: u32) -> ChatBatch { + ChatBatch { + platform: "slack".into(), + channel_label: "#eng".into(), + messages: vec![ + ChatMessage { + author: "alice".into(), + timestamp: Utc + .timestamp_millis_opt(1_700_000_000_000 + (seq as i64) * 10_000) + .unwrap(), + text: format!( + "Phoenix migration status update {seq}: the runbook review is \ + proceeding. alice@example.com is coordinating. We land \ + Friday evening." + ), + source_ref: Some(format!("slack://phoenix/{seq}")), + }, + ChatMessage { + author: "bob".into(), + timestamp: Utc + .timestamp_millis_opt(1_700_000_001_000 + (seq as i64) * 10_000) + .unwrap(), + text: "Confirmed. I'll handle coordination. #launch-q2 tracked in \ + Notion. bob@example.com will cut the release." + .to_string(), + source_ref: Some(format!("slack://phoenix/{seq}-reply")), + }, + ], + } +} + +#[tokio::test] +async fn end_to_end_three_chat_batches() { + let (_tmp, cfg) = test_config(); + + // Ingest three batches in distinct slack channels. + for (i, scope) in ["slack:#eng", "slack:#ops", "slack:#product"] + .iter() + .enumerate() + { + ingest_chat(&cfg, scope, "alice", vec![], chat_about_phoenix(i as u32)) + .await + .unwrap(); + } + + // ── search_entities should surface alice under her canonical email id. + let matches = search_entities(&cfg, "alice", None, 10).await.unwrap(); + let alice = matches + .iter() + .find(|m| m.canonical_id == "email:alice@example.com") + .expect("alice should be discoverable via search"); + assert!(alice.mention_count >= 1); + + // ── query_source by source_id returns what we put in (chunks get + // surfaced directly since none of the channels seal — 2 short msgs + // per channel is under the seal budget). + let by_source_kind = query_source(&cfg, None, Some(SourceKind::Chat), None, None, 20) + .await + .unwrap(); + // query_source returns summaries from sealed source trees only. With two + // messages per channel the seal budget is not reached, so sealed + // summaries may not exist yet. The invariant we lock in is that the + // response is well-formed: total accurately reflects hits.len() (or + // exceeds it when truncated) and never reports more hits than total. + assert!( + by_source_kind.total >= by_source_kind.hits.len(), + "query_source total must be >= hits.len()" + ); + + // ── drill_down on a bogus id returns empty (no error). + let empty_drill = drill_down(&cfg, "bogus:id", 1, None, None).await.unwrap(); + assert!(empty_drill.is_empty()); + + // ── fetch_leaves on a bogus id hydrates nothing (no error). + let none = fetch_leaves(&cfg, &["ghost:nonexistent".to_string()]) + .await + .unwrap(); + assert!(none.is_empty()); +} + +// ── Phase 4 (#710): embedding + semantic rerank tests ─────────────────── + +/// Ingest with an inert embedder must populate every kept chunk's +/// `embedding` column. Embeddings are written by the async `extract_chunk` +/// handler, so the test drains the queue before inspecting. +#[tokio::test] +async fn ingest_populates_chunk_embeddings() { + use crate::queue::drain_until_idle; + use crate::store::chunks::store::get_chunk_embedding; + use crate::tree::score::embed::EMBEDDING_DIM; + + let (_tmp, cfg) = test_config(); + let out = ingest_chat(&cfg, "slack:#eng", "alice", vec![], chat_about_phoenix(0)) + .await + .unwrap(); + assert!( + out.chunks_written >= 1, + "expected at least one persisted chunk" + ); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(60); + loop { + drain_until_idle(&cfg).await.unwrap(); + let all_embedded = out + .chunk_ids + .iter() + .all(|id| get_chunk_embedding(&cfg, id).ok().flatten().is_some()); + if all_embedded { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "chunk embeddings were not persisted before timeout" + ); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + for id in &out.chunk_ids { + let emb = get_chunk_embedding(&cfg, id).unwrap(); + let v = emb.unwrap_or_else(|| panic!("embedding missing for chunk_id={id}")); + assert_eq!(v.len(), EMBEDDING_DIM, "embedding for {id} has wrong dim"); + } +} + +/// Seal through the source-tree cascade must populate the summary's +/// embedding column. We drive large chunks directly through `append_leaf` +/// to cross the 10k-token seal budget, then inspect the L1 summary row. +/// This mirrors the bucket-seal unit test pattern — the ingest-driven +/// path uses the chunker, which caps individual chunk tokens and keeps +/// the seal from firing on short batches. +#[tokio::test] +async fn seal_populates_summary_embedding() { + use crate::chat::{test_override, ChatProvider, StaticChatProvider}; + use crate::store::chunks::store::upsert_chunks; + use crate::store::chunks::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; + use crate::store::content as content_store; + use crate::tree::score::embed::EMBEDDING_DIM; + use crate::tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef}; + use crate::tree::tree::store as src_store; + use crate::tree_source::registry::get_or_create_source_tree; + use std::sync::Arc; + + let (_tmp, cfg) = test_config(); + let tree = get_or_create_source_tree(&cfg, "slack:#seal-test").unwrap(); + let provider: Arc = Arc::new(StaticChatProvider::new("test summary content")); + let ts = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(); + + let mk_chunk = |seq: u32, tokens: u32| Chunk { + id: chunk_id(SourceKind::Chat, "slack:#seal-test", seq, "test-content"), + content: format!("substantive chunk content {seq}"), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: "slack:#seal-test".into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: vec![], + source_ref: Some(SourceRef::new("slack://x")), + path_scope: None, + }, + token_count: tokens, + seq_in_source: seq, + created_at: ts, + partial_message: false, + }; + let c1 = mk_chunk(0, 30_000); + let c2 = mk_chunk(1, 30_000); + upsert_chunks(&cfg, &[c1.clone(), c2.clone()]).unwrap(); + { + let content_root = cfg.memory_tree_content_root(); + std::fs::create_dir_all(&content_root).expect("create content_root for test"); + let staged = content_store::stage_chunks(&content_root, &[c1.clone(), c2.clone()]) + .expect("stage_chunks for test chunks"); + crate::store::chunks::store::with_connection(&cfg, |conn| { + let tx = conn.unchecked_transaction()?; + crate::store::chunks::store::upsert_staged_chunks_tx(&tx, &staged)?; + tx.commit()?; + Ok(()) + }) + .expect("persist staged chunk pointers"); + } + + let leaf_of = |c: &Chunk| LeafRef { + chunk_id: c.id.clone(), + token_count: c.token_count, + timestamp: c.metadata.timestamp, + content: c.content.clone(), + entities: vec![], + topics: vec![], + score: 0.5, + }; + test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf_of(&c1), &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; + let sealed = test_override::with_provider(Arc::clone(&provider), async { + append_leaf(&cfg, &tree, &leaf_of(&c2), &LabelStrategy::Empty) + .await + .unwrap() + }) + .await; + assert_eq!(sealed.len(), 1, "expected one seal at the budget crossing"); + + // #1574 cutover: the seal path no longer writes the legacy + // `mem_tree_summaries.embedding` column — the vector is persisted to the + // per-model sidecar at the active signature inside the seal tx. Assert + // it round-trips through the public accessor (which reads the sidecar at + // the active signature), validating the write-side cutover end to end. + let summary = src_store::get_summary(&cfg, &sealed[0]).unwrap().unwrap(); + assert!( + summary.embedding.is_none(), + "legacy summary embedding column must be NULL post-cutover" + ); + let emb = src_store::get_summary_embedding(&cfg, &sealed[0]) + .unwrap() + .expect("sealed summary must have an embedding in the per-model sidecar"); + assert_eq!(emb.len(), EMBEDDING_DIM); +} + +/// Setting `query = Some(...)` changes ordering relative to the default +/// recency sort. We can't easily assert specific similarity scores when +/// using the inert embedder (all zero vectors → all similarities are 0), +/// so we instead verify that (a) the path doesn't error out and (b) the +/// response total/hit counts match the non-semantic path. Semantic +/// reranking correctness is covered in the per-tool unit tests below. +#[tokio::test] +async fn query_source_with_query_returns_same_count() { + let (_tmp, cfg) = test_config(); + ingest_chat(&cfg, "slack:#eng", "alice", vec![], chat_about_phoenix(0)) + .await + .unwrap(); + + let recency = query_source(&cfg, None, Some(SourceKind::Chat), None, None, 20) + .await + .unwrap(); + let semantic = query_source( + &cfg, + None, + Some(SourceKind::Chat), + None, + Some("phoenix migration"), + 20, + ) + .await + .unwrap(); + assert_eq!(recency.total, semantic.total); + assert_eq!(recency.hits.len(), semantic.hits.len()); +} diff --git a/core/src/tree/retrieval/mod.rs b/core/src/tree/retrieval/mod.rs new file mode 100644 index 0000000..eb6910b --- /dev/null +++ b/core/src/tree/retrieval/mod.rs @@ -0,0 +1,42 @@ +//! Retrieval tools for the hierarchical memory tree (#710). +//! +//! Exposes the **source** trees as LLM-callable primitives. Each tool is +//! deterministic and scope-specific; orchestration (which tool to call, how +//! to combine results) is left to the calling LLM — there is no classifier, +//! gate, or composer here. The global (time-axis) and topic (subject-axis) +//! trees were removed: source trees hold all the content, and walking the +//! source hierarchy plus the entity index reconstructs both projections. +//! +//! Public JSON-RPC surface (see `schemas.rs`): +//! - `openhuman.memory_tree_query_source` — per-source summary retrieval +//! - `openhuman.memory_tree_search_entities` — fuzzy canonical-id lookup +//! - `openhuman.memory_tree_drill_down` — walk summary children +//! - `openhuman.memory_tree_fetch_leaves` — batch chunk hydration +//! - `openhuman.memory_tree_cover_window` — minimum-node cover of a window +//! +//! All tools share the [`types::RetrievalHit`] / [`types::QueryResponse`] +//! shape so the LLM sees a uniform schema regardless of which tool ran. + +pub mod cover; +pub mod drill_down; +mod engine; +pub mod fast; +pub mod fetch; +pub mod search; +pub mod source; +pub mod types; + +#[cfg(test)] +mod benchmarks; +#[cfg(test)] +mod integration_tests; +#[cfg(test)] +mod source_scope_tests; + +pub use cover::cover_window; +pub use drill_down::drill_down; +pub use fast::{fast_retrieve, FastRetrieveOptions}; +pub use fetch::fetch_leaves; +pub use search::search_entities; +pub use source::query_source; +pub use types::{EntityMatch, NodeKind, QueryResponse, RetrievalHit}; diff --git a/core/src/tree/retrieval/search.rs b/core/src/tree/retrieval/search.rs new file mode 100644 index 0000000..9b379d8 --- /dev/null +++ b/core/src/tree/retrieval/search.rs @@ -0,0 +1,26 @@ +use anyhow::Result; + +use crate::tinycortex::engine_config; +use crate::tree::retrieval::types::EntityMatch; +use crate::tree::score::extract::EntityKind; +use crate::Config; + +pub async fn search_entities( + config: &Config, + query: &str, + kinds: Option>, + limit: usize, +) -> Result> { + log::debug!( + "[retrieval::search] tinycortex query_len={} kinds={} limit={}", + query.len(), + kinds.as_ref().map_or(0, Vec::len), + limit + ); + tinycortex::memory::retrieval::search_entities( + &engine_config(config), + query, + kinds.as_deref(), + limit, + ) +} diff --git a/core/src/tree/retrieval/source.rs b/core/src/tree/retrieval/source.rs new file mode 100644 index 0000000..fa91a6a --- /dev/null +++ b/core/src/tree/retrieval/source.rs @@ -0,0 +1,64 @@ +use anyhow::Result; + +use crate::source_scope::current_source_scope; +use crate::store::chunks::types::SourceKind; +use crate::tinycortex::engine_config; +use crate::tree::retrieval::engine::EmbedderBridge; +use crate::tree::retrieval::types::QueryResponse; +use crate::tree::score::embed::build_embedder_from_config; +use crate::Config; + +const DEFAULT_LIMIT: usize = 10; + +pub async fn query_source( + config: &Config, + source_id: Option<&str>, + source_kind: Option, + time_window_days: Option, + query: Option<&str>, + limit: usize, +) -> Result { + let limit = if limit == 0 { DEFAULT_LIMIT } else { limit }; + let scope = current_source_scope(); + if source_id.is_some_and(|id| scope.as_ref().is_some_and(|set| !set.contains(id))) { + log::debug!("[retrieval::source] explicit source excluded by active scope"); + return Ok(QueryResponse::empty()); + } + + log::debug!( + "[retrieval::source] tinycortex query has_source_id={} source_kind={:?} window_days={:?} has_query={} limit={}", + source_id.is_some(), source_kind.map(|k| k.as_str()), time_window_days, query.is_some(), limit + ); + let semantic_query = query.filter(|value| !value.trim().is_empty()); + let mut response = if let Some(query) = semantic_query { + let embedder = build_embedder_from_config(config)?; + let bridge = EmbedderBridge(embedder.as_ref()); + tinycortex::memory::retrieval::query_source( + &engine_config(config), + source_id, + source_kind, + time_window_days, + Some(query), + &bridge, + usize::MAX, + ) + .await? + } else { + tinycortex::memory::retrieval::query_source( + &engine_config(config), + source_id, + source_kind, + time_window_days, + None, + &tinycortex::memory::score::embed::InertEmbedder::new(), + usize::MAX, + ) + .await? + }; + if let Some(set) = scope { + response.hits.retain(|hit| set.contains(&hit.tree_scope)); + } + let total = response.hits.len(); + response.hits.truncate(limit); + Ok(QueryResponse::new(response.hits, total)) +} diff --git a/core/src/tree/retrieval/source_scope_tests.rs b/core/src/tree/retrieval/source_scope_tests.rs new file mode 100644 index 0000000..a36c4ea --- /dev/null +++ b/core/src/tree/retrieval/source_scope_tests.rs @@ -0,0 +1,664 @@ +//! Characterization tests for the THREE distinct `source_scope` predicates. +//! +//! These pin **current** behaviour — including behaviour that looks wrong. Do +//! not "fix" anything asserted here: a failure means a refactor changed one of +//! the predicates, which is exactly what these tests exist to catch. +//! +//! 1. `fetch.rs` → `source_scope::chunk_source_allowed_in`: fail-OPEN for +//! chunks without the `memory_sources` tag, otherwise equality on +//! `source_id` OR the `mem_src:{id}:` composite rule via +//! `sync_events::extract_mem_src_id` (which returns `None` for an EMPTY +//! item id, so `mem_src:src-abc:` is BLOCKED host-side). +//! 2. `source.rs` / `drill_down.rs` → `hits.retain(|h| set.contains(&h.tree_scope))`: +//! PLAIN EQUALITY on a DIFFERENT field. No tag fail-open, no `mem_src:` +//! prefix rule. For leaf hits `tree_scope` *is* the chunk's `source_id` +//! (`tinycortex` `retrieval::{fetch,drill_down}`), so on leaves this is +//! strictly narrower than predicate 1. `source.rs` / `cover.rs` additionally +//! carry a *pre-filter* short circuit on the explicit `source_id` argument. +//! 3. `tinycortex::memory::chunks::store_list::append_source_scope` — a SQL +//! predicate applied BEFORE `LIMIT`. Reached via `cover_window_scoped` and +//! the raw `list_chunks` callers. It admits `mem_src:src-abc:` (empty item +//! id), diverging from predicate 1. +//! +//! Note: `fast_retrieve` does NOT reach predicate 3 — it threads the scope into +//! `resolve_local` / `dense`, which apply the predicate-2 `tree_scope` retain. + +#![cfg(test)] + +use std::collections::HashSet; + +use chrono::{TimeZone, Utc}; +use tempfile::TempDir; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::source_scope::{chunk_source_allowed_in, with_source_scope}; +use crate::store::chunks::store::{ + list_chunks, upsert_chunks, upsert_staged_chunks_tx, with_connection, ListChunksQuery, +}; +use crate::store::chunks::types::{chunk_id, Chunk, Metadata, SourceKind, SourceRef}; +use crate::store::content as content_store; +use crate::store::trees::store::{insert_summary_tx, insert_tree}; +use crate::store::trees::types::{SummaryNode, Tree, TreeKind, TreeStatus}; +use crate::tree::retrieval::{cover_window, drill_down, fetch_leaves, query_source}; +use crate::Config; + +const BASE_MS: i64 = 1_700_000_000_000; +const MEMORY_SOURCES: &str = "memory_sources"; + +// ── fixtures ───────────────────────────────────────────────────────────── + +fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + // Inert embedder keeps these deterministic and avoids any real provider + // call. Every retrieval call below passes `query: None`, so no embedder is + // ever built. + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + (tmp, cfg) +} + +/// A chunk in `source`, tagged with `tags`, timestamped `ts_ms`. +fn src_chunk(source: &str, seq: u32, tags: &[&str], ts_ms: i64) -> Chunk { + let ts = Utc.timestamp_millis_opt(ts_ms).unwrap(); + Chunk { + id: chunk_id(SourceKind::Chat, source, seq, "test-content"), + content: format!("content-{source}-{seq}"), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: source.into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: tags.iter().map(|t| (*t).to_string()).collect(), + source_ref: Some(SourceRef::new(format!("slack://{source}/{seq}"))), + path_scope: None, + }, + token_count: 20, + seq_in_source: seq, + created_at: ts, + partial_message: false, + } +} + +/// Persist chunk rows AND their staged content bodies, mirroring `rpc.rs`. +fn seed_chunks(cfg: &Config, chunks: &[Chunk]) { + upsert_chunks(cfg, chunks).expect("upsert_chunks"); + let content_root = cfg.memory_tree_content_root(); + std::fs::create_dir_all(&content_root).expect("create content_root for test"); + let staged = content_store::stage_chunks(&content_root, chunks).expect("stage_chunks"); + with_connection(cfg, |conn| { + let tx = conn.unchecked_transaction()?; + upsert_staged_chunks_tx(&tx, &staged)?; + tx.commit()?; + Ok(()) + }) + .expect("persist staged chunk pointers"); +} + +fn seed_tree(cfg: &Config, id: &str, scope: &str, root_id: &str, max_level: u32) { + let ts = Utc.timestamp_millis_opt(BASE_MS).unwrap(); + let tree = Tree { + id: id.to_string(), + kind: TreeKind::Source, + scope: scope.to_string(), + ask: None, + root_id: Some(root_id.to_string()), + max_level, + status: TreeStatus::Active, + created_at: ts, + last_sealed_at: Some(ts), + }; + insert_tree(cfg, &tree).expect("insert_tree"); +} + +fn seed_summary(cfg: &Config, id: &str, tree_id: &str, level: u32, children: &[&str]) { + let ts = Utc.timestamp_millis_opt(BASE_MS).unwrap(); + let node = SummaryNode { + id: id.to_string(), + tree_id: tree_id.to_string(), + tree_kind: TreeKind::Source, + level, + parent_id: None, + child_ids: children.iter().map(|c| (*c).to_string()).collect(), + content: format!("seal-{id}"), + token_count: 100, + entities: vec![], + topics: vec![], + time_range_start: ts, + time_range_end: ts, + score: 0.5, + sealed_at: ts, + deleted: false, + embedding: None, + doc_id: None, + version_ms: None, + }; + with_connection(cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &node, None, "test")?; + tx.commit()?; + Ok(()) + }) + .expect("insert summary"); +} + +fn set_of(items: &[&str]) -> HashSet { + items.iter().map(|s| (*s).to_string()).collect() +} + +fn scoped_query(scope: Option<&[&str]>) -> ListChunksQuery { + ListChunksQuery { + source_scope: scope.map(set_of), + exclude_dropped: false, + ..Default::default() + } +} + +fn ids_of(chunks: &[Chunk]) -> Vec { + chunks.iter().map(|c| c.id.clone()).collect() +} + +// ═════════════════════════════════════════════════════════════════════════ +// Group 1 — predicate 1: `chunk_source_allowed_in`, via `fetch_leaves`. +// ═════════════════════════════════════════════════════════════════════════ + +/// Every group-1 fixture at once: one chunk per interesting source shape. +fn group1_chunks() -> Vec { + vec![ + // Untagged → fail-open under predicate 1. + src_chunk("gmail:alice", 0, &[], BASE_MS), + // Tagged, exact source-id match. + src_chunk("slack:#eng", 1, &[MEMORY_SOURCES], BASE_MS + 1_000), + // Tagged, `mem_src:` composite with a non-empty item id. + src_chunk( + "mem_src:src-abc:item-1", + 2, + &[MEMORY_SOURCES], + BASE_MS + 2_000, + ), + // Tagged, longer registry id — must NOT be smeared into by `src-abc`. + src_chunk( + "mem_src:src-abcdef:item-1", + 3, + &[MEMORY_SOURCES], + BASE_MS + 3_000, + ), + // Tagged, EMPTY item id — `extract_mem_src_id` returns None here. + src_chunk("mem_src:src-abc:", 4, &[MEMORY_SOURCES], BASE_MS + 4_000), + ] +} + +async fn fetch_ids_under( + cfg: &Config, + chunks: &[Chunk], + scope: Option>, +) -> Vec { + let ids = ids_of(chunks); + let hits = with_source_scope(scope, async { fetch_leaves(cfg, &ids).await }) + .await + .expect("fetch_leaves"); + hits.into_iter().map(|h| h.node_id).collect() +} + +#[tokio::test] +async fn fetch_leaves_fails_open_for_untagged_chunk() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["src-abc".into()])).await; + assert!( + got.contains(&chunks[0].id), + "untagged chunk must fail OPEN through predicate 1: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_allows_exact_source_id_match() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["slack:#eng".into()])).await; + assert!( + got.contains(&chunks[1].id), + "exact source_id match: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_allows_mem_src_prefix_match() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["src-abc".into()])).await; + assert!( + got.contains(&chunks[2].id), + "mem_src:src-abc:item-1 must resolve to registry id src-abc: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_prefix_does_not_smear_to_longer_source_id() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["src-abc".into()])).await; + assert!( + !got.contains(&chunks[3].id), + "src-abc must not smear into src-abcdef: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_blocks_mem_src_with_empty_item_id() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + // `extract_mem_src_id` bails when nothing follows the registry-id colon + // (`colon_pos + 1 >= rest.len()`), so the composite never resolves and the + // tagged chunk is blocked — even though the SQL predicate admits it (see + // `list_chunks_scope_admits_empty_item_id_unlike_the_host_predicate`). + let set = set_of(&["src-abc"]); + let tags = vec![MEMORY_SOURCES.to_string()]; + assert!(!chunk_source_allowed_in(&set, &tags, "mem_src:src-abc:")); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["src-abc".into()])).await; + assert!( + !got.contains(&chunks[4].id), + "empty-item-id composite must be blocked host-side: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_empty_allowlist_blocks_tagged_but_not_untagged() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec![])).await; + assert_eq!( + got, + vec![chunks[0].id.clone()], + "an empty allowlist keeps only the fail-open untagged chunk: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_without_scope_returns_every_chunk() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let ids = ids_of(&chunks); + let hits = fetch_leaves(&cfg, &ids).await.expect("fetch_leaves"); + assert_eq!(hits.len(), chunks.len(), "absent scope is unrestricted"); +} + +// ═════════════════════════════════════════════════════════════════════════ +// Group 2 — predicate 2: plain equality on `tree_scope`. +// ═════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +async fn query_source_retains_only_exact_tree_scope_matches() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-eng", 1); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-secret", 1); + seed_summary(&cfg, "s-eng", "tree-eng", 1, &["leaf-a"]); + seed_summary(&cfg, "s-secret", "tree-secret", 1, &["leaf-b"]); + + let resp = with_source_scope(Some(vec!["slack:#eng".into()]), async { + query_source(&cfg, None, None, None, None, 10).await + }) + .await + .expect("query_source"); + + assert_eq!(resp.hits.len(), 1, "hits: {:?}", resp.hits); + assert_eq!(resp.hits[0].tree_scope, "slack:#eng"); + assert_eq!(resp.hits[0].node_id, "s-eng"); +} + +#[tokio::test] +async fn query_source_tree_scope_filter_has_no_mem_src_prefix_rule() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-m", "mem_src:src-abc:item-1", "s-m", 1); + seed_summary(&cfg, "s-m", "tree-m", 1, &["leaf-a"]); + + // Predicate 1 WOULD admit this identifier… + let set = set_of(&["src-abc"]); + let tags = vec![MEMORY_SOURCES.to_string()]; + assert!(chunk_source_allowed_in( + &set, + &tags, + "mem_src:src-abc:item-1" + )); + + // …but predicate 2 is plain equality on `tree_scope`, so it does not. + let resp = with_source_scope(Some(vec!["src-abc".into()]), async { + query_source(&cfg, None, None, None, None, 10).await + }) + .await + .expect("query_source"); + assert!( + resp.hits.is_empty(), + "tree_scope retain has no mem_src rule: {:?}", + resp.hits + ); +} + +#[tokio::test] +async fn query_source_empty_allowlist_returns_no_hits() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-eng", 1); + seed_summary(&cfg, "s-eng", "tree-eng", 1, &["leaf-a"]); + + let resp = with_source_scope(Some(vec![]), async { + query_source(&cfg, None, None, None, None, 10).await + }) + .await + .expect("query_source"); + assert!(resp.hits.is_empty()); + assert_eq!(resp.total, 0); +} + +#[tokio::test] +async fn query_source_without_scope_returns_every_tree() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-eng", 1); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-secret", 1); + seed_summary(&cfg, "s-eng", "tree-eng", 1, &["leaf-a"]); + seed_summary(&cfg, "s-secret", "tree-secret", 1, &["leaf-b"]); + + let resp = query_source(&cfg, None, None, None, None, 10) + .await + .expect("query_source"); + assert_eq!(resp.hits.len(), 2, "absent scope is unrestricted"); +} + +#[tokio::test] +async fn query_source_explicit_source_id_outside_scope_short_circuits() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-secret", 1); + seed_summary(&cfg, "s-secret", "tree-secret", 1, &["leaf-b"]); + + // The `source.rs` PRE-filter: plain equality on the request argument, + // returning `QueryResponse::empty()` before the engine is even called. + // This is a fourth predicate, distinct from the post-filter retain. + let resp = with_source_scope(Some(vec!["slack:#eng".into()]), async { + query_source(&cfg, Some("slack:#secret"), None, None, None, 10).await + }) + .await + .expect("query_source"); + assert!(resp.hits.is_empty()); + assert_eq!(resp.total, 0); + assert!(!resp.truncated); +} + +#[tokio::test] +async fn drill_down_retains_only_exact_tree_scope_matches() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-root", 2); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-b", 1); + seed_summary(&cfg, "s-root", "tree-eng", 2, &["s-a", "s-b"]); + seed_summary(&cfg, "s-a", "tree-eng", 1, &["leaf-a"]); + seed_summary(&cfg, "s-b", "tree-secret", 1, &["leaf-b"]); + + let hits = with_source_scope(Some(vec!["slack:#eng".into()]), async { + drill_down(&cfg, "s-root", 1, None, None).await + }) + .await + .expect("drill_down"); + + let ids: Vec<&str> = hits.iter().map(|h| h.node_id.as_str()).collect(); + assert_eq!(ids, vec!["s-a"], "hits: {ids:?}"); +} + +#[tokio::test] +async fn drill_down_without_scope_keeps_every_hit() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-root", 2); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-b", 1); + seed_summary(&cfg, "s-root", "tree-eng", 2, &["s-a", "s-b"]); + seed_summary(&cfg, "s-a", "tree-eng", 1, &["leaf-a"]); + seed_summary(&cfg, "s-b", "tree-secret", 1, &["leaf-b"]); + + let hits = drill_down(&cfg, "s-root", 1, None, None) + .await + .expect("drill_down"); + let ids: Vec<&str> = hits.iter().map(|h| h.node_id.as_str()).collect(); + assert_eq!(ids, vec!["s-a", "s-b"], "hits: {ids:?}"); +} + +#[tokio::test] +async fn drill_down_chunk_leaves_are_scoped_by_source_id_not_by_tag() { + let (_tmp, cfg) = test_config(); + // An UNTAGGED chunk and a TAGGED `mem_src:` chunk hanging off one L1 node. + let untagged = src_chunk("gmail:alice", 0, &[], BASE_MS); + let tagged = src_chunk( + "mem_src:src-abc:item-1", + 1, + &[MEMORY_SOURCES], + BASE_MS + 1_000, + ); + seed_chunks(&cfg, &[untagged.clone(), tagged.clone()]); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-leaves", 1); + seed_summary( + &cfg, + "s-leaves", + "tree-eng", + 1, + &[untagged.id.as_str(), tagged.id.as_str()], + ); + + // Leaves carry `tree_scope = chunk.metadata.source_id`, so an allowlist + // naming that source id keeps the chunk — with NO tag fail-open for the + // untagged one, which is why the tagged sibling drops out here. + let hits = with_source_scope(Some(vec!["gmail:alice".into()]), async { + drill_down(&cfg, "s-leaves", 1, None, None).await + }) + .await + .expect("drill_down"); + let ids: Vec<&str> = hits.iter().map(|h| h.node_id.as_str()).collect(); + assert_eq!(ids, vec![untagged.id.as_str()], "hits: {ids:?}"); + + // And the `mem_src:` prefix rule does NOT apply on this path either: + // predicate 1 would admit `mem_src:src-abc:item-1` under `src-abc`. + let hits = with_source_scope(Some(vec!["src-abc".into()]), async { + drill_down(&cfg, "s-leaves", 1, None, None).await + }) + .await + .expect("drill_down"); + assert!( + hits.is_empty(), + "leaf retain is plain equality on source_id: {hits:?}" + ); +} + +#[tokio::test] +async fn drill_down_scope_widens_engine_limit_so_a_blocked_prefix_cannot_starve_results() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-root", 2); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-b1", 1); + // BFS order puts the two blocked children FIRST. + seed_summary(&cfg, "s-root", "tree-eng", 2, &["s-b1", "s-b2", "s-a"]); + seed_summary(&cfg, "s-b1", "tree-secret", 1, &["leaf-1"]); + seed_summary(&cfg, "s-b2", "tree-secret", 1, &["leaf-2"]); + seed_summary(&cfg, "s-a", "tree-eng", 1, &["leaf-3"]); + + // `drill_down.rs` forces the ENGINE limit to `None` whenever a scope is + // active, then applies the caller's limit after the retain. Without that, + // the engine would return only `s-b1` and the retain would empty it. + let hits = with_source_scope(Some(vec!["slack:#eng".into()]), async { + drill_down(&cfg, "s-root", 1, None, Some(1)).await + }) + .await + .expect("drill_down"); + let ids: Vec<&str> = hits.iter().map(|h| h.node_id.as_str()).collect(); + assert_eq!(ids, vec!["s-a"], "hits: {ids:?}"); +} + +// ═════════════════════════════════════════════════════════════════════════ +// Group 3 — predicate 3: the SQL `append_source_scope`, applied before LIMIT. +// ═════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +async fn list_chunks_scope_fails_open_for_untagged_chunk() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&["src-abc"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!( + ids.contains(&chunks[0].id.as_str()), + "SQL `NOT EXISTS json_each(...)` fail-open: {ids:?}" + ); +} + +#[tokio::test] +async fn list_chunks_scope_matches_exact_source_id() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&["slack:#eng"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!(ids.contains(&chunks[1].id.as_str()), "ids: {ids:?}"); +} + +#[tokio::test] +async fn list_chunks_scope_matches_mem_src_prefix() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&["src-abc"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!(ids.contains(&chunks[2].id.as_str()), "ids: {ids:?}"); +} + +#[tokio::test] +async fn list_chunks_scope_does_not_smear_prefix() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&["src-abc"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!( + !ids.contains(&chunks[3].id.as_str()), + "substr(source_id, 1, length('mem_src:src-abc:')) must not match \ + mem_src:src-abcdef:item-1: {ids:?}" + ); +} + +#[tokio::test] +async fn list_chunks_scope_admits_empty_item_id_unlike_the_host_predicate() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + // THE headline divergence, characterized as observed (not endorsed): + // the SQL prefix test is a pure `substr` compare with no "item id must be + // non-empty" rule, so `mem_src:src-abc:` passes here… + let got = list_chunks(&cfg, &scoped_query(Some(&["src-abc"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!( + ids.contains(&chunks[4].id.as_str()), + "SQL admits mem_src:src-abc: : {ids:?}" + ); + + // …while the host predicate blocks the very same source_id. + let set = set_of(&["src-abc"]); + let tags = vec![MEMORY_SOURCES.to_string()]; + assert!(!chunk_source_allowed_in(&set, &tags, "mem_src:src-abc:")); +} + +#[tokio::test] +async fn list_chunks_empty_allowlist_keeps_only_untagged_chunks() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&[]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert_eq!(ids, vec![chunks[0].id.as_str()], "ids: {ids:?}"); +} + +#[tokio::test] +async fn list_chunks_absent_scope_returns_everything() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(None)).expect("list_chunks"); + assert_eq!(got.len(), chunks.len()); +} + +#[tokio::test] +async fn list_chunks_scope_is_applied_before_limit() { + let (_tmp, cfg) = test_config(); + // Three blocked chunks NEWER than the single allowed one. Ordering is + // `timestamp_ms DESC`, so a post-filter with LIMIT 1 would return nothing. + let blocked: Vec = (0..3) + .map(|i| { + src_chunk( + "slack:#secret", + i, + &[MEMORY_SOURCES], + BASE_MS + 10_000 + i64::from(i) * 1_000, + ) + }) + .collect(); + let allowed = src_chunk("slack:#eng", 9, &[MEMORY_SOURCES], BASE_MS); + let mut all = blocked.clone(); + all.push(allowed.clone()); + seed_chunks(&cfg, &all); + + let got = list_chunks( + &cfg, + &ListChunksQuery { + source_scope: Some(set_of(&["slack:#eng"])), + limit: Some(1), + exclude_dropped: false, + ..Default::default() + }, + ) + .expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert_eq!(ids, vec![allowed.id.as_str()], "ids: {ids:?}"); +} + +#[tokio::test] +async fn cover_window_scope_matches_mem_src_prefix() { + let (_tmp, cfg) = test_config(); + let allowed = src_chunk("mem_src:src-abc:item-1", 0, &[MEMORY_SOURCES], BASE_MS); + let blocked = src_chunk( + "mem_src:src-zzz:item-1", + 1, + &[MEMORY_SOURCES], + BASE_MS + 1_000, + ); + seed_chunks(&cfg, &[allowed.clone(), blocked.clone()]); + + // `cover_window` hands the allowlist straight to `cover_window_scoped`, + // which applies predicate 3 in SQL — so the `mem_src:` prefix rule holds + // here, unlike on the `tree_scope` paths above. + let resp = with_source_scope(Some(vec!["src-abc".into()]), async { + cover_window(&cfg, 0, 4_000_000_000_000, None, None, 0).await + }) + .await + .expect("cover_window"); + let ids: Vec<&str> = resp.hits.iter().map(|h| h.node_id.as_str()).collect(); + assert!(ids.contains(&allowed.id.as_str()), "ids: {ids:?}"); + assert!(!ids.contains(&blocked.id.as_str()), "ids: {ids:?}"); +} diff --git a/core/src/tree/retrieval/types.rs b/core/src/tree/retrieval/types.rs new file mode 100644 index 0000000..91b48fa --- /dev/null +++ b/core/src/tree/retrieval/types.rs @@ -0,0 +1,6 @@ +//! Stable host path for tinycortex-owned retrieval wire types and converters. + +pub use tinycortex::memory::retrieval::{ + hit_from_chunk, hit_from_summary, hit_from_summary_with_tree, leaf_tree_placeholder, + EntityMatch, NodeKind, QueryResponse, RetrievalHit, +}; diff --git a/core/src/tree/score/README.md b/core/src/tree/score/README.md new file mode 100644 index 0000000..8c8785b --- /dev/null +++ b/core/src/tree/score/README.md @@ -0,0 +1,23 @@ +# Memory tree — score (Phase 2 / #708) + +Per-chunk admission, enrichment, and entity indexing for the bucket-seal-ready memory tree. Sits between leaf chunking and L0 buffer append: every chunk passes through `score_chunk` which decides whether to keep it, runs entity extraction, and persists score rationale + an inverted entity index used by retrieval. + +## Public surface + +- `pub fn score_chunk` / `pub fn score_chunks` / `pub fn score_chunks_fast` — `mod.rs` — scoring pipeline entry points (full / batch / cheap-only batch). +- `pub struct ScoreResult` / `pub struct ScoringConfig` — `mod.rs` — outcome and configuration of one scoring pass. +- `pub fn persist_score` / `persist_score_tx` — `mod.rs` — write the score row + entity-index rows for one kept chunk. +- `pub const DEFAULT_DROP_THRESHOLD` / `DEFAULT_DEFINITE_KEEP` / `DEFAULT_DEFINITE_DROP` — `mod.rs` — admission band defaults. + +## Subdirectories + +- `signals/` — per-signal feature computation (token count, unique words, metadata weight, source weight, interaction tags, entity density, LLM importance) plus the weighted combine that produces the final `[0.0, 1.0]` total. +- `extract/` — entity extraction: `EntityExtractor` trait, `RegexEntityExtractor` for mechanical identifiers (email, URL, handle, hashtag), `LlmEntityExtractor` for semantic NER + importance rating, `CompositeExtractor` for chaining them. +- `embed/` — Phase 4 vector embedder: `Embedder` trait, `OllamaEmbedder` (default), `InertEmbedder` (tests), pack/unpack helpers for the SQLite BLOB storage layout. + +## Files + +- `mod.rs` — orchestration: `score_chunk` runs extraction → cheap signals → optional borderline LLM call → admission gate → canonicalisation. +- `store.rs` — SQLite CRUD for `mem_tree_score` (per-chunk rationale) and `mem_tree_entity_index` (inverted index `entity_id → node_id`). +- `resolver.rs` — entity canonicalisation: normalises surface forms (lowercase emails, strip leading `@`/`#`) and assigns stable `canonical_id` strings; promotes extracted topics into the canonical entity stream. +- `mod_tests.rs` / `store_tests.rs` — unit tests. diff --git a/core/src/tree/score/embed/README.md b/core/src/tree/score/embed/README.md new file mode 100644 index 0000000..43b71bd --- /dev/null +++ b/core/src/tree/score/embed/README.md @@ -0,0 +1,17 @@ +# Memory tree embedding bridge + +The memory tree stores fixed 1024-dimensional vectors. Concrete provider +transports live in TinyAgents; this directory keeps only memory-tree policy and +compatibility: + +- `factory.rs`: read/write provider resolution, cloud-session policy, and + degraded-state behavior; +- `openai_compat.rs`: OpenHuman config/credential and custom-slug resolution; +- `inert.rs`: deterministic 1024-element zero vectors for opt-out/tests; +- `mod.rs`: the legacy `Embedder` contract, `ProviderEmbedder` bridge, batch + fallback/dimension checks, cosine math, and SQLite f32 packing helpers. + +Ollama uses TinyAgents `OllamaEmbeddingModel` and `/api/embed`, with the shared +8192-token context and batch window. Managed cloud uses the host credential and +privacy wrapper around TinyAgents `CloudEmbeddingModel`. Do not add provider +HTTP clients in this directory. diff --git a/core/src/tree/score/embed/factory.rs b/core/src/tree/score/embed/factory.rs new file mode 100644 index 0000000..b9758d3 --- /dev/null +++ b/core/src/tree/score/embed/factory.rs @@ -0,0 +1,872 @@ +//! Build an [`Embedder`] from [`Config`] settings. +//! +//! Resolution order: +//! 1. **Explicit override** — `memory_tree.embedding_endpoint` + +//! `memory_tree.embedding_model` both Some → `OllamaEmbedder` with +//! those exact values. For power users / E2E test rigs that want to +//! point at a non-default Ollama endpoint. +//! 2. **Local-AI usage flag** — `config.local_ai().use_local_for_embeddings()` +//! (i.e. `runtime_enabled && usage.embeddings`) → `OllamaEmbedder` +//! against `ollama_base_url` with the user's chosen +//! `config.local_ai().embedding_model_id`. This is the path driven by +//! the "Memory embeddings" checkbox in Local AI Settings. +//! 3. **Default** — `CloudEmbedder` (OpenHuman backend / Voyage, +//! 1024 dims). Auth failures surface at the first `embed()` call so +//! ingest's existing retry-with-backoff logic handles them. +//! +//! NOTE on dimensions: the memory tree on-disk format is hard-coded at +//! [`EMBEDDING_DIM`] (1024). If the user picks a +//! local embedding model whose output is a different dimensionality, +//! the trait's post-call validator rejects each embed with a clear +//! `expected N dims, got M` error. Switching the local model picker in +//! Local AI Settings is the fix. +//! +//! The historical `InertEmbedder` (zero vectors) path is retained for +//! tests only — it is no longer the production lax-mode fallback. +//! +//! Env var overrides applied in the host's `config::load`: +//! - `OPENHUMAN_MEMORY_EMBED_ENDPOINT` +//! - `OPENHUMAN_MEMORY_EMBED_MODEL` +//! - `OPENHUMAN_MEMORY_EMBED_TIMEOUT_MS` + +use anyhow::{Context, Result}; + +use std::time::Duration; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use super::{Embedder, InertEmbedder, ProviderEmbedder, EMBEDDING_DIM}; +use crate::embedding_host::require_embedding_host; +use crate::Config; +use tinyagents::harness::embeddings::{OllamaEmbeddingModel, RECOMMENDED_OLLAMA_CONTEXT_TOKENS}; + +/// Cheap heuristic for "is a backend session reachable?" — the cloud +/// embedder needs one and bails on first embed call without it. We use +/// the *presence* of `auth-profiles.json` next to the config file as a +/// proxy: production after login has it, test harnesses and fresh +/// pre-login installs don't. The CloudEmbedder still re-validates the +/// JWT at every embed call, so a stale file just surfaces at embed +/// time (not factory build), preserving the prior failure behavior. +fn cloud_session_available(config: &Config) -> bool { + config + .config_path() + .parent() + .map(|dir| dir.join("auth-profiles.json").exists()) + .unwrap_or(false) +} + +/// Construct the active embedder for this process, honouring +/// `config.memory_tree().*` and `embedding_strict`. +/// +/// Returns a boxed trait object so ingest / seal can call one code path +/// regardless of which provider is active. The returned box is created +/// per call — cheap because `OllamaEmbedder` owns a cloned `reqwest::Client` +/// internally and `InertEmbedder` is a ZST. +pub fn build_embedder_from_config(config: &Config) -> Result> { + // Read path: walk the shared ladder, then terminate at InertEmbedder (zero + // vectors) so retrieval / semantic rerank can still run with no provider. + Ok(match resolve_embedder_choice(config)? { + EmbedderChoice::Ollama { + endpoint, + model, + timeout_ms, + } => { + log::debug!( + "[memory_tree::embed::factory] read → Ollama endpoint={endpoint} model={model} timeout_ms={timeout_ms}" + ); + Box::new(build_ollama_embedder(&endpoint, &model, timeout_ms)?) + } + EmbedderChoice::OptOut => { + log::info!( + "[memory_tree::embed::factory] embeddings_provider=none — \ + using InertEmbedder (vector search disabled)" + ); + Box::new(InertEmbedder::new()) + } + EmbedderChoice::OpenAiCompat(openai) => { + log::debug!( + "[memory_tree::embed::factory] read → user OpenAI-compatible embeddings ({})", + openai.name() + ); + Box::new(openai) + } + EmbedderChoice::Cloud => { + log::debug!( + "[memory_tree::embed::factory] read → cloud (Voyage) — flip \ + 'Memory embeddings' in Local AI Settings to switch to local" + ); + Box::new(build_cloud_embedder(config)) + } + EmbedderChoice::NoProvider => { + log::warn!( + "[memory_tree::embed::factory] no backend session found — \ + using InertEmbedder (zero vectors). Log in to OpenHuman, or \ + enable 'Memory embeddings' in Local AI Settings, to fix." + ); + Box::new(InertEmbedder::new()) + } + }) +} + +/// The embedder the resolution ladder selects, independent of whether the +/// caller is a read path (retrieval) or a write path (ingest/seal). Both +/// public factories walk [`resolve_embedder_choice`] and differ ONLY at the +/// terminal + degraded-flag side-effects — so "identical resolution for every +/// real provider" is a structural guarantee, not two hand-maintained copies +/// that could drift (reviewer sanil-23, #3076: a read/write provider mismatch +/// would silently corrupt recall). +enum EmbedderChoice { + /// Explicit Ollama override, or the unified `ollama:` workload setting. + Ollama { + endpoint: String, + model: String, + timeout_ms: u64, + }, + /// `embeddings_provider = "none"` — vector search off by deliberate user + /// choice (NOT a degradation). Both paths use `InertEmbedder`. + OptOut, + /// User-configured OpenAI / custom OpenAI-compatible endpoint (#002 FR-015). + OpenAiCompat(super::openai_compat::OpenAiCompatEmbedder), + /// Logged-in managed cloud (Voyage). + Cloud, + /// No usable provider. Read path → `InertEmbedder` (zero vectors); write + /// path → `None` (skip) + mark `semantic_recall` degraded. + NoProvider, +} + +/// Walk the provider-resolution ladder once. The order is the single source of +/// truth for both factories; the only read/write differences are encoded by the +/// callers at the terminal, never here. +fn resolve_embedder_choice(config: &Config) -> Result { + let tree_cfg = &config.memory_tree(); + + // 1. Explicit Ollama override (power-user / E2E rig). + if let (Some(endpoint), Some(model)) = ( + tree_cfg.embedding_endpoint.as_deref(), + tree_cfg.embedding_model.as_deref(), + ) { + if !endpoint.trim().is_empty() && !model.trim().is_empty() { + return Ok(EmbedderChoice::Ollama { + endpoint: endpoint.to_string(), + model: model.to_string(), + timeout_ms: tree_cfg.embedding_timeout_ms.unwrap_or(0), + }); + } + } + + // 2. Deliberate opt-out — vector search off by user choice. + if config + .embeddings_provider() + .map(str::trim) + .is_some_and(|s| s == "none") + { + return Ok(EmbedderChoice::OptOut); + } + + // 3. Local Ollama via the unified workload setting. + if let Some(model) = config.workload_local_model("embeddings") { + return Ok(EmbedderChoice::Ollama { + endpoint: require_embedding_host() + .map_err(|e| anyhow::anyhow!(e))? + .ollama_base_url(), + model, + timeout_ms: tree_cfg.embedding_timeout_ms.unwrap_or(0), + }); + } + + // 4. #002 FR-015: user-configured OpenAI / custom OpenAI-compatible. + if let Some(openai) = super::openai_compat::OpenAiCompatEmbedder::try_from_config(config)? { + return Ok(EmbedderChoice::OpenAiCompat(openai)); + } + + // 5. Logged-in managed cloud (Voyage). + if cloud_session_available(config) { + return Ok(EmbedderChoice::Cloud); + } + + // 6. Nothing usable. + Ok(EmbedderChoice::NoProvider) +} + +/// Build the embedder used by **write** paths (ingest extract + seal), with an +/// explicit "no usable embedder" signal (#002 FR-002). +/// +/// Identical resolution to [`build_embedder_from_config`] for every real +/// provider (explicit Ollama override, local Ollama, cloud session). The one +/// difference is the terminal fallback: where the read-path factory returns an +/// [`InertEmbedder`] (zero vectors) so retrieval can still run, the write path +/// returns **`Ok(None)`** so callers **skip** embedding instead of persisting a +/// fake all-zero vector that would silently poison semantic recall and present +/// a degraded result as success. The chunk/summary is written embedding-less +/// (re-embeddable later once a provider is configured), and the process-global +/// `semantic_recall` degraded flag is set with a typed cause so the status / +/// doctor surface can name the fix. +/// +/// `embeddings_provider = "none"` is treated as a deliberate opt-out, not a +/// degradation: it returns the [`InertEmbedder`] (vector search intentionally +/// off) without setting the degraded flag — same as the read path. +pub fn build_write_embedder(config: &Config) -> Result>> { + use crate::tree::health::{ + clear_semantic_recall_degraded, mark_semantic_recall_degraded, FailureCode, + }; + + // Write path: same ladder as the read factory, terminating at `None` (skip, + // don't persist zero vectors) + a typed degraded flag when no provider is + // usable. Every real-provider branch clears the flag; the deliberate + // "none" opt-out leaves it untouched (off by choice, not degradation). + Ok(match resolve_embedder_choice(config)? { + EmbedderChoice::Ollama { + endpoint, + model, + timeout_ms, + } => { + clear_semantic_recall_degraded(); + Some(Box::new(build_ollama_embedder( + &endpoint, &model, timeout_ms, + )?)) + } + EmbedderChoice::OptOut => { + clear_semantic_recall_degraded(); + log::info!( + "[memory_tree::embed::factory] embeddings_provider=none — write path \ + uses InertEmbedder (vector search disabled by choice)" + ); + Some(Box::new(InertEmbedder::new())) + } + EmbedderChoice::OpenAiCompat(openai) => { + clear_semantic_recall_degraded(); + Some(Box::new(openai)) + } + EmbedderChoice::Cloud => { + clear_semantic_recall_degraded(); + Some(Box::new(build_cloud_embedder(config))) + } + EmbedderChoice::NoProvider => { + log::warn!( + "[memory_tree::embed::factory] no usable embeddings provider — skipping \ + embedding (chunk persists embedding-less, re-embeddable later). Set up \ + local Ollama embeddings or log in to OpenHuman to enable semantic recall." + ); + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + None + } + }) +} + +/// Render a ladder-resolution error safely for a log line. +/// +/// The only user-controlled values these errors interpolate are the configured +/// provider string and model (see `openai_compat::try_from_config`), and one of +/// them is an endpoint: in its `custom:` form `memory.embedding_provider` +/// *is* a URL, which may carry `user:pass@` userinfo. Configured +/// `cloud_providers` endpoints can reach the message the same way through the +/// underlying constructor's own context. +/// +/// So rather than dropping the reason — which would cost the diagnostic that +/// makes this log worth having ("dimension mismatch", "build failed") — replace +/// each known endpoint substring with its [`redact_endpoint`] form. Scrubbing +/// the exact strings we already hold is precise, where a generic URL-matching +/// pass over free text would be guesswork (CodeRabbit, #5402 / CWE-532). +fn redact_ladder_error(config: &Config, err: &anyhow::Error) -> String { + use crate::util::redact::redact_endpoint; + + // Candidates: the inline `custom:` endpoint (when that is the + // configured form) plus every configured OpenAI-compatible endpoint + // (LM Studio, vLLM, …), any of which the ladder may have been resolving. + let mut endpoints: Vec<&str> = config + .memory() + .embedding_provider + .trim() + .strip_prefix("custom:") + .into_iter() + .chain(config.cloud_providers().iter().map(|e| e.endpoint.as_str())) + .map(str::trim) + .filter(|e| !e.is_empty()) + .collect(); + + // Longest first. Substring replacement is order-sensitive: if a short + // endpoint is a strict prefix of a longer one (`https://host` vs + // `https://host/v1?key=…`), scrubbing the short one first rewrites the + // longer one's prefix, so its own replacement no longer matches and the + // credential-bearing suffix survives in the log (CodeRabbit, #5402). + endpoints.sort_by_key(|e| std::cmp::Reverse(e.len())); + endpoints.dedup(); + + let mut msg = format!("{err:#}"); + for endpoint in endpoints { + msg = msg.replace(endpoint, &redact_endpoint(endpoint)); + } + msg +} + +/// Slug naming the embedder ingestion will **actually** use, walking the same +/// `resolve_embedder_choice` ladder the read and write factories walk. +/// +/// This exists because `config.memory().embedding_provider` is *not* authoritative +/// for how embeddings are funded, and reading it as if it were produces a false +/// alarm. The ladder resolves local Ollama from `memory_tree.embedding_endpoint` +/// or from the unified `workload_local_model("embeddings")` setting (the "Memory +/// embeddings" toggle in Local AI Settings), and **neither path rewrites +/// `memory.embedding_provider`** — so a user running fully local still reads as +/// `"cloud"` there. Any surface that asks "do these embeddings bill against the +/// managed budget?" must ask this function, not that field (reviewer M3gA-Mind, +/// #5402: otherwise a local-embeddings user whose *chat* budget crosses 90% is +/// told memory has stopped growing while it is growing fine). +/// +/// Slugs are stable wire values consumed by the frontend: +/// - `"ollama"` — local daemon; user-funded. +/// - `"custom"` — user's own OpenAI-compatible endpoint / key; user-funded. +/// - `"cloud"` — managed OpenHuman backend; **bills the managed cycle budget**. +/// - `"none"` — deliberate opt-out (`embeddings_provider = "none"`). +/// - `"unconfigured"` — no usable provider (signed out); nothing is billed. +/// - `"unknown"` — the ladder itself failed to resolve. Deliberately not +/// `"cloud"`: an unresolvable config must never manufacture a budget warning. +pub fn effective_embedder_slug(config: &Config) -> &'static str { + let slug = match resolve_embedder_choice(config) { + Ok(EmbedderChoice::Ollama { .. }) => "ollama", + Ok(EmbedderChoice::OptOut) => "none", + Ok(EmbedderChoice::OpenAiCompat(_)) => "custom", + Ok(EmbedderChoice::Cloud) => "cloud", + Ok(EmbedderChoice::NoProvider) => "unconfigured", + Err(err) => { + log::warn!( + "[memory_tree::embed::factory] effective_embedder_slug: ladder failed to \ + resolve ({}) — reporting 'unknown' (treated as NOT managed)", + redact_ladder_error(config, &err) + ); + "unknown" + } + }; + log::debug!("[memory_tree::embed::factory] effective_embedder_slug → {slug}"); + slug +} + +fn build_ollama_embedder(endpoint: &str, model: &str, timeout_ms: u64) -> Result { + let timeout = Duration::from_millis(if timeout_ms == 0 { 10_000 } else { timeout_ms }); + let client = reqwest::Client::builder() + .connect_timeout(timeout) + .build() + .context("build Ollama embeddings HTTP client")?; + let model = OllamaEmbeddingModel::try_new(endpoint, model, EMBEDDING_DIM)? + .with_context_options( + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, + ) + .with_client(client); + Ok(ProviderEmbedder::new( + crate::embedding_adapter::TinyAgentsEmbeddingProvider::boxed(model), + "ollama", + )) +} + +fn build_cloud_embedder(config: &Config) -> ProviderEmbedder { + let openhuman_dir = config.config_path().parent().map(std::path::PathBuf::from); + // The managed cloud embedder resolves a session JWT and the backend API + // URL, both host concerns — it is built through the seam rather than here. + // `openhuman_dir` stays part of the caller's contract: the host reads the + // encrypted-secrets material relative to it. + let _ = openhuman_dir; + let host = require_embedding_host().expect("embedding host installed"); + let provider = host + .cloud_embedding_provider( + host.default_cloud_embedding_model(), + host.default_cloud_embedding_dimensions(), + ) + .expect("cloud embedding provider"); + ProviderEmbedder::new(provider, "cloud") +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + // Plant config_path in the tempdir so cloud_session_available() + // checks a writable directory; tests that need to simulate a + // logged-in user just `touch` auth-profiles.json next to it. + cfg.config_path = tmp.path().join("config.toml"); + (tmp, cfg) + } + + /// Drop a stub `auth-profiles.json` next to the test config so + /// `cloud_session_available()` returns true. Contents don't matter + /// — the factory only checks presence. + fn touch_auth_profile(cfg: &Config) { + let path = cfg + .config_path() + .parent() + .map(|p| p.join("auth-profiles.json")) + .expect("config_path has a parent"); + std::fs::write(&path, "{}").expect("write stub auth-profiles.json"); + } + + #[test] + fn ollama_chosen_when_endpoint_and_model_set() { + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = Some("http://localhost:11434".into()); + cfg.memory_tree.embedding_model = Some("bge-m3".into()); + cfg.memory_tree.embedding_timeout_ms = Some(5000); + let e = build_embedder_from_config(&cfg).expect("Ollama path should build"); + assert_eq!(e.name(), "ollama"); + } + + // ── build_write_embedder (T010, #002 FR-002) ───────────────────────── + // + // These assert the write-path factory's "skip vs embed" contract. The + // degraded flag is a process-global atomic, so the flag-sensitive tests + // serialize on a shared mutex to avoid stomping each other under cargo's + // parallel test runner. + // Delegate to the health module's shared guard so factory tests serialise + // against the rpc/extract tests that touch the SAME process-global flags + // (a factory-local mutex would only serialise within this module, leaving + // a cross-module race). The guard also resets the flags on entry. + fn degraded_flag_lock() -> std::sync::MutexGuard<'static, ()> { + crate::tree::health::test_guard() + } + + #[test] + fn write_embedder_none_when_no_provider_and_marks_degraded() { + use crate::tree::health::{ + clear_semantic_recall_degraded, current_degraded_state, FailureCode, + }; + let _guard = degraded_flag_lock(); + clear_semantic_recall_degraded(); + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + // No auth-profiles.json, no local workload model → no usable provider. + let e = build_write_embedder(&cfg).expect("factory must not error"); + assert!( + e.is_none(), + "no provider → skip embedding (None), not inert" + ); + let d = current_degraded_state(); + assert!( + d.semantic_recall, + "semantic recall must be flagged degraded" + ); + assert_eq!( + d.cause.map(|c| c.code), + Some(FailureCode::EmbeddingsUnconfigured) + ); + clear_semantic_recall_degraded(); + } + + #[test] + fn write_embedder_some_cloud_with_session_and_clears_degraded() { + use crate::tree::health::{ + current_degraded_state, mark_semantic_recall_degraded, FailureCode, + }; + let _guard = degraded_flag_lock(); + // Pretend a prior run left recall degraded; a working provider clears it. + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + touch_auth_profile(&cfg); + let e = build_write_embedder(&cfg) + .expect("factory must not error") + .expect("cloud session → Some(embedder)"); + assert_eq!(e.name(), "cloud"); + assert!( + !current_degraded_state().semantic_recall, + "a usable provider must clear the degraded flag" + ); + } + + #[test] + fn write_embedder_some_ollama_override() { + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = Some("http://localhost:11434".into()); + cfg.memory_tree.embedding_model = Some("bge-m3".into()); + let e = build_write_embedder(&cfg) + .expect("factory must not error") + .expect("override → Some(embedder)"); + assert_eq!(e.name(), "ollama"); + } + + #[test] + fn write_embedder_none_provider_is_inert_not_skip() { + use crate::tree::health::{clear_semantic_recall_degraded, current_degraded_state}; + let _guard = degraded_flag_lock(); + clear_semantic_recall_degraded(); + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = Some("none".into()); + // Deliberate opt-out → InertEmbedder (vector search off by choice), + // and NOT flagged as a degradation. + let e = build_write_embedder(&cfg) + .expect("factory must not error") + .expect("provider=none → Some(inert), not skip"); + assert_eq!(e.name(), "inert"); + assert!( + !current_degraded_state().semantic_recall, + "explicit opt-out is not a degradation" + ); + } + + #[test] + fn unset_endpoint_with_session_routes_to_cloud() { + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + touch_auth_profile(&cfg); + let e = build_embedder_from_config(&cfg).expect("cloud default should build"); + assert_eq!(e.name(), "cloud"); + } + + #[test] + fn unset_endpoint_without_session_falls_back_to_inert() { + // Test harness / pre-login: no auth-profiles.json on disk, + // factory degrades to InertEmbedder so callers don't crash on + // first embed call. + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + let e = build_embedder_from_config(&cfg).expect("inert fallback should build"); + assert_eq!(e.name(), "inert"); + } + + #[test] + fn empty_strings_count_as_unset_with_session() { + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = Some("".into()); + cfg.memory_tree.embedding_model = Some("".into()); + cfg.memory_tree.embedding_strict = false; + touch_auth_profile(&cfg); + let e = build_embedder_from_config(&cfg).expect("cloud default should build"); + assert_eq!(e.name(), "cloud"); + } + + #[test] + fn strict_mode_no_longer_bails_with_cloud_default() { + // Strict mode used to bail when endpoint/model were unset because + // the only fallback was InertEmbedder. Now the lax-and-strict + // paths share the cloud fallback; strict bail is a no-op here + // and auth failures surface at first embed() call instead. + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = true; + touch_auth_profile(&cfg); + let e = build_embedder_from_config(&cfg).expect("cloud default should build"); + assert_eq!(e.name(), "cloud"); + } + + #[test] + fn local_ai_usage_embeddings_routes_to_ollama() { + // After #1710 the local-vs-cloud decision for embeddings is + // driven by `embeddings_provider` (via + // `Config::workload_uses_local("embeddings")`), not the legacy + // `local_ai.usage.embeddings` flag. Set the new workload field + // so the local branch is taken; `embedding_model_id` is still + // the model name source for the Ollama provider. + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.embeddings_provider = Some("ollama:all-minilm:latest".into()); + cfg.local_ai.runtime_enabled = true; + cfg.local_ai.embedding_model_id = "all-minilm:latest".to_string(); + let e = build_embedder_from_config(&cfg).expect("ollama path should build"); + assert_eq!(e.name(), "ollama"); + } + + #[test] + fn local_ai_usage_off_with_session_falls_back_to_cloud() { + // runtime_enabled=true but usage.embeddings=false → cloud (with session). + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.local_ai.runtime_enabled = true; + cfg.local_ai.usage.embeddings = false; + touch_auth_profile(&cfg); + let e = build_embedder_from_config(&cfg).expect("cloud default should build"); + assert_eq!(e.name(), "cloud"); + } + + #[test] + fn none_provider_returns_inert() { + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = Some("none".into()); + touch_auth_profile(&cfg); + let e = build_embedder_from_config(&cfg).expect("none should build"); + assert_eq!(e.name(), "inert"); + } + + #[test] + fn write_embedder_routes_to_openai_when_memory_provider_is_openai() { + // #002 FR-015 regression: the headline bug was that a user-configured + // OpenAI embeddings provider (`config.memory().embedding_provider = + // "openai"`) matched no factory branch and silently fell through to the + // managed-budget backend. Lock the routing in at the FACTORY level — + // `openai_compat`'s own tests only cover `try_from_config` in isolation, + // so a factory refactor could re-break this with those tests still green. + // + // Note the two distinct config fields the factory reads: the top-level + // `embeddings_provider` (here unset, so the "none"/`ollama:` branches do + // not match) vs `memory.embedding_provider` (the unified Embeddings- + // settings field that drives the OpenAI/custom detection). + let _guard = degraded_flag_lock(); + use crate::tree::health::{ + current_degraded_state, mark_semantic_recall_degraded, FailureCode, + }; + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.embeddings_provider = None; // top-level workload routing: unset + cfg.memory.embedding_provider = "openai".to_string(); + cfg.memory.embedding_model = "text-embedding-3-large".to_string(); + let e = build_write_embedder(&cfg) + .expect("factory must not error") + .expect("openai provider → Some(embedder), must NOT fall through to skip/cloud"); + assert_eq!( + e.name(), + "openai", + "must route to the user's OpenAI embeddings, not the managed backend" + ); + assert!( + !current_degraded_state().semantic_recall, + "a usable OpenAI provider must clear the degraded flag" + ); + } + + #[test] + fn write_embedder_routes_to_lmstudio_local_endpoint() { + // #3781 regression at the factory/seal level: a configured local + // OpenAI-compatible embeddings backend (LM Studio at localhost:1234, + // registered as a `cloud_providers` slug) must drive bucket sealing — + // the same way the LLM extractor already resolves the `lmstudio` slug — + // and NOT fall through to the managed cloud budget (which 400s with + // "Insufficient budget" and fails the seal job unrecoverably). + use crate::tree::health::{ + current_degraded_state, mark_semantic_recall_degraded, FailureCode, + }; + use tinymemory_api::host::cloud_providers::CloudProviderCreds; + let _guard = degraded_flag_lock(); + mark_semantic_recall_degraded(FailureCode::EmbeddingsUnconfigured); + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.embeddings_provider = None; // top-level workload routing: unset + cfg.memory.embedding_provider = "lmstudio".to_string(); + cfg.memory.embedding_model = "bge-m3".to_string(); + cfg.cloud_providers = vec![CloudProviderCreds { + id: "p_lmstudio".to_string(), + slug: "lmstudio".to_string(), + endpoint: "http://localhost:1234/v1".to_string(), + ..Default::default() + }]; + let e = build_write_embedder(&cfg) + .expect("factory must not error") + .expect("lmstudio backend → Some(embedder), must NOT fall through to cloud"); + assert_eq!( + e.name(), + "custom", + "must route to the local OpenAI-compatible endpoint, not the managed backend" + ); + assert!( + !current_degraded_state().semantic_recall, + "a usable local provider must clear the degraded flag" + ); + } + + #[test] + fn read_embedder_routes_to_openai_when_memory_provider_is_openai() { + // Same FR-015 routing, read path (`build_embedder_from_config`). + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.embeddings_provider = None; + cfg.memory.embedding_provider = "openai".to_string(); + cfg.memory.embedding_model = "text-embedding-3-large".to_string(); + let e = build_embedder_from_config(&cfg).expect("openai path should build"); + assert_eq!(e.name(), "openai"); + } + + #[test] + fn explicit_endpoint_override_wins_over_local_ai_flag() { + // Power-user override beats the checkbox. + let (_tmp, mut cfg) = test_config(); + cfg.memory_tree.embedding_endpoint = Some("http://staging-embed:11434".into()); + cfg.memory_tree.embedding_model = Some("bge-m3".into()); + cfg.local_ai.runtime_enabled = true; + cfg.local_ai.usage.embeddings = true; + let e = build_embedder_from_config(&cfg).expect("override path should build"); + assert_eq!(e.name(), "ollama"); + } + + /// The regression this whole helper exists for (reviewer M3gA-Mind, #5402): + /// a user who enabled local embeddings through Local AI Settings still has + /// `memory.embedding_provider == "cloud"` (nothing rewrites it), so any + /// surface reading that field concludes they bill against the managed + /// budget and warns them their memory has stopped growing — while it is + /// growing fine, fully locally, costing them nothing. + #[test] + fn effective_slug_reports_ollama_when_local_ai_overrides_cloud_setting() { + let (_tmp, mut cfg) = test_config(); + cfg.memory.embedding_provider = "cloud".to_string(); + cfg.embeddings_provider = Some("ollama:all-minilm:latest".into()); + cfg.local_ai.runtime_enabled = true; + cfg.local_ai.embedding_model_id = "all-minilm:latest".to_string(); + touch_auth_profile(&cfg); + + // The stale per-section field still says cloud … + assert_eq!(cfg.memory.embedding_provider, "cloud"); + // … but the ladder — and therefore the wire field — says local. + assert_eq!(effective_embedder_slug(&cfg), "ollama"); + } + + #[test] + fn effective_slug_reports_ollama_for_explicit_endpoint_override() { + let (_tmp, mut cfg) = test_config(); + cfg.memory.embedding_provider = "cloud".to_string(); + cfg.memory_tree.embedding_endpoint = Some("http://localhost:11434".into()); + cfg.memory_tree.embedding_model = Some("bge-m3".into()); + touch_auth_profile(&cfg); + assert_eq!(effective_embedder_slug(&cfg), "ollama"); + } + + #[test] + fn effective_slug_reports_cloud_only_for_a_real_managed_session() { + let (_tmp, mut cfg) = test_config(); + cfg.memory.embedding_provider = "cloud".to_string(); + touch_auth_profile(&cfg); + assert_eq!(effective_embedder_slug(&cfg), "cloud"); + } + + #[test] + fn effective_slug_reports_unconfigured_without_a_session() { + // No auth-profiles.json → nothing is billed, so this must not read as + // managed even though the per-section field defaults to cloud. + let (_tmp, mut cfg) = test_config(); + cfg.memory.embedding_provider = "cloud".to_string(); + assert_eq!(effective_embedder_slug(&cfg), "unconfigured"); + } + + #[test] + fn effective_slug_reports_none_for_deliberate_opt_out() { + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = Some("none".into()); + touch_auth_profile(&cfg); + assert_eq!(effective_embedder_slug(&cfg), "none"); + } + + /// The ladder error quotes `memory.embedding_provider` verbatim, and in the + /// `custom:` form that string is a full endpoint URL — potentially with + /// `user:pass@` userinfo. Logging it raw would write credentials to disk + /// (CodeRabbit, #5402 / CWE-532). Scrub the endpoint, keep the reason. + #[test] + fn ladder_error_log_redacts_custom_endpoint_credentials() { + let (_tmp, mut cfg) = test_config(); + // No model + a non-tree dimension → `try_from_config` bails, and its + // message interpolates the provider string. + cfg.memory.embedding_provider = "custom:https://user:pass@embed.example.com/v1".to_string(); + cfg.memory.embedding_model = String::new(); + cfg.memory.embedding_dimensions = 512; + + // `EmbedderChoice` is not `Debug` (it holds a live embedder), so unwrap + // the error by hand rather than via `expect_err`. + let err = match resolve_embedder_choice(&cfg) { + Err(e) => e, + Ok(_) => panic!("a non-tree dimension with no model must fail to resolve"), + }; + let raw = format!("{err:#}"); + assert!( + raw.contains("user:pass"), + "precondition: the unredacted error really does carry the credentials — \ + otherwise this test proves nothing. Got: {raw}" + ); + + let rendered = redact_ladder_error(&cfg, &err); + assert!( + !rendered.contains("user:pass"), + "userinfo must not reach the log: {rendered}" + ); + assert!( + !rendered.contains("/v1"), + "path must not reach the log: {rendered}" + ); + assert!( + rendered.contains("embed.example.com"), + "host is kept so the line stays diagnosable: {rendered}" + ); + assert!( + rendered.contains("1024"), + "the failure reason must survive redaction: {rendered}" + ); + + // And the caller degrades to not-managed rather than to `cloud`. + assert_eq!(effective_embedder_slug(&cfg), "unknown"); + } + + /// Substring replacement is order-sensitive. With a short endpoint that is a + /// strict prefix of the long one, scrubbing shortest-first rewrites the long + /// endpoint's prefix, its own replacement then fails to match, and the + /// credential-bearing suffix survives in the log. Longest-first is the fix + /// (CodeRabbit, #5402). + #[test] + fn ladder_error_redaction_handles_prefix_overlapping_endpoints() { + use tinymemory_api::host::cloud_providers::CloudProviderCreds; + let (_tmp, mut cfg) = test_config(); + // The SHORT endpoint is the one the old code scrubbed first (the inline + // `custom:` form led the list), and it is a strict prefix of the long + // one. That ordering is what let the long endpoint's secret survive: + // scrubbing `https://embed.example.com` first rewrote the long string's + // prefix, so the long string's own replacement no longer matched. + cfg.memory.embedding_provider = "custom:https://embed.example.com".to_string(); + cfg.cloud_providers = vec![CloudProviderCreds { + id: "p_long".to_string(), + slug: "longpfx".to_string(), + endpoint: "https://embed.example.com/v1?key=super-secret".to_string(), + ..Default::default() + }]; + + // Synthesize the error rather than driving the ladder: this pins the + // redaction function's ordering contract for ANY message carrying both + // endpoints, which is the property at risk. Which ladder branch happens + // to surface a `cloud_providers` endpoint today is beside the point. + let err = anyhow::anyhow!( + "build custom embedder failed (provider='custom:https://embed.example.com', \ + endpoint='https://embed.example.com/v1?key=super-secret')" + ); + let rendered = redact_ladder_error(&cfg, &err); + + assert!( + !rendered.contains("super-secret"), + "the long endpoint's query must not survive the short endpoint's scrub: {rendered}" + ); + assert!( + !rendered.contains("/v1"), + "the long endpoint's path must not survive either: {rendered}" + ); + assert!( + rendered.contains("embed.example.com"), + "host is still kept: {rendered}" + ); + } + + #[test] + fn effective_slug_reports_custom_for_byo_openai_compatible() { + use tinymemory_api::host::cloud_providers::CloudProviderCreds; + let (_tmp, mut cfg) = test_config(); + cfg.embeddings_provider = None; + cfg.memory.embedding_provider = "lmstudio".to_string(); + cfg.memory.embedding_model = "bge-m3".to_string(); + cfg.cloud_providers = vec![CloudProviderCreds { + id: "p_lmstudio".to_string(), + slug: "lmstudio".to_string(), + endpoint: "http://localhost:1234/v1".to_string(), + ..Default::default() + }]; + touch_auth_profile(&cfg); + assert_eq!(effective_embedder_slug(&cfg), "custom"); + } +} diff --git a/core/src/tree/score/embed/inert.rs b/core/src/tree/score/embed/inert.rs new file mode 100644 index 0000000..a122043 --- /dev/null +++ b/core/src/tree/score/embed/inert.rs @@ -0,0 +1,64 @@ +//! Deterministic zero-vector embedder for tests. +//! +//! `InertEmbedder::embed` always returns a fresh `Vec` of length +//! [`super::EMBEDDING_DIM`] filled with zeros — no network, no randomness, +//! no per-text variation. Useful in tests that want to exercise the +//! ingest/seal embedding plumbing without standing up Ollama. +//! +//! Note: because every chunk and summary ends up with the same +//! zero-vector embedding, cosine similarity between them is always 0.0 +//! (see [`super::cosine_similarity`] — zero-magnitude vectors short to +//! 0.0 instead of NaN). Retrieval tests that want to see reranking work +//! should hand-stitch embeddings via the store accessors rather than +//! rely on the inert path. + +use anyhow::Result; +use async_trait::async_trait; + +use super::{Embedder, EMBEDDING_DIM}; + +/// Zero-vector embedder. Returns `vec![0.0; EMBEDDING_DIM]` for every call. +#[derive(Clone, Copy, Debug, Default)] +pub struct InertEmbedder; + +impl InertEmbedder { + /// Construct an inert embedder. Free — `InertEmbedder` is a ZST. + pub fn new() -> Self { + Self + } +} + +#[async_trait] +impl Embedder for InertEmbedder { + fn name(&self) -> &'static str { + "inert" + } + + async fn embed(&self, _text: &str) -> Result> { + Ok(vec![0.0; EMBEDDING_DIM]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn returns_768_zero_vector() { + let e = InertEmbedder::new(); + let v = e.embed("anything").await.unwrap(); + assert_eq!(v.len(), EMBEDDING_DIM); + assert!(v.iter().all(|f| *f == 0.0)); + } + + #[tokio::test] + async fn name_is_inert() { + assert_eq!(InertEmbedder::new().name(), "inert"); + } + + #[tokio::test] + async fn empty_input_still_returns_full_vector() { + let v = InertEmbedder::new().embed("").await.unwrap(); + assert_eq!(v.len(), EMBEDDING_DIM); + } +} diff --git a/core/src/tree/score/embed/mod.rs b/core/src/tree/score/embed/mod.rs new file mode 100644 index 0000000..a096407 --- /dev/null +++ b/core/src/tree/score/embed/mod.rs @@ -0,0 +1,629 @@ +//! Phase 4 embedding layer (#710). +//! +//! Produces a fixed-dimension vector per chunk / summary so retrieval can +//! rerank candidates by semantic similarity. Phase 4's default backend is a +//! local [Ollama](https://ollama.com) endpoint running `bge-m3`; +//! tests use the deterministic [`InertEmbedder`] so no network is required. +//! +//! Dimension is hard-coded at [`EMBEDDING_DIM`] (1024) — matches the +//! bge-m3 output and keeps the blob layout on `mem_tree_chunks` / +//! `mem_tree_summaries` consistent across providers. Mixing dimensions +//! mid-run would corrupt cosine comparisons; we catch that at the trait +//! level rather than deferring to retrieval-time diagnostics. +//! +//! NOTE: bge-m3 replaces the prior `nomic-embed-text` (768-dim, 2048 +//! token context). Migration was driven by nomic's hard 2048-token +//! context cap causing long-chunk embed failures (chunker estimates +//! undercount BERT-WordPiece tokens by ~1.5-2× for HTML-derived +//! markdown, so 1500 chunker-tokens routinely exceed nomic's cap). +//! bge-m3 has a native 8192-token context. Existing `embedding` blobs +//! from the 768-dim era are invalid against the new dimension and +//! must be wiped or re-embedded. +//! +//! Write-time semantics: ingest + seal call [`Embedder::embed`] **before** +//! persisting the new row, so a provider error cascades into "don't write +//! this row". Legacy rows from Phases 1-3 predate embeddings and read back +//! with `Option::None`; retrieval tolerates that by dropping legacy rows +//! to the bottom of a semantic rerank. + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +pub mod factory; +pub mod inert; +pub mod openai_compat; + +pub use factory::{build_embedder_from_config, build_write_embedder, effective_embedder_slug}; +pub use inert::InertEmbedder; +pub use openai_compat::OpenAiCompatEmbedder; + +/// Embedding dimensionality used across the memory tree. +/// +/// Hard-coded to match `bge-m3`; swapping providers requires a matching +/// dimension or the trait's post-call validation will bail. Any change +/// to this constant breaks on-disk compatibility with existing +/// `mem_tree_chunks.embedding` / `mem_tree_summaries.embedding` blobs. +pub const EMBEDDING_DIM: usize = 1024; + +/// Trait backing all Phase 4 embedders. Implementations MUST produce +/// exactly [`EMBEDDING_DIM`] floats per call — callers that persist the +/// result rely on the fixed layout. +#[async_trait] +pub trait Embedder: Send + Sync { + /// Stable short name, used in debug logs and provider diagnostics. + fn name(&self) -> &'static str; + + /// Embed one text. Must return a `Vec` of length + /// [`EMBEDDING_DIM`]. Hard failure — ingest / seal treat `Err` as + /// "don't persist the row" so retries stay idempotent on `chunk_id`. + async fn embed(&self, text: &str) -> Result>; + + /// Embed many texts, returning **one [`Result`] per input position** + /// aligned by index. A single failing text does not strand the rest of + /// the batch — its slot carries the `Err` while the others succeed — + /// which lets bulk callers (e.g. the re-embed backfill) attribute and + /// skip individual rows exactly as a per-text loop would. + /// + /// The default implementation issues one sequential [`Embedder::embed`] + /// call per text: correct for any provider, but with no batching win. + /// Providers whose backend accepts many texts in a single request + /// (cloud / OpenAI-compatible) override this to collapse N network + /// round-trips into one — see `embed_batch_via_provider`. + /// + /// The returned vector always has `texts.len()` elements. + async fn embed_batch(&self, texts: &[&str]) -> Vec>> { + log::debug!( + "[memory_tree::embed::{}] embed_batch:enter sequential texts={}", + self.name(), + texts.len() + ); + let mut out = Vec::with_capacity(texts.len()); + for text in texts { + out.push(self.embed(text).await); + } + out + } +} + +/// Adapts the canonical host embedding-provider contract to the legacy +/// memory-tree embedder shape. Concrete network implementations live in +/// `tinyagents::harness::embeddings`; this bridge owns only dimension checks +/// and the memory tree's per-position batch fallback contract. +pub struct ProviderEmbedder { + inner: Box, + label: &'static str, +} + +impl ProviderEmbedder { + pub fn new( + inner: Box, + label: &'static str, + ) -> Self { + Self { inner, label } + } +} + +#[async_trait] +impl Embedder for ProviderEmbedder { + fn name(&self) -> &'static str { + self.label + } + + async fn embed(&self, text: &str) -> Result> { + self.inner + .embed_one(text) + .await + .with_context(|| format!("{} embeddings failed", self.label)) + .and_then(|vector| check_embed_dim(vector, self.label)) + } + + async fn embed_batch(&self, texts: &[&str]) -> Vec>> { + embed_batch_via_provider(self.inner.as_ref(), self.label, texts).await + } +} + +/// Validate that a freshly-produced embedding has exactly [`EMBEDDING_DIM`] +/// floats, returning a labelled error otherwise. Shared by the per-text and +/// batched provider adapters so the "wrong dims" diagnostic is identical +/// regardless of path. +pub(crate) fn check_embed_dim(v: Vec, label: &str) -> Result> { + if v.len() != EMBEDDING_DIM { + anyhow::bail!( + "{label} embedder returned {} dims, expected {}", + v.len(), + EMBEDDING_DIM + ); + } + Ok(v) +} + +/// Voyage batch API limits (conservative estimates). +const MAX_BATCH_ITEMS: usize = 1000; +const MAX_BATCH_TOKENS: usize = 1_000_000; +const CHARS_PER_TOKEN_ESTIMATE: usize = 4; + +fn estimate_tokens(text: &str) -> usize { + text.len().div_ceil(CHARS_PER_TOKEN_ESTIMATE) +} + +/// Split `texts` into sub-batches that respect the batch API limits: +/// at most `MAX_BATCH_ITEMS` items per batch and at most +/// `MAX_BATCH_TOKENS` estimated tokens per batch. +fn split_into_sub_batches<'a>(texts: &[&'a str]) -> Vec> { + let mut batches: Vec> = Vec::new(); + let mut current: Vec<&'a str> = Vec::new(); + let mut current_tokens: usize = 0; + + for &text in texts { + let tokens = estimate_tokens(text); + if !current.is_empty() + && (current.len() >= MAX_BATCH_ITEMS || current_tokens + tokens > MAX_BATCH_TOKENS) + { + batches.push(std::mem::take(&mut current)); + current_tokens = 0; + } + current.push(text); + current_tokens += tokens; + } + if !current.is_empty() { + batches.push(current); + } + batches +} + +/// Batch-embed `texts` through a unified [`EmbeddingProvider`], splitting +/// into sub-batches that respect the batch API limits (1000 items, ~1M +/// tokens per request). +/// +/// Each sub-batch is sent as a single provider `embed()` call. On a +/// wholesale batch failure **or** a length-contract violation, the failing +/// sub-batch falls back to per-text [`EmbeddingProvider::embed_one`] so a +/// single transient blip cannot fail — and, in the backfill, *tombstone* — +/// every row in the batch. +pub(crate) async fn embed_batch_via_provider( + inner: &dyn tinymemory_api::host::EmbeddingProvider, + label: &str, + texts: &[&str], +) -> Vec>> { + if texts.is_empty() { + return Vec::new(); + } + + let sub_batches = split_into_sub_batches(texts); + log::debug!( + "[memory_tree::embed::{label}] embed_batch:enter texts={} sub_batches={}", + texts.len(), + sub_batches.len() + ); + + let mut all_results: Vec>> = Vec::with_capacity(texts.len()); + + for (batch_idx, batch) in sub_batches.iter().enumerate() { + let batch_results = embed_one_sub_batch(inner, label, batch, batch_idx).await; + all_results.extend(batch_results); + } + + all_results +} + +/// Embed a single sub-batch via the provider, with per-text fallback on +/// batch failure. +async fn embed_one_sub_batch( + inner: &dyn tinymemory_api::host::EmbeddingProvider, + label: &str, + texts: &[&str], + batch_idx: usize, +) -> Vec>> { + match inner.embed(texts).await { + Ok(vectors) if vectors.len() == texts.len() => { + log::debug!( + "[memory_tree::embed::{label}] embed_batch:success sub_batch={batch_idx} \ + collapsed {} texts into one provider call", + texts.len() + ); + vectors + .into_iter() + .map(|v| check_embed_dim(v, label)) + .collect() + } + Ok(vectors) => { + log::warn!( + "[memory_tree::embed::{label}] embed_batch:fallback sub_batch={batch_idx} \ + returned {} vectors for {} texts; falling back to per-text embedding", + vectors.len(), + texts.len() + ); + embed_each_via_provider(inner, label, texts).await + } + Err(e) => { + log::warn!( + "[memory_tree::embed::{label}] embed_batch:fallback sub_batch={batch_idx} \ + batch embed failed ({e:#}); falling back to per-text embedding" + ); + embed_each_via_provider(inner, label, texts).await + } + } +} + +/// Sequential per-text fallback used when a provider's native batch call is +/// unavailable or fails wholesale. Each slot is dimension-checked so the +/// result is interchangeable with the happy-path mapping in +/// [`embed_batch_via_provider`]. +async fn embed_each_via_provider( + inner: &dyn tinymemory_api::host::EmbeddingProvider, + label: &str, + texts: &[&str], +) -> Vec>> { + let mut out = Vec::with_capacity(texts.len()); + for text in texts { + let result = inner + .embed_one(text) + .await + .with_context(|| format!("{label} embeddings failed")) + .and_then(|v| check_embed_dim(v, label)); + out.push(result); + } + out +} + +/// Cosine similarity between two equal-length vectors. +/// +/// Returns `0.0` when either vector has zero magnitude (including empty +/// vectors) to keep the rerank sort stable instead of surfacing `NaN`. +/// Length mismatch also returns `0.0` — callers upstream of the +/// comparison should normalise to [`EMBEDDING_DIM`] before calling. +pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { + if a.len() != b.len() || a.is_empty() { + return 0.0; + } + let mut dot = 0.0_f32; + let mut na = 0.0_f32; + let mut nb = 0.0_f32; + for (x, y) in a.iter().zip(b.iter()) { + dot += x * y; + na += x * x; + nb += y * y; + } + if na == 0.0 || nb == 0.0 { + return 0.0; + } + dot / (na.sqrt() * nb.sqrt()) +} + +/// Pack a `Vec` into little-endian bytes for SQLite BLOB storage. +/// +/// Output length is `v.len() * 4`. The inverse is [`unpack_embedding`]. +pub fn pack_embedding(v: &[f32]) -> Vec { + let mut out = Vec::with_capacity(v.len() * 4); + for f in v { + out.extend_from_slice(&f.to_le_bytes()); + } + out +} + +/// Unpack little-endian bytes into a `Vec`. +/// +/// Errors when the byte length isn't a multiple of 4 or doesn't match +/// [`EMBEDDING_DIM`] (after decoding). The latter guards against rows +/// written with a mismatched-provider blob silently passing as valid. +pub fn unpack_embedding(b: &[u8]) -> Result> { + if b.len() % 4 != 0 { + anyhow::bail!( + "embedding blob length {} not a multiple of 4 — corrupt row", + b.len() + ); + } + let floats: Vec = b + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect(); + if floats.len() != EMBEDDING_DIM { + anyhow::bail!( + "embedding blob length {} floats, expected {}", + floats.len(), + EMBEDDING_DIM + ); + } + Ok(floats) +} + +/// Pack helper that also validates the input dimension before storing. +/// Used by write-time call sites where we want a loud error if a provider +/// misbehaves rather than writing a differently-shaped blob. +pub fn pack_checked(v: &[f32]) -> Result> { + if v.len() != EMBEDDING_DIM { + anyhow::bail!( + "embedding vector has {} dims, expected {}", + v.len(), + EMBEDDING_DIM + ); + } + Ok(pack_embedding(v)) +} + +/// Decode a possibly-NULL embedding blob straight from a query row. +/// Returns `Ok(None)` for NULL (legacy rows predating Phase 4) and +/// surfaces decoding errors with context so the caller sees which row +/// was malformed. +pub fn decode_optional_blob( + blob: Option>, + context_label: &str, +) -> Result>> { + match blob { + None => Ok(None), + Some(bytes) => { + let v = unpack_embedding(&bytes) + .with_context(|| format!("decode embedding for {context_label}"))?; + Ok(Some(v)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cosine_identical_vectors_is_one() { + let a = vec![0.1_f32, 0.2, 0.3, 0.4]; + assert!((cosine_similarity(&a, &a) - 1.0).abs() < 1e-6); + } + + #[test] + fn cosine_orthogonal_vectors_is_zero() { + let a = vec![1.0_f32, 0.0, 0.0]; + let b = vec![0.0_f32, 1.0, 0.0]; + assert!(cosine_similarity(&a, &b).abs() < 1e-6); + } + + #[test] + fn cosine_opposite_vectors_is_minus_one() { + let a = vec![1.0_f32, 2.0, 3.0]; + let b = vec![-1.0_f32, -2.0, -3.0]; + assert!((cosine_similarity(&a, &b) + 1.0).abs() < 1e-6); + } + + #[test] + fn cosine_zero_vector_returns_zero_not_nan() { + let a = vec![0.0_f32; 4]; + let b = vec![1.0_f32, 2.0, 3.0, 4.0]; + let s = cosine_similarity(&a, &b); + assert_eq!(s, 0.0, "expected 0.0, got {s}"); + assert!(!s.is_nan()); + } + + #[test] + fn cosine_empty_returns_zero() { + assert_eq!(cosine_similarity(&[], &[]), 0.0); + } + + #[test] + fn cosine_length_mismatch_returns_zero() { + let a = vec![1.0_f32, 2.0]; + let b = vec![1.0_f32, 2.0, 3.0]; + assert_eq!(cosine_similarity(&a, &b), 0.0); + } + + #[test] + fn pack_unpack_round_trip() { + let v: Vec = (0..EMBEDDING_DIM).map(|i| (i as f32) / 100.0).collect(); + let packed = pack_embedding(&v); + assert_eq!(packed.len(), EMBEDDING_DIM * 4); + let back = unpack_embedding(&packed).unwrap(); + assert_eq!(back, v); + } + + #[test] + fn unpack_wrong_byte_count_errors() { + let bad = vec![0u8, 0, 0]; // not multiple of 4 + assert!(unpack_embedding(&bad).is_err()); + } + + #[test] + fn unpack_wrong_dim_errors() { + // Correct byte multiple, but wrong float count. + let bad = vec![0u8; 16]; // 4 floats, expected EMBEDDING_DIM (1024) + let err = unpack_embedding(&bad).unwrap_err().to_string(); + assert!( + err.contains(&format!("expected {EMBEDDING_DIM}")), + "got {err}" + ); + } + + #[test] + fn pack_checked_rejects_wrong_dim() { + let too_short = vec![0.0_f32; 5]; + assert!(pack_checked(&too_short).is_err()); + let correct = vec![0.0_f32; EMBEDDING_DIM]; + assert!(pack_checked(&correct).is_ok()); + } + + // --- batch-embedding (variant B) scaffolding + tests --- + + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + use tinymemory_api::host::EmbeddingProvider; + + fn ok_vec() -> Vec { + vec![0.5_f32; EMBEDDING_DIM] + } + + #[derive(Clone)] + enum ProviderMode { + /// One correct-dim vector per text (single batch call succeeds). + Ok, + /// Batch (`len > 1`) call errors, per-text (`len == 1`) succeeds — + /// exercises the whole-batch-error fallback path. + BatchFailsPerTextOk, + /// Batch (`len > 1`) returns one extra vector, per-text is fine — + /// exercises the length-mismatch fallback path. + WrongCount, + /// Returns `len` vectors but the one at `idx` has the wrong dim — + /// length matches so no fallback; that position must map to `Err`. + OneWrongDim(usize), + } + + struct FakeProvider { + calls: Arc, + mode: ProviderMode, + } + + #[async_trait::async_trait] + impl EmbeddingProvider for FakeProvider { + fn name(&self) -> &str { + "fake" + } + fn model_id(&self) -> &str { + "fake-model" + } + fn dimensions(&self) -> usize { + EMBEDDING_DIM + } + async fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + self.calls.fetch_add(1, Ordering::SeqCst); + match self.mode { + ProviderMode::Ok => Ok(texts.iter().map(|_| ok_vec()).collect()), + ProviderMode::BatchFailsPerTextOk => { + if texts.len() > 1 { + anyhow::bail!("simulated batch endpoint failure") + } else { + Ok(texts.iter().map(|_| ok_vec()).collect()) + } + } + ProviderMode::WrongCount => { + if texts.len() > 1 { + Ok((0..texts.len() + 1).map(|_| ok_vec()).collect()) + } else { + Ok(texts.iter().map(|_| ok_vec()).collect()) + } + } + ProviderMode::OneWrongDim(idx) => Ok(texts + .iter() + .enumerate() + .map(|(i, _)| if i == idx { vec![0.0_f32; 3] } else { ok_vec() }) + .collect()), + } + } + } + + #[tokio::test] + async fn embed_batch_via_provider_happy_is_single_call() { + let calls = Arc::new(AtomicUsize::new(0)); + let p = FakeProvider { + calls: calls.clone(), + mode: ProviderMode::Ok, + }; + let out = embed_batch_via_provider(&p, "test", &["a", "b", "c"]).await; + assert_eq!(out.len(), 3); + assert!(out.iter().all(|r| r.is_ok())); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "happy path must collapse to exactly one batch call" + ); + } + + #[tokio::test] + async fn embed_batch_via_provider_empty_makes_no_call() { + let calls = Arc::new(AtomicUsize::new(0)); + let p = FakeProvider { + calls: calls.clone(), + mode: ProviderMode::Ok, + }; + let texts: [&str; 0] = []; + let out = embed_batch_via_provider(&p, "test", &texts).await; + assert!(out.is_empty()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn embed_batch_via_provider_falls_back_on_batch_error() { + let calls = Arc::new(AtomicUsize::new(0)); + let p = FakeProvider { + calls: calls.clone(), + mode: ProviderMode::BatchFailsPerTextOk, + }; + let out = embed_batch_via_provider(&p, "test", &["a", "b", "c"]).await; + assert_eq!(out.len(), 3); + assert!( + out.iter().all(|r| r.is_ok()), + "per-text fallback should still produce all vectors" + ); + // 1 failed batch call + 3 per-text calls. + assert_eq!(calls.load(Ordering::SeqCst), 4); + } + + #[tokio::test] + async fn embed_batch_via_provider_falls_back_on_length_mismatch() { + let calls = Arc::new(AtomicUsize::new(0)); + let p = FakeProvider { + calls: calls.clone(), + mode: ProviderMode::WrongCount, + }; + let out = embed_batch_via_provider(&p, "test", &["a", "b"]).await; + assert_eq!(out.len(), 2); + assert!(out.iter().all(|r| r.is_ok())); + // 1 mismatched batch call + 2 per-text calls. + assert_eq!(calls.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn embed_batch_via_provider_maps_wrong_dim_per_position() { + let calls = Arc::new(AtomicUsize::new(0)); + let p = FakeProvider { + calls: calls.clone(), + mode: ProviderMode::OneWrongDim(1), + }; + let out = embed_batch_via_provider(&p, "test", &["a", "b", "c"]).await; + assert_eq!(out.len(), 3); + assert!(out[0].is_ok()); + assert!(out[1].is_err(), "wrong-dim vector maps to Err at its slot"); + assert!(out[2].is_ok()); + // Length matched, so no fallback — a single batch call. + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + struct SeqEmbedder { + calls: Arc, + } + + #[async_trait::async_trait] + impl Embedder for SeqEmbedder { + fn name(&self) -> &'static str { + "seq" + } + async fn embed(&self, text: &str) -> Result> { + self.calls.fetch_add(1, Ordering::SeqCst); + if text == "bad" { + anyhow::bail!("simulated per-text failure") + } + Ok(ok_vec()) + } + // Uses the default `embed_batch`. + } + + #[tokio::test] + async fn default_embed_batch_calls_embed_per_text() { + let calls = Arc::new(AtomicUsize::new(0)); + let e = SeqEmbedder { + calls: calls.clone(), + }; + let out = e.embed_batch(&["a", "b", "c"]).await; + assert_eq!(out.len(), 3); + assert!(out.iter().all(|r| r.is_ok())); + assert_eq!(calls.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn default_embed_batch_preserves_per_position_errors() { + let calls = Arc::new(AtomicUsize::new(0)); + let e = SeqEmbedder { + calls: calls.clone(), + }; + let out = e.embed_batch(&["ok", "bad", "ok"]).await; + assert_eq!(out.len(), 3); + assert!(out[0].is_ok()); + assert!(out[1].is_err()); + assert!(out[2].is_ok()); + } +} diff --git a/core/src/tree/score/embed/openai_compat.rs b/core/src/tree/score/embed/openai_compat.rs new file mode 100644 index 0000000..a0cda68 --- /dev/null +++ b/core/src/tree/score/embed/openai_compat.rs @@ -0,0 +1,415 @@ +//! Memory-tree [`Embedder`] backed by a user-configured OpenAI-compatible +//! embeddings provider (#002 FR-015). +//! +//! ## Why this exists +//! +//! The memory-tree embedder factory historically resolved only: explicit +//! Ollama override → `ollama:` workload prefix → managed `CloudEmbedder` +//! (backend→Voyage) → skip. So a user who configured **OpenAI** (or any +//! custom OpenAI-compatible endpoint) in Connections → API keys → Embeddings was +//! silently ignored: their `embeddings_provider = "openai"` matched no branch +//! and fell through to the managed backend, which then hit "managed budget" +//! while the user's own key sat unused. This adapter closes that gap. +//! +//! ## How +//! +//! It wraps the unified [`EmbeddingProvider`] built by +//! `create_embedding_provider_with_credentials` (the same construction the +//! Settings "Test connection" + main embed RPC use, so there is one source of +//! truth for OpenAI/custom embeddings) and adapts it to the memory-tree +//! [`Embedder`] trait. Dimensions are pinned to [`EMBEDDING_DIM`] (1024) — the +//! tree's on-disk format is fixed there — and the OpenAI request path now +//! sends the `dimensions` parameter (see `embeddings::openai`) so a reducible +//! model (`text-embedding-3-large`) returns 1024 instead of its native 3072. +//! A returned vector of the wrong size surfaces as the trait's standard +//! "expected N dims" error, which the worker classifies as +//! `embedding_dim_mismatch`. + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use super::{Embedder, EMBEDDING_DIM}; +use crate::Config; +use tinymemory_api::host::EmbeddingProvider; + +/// Adapter from the unified [`EmbeddingProvider`] to the memory-tree +/// [`Embedder`] trait for the OpenAI / custom-OpenAI providers. +pub struct OpenAiCompatEmbedder { + inner: Box, + /// Short label for logs (e.g. "openai", "custom"). + label: &'static str, +} + +impl OpenAiCompatEmbedder { + /// Try to build the adapter from the user's configured embeddings settings. + /// + /// Returns `Ok(None)` when `config.memory().embedding_provider` is **not** an + /// OpenAI-compatible provider (so the caller's resolution chain continues + /// to the next branch), and `Ok(Some(_))` when it is. Errors only on an + /// actual construction failure (which the caller can treat as + /// fail-fast-worthy). + /// + /// Always requests [`EMBEDDING_DIM`] regardless of the user's configured + /// dimensions — the tree format is fixed at 1024, and the OpenAI path now + /// honours the `dimensions` param so 3-large complies. + pub fn try_from_config(config: &Config) -> Result> { + let provider = config.memory().embedding_provider.trim(); + + // Decide which OpenAI-compatible endpoint to route to, if any: + // * `openai` → OpenAI's hosted API. + // * `custom` / `custom:` → an inline custom endpoint. + // * any configured `cloud_providers` slug (e.g. `lmstudio`, `vllm`) + // → that entry's endpoint, treated as a custom OpenAI-compatible + // server. + // + // The third case is the #3781 fix. The chat/LLM factory already + // resolves these slugs via `config.cloud_providers()` + // (`make_cloud_provider_by_slug`), so the memory_tree LLM extractor + // honours a local `lmstudio` backend. The embedder, however, only knew + // `openai`/`custom` — so a local LM Studio embeddings backend + // (`[memory] embedding_provider = "lmstudio"`) was silently ignored and + // bucket sealing fell through to the managed cloud budget, 400ing with + // "Insufficient budget" and failing jobs as unrecoverable. Mirroring the + // chat factory's slug resolution here gives sealing/ingest embeddings + // the same local-endpoint parity the extractor already has. + // + // Anything else returns `Ok(None)` so the caller's resolution ladder + // continues to the managed cloud default. + let (slug, label, custom_endpoint): (&str, &'static str, Option<&str>) = + if provider == "openai" { + ("openai", "openai", None) + } else if provider == "custom" || provider.starts_with("custom:") { + ("custom", "custom", provider.strip_prefix("custom:")) + } else { + // Bare slug, tolerating a trailing `:model` for symmetry with the + // top-level `embeddings_provider = "slug:model"` form. + let bare = provider.split(':').next().unwrap_or(provider).trim(); + // Reserved / managed / native-API slugs are owned by other + // branches of the resolution ladder (managed cloud, Voyage, + // Cohere, native Ollama, deliberate opt-out) — never the + // OpenAI-compatible adapter. Let the caller fall through. + if bare.is_empty() + || matches!( + bare, + "managed" | "cloud" | "openhuman" | "voyage" | "cohere" | "ollama" | "none" + ) + { + return Ok(None); + } + match config + .cloud_providers() + .iter() + .find(|e| e.slug == bare) + .map(|e| e.endpoint.trim()) + .filter(|ep| !ep.is_empty()) + { + // A configured OpenAI-compatible provider (LM Studio, vLLM, + // text-embeddings-inference, …) → route as `custom` against + // its endpoint. + Some(endpoint) => ("custom", "custom", Some(endpoint)), + // Unknown slug, or one with no endpoint configured — fall + // through to the managed cloud default rather than erroring. + None => return Ok(None), + } + }; + + // Credential lookup keys on the bare slug (`embeddings:`); local + // servers like LM Studio usually need no key, so an empty result is fine + // and matches the existing `custom` behaviour. `resolve_api_key` already + // normalises a `custom:` argument down to the `custom` slug. + let cred_slug = provider.split(':').next().unwrap_or(provider).trim(); + let api_key = crate::embedding_host::require_embedding_host() + .map_err(|e| anyhow::anyhow!(e))? + .resolve_api_key(cred_slug) + .unwrap_or_default(); + + // Model: prefer the explicit `embedding_model`; otherwise fall back to an + // inline `slug:model` suffix on the provider string. The `custom:` + // form is exempt — its suffix is an endpoint URL, not a model name, so + // splitting it would mis-route the URL as the model. Leave the model + // empty in that case and let the endpoint default apply. + let model = { + let explicit = config.memory().embedding_model.trim(); + if !explicit.is_empty() { + explicit + } else if provider.starts_with("custom:") { + "" + } else { + provider + .split_once(':') + .map(|(_, m)| m.trim()) + .unwrap_or("") + } + }; + + // The memory tree's on-disk format is fixed at [`EMBEDDING_DIM`]. Models + // that don't honour the OpenAI `dimensions` request param (everything + // outside `text-embedding-3-*`) return their own native length, so a + // config whose stored dimension isn't `EMBEDDING_DIM` can never satisfy + // the tree. Building the adapter anyway would only defer the failure to + // the first embed ("expected 1024, got N") — refuse it here with an + // actionable message instead (Codex review on #4056). `text-embedding-3-*` + // is exempt: we request `EMBEDDING_DIM` below and the server reduces to it. + if !crate::embedding_host::require_embedding_host() + .map_err(|e| anyhow::anyhow!(e))? + .model_supports_dimensions(model) + && config.memory().embedding_dimensions != EMBEDDING_DIM + { + anyhow::bail!( + "embeddings provider '{provider}' (model '{model}') produces \ + {}-dimensional vectors, but the memory tree requires {EMBEDDING_DIM}. \ + Choose a {EMBEDDING_DIM}-dimension model — an OpenAI `text-embedding-3-*` \ + model, or a {EMBEDDING_DIM}-dim model such as `mxbai-embed-large` or `bge-large`.", + config.memory().embedding_dimensions + ); + } + + let inner = crate::embedding_host::require_embedding_host() + .map_err(|e| anyhow::anyhow!(e))? + .create_embedding_provider_with_credentials( + slug, + model, + EMBEDDING_DIM, + &api_key, + custom_endpoint, + ) + .map_err(|e| anyhow::anyhow!(e)) + .with_context(|| { + format!("build {label} embedder for memory tree (provider='{provider}')") + })?; + + log::debug!( + "[memory_tree::embed::openai_compat] using {label} provider (config='{}') \ + endpoint={:?} model={} dims={}", + provider, + custom_endpoint, + model, + EMBEDDING_DIM + ); + Ok(Some(Self { inner, label })) + } +} + +#[async_trait] +impl Embedder for OpenAiCompatEmbedder { + fn name(&self) -> &'static str { + self.label + } + + async fn embed(&self, text: &str) -> Result> { + let v = self + .inner + .embed_one(text) + .await + .with_context(|| format!("{} embeddings failed", self.label))?; + if v.len() != EMBEDDING_DIM { + anyhow::bail!( + "{} embedder returned {} dims, expected {}", + self.label, + v.len(), + EMBEDDING_DIM + ); + } + Ok(v) + } + + /// Collapse N per-text round-trips into a single batched request by + /// delegating to the inner provider's native batch `embed`. Falls back to + /// per-text embedding (preserving per-position error attribution) on a + /// whole-batch failure or a length mismatch. + async fn embed_batch(&self, texts: &[&str]) -> Vec>> { + super::embed_batch_via_provider(self.inner.as_ref(), self.label, texts).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn cfg_with_provider(p: &str) -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + cfg.config_path = tmp.path().join("config.toml"); + cfg.memory.embedding_provider = p.to_string(); + cfg.memory.embedding_model = "text-embedding-3-large".to_string(); + (tmp, cfg) + } + + #[test] + fn none_for_non_openai_providers() { + // managed / voyage / ollama / none must fall through (Ok(None)). + for p in ["managed", "cloud", "voyage", "ollama:bge-m3", "none"] { + let (_tmp, cfg) = cfg_with_provider(p); + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + assert!(got.is_none(), "{p} should fall through, got Some"); + } + } + + #[test] + fn some_for_openai() { + let (_tmp, cfg) = cfg_with_provider("openai"); + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + let e = got.expect("openai should build an adapter"); + assert_eq!(e.name(), "openai"); + } + + #[test] + fn some_for_custom() { + let (_tmp, cfg) = cfg_with_provider("custom:https://embed.example/v1"); + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + let e = got.expect("custom should build an adapter"); + assert_eq!(e.name(), "custom"); + } + + /// When `embedding_model` is unset, a `custom:` provider must NOT treat + /// the endpoint URL suffix as an inline model name (CodeRabbit #3781). The + /// adapter still builds; the model is simply left empty. + #[test] + fn some_for_custom_endpoint_does_not_use_url_as_model() { + let (_tmp, mut cfg) = cfg_with_provider("custom:https://embed.example/v1"); + cfg.memory.embedding_model = String::new(); // force the inline fallback path + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + let e = got.expect("custom endpoint with no model should still build"); + assert_eq!(e.name(), "custom"); + } + + /// Build a `cloud_providers` entry the way AI Settings persists a local + /// OpenAI-compatible server. + fn lmstudio_entry(endpoint: &str) -> tinymemory_api::host::cloud_providers::CloudProviderCreds { + tinymemory_api::host::cloud_providers::CloudProviderCreds { + id: "p_lmstudio_test".to_string(), + slug: "lmstudio".to_string(), + endpoint: endpoint.to_string(), + ..Default::default() + } + } + + /// #3781: a configured `lmstudio` slug (OpenAI-compatible, like LM Studio at + /// localhost:1234) must resolve to its `cloud_providers` endpoint and route + /// as a `custom` OpenAI-compatible embedder — NOT fall through to managed + /// cloud. This is the headline bug: sealing ignored the local backend. + #[test] + fn some_for_configured_lmstudio_slug() { + let (_tmp, mut cfg) = cfg_with_provider("lmstudio"); + cfg.memory.embedding_model = "bge-m3".to_string(); + cfg.cloud_providers = vec![lmstudio_entry("http://localhost:1234/v1")]; + + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + let e = got.expect("configured lmstudio slug must build an adapter, not fall through"); + assert_eq!(e.name(), "custom"); + } + + /// The `slug:model` form (mirroring the top-level + /// `embeddings_provider = "lmstudio:bge-m3"` shape) also resolves, taking the + /// model from the inline suffix when `embedding_model` is unset. + #[test] + fn some_for_lmstudio_slug_with_inline_model() { + let (_tmp, mut cfg) = cfg_with_provider("lmstudio:bge-m3"); + cfg.memory.embedding_model = String::new(); // force inline-suffix fallback + cfg.cloud_providers = vec![lmstudio_entry("http://localhost:1234/v1")]; + + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + let e = got.expect("lmstudio:model slug must resolve"); + assert_eq!(e.name(), "custom"); + } + + /// A custom slug with no matching `cloud_providers` entry must fall through + /// (Ok(None)) so the caller's ladder continues to the managed default — + /// rather than erroring or hijacking the resolution. + #[test] + fn none_for_unconfigured_custom_slug() { + let (_tmp, cfg) = cfg_with_provider("lmstudio"); // no cloud_providers entry + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + assert!( + got.is_none(), + "unconfigured slug should fall through, not build an adapter" + ); + } + + /// An entry that exists but has a blank endpoint is unusable → fall through. + #[test] + fn none_for_configured_slug_with_blank_endpoint() { + let (_tmp, mut cfg) = cfg_with_provider("lmstudio"); + cfg.cloud_providers = vec![lmstudio_entry(" ")]; + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + assert!(got.is_none(), "blank endpoint should fall through"); + } + + /// Reserved/managed/native slugs must keep falling through even if a stray + /// `cloud_providers` entry exists for them — they are owned by other ladder + /// branches, not the OpenAI-compatible adapter. + #[test] + fn reserved_slugs_still_fall_through() { + use tinymemory_api::host::cloud_providers::CloudProviderCreds; + for p in ["managed", "cloud", "voyage", "cohere", "ollama", "none"] { + let (_tmp, mut cfg) = cfg_with_provider(p); + cfg.cloud_providers = vec![CloudProviderCreds { + id: format!("p_{p}"), + slug: p.to_string(), + endpoint: "http://localhost:1234/v1".to_string(), + ..Default::default() + }]; + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + assert!(got.is_none(), "{p} must fall through, got Some"); + } + } + + /// Codex review on #4056: a custom config whose stored dimension isn't the + /// tree's fixed [`EMBEDDING_DIM`] (and whose model can't reduce to it via the + /// OpenAI `dimensions` param) must be refused at construction with a clear, + /// actionable error — not built and then failed at the first embed with a raw + /// "expected 1024, got N". This is what keeps an auto-detected non-1024 custom + /// endpoint (which the embeddings RPC still accepts) out of the 1024-only tree. + #[test] + fn err_for_non_reducible_model_with_incompatible_dimension() { + let (_tmp, mut cfg) = cfg_with_provider("custom:https://embed.example/v1"); + cfg.memory.embedding_model = "nomic-embed-text".to_string(); // not text-embedding-3-* + cfg.memory.embedding_dimensions = 768; // != EMBEDDING_DIM (1024) + // `expect_err` would require the Ok type (the embedder) to impl Debug, + // which it can't (boxed trait object) — match instead. + let err = match OpenAiCompatEmbedder::try_from_config(&cfg) { + Err(e) => e, + Ok(_) => panic!("768 != tree dim must error, got Ok"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("768") && msg.contains(&EMBEDDING_DIM.to_string()), + "error must name both the model's dim and the required dim: {msg}" + ); + } + + /// A non-reducible model that natively matches [`EMBEDDING_DIM`] still builds — + /// only an incompatible dimension is refused. + #[test] + fn some_for_non_reducible_model_at_tree_dimension() { + let (_tmp, mut cfg) = cfg_with_provider("custom:https://embed.example/v1"); + cfg.memory.embedding_model = "mxbai-embed-large".to_string(); + cfg.memory.embedding_dimensions = EMBEDDING_DIM; // 1024 + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + assert!( + got.is_some(), + "a 1024-native custom model must build the tree adapter" + ); + } + + /// `text-embedding-3-*` is exempt from the dimension guard: the adapter + /// requests `EMBEDDING_DIM` and the server reduces to it, so even a config + /// stored at a different dimension still builds. + #[test] + fn some_for_reducible_model_regardless_of_stored_dimension() { + let (_tmp, mut cfg) = cfg_with_provider("openai"); + cfg.memory.embedding_model = "text-embedding-3-large".to_string(); + cfg.memory.embedding_dimensions = 256; // reducible — tree still requests 1024 + let got = OpenAiCompatEmbedder::try_from_config(&cfg).expect("no error"); + assert!( + got.is_some(), + "reducible model must build regardless of stored dim" + ); + } +} diff --git a/core/src/tree/score/extract/README.md b/core/src/tree/score/extract/README.md new file mode 100644 index 0000000..55b215a --- /dev/null +++ b/core/src/tree/score/extract/README.md @@ -0,0 +1,20 @@ +# Memory tree — score extract + +Entity extraction for the scoring pipeline. Pluggable via the `EntityExtractor` trait so the scorer can run a deterministic regex pass plus an optional LLM pass and merge their outputs. Also surfaces the LLM-derived importance rating consumed by the `llm_importance` signal. + +## Public surface + +- `pub trait EntityExtractor` — `extractor.rs` — async `extract(text) -> ExtractedEntities` contract. +- `pub struct RegexEntityExtractor` / `pub struct CompositeExtractor` — `extractor.rs` — built-in implementations. +- `pub struct LlmEntityExtractor` / `pub struct LlmExtractorConfig` — `llm.rs` — Ollama-backed semantic NER + importance rater. +- `pub fn build_summary_extractor` — `mod.rs` — composes regex + LLM (with `emit_topics: true`) for seal-time summary labelling. +- `pub enum EntityKind` / `pub struct ExtractedEntity` / `pub struct ExtractedTopic` / `pub struct ExtractedEntities` — `types.rs`. + +## Files + +- `mod.rs` — module surface and `build_summary_extractor` for the seal path. +- `types.rs` — output types and the `EntityKind` enum (mechanical kinds `Email/Url/Handle/Hashtag` + semantic kinds `Person/Organization/Location/...` + `Topic`). `ExtractedEntities::merge` deduplicates entities and combines LLM importance by max. +- `extractor.rs` — `EntityExtractor` trait, `RegexEntityExtractor` adapter, `CompositeExtractor` (runs a sequence of extractors and tolerates per-extractor failures). +- `regex.rs` — once-compiled regex patterns for email, URL, handle (`@alice` and Discord-style `alice#1234`), and hashtag. UTF-8 safe — spans are char offsets, not bytes. +- `llm.rs` — Ollama `/api/chat` client that asks the model for NER + an importance rating in one structured-JSON call, with span recovery via `text.find(...)` and a soft fallback (warn + empty) on transport failure. +- `llm_tests.rs` — unit tests for the LLM extractor. diff --git a/core/src/tree/score/extract/mod.rs b/core/src/tree/score/extract/mod.rs new file mode 100644 index 0000000..4768ebf --- /dev/null +++ b/core/src/tree/score/extract/mod.rs @@ -0,0 +1,62 @@ +//! Product construction over tinycortex entity extraction. + +use std::sync::Arc; + +use crate::Config; +use async_trait::async_trait; + +pub use tinycortex::memory::score::extract::{ + ChatPrompt, ChatProvider, CompositeExtractor, EntityExtractor, EntityKind, ExtractedEntities, + ExtractedEntity, ExtractedTopic, LlmExtractorConfig, RegexEntityExtractor, +}; + +pub mod regex { + pub use tinycortex::memory::score::extract::regex::extract; +} + +pub struct LlmEntityExtractor(tinycortex::memory::score::extract::LlmEntityExtractor); + +impl LlmEntityExtractor { + pub fn new(config: LlmExtractorConfig, provider: Arc) -> Self { + let provider = Arc::new(crate::tinycortex::SeamChatProvider::new(provider)); + Self(tinycortex::memory::score::extract::LlmEntityExtractor::new( + config, provider, + )) + } +} + +#[async_trait] +impl EntityExtractor for LlmEntityExtractor { + fn name(&self) -> &'static str { + self.0.name() + } + + async fn extract(&self, text: &str) -> anyhow::Result { + self.0.extract(text).await + } +} + +pub fn build_summary_extractor(config: &Config) -> Arc { + let (provider, model) = match crate::chat::build_chat_runtime(config) { + Ok(runtime) => runtime, + Err(error) => { + log::warn!( + "[memory_tree::extract] chat provider unavailable; using regex-only extraction: {error:#}" + ); + return Arc::new(CompositeExtractor::regex_only()); + } + }; + let extractor = LlmEntityExtractor::new( + LlmExtractorConfig { + model, + emit_topics: true, + output_language: config.output_language().map(str::to_string), + ..Default::default() + }, + provider, + ); + Arc::new(CompositeExtractor::new(vec![ + Box::new(RegexEntityExtractor), + Box::new(extractor), + ])) +} diff --git a/core/src/tree/score/mod.rs b/core/src/tree/score/mod.rs new file mode 100644 index 0000000..136aeeb --- /dev/null +++ b/core/src/tree/score/mod.rs @@ -0,0 +1,41 @@ +//! Product adapters over tinycortex scoring and admission. + +pub mod embed; +pub mod extract; +pub mod store; + +use std::sync::Arc; + +pub use anyhow::Result; +pub use tinycortex::memory::score::{ + persist_score_tx, score_chunk, score_chunks, score_chunks_fast, ScoreResult, ScoringConfig, + DEFAULT_DEFINITE_DROP, DEFAULT_DEFINITE_KEEP, DEFAULT_DROP_THRESHOLD, PRIORITY_BOOST, + PRIORITY_TAG, +}; +pub use tinycortex::memory::score::{resolver, signals}; + +/// Build crate scoring policy from product inference routing. +pub fn scoring_config_from(config: &crate::Config) -> ScoringConfig { + let (provider, model) = match crate::chat::build_chat_runtime(config) { + Ok((provider, model)) => ( + Arc::new(crate::tinycortex::SeamChatProvider::new(provider)) + as Arc, + model, + ), + Err(error) => { + log::warn!( + "[memory::score] chat provider unavailable; using regex-only scoring: {error:#}" + ); + return ScoringConfig::default_regex_only(); + } + }; + let extractor = tinycortex::memory::score::extract::LlmEntityExtractor::new( + tinycortex::memory::score::extract::LlmExtractorConfig { + model, + output_language: config.output_language().map(str::to_string), + ..Default::default() + }, + provider, + ); + ScoringConfig::with_llm_extractor(Arc::new(extractor)) +} diff --git a/core/src/tree/score/signals/README.md b/core/src/tree/score/signals/README.md new file mode 100644 index 0000000..119fbaf --- /dev/null +++ b/core/src/tree/score/signals/README.md @@ -0,0 +1,14 @@ +# Memory tree — score signals + +Per-chunk scoring features. Each submodule computes one signal in `[0.0, 1.0]`; `ops::combine` aggregates them via `SignalWeights` into the final admission total. Signals are stored alongside the total in `mem_tree_score` so admit/drop decisions remain auditable. + +## Files + +- `mod.rs` — module surface: re-exports `compute`, `combine`, `combine_cheap_only`, `entity_density_score`, `ScoreSignals`, `SignalWeights`. +- `types.rs` — `ScoreSignals` (per-signal breakdown) and `SignalWeights` (per-signal multipliers, with `with_llm_enabled()` builder). +- `ops.rs` — `compute(meta, content, token_count, extracted)` populates a `ScoreSignals`; `combine` and `combine_cheap_only` produce the weighted total (the latter excludes the LLM-importance term used by the borderline-band short-circuit). +- `token_count.rs` — plateau-shaped score over chunk token count; scores 0 below `TOKEN_MIN`, ramps to 1 by `TOKEN_RAMP_LOW`, ramps back to 0.5 between `TOKEN_RAMP_HIGH` and `TOKEN_MAX`. +- `unique_words.rs` — type-token-ratio noise detector: low diversity scores low; messages under `MIN_TOTAL_WORDS` return a neutral 0.5. +- `metadata_weight.rs` — base weight per `SourceKind` (Email > Document > Chat). +- `source_weight.rs` — per-`DataSource` weight inferred from `provider:` tags, with `SourceKind` defaults as fallback. +- `interaction.rs` — engagement-tag bonus (`sent`, `reply`, `dm`, `mention`); absent tags return 0.5 so silent content isn't penalised. diff --git a/core/src/tree/score/store.rs b/core/src/tree/score/store.rs new file mode 100644 index 0000000..0a757e8 --- /dev/null +++ b/core/src/tree/score/store.rs @@ -0,0 +1,78 @@ +//! Product Config adapters over tinycortex score and entity-index persistence. + +use std::collections::HashMap; + +use anyhow::Result; + +use crate::tinycortex::engine_config; +use crate::Config; + +pub use tinycortex::memory::score::store::{EntityHit, ScoreRow}; + +pub fn upsert_score(config: &Config, row: &ScoreRow) -> Result<()> { + tinycortex::memory::score::store::upsert_score(&engine_config(config), row) +} + +pub fn get_score(config: &Config, chunk_id: &str) -> Result> { + tinycortex::memory::score::store::get_score(&engine_config(config), chunk_id) +} + +pub fn get_scores_batch(config: &Config, chunk_ids: &[String]) -> Result> { + tinycortex::memory::score::store::get_scores_batch(&engine_config(config), chunk_ids) +} + +pub use crate::store::entities::{ + clear_entity_index_for_node, count_entity_index, list_entity_ids_for_node, lookup_entity, +}; + +pub fn index_entity( + config: &Config, + entity: &tinycortex::memory::score::resolver::CanonicalEntity, + node_id: &str, + node_kind: &str, + timestamp_ms: i64, + tree_id: Option<&str>, +) -> Result<()> { + let entity = to_store_entity(entity)?; + crate::store::entities::index_entity(config, &entity, node_id, node_kind, timestamp_ms, tree_id) +} + +pub fn index_entities( + config: &Config, + entities: &[tinycortex::memory::score::resolver::CanonicalEntity], + node_id: &str, + node_kind: &str, + timestamp_ms: i64, + tree_id: Option<&str>, +) -> Result { + let entities: Vec = entities + .iter() + .map(to_store_entity) + .collect::>()?; + crate::store::entities::index_entities( + config, + &entities, + node_id, + node_kind, + timestamp_ms, + tree_id, + ) +} + +fn to_store_entity( + entity: &tinycortex::memory::score::resolver::CanonicalEntity, +) -> Result { + Ok(tinycortex::memory::store::CanonicalEntity { + canonical_id: entity.canonical_id.clone(), + kind: tinycortex::memory::store::EntityKind::parse(entity.kind.as_str()) + .map_err(anyhow::Error::msg)?, + surface: entity.surface.clone(), + span_start: entity.span_start, + span_end: entity.span_end, + score: entity.score, + }) +} + +pub fn count_scores(config: &Config) -> Result { + tinycortex::memory::score::store::count_scores(&engine_config(config)) +} diff --git a/core/src/tree/summarise.rs b/core/src/tree/summarise.rs new file mode 100644 index 0000000..fc69fdd --- /dev/null +++ b/core/src/tree/summarise.rs @@ -0,0 +1,86 @@ +//! OpenHuman chat-provider adapter for tinycortex summary preparation. + +use anyhow::{Context, Result}; + +use crate::chat::{build_chat_provider, ChatPrompt}; +use crate::Config; + +pub use tinycortex::memory::tree::{SummaryContext, SummaryInput}; + +/// Compatibility result carrying provider usage alongside the crate-owned +/// summary output fields. +#[derive(Clone, Debug, Default)] +pub struct SummaryOutput { + pub content: String, + pub token_count: u32, + pub entities: Vec, + pub topics: Vec, + pub input_tokens: u64, + pub output_tokens: u64, + pub charged_amount_usd: Option, +} + +pub async fn summarise( + config: &Config, + inputs: &[SummaryInput], + context: &SummaryContext<'_>, +) -> Result { + let Some(prepared) = + tinycortex::memory::tree::prepare_summary_prompt(inputs, context, config.output_language()) + else { + return Ok(SummaryOutput::default()); + }; + let provider = + build_chat_provider(config).context("memory_tree::summarise: build chat provider")?; + log::debug!( + "[memory_tree::summarise] provider={} level={} inputs={} budget={}", + provider.name(), + context.target_level, + inputs.len(), + prepared.effective_budget + ); + let (text, usage) = provider + .chat_for_text_with_usage(&ChatPrompt { + system: prepared.system, + user: prepared.user, + temperature: 0.0, + kind: "memory_tree::summarise", + max_tokens: None, + }) + .await + .with_context(|| format!("memory_tree::summarise: provider={}", provider.name()))?; + let output = + tinycortex::memory::tree::finish_provider_summary(&text, prepared.effective_budget); + let input_tokens = usage.as_ref().map_or(0, |usage| usage.input_tokens); + let output_tokens = usage.as_ref().map_or(0, |usage| usage.output_tokens); + let charged_amount_usd = usage + .as_ref() + .map(|usage| usage.charged_amount_usd) + .filter(|amount| *amount > 0.0); + log::debug!( + "[memory_tree::summarise] complete tokens={} usage_input={} usage_output={}", + output.token_count, + input_tokens, + output_tokens + ); + Ok(SummaryOutput { + content: output.content, + token_count: output.token_count, + entities: output.entities, + topics: output.topics, + input_tokens, + output_tokens, + charged_amount_usd, + }) +} + +pub fn fallback_summary(inputs: &[SummaryInput], budget: u32) -> SummaryOutput { + let output = tinycortex::memory::tree::fallback_summary(inputs, budget); + SummaryOutput { + content: output.content, + token_count: output.token_count, + entities: output.entities, + topics: output.topics, + ..SummaryOutput::default() + } +} diff --git a/core/src/tree/tree/bucket_seal.rs b/core/src/tree/tree/bucket_seal.rs new file mode 100644 index 0000000..6903832 --- /dev/null +++ b/core/src/tree/tree/bucket_seal.rs @@ -0,0 +1,81 @@ +//! Product adapters for tinycortex-owned bucket and document sealing. + +use anyhow::Result; +use chrono::{DateTime, Utc}; + +use crate::store::trees::types::{Buffer, Tree}; +use crate::tinycortex::engine_config; +use crate::Config; + +pub use tinycortex::memory::tree::{LabelStrategy, LeafRef, MERGE_LEVEL_BASE}; + +pub async fn append_leaf( + config: &Config, + tree: &Tree, + leaf: &LeafRef, + strategy: &LabelStrategy, +) -> Result> { + append_to_buffer( + config, + &tree.id, + 0, + &leaf.chunk_id, + leaf.token_count as i64, + leaf.timestamp, + )?; + crate::tinycortex::cascade_tree(config, tree, 0, false, strategy).await +} + +pub fn append_leaf_deferred(config: &Config, tree: &Tree, leaf: &LeafRef) -> Result { + tinycortex::memory::tree::append_leaf_deferred(&engine_config(config), tree, leaf) +} + +pub fn append_to_buffer( + config: &Config, + tree_id: &str, + level: u32, + item_id: &str, + token_delta: i64, + item_ts: DateTime, +) -> Result<()> { + tinycortex::memory::tree::append_to_buffer( + &engine_config(config), + tree_id, + level, + item_id, + token_delta, + item_ts, + ) +} + +pub async fn cascade_all_from( + config: &Config, + tree: &Tree, + start_level: u32, + force_now: Option>, + strategy: &LabelStrategy, +) -> Result> { + crate::tinycortex::cascade_tree(config, tree, start_level, force_now.is_some(), strategy).await +} + +pub async fn seal_document_subtree( + config: &Config, + tree: &Tree, + doc_id: &str, + version_ms: Option, + chunk_ids: &[String], + strategy: &LabelStrategy, +) -> Result { + crate::tinycortex::seal_document_subtree(config, tree, doc_id, version_ms, chunk_ids, strategy) + .await +} + +pub async fn seal_one_level( + config: &Config, + tree: &Tree, + buffer: &Buffer, + strategy: &LabelStrategy, + enqueue_follow_ups: bool, +) -> Result { + crate::tinycortex::seal_tree_level(config, tree, buffer, strategy, enqueue_follow_ups).await +} diff --git a/core/src/tree/tree/factory.rs b/core/src/tree/tree/factory.rs new file mode 100644 index 0000000..fbb738e --- /dev/null +++ b/core/src/tree/tree/factory.rs @@ -0,0 +1,166 @@ +//! Kind/profile factory for memory-tree instances. +//! +//! Centralizes the flavor-specific bits so callers get a uniform API: +//! - underlying [`TreeKind`] +//! - canonical scope +//! - summary-file kind +//! - scope-slug rules +//! - default seal-time label strategy + +use std::borrow::Cow; + +use anyhow::Result; + +use crate::store::content::paths::slugify_source_id; +use crate::store::content::SummaryTreeKind; +use crate::store::trees::archive_tree; +use crate::store::trees::types::{Tree, TreeKind}; +use crate::tree::score::extract::build_summary_extractor; +use crate::tree::tree::bucket_seal::{append_leaf, LabelStrategy, LeafRef}; +use crate::tree::tree::flush::force_flush_tree; +use crate::tree::tree::registry::get_or_create_tree; +use crate::Config; + +pub use tinycortex::memory::tree::{TreeProfile, GLOBAL_SCOPE}; + +/// Factory/config object for one tree instance. +#[derive(Debug, Clone)] +pub struct TreeFactory<'a> { + inner: tinycortex::memory::tree::TreeFactory<'a>, +} + +impl<'a> TreeFactory<'a> { + pub fn source(scope: impl Into>) -> Self { + Self { + inner: tinycortex::memory::tree::TreeFactory::source(scope), + } + } + + pub fn topic(scope: impl Into>) -> Self { + Self { + inner: tinycortex::memory::tree::TreeFactory::topic(scope), + } + } + + pub fn global() -> Self { + Self { + inner: tinycortex::memory::tree::TreeFactory::global(), + } + } + + pub fn from_tree(tree: &'a Tree) -> Self { + Self { + inner: tinycortex::memory::tree::TreeFactory::from_tree(tree), + } + } + + pub fn profile(&self) -> TreeProfile { + self.inner.profile() + } + + pub fn kind(&self) -> TreeKind { + self.inner.kind() + } + + pub fn scope(&self) -> &str { + self.inner.scope() + } + + pub fn summary_tree_kind(&self) -> SummaryTreeKind { + match self.kind() { + TreeKind::Source => SummaryTreeKind::Source, + TreeKind::Topic => SummaryTreeKind::Topic, + TreeKind::Global => SummaryTreeKind::Global, + _ => SummaryTreeKind::Source, + } + } + + pub fn scope_slug(&self) -> String { + let scope = self.scope(); + match self.kind() { + TreeKind::Topic | TreeKind::Global => slugify_source_id(scope), + TreeKind::Source => { + if let Some(gmail_scope) = scope.strip_prefix("gmail:") { + slugify_source_id(gmail_scope) + } else { + slugify_source_id(scope) + } + } + _ => slugify_source_id(scope), + } + } + + pub fn label_strategy(&self, config: &Config) -> LabelStrategy { + match self.kind() { + TreeKind::Source => LabelStrategy::ExtractFromContent(build_summary_extractor(config)), + TreeKind::Topic | TreeKind::Global => LabelStrategy::Empty, + _ => LabelStrategy::ExtractFromContent(build_summary_extractor(config)), + } + } + + /// Look up or create the tree row in the database. Instance-specific + /// side-effects (e.g. `_source.md` mirror) are handled by the + /// per-instance registry wrappers in `memory::tree_source` etc. + pub fn get_or_create(&self, config: &Config) -> Result { + get_or_create_tree(config, self.kind(), self.scope()) + } + + /// Append one leaf to this tree profile using its default labeling policy. + pub async fn insert_leaf(&self, config: &Config, leaf: &LeafRef) -> Result> { + let tree = self.get_or_create(config)?; + let strategy = self.label_strategy(config); + append_leaf(config, &tree, leaf, &strategy).await + } + + /// Force-flush/seal this tree profile's currently loaded tree. + pub async fn seal_now(&self, config: &Config) -> Result> { + let tree = self.get_or_create(config)?; + let strategy = self.label_strategy(config); + force_flush_tree(config, &tree.id, None, &strategy).await + } + + /// Archive this tree profile's current tree. + pub fn archive(&self, config: &Config) -> Result<()> { + let tree = self.get_or_create(config)?; + archive_tree(config, &tree.id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_factory_uses_source_kind_and_full_scope() { + let f = TreeFactory::source("slack:#eng"); + assert_eq!(f.kind(), TreeKind::Source); + assert_eq!(f.scope(), "slack:#eng"); + assert_eq!(f.summary_tree_kind(), SummaryTreeKind::Source); + } + + #[test] + fn global_uses_global_scope_and_kind() { + let global = TreeFactory::global(); + assert_eq!(global.kind(), TreeKind::Global); + assert_eq!(global.scope(), GLOBAL_SCOPE); + } + + #[test] + fn source_scope_slug_preserves_non_gmail_prefix() { + let f = TreeFactory::source("slack:#eng"); + assert_eq!(f.scope_slug(), "slack-eng"); + } + + #[test] + fn source_scope_slug_strips_gmail_prefix_only() { + let f = TreeFactory::source("gmail:alice@example.com|bob@example.com"); + assert_eq!(f.scope_slug(), "alice-example-com-bob-example-com"); + } + + #[test] + fn topic_scope_slug_keeps_canonical_prefix() { + let f = TreeFactory::topic("email:alice@example.com"); + assert_eq!(f.scope_slug(), "email-alice-example-com"); + assert_eq!(f.summary_tree_kind(), SummaryTreeKind::Topic); + } +} diff --git a/core/src/tree/tree/flush.rs b/core/src/tree/tree/flush.rs new file mode 100644 index 0000000..9c781e5 --- /dev/null +++ b/core/src/tree/tree/flush.rs @@ -0,0 +1,34 @@ +//! Product adapters for tinycortex-owned stale-buffer flushing. + +use anyhow::Result; +use chrono::{DateTime, Duration, Utc}; + +use crate::store::trees::types::DEFAULT_FLUSH_AGE_SECS; +use crate::tree::tree::bucket_seal::{cascade_all_from, LabelStrategy}; +use crate::Config; + +pub async fn flush_stale_buffers( + config: &Config, + max_age: Duration, + strategy: &LabelStrategy, +) -> Result { + crate::tinycortex::flush_stale_tree_buffers(config, max_age, strategy).await +} + +pub async fn flush_stale_buffers_default( + config: &Config, + strategy: &LabelStrategy, +) -> Result { + flush_stale_buffers(config, Duration::seconds(DEFAULT_FLUSH_AGE_SECS), strategy).await +} + +pub async fn force_flush_tree( + config: &Config, + tree_id: &str, + now: Option>, + strategy: &LabelStrategy, +) -> Result> { + let tree = crate::store::trees::store::get_tree(config, tree_id)? + .ok_or_else(|| anyhow::anyhow!("no tree with id {tree_id}"))?; + cascade_all_from(config, &tree, 0, now.or_else(|| Some(Utc::now())), strategy).await +} diff --git a/core/src/tree/tree/mod.rs b/core/src/tree/tree/mod.rs new file mode 100644 index 0000000..4b9c1cc --- /dev/null +++ b/core/src/tree/tree/mod.rs @@ -0,0 +1,33 @@ +//! Generic summary-tree mechanics shared by all tree flavors. +//! +//! Covers storage, buffer management, bucket-seal cascade, time-based +//! flush, the get-or-create registry primitive, and the kind/profile +//! factory. +//! +//! Flavor-specific policy (global digest, topic hotness, source file +//! mirror) lives in `crate::tree_global`, +//! `crate::tree_topic`, and +//! [`crate::tree_source`] respectively. +//! +//! Persistence (store + types) has moved to `memory_store::trees`. + +pub mod bucket_seal; +pub mod factory; +pub mod flush; +pub mod registry; + +// Re-export persistence from memory_store so callers using tree::store / tree::types still work. +pub use crate::store::trees::store; +pub use crate::store::trees::types; + +pub use crate::store::trees::{get_summary_embedding, set_summary_embedding}; +pub use crate::store::trees::{ + Buffer, SummaryNode, Tree, TreeKind, TreeStatus, INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, + SUMMARY_FANOUT, +}; +pub use bucket_seal::{ + append_leaf, append_leaf_deferred, seal_document_subtree, LabelStrategy, LeafRef, + MERGE_LEVEL_BASE, +}; +pub use factory::{TreeFactory, TreeProfile, GLOBAL_SCOPE}; +pub use registry::{get_or_create_tree, new_summary_id, new_tree_id}; diff --git a/core/src/tree/tree/registry.rs b/core/src/tree/tree/registry.rs new file mode 100644 index 0000000..e060cdc --- /dev/null +++ b/core/src/tree/tree/registry.rs @@ -0,0 +1,231 @@ +//! Generic tree registry — get-or-create for any tree kind (#709). +//! +//! All three tree flavors (Source, Global, Topic) share `UNIQUE(kind, scope)` +//! and the same race-recovery dance — there is no reason for three copies. +//! Source-specific side-effects (writing the `_source.md` mirror) live in +//! the `sources::registry` wrapper rather than here. + +use anyhow::Result; +use chrono::Utc; +use uuid::Uuid; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::store::trees::types::{Tree, TreeKind, TreeStatus}; +use crate::tree::tree::store; +use crate::Config; + +/// Generic get-or-create. All three tree flavors (Source, Global, Topic) +/// share UNIQUE(kind, scope) and the same race-recovery dance — there's +/// no reason for three copies. +/// +/// Source-specific side-effects (writing the `_source.md` on-disk mirror) +/// are NOT performed here; callers that need them should go through +/// [`crate::tree_source::registry::get_or_create_source_tree`]. +pub fn get_or_create_tree(config: &Config, kind: TreeKind, scope: &str) -> Result { + if let Some(existing) = store::get_tree_by_scope(config, kind, scope)? { + log::debug!( + "[tree::registry] found tree id={} kind={} scope={}", + existing.id, + kind.as_str(), + scope + ); + return Ok(existing); + } + + let tree = Tree { + id: new_tree_id(kind), + kind, + scope: scope.to_string(), + ask: None, + root_id: None, + max_level: 0, + status: TreeStatus::Active, + created_at: Utc::now(), + last_sealed_at: None, + }; + match store::insert_tree(config, &tree) { + Ok(()) => { + log::info!( + "[tree::registry] created tree id={} kind={} scope={}", + tree.id, + kind.as_str(), + scope + ); + Ok(tree) + } + Err(err) if is_unique_violation(&err) => { + // Race: another caller created a tree for the same (kind, scope) + // between our initial lookup and this insert. UNIQUE(kind, scope) + // rejected our row; re-query and return the winner. + log::debug!( + "[tree::registry] UNIQUE race for kind={} scope={} — re-querying", + kind.as_str(), + scope + ); + store::get_tree_by_scope(config, kind, scope)?.ok_or_else(|| { + anyhow::anyhow!( + "UNIQUE violation on insert but no row found on re-query for kind={} scope={}", + kind.as_str(), + scope + ) + }) + } + Err(err) => Err(err), + } +} + +/// Return true if `err` represents a SQLite UNIQUE constraint violation. +/// Matches both the anyhow-wrapped rusqlite error text and the raw SQLite +/// error codes in case the wrapping chain is shorter. +pub fn is_unique_violation(err: &anyhow::Error) -> bool { + if let Some(rusqlite::Error::SqliteFailure(sqlite_err, _)) = + err.downcast_ref::() + { + return sqlite_err.code == rusqlite::ErrorCode::ConstraintViolation; + } + // Fallback for chained/wrapped errors: scan the rendered message. + let msg = format!("{err:#}"); + msg.contains("UNIQUE constraint failed") +} + +/// Generate a stable id for a new tree row, prefixed with the kind discriminator. +pub fn new_tree_id(kind: TreeKind) -> String { + format!("{}:{}", kind.as_str(), Uuid::new_v4()) +} + +/// Public id generator for summary nodes — exported so `bucket_seal` can +/// share the same format. The Unix-ms timestamp is the leading sort +/// key so `ORDER BY id` is globally chronological across all levels +/// (a level-first layout grouped L1, L2, … together, breaking that). +/// `:013` zero-pads the millisecond field to 13 digits so the +/// lexicographic order matches numeric order through year 2286 — well +/// outside any reasonable retention window. Level is suffixed for +/// filter-by-level queries (`LIKE '%:L1-%'`). 8-hex of `u32` entropy +/// shrinks same-millisecond collision probability to ~2⁻³² per pair, +/// sized for uniqueness across the file-system and Obsidian wikilink +/// namespaces. +pub fn new_summary_id(level: u32) -> String { + let ms = chrono::Utc::now().timestamp_millis() as u64; + let rand_tail: u32 = rand::random(); + format!("summary:{:013}:L{}-{:08x}", ms, level, rand_tail) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + #[test] + fn get_or_create_is_idempotent_on_scope() { + let (_tmp, cfg) = test_config(); + let first = get_or_create_tree(&cfg, TreeKind::Source, "slack:#eng").unwrap(); + let second = get_or_create_tree(&cfg, TreeKind::Source, "slack:#eng").unwrap(); + assert_eq!(first.id, second.id); + assert_eq!(first.kind, TreeKind::Source); + assert_eq!(first.status, TreeStatus::Active); + } + + #[test] + fn different_scopes_yield_different_trees() { + let (_tmp, cfg) = test_config(); + let a = get_or_create_tree(&cfg, TreeKind::Source, "slack:#eng").unwrap(); + let b = get_or_create_tree(&cfg, TreeKind::Source, "gmail:user@example.com").unwrap(); + assert_ne!(a.id, b.id); + assert_ne!(a.scope, b.scope); + } + + #[test] + fn different_kinds_same_scope_yield_different_trees() { + let (_tmp, cfg) = test_config(); + let source = get_or_create_tree(&cfg, TreeKind::Source, "shared:scope").unwrap(); + let topic = get_or_create_tree(&cfg, TreeKind::Topic, "shared:scope").unwrap(); + assert_ne!(source.id, topic.id); + assert_eq!(source.kind, TreeKind::Source); + assert_eq!(topic.kind, TreeKind::Topic); + } + + #[test] + fn global_tree_is_singleton() { + let (_tmp, cfg) = test_config(); + let first = get_or_create_tree(&cfg, TreeKind::Global, "global").unwrap(); + let second = get_or_create_tree(&cfg, TreeKind::Global, "global").unwrap(); + assert_eq!(first.id, second.id); + assert_eq!(first.kind, TreeKind::Global); + } + + #[test] + fn tree_id_has_expected_prefix() { + let source_id = new_tree_id(TreeKind::Source); + assert!(source_id.starts_with("source:")); + let topic_id = new_tree_id(TreeKind::Topic); + assert!(topic_id.starts_with("topic:")); + let global_id = new_tree_id(TreeKind::Global); + assert!(global_id.starts_with("global:")); + + let sum_id = new_summary_id(3); + assert!(sum_id.starts_with("summary:")); + assert!(sum_id.contains(":L3-"), "expected level suffix in {sum_id}"); + } + + #[test] + fn summary_id_format_is_lexicographically_chronological() { + let earlier_ms: u64 = 1_700_000_000_000; + let later_ms: u64 = 1_700_000_000_001; + let earlier = format!("summary:{:013}:L1-{:08x}", earlier_ms, u32::MAX); + let later = format!("summary:{:013}:L9-{:08x}", later_ms, 0u32); + assert!( + earlier < later, + "expected {earlier} < {later} (ms must outrank level + tail)" + ); + + let live = new_summary_id(2); + assert!(live.starts_with("summary:"), "live: {live}"); + let rest = &live["summary:".len()..]; + let ms_part = rest.split(':').next().expect("ms segment"); + assert_eq!(ms_part.len(), 13, "ms must be 13 digits in {live}"); + assert!( + ms_part.chars().all(|c| c.is_ascii_digit()), + "ms must be all digits in {live}" + ); + } + + #[test] + fn get_or_create_recovers_from_unique_race() { + let (_tmp, cfg) = test_config(); + let pre_existing = Tree { + id: "source:preexisting".into(), + kind: TreeKind::Source, + scope: "slack:#eng".into(), + ask: None, + root_id: None, + max_level: 0, + status: TreeStatus::Active, + created_at: Utc::now(), + last_sealed_at: None, + }; + store::insert_tree(&cfg, &pre_existing).unwrap(); + + let got = get_or_create_tree(&cfg, TreeKind::Source, "slack:#eng").unwrap(); + assert_eq!(got.id, "source:preexisting"); + + let dup = Tree { + id: "source:would-collide".into(), + ..pre_existing.clone() + }; + let err = store::insert_tree(&cfg, &dup).unwrap_err(); + assert!( + is_unique_violation(&err), + "expected UNIQUE violation, got: {err:#}" + ); + } +} diff --git a/core/src/tree/tree_runtime/engine.rs b/core/src/tree/tree_runtime/engine.rs new file mode 100644 index 0000000..ca3ec68 --- /dev/null +++ b/core/src/tree/tree_runtime/engine.rs @@ -0,0 +1,153 @@ +//! Product adapters around the tinycortex markdown time-tree engine. + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use chrono::{DateTime, Timelike, Utc}; +use tinyagents::harness::message::Message; +use tinyagents::harness::model::{ChatModel, ModelRequest}; +use tinycortex::memory::tree::runtime::{ + NodeLevel, RuntimeObserver, Summariser, TreeNode, TreeStatus, +}; + +use crate::tinycortex::engine_config; +use crate::Config; + +const SUMMARIZATION_TEMP: f64 = 0.3; + +struct ChatSummariser<'a>(&'a dyn ChatModel<()>); + +#[async_trait] +impl Summariser for ChatSummariser<'_> { + async fn summarise(&self, system: Option<&str>, content: &str) -> Result { + log::debug!( + "[tree_summarizer] provider call content_chars={} has_system={}", + content.len(), + system.is_some() + ); + let mut messages = Vec::with_capacity(2); + if let Some(system) = system { + messages.push(Message::system(system.to_string())); + } + messages.push(Message::user(content.to_string())); + let response = self + .0 + .invoke( + &(), + ModelRequest::new(messages).with_temperature(SUMMARIZATION_TEMP), + ) + .await + .context("time-tree summarization provider call failed")? + .text(); + log::debug!( + "[tree_summarizer] provider call complete response_chars={}", + response.len() + ); + Ok(response) + } +} + +struct EventObserver; + +impl RuntimeObserver for EventObserver { + fn hour_completed(&self, namespace: &str, node_id: &str, token_count: u32) { + crate::events::publish(crate::events::MemoryEvent::TreeSummarizerHourCompleted { + namespace: namespace.to_string(), + node_id: node_id.to_string(), + token_count, + }); + } + + fn node_propagated(&self, namespace: &str, node_id: &str, level: NodeLevel, token_count: u32) { + crate::events::publish(crate::events::MemoryEvent::TreeSummarizerPropagated { + namespace: namespace.to_string(), + node_id: node_id.to_string(), + level: level.as_str().to_string(), + token_count, + }); + } + + fn rebuild_completed(&self, namespace: &str, total_nodes: u64) { + crate::events::publish(crate::events::MemoryEvent::TreeSummarizerRebuildCompleted { + namespace: namespace.to_string(), + total_nodes, + }); + } +} + +pub async fn run_summarization( + config: &Config, + provider: &dyn ChatModel<()>, + namespace: &str, + ts: DateTime, +) -> Result> { + log::debug!("[tree_summarizer] tinycortex run namespace={namespace}"); + let result = tinycortex::memory::tree::runtime::run_summarization_observed( + &engine_config(config), + &ChatSummariser(provider), + namespace, + ts, + &EventObserver, + ) + .await; + log::debug!( + "[tree_summarizer] tinycortex run complete namespace={} success={}", + namespace, + result.is_ok() + ); + result +} + +pub async fn rebuild_tree( + config: &Config, + provider: &dyn ChatModel<()>, + namespace: &str, +) -> Result { + log::debug!("[tree_summarizer] tinycortex rebuild namespace={namespace}"); + tinycortex::memory::tree::runtime::rebuild_tree_observed( + &engine_config(config), + &ChatSummariser(provider), + namespace, + &EventObserver, + ) + .await +} + +pub async fn run_hourly_loop(config: Arc, provider: Arc>) { + log::debug!("[tree_summarizer] hourly loop started"); + loop { + let now = Utc::now(); + let base = now + .date_naive() + .and_hms_opt(now.hour(), 0, 0) + .unwrap_or(now.naive_utc()); + let next_hour = + DateTime::::from_naive_utc_and_offset(base + chrono::Duration::hours(1), Utc); + let sleep_duration = (next_hour - now) + .to_std() + .unwrap_or(std::time::Duration::from_secs(3600)); + log::debug!( + "[tree_summarizer] sleeping seconds={}", + sleep_duration.as_secs() + ); + tokio::time::sleep(sleep_duration).await; + + let ts = Utc::now(); + let namespaces = + tinycortex::memory::tree::runtime::discover_active_namespaces(&engine_config(&*config)); + log::debug!( + "[tree_summarizer] hourly tick active_namespaces={}", + namespaces.len() + ); + for namespace in namespaces { + if let Err(error) = run_summarization(&*config, provider.as_ref(), &namespace, ts).await + { + log::error!( + "[tree_summarizer] hourly run failed namespace={} error={error:#}", + namespace + ); + } + } + } +} diff --git a/core/src/tree/tree_runtime/mod.rs b/core/src/tree/tree_runtime/mod.rs new file mode 100644 index 0000000..5a9fcfe --- /dev/null +++ b/core/src/tree/tree_runtime/mod.rs @@ -0,0 +1,17 @@ +//! Hierarchical time-based summary tree. +//! +//! Organizes summaries as a tree: root → year → month → day → hour (leaf). +//! Each hour, a background job drains buffered raw content, summarizes it into +//! the hour leaf, and propagates updated summaries upward through the tree. +//! Stored as markdown files in `memory/namespaces/{ns}/tree/`. +//! +//! This module was renamed from `memory::summarizer` to +//! `memory_tree::tree_runtime` so it no longer collides conceptually with +//! [`crate::tree::summarise`], which is only the single-call +//! LLM fold primitive used during seals. + +pub mod engine; +pub mod store; + +// Runtime tree types are engine-owned. +pub use tinycortex::memory::tree::runtime::*; diff --git a/core/src/tree/tree_runtime/store.rs b/core/src/tree/tree_runtime/store.rs new file mode 100644 index 0000000..8923616 --- /dev/null +++ b/core/src/tree/tree_runtime/store.rs @@ -0,0 +1,117 @@ +//! `Config` adapters for tinycortex-owned markdown tree persistence. + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use serde_json::Value; + +use crate::tinycortex::engine_config; +use crate::Config; +use tinycortex::memory::tree::runtime::{TreeNode, TreeStatus}; + +pub fn tree_dir(config: &Config, namespace: &str) -> PathBuf { + tinycortex::memory::tree::runtime::store::tree_dir(&engine_config(config), namespace) +} + +pub fn buffer_dir(config: &Config, namespace: &str) -> PathBuf { + tinycortex::memory::tree::runtime::store::buffer_dir(&engine_config(config), namespace) +} + +pub fn node_file_path(config: &Config, namespace: &str, node_id: &str) -> PathBuf { + tinycortex::memory::tree::runtime::store::node_file_path( + &engine_config(config), + namespace, + node_id, + ) +} + +pub use tinycortex::memory::tree::runtime::store::{validate_namespace, validate_node_id}; + +pub fn write_node(config: &Config, node: &TreeNode) -> Result<()> { + tinycortex::memory::tree::runtime::store::write_node(&engine_config(config), node) +} + +pub fn read_node(config: &Config, namespace: &str, node_id: &str) -> Result> { + tinycortex::memory::tree::runtime::store::read_node(&engine_config(config), namespace, node_id) +} + +pub fn read_children(config: &Config, namespace: &str, parent_id: &str) -> Result> { + tinycortex::memory::tree::runtime::store::read_children( + &engine_config(config), + namespace, + parent_id, + ) +} + +pub fn read_ancestors(config: &Config, namespace: &str, node_id: &str) -> Result> { + tinycortex::memory::tree::runtime::store::read_ancestors( + &engine_config(config), + namespace, + node_id, + ) +} + +pub fn count_nodes(config: &Config, namespace: &str) -> Result { + tinycortex::memory::tree::runtime::store::count_nodes(&engine_config(config), namespace) +} + +pub fn get_tree_status(config: &Config, namespace: &str) -> Result { + tinycortex::memory::tree::runtime::store::get_tree_status(&engine_config(config), namespace) +} + +pub fn collect_root_summaries_with_caps( + workspace_dir: &Path, + per_namespace_cap: usize, + total_cap: usize, +) -> Vec<(String, String, DateTime)> { + tinycortex::memory::tree::runtime::store::collect_root_summaries_with_caps( + workspace_dir, + per_namespace_cap, + total_cap, + ) +} + +pub fn list_namespaces_with_root(config: &Config) -> Result> { + tinycortex::memory::tree::runtime::store::list_namespaces_with_root(&engine_config(config)) +} + +pub fn delete_tree(config: &Config, namespace: &str) -> Result { + tinycortex::memory::tree::runtime::store::delete_tree(&engine_config(config), namespace) +} + +pub fn buffer_write( + config: &Config, + namespace: &str, + content: &str, + ts: &DateTime, + metadata: Option<&Value>, +) -> Result { + tinycortex::memory::tree::runtime::store::buffer_write( + &engine_config(config), + namespace, + content, + ts, + metadata, + ) +} + +pub fn buffer_read(config: &Config, namespace: &str) -> Result> { + tinycortex::memory::tree::runtime::store::buffer_read(&engine_config(config), namespace) +} + +pub fn buffer_delete(config: &Config, namespace: &str, filenames: &[String]) -> Result<()> { + tinycortex::memory::tree::runtime::store::buffer_delete( + &engine_config(config), + namespace, + filenames, + ) +} + +pub fn buffer_drain(config: &Config, namespace: &str) -> Result> { + tinycortex::memory::tree::runtime::store::buffer_drain(&engine_config(config), namespace) +} + +pub fn parse_node_markdown_pub(raw: &str, namespace: &str, node_id: &str) -> Result { + tinycortex::memory::tree::runtime::store::parse_node_markdown_pub(raw, namespace, node_id) +} diff --git a/core/src/tree_policy.rs b/core/src/tree_policy.rs new file mode 100644 index 0000000..83205a6 --- /dev/null +++ b/core/src/tree_policy.rs @@ -0,0 +1,320 @@ +//! Tree policy layer. +//! +//! `tree` itself stays generic: summaries, buffers, sealing, and storage. +//! Flavor-specific tuning (global cadence, topic hotness thresholds, source +//! label policy) is centralized here so per-flavor modules don't each own +//! their own scattered constants and arithmetic. + +use crate::store::trees::types::{ + EntityIndexStats, TOPIC_ARCHIVE_THRESHOLD, TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TreePolicy { + Source, + Topic, + Global, +} + +impl TreePolicy { + pub fn global() -> Self { + Self::Global + } + + pub fn topic() -> Self { + Self::Topic + } + + pub fn source() -> Self { + Self::Source + } + + pub fn topic_creation_threshold(self) -> f32 { + let _ = self; + TOPIC_CREATION_THRESHOLD + } + + pub fn topic_archive_threshold(self) -> f32 { + let _ = self; + TOPIC_ARCHIVE_THRESHOLD + } + + pub fn topic_recheck_every(self) -> u32 { + let _ = self; + TOPIC_RECHECK_EVERY + } + + pub fn topic_hotness(self, entity_id: &str, idx: &EntityIndexStats, now_ms: i64) -> f32 { + let _ = self; + let mention_weight = ((idx.mention_count_30d as f32) + 1.0).ln(); + let source_weight = (idx.distinct_sources as f32) * 0.5; + let recency_weight = self.topic_recency_decay(idx.last_seen_ms, now_ms); + let centrality = idx.graph_centrality.unwrap_or(0.0); + let query_weight = (idx.query_hits_30d as f32) * 2.0; + + let total = mention_weight + source_weight + recency_weight + centrality + query_weight; + log::debug!( + "[tree_topic::hotness] id={} mentions={} sources={} recency={:.3} centrality={:.3} \ + queries={} total={:.3}", + crate::util::redact::redact(entity_id), + idx.mention_count_30d, + idx.distinct_sources, + recency_weight, + centrality, + idx.query_hits_30d, + total + ); + total + } + + pub fn topic_recency_decay(self, last_seen_ms: Option, now_ms: i64) -> f32 { + let _ = self; + let Some(last_seen) = last_seen_ms else { + return 0.0; + }; + let age_ms = (now_ms - last_seen).max(0); + const DAY_MS: i64 = 24 * 60 * 60 * 1_000; + let age_days = (age_ms as f32) / (DAY_MS as f32); + + if age_days <= 1.0 { + 1.0 + } else if age_days <= 7.0 { + let frac = (age_days - 1.0) / 6.0; + 1.0 - 0.5 * frac + } else if age_days <= 30.0 { + let frac = (age_days - 7.0) / 23.0; + 0.5 - 0.5 * frac + } else { + 0.0 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::trees::types::EntityIndexStats; + + const DAY_MS: i64 = 86_400_000; + const NOW_MS: i64 = 1_700_000_000_000; + + // ── helpers ────────────────────────────────────────────────────────────── + + fn zero_stats() -> EntityIndexStats { + EntityIndexStats { + mention_count_30d: 0, + distinct_sources: 0, + last_seen_ms: None, + query_hits_30d: 0, + graph_centrality: None, + } + } + + // ── 1. Constructors ─────────────────────────────────────────────────────── + + #[test] + fn constructors_return_expected_variants() { + assert_eq!(TreePolicy::global(), TreePolicy::Global); + assert_eq!(TreePolicy::topic(), TreePolicy::Topic); + assert_eq!(TreePolicy::source(), TreePolicy::Source); + } + + // ── 2. Threshold constants ──────────────────────────────────────────────── + + #[test] + fn threshold_constants_are_positive() { + let p = TreePolicy::Topic; + assert!( + p.topic_creation_threshold() > 0.0, + "creation threshold must be positive" + ); + assert!( + p.topic_archive_threshold() > 0.0, + "archive threshold must be positive" + ); + assert!( + p.topic_recheck_every() > 0, + "recheck cadence must be positive" + ); + } + + #[test] + fn creation_threshold_exceeds_archive_threshold() { + let p = TreePolicy::Topic; + assert!( + p.topic_creation_threshold() > p.topic_archive_threshold(), + "creation threshold ({}) must exceed archive threshold ({})", + p.topic_creation_threshold(), + p.topic_archive_threshold() + ); + } + + // ── 3. Recency decay boundary values ────────────────────────────────────── + + #[test] + fn recency_decay_none_last_seen_is_zero() { + let decay = TreePolicy::Topic.topic_recency_decay(None, NOW_MS); + assert_eq!(decay, 0.0); + } + + #[test] + fn recency_decay_age_zero_is_one() { + // Seen exactly at now — age = 0. + let decay = TreePolicy::Topic.topic_recency_decay(Some(NOW_MS), NOW_MS); + assert_eq!(decay, 1.0); + } + + #[test] + fn recency_decay_age_one_day_is_one() { + let last_seen = NOW_MS - DAY_MS; + let decay = TreePolicy::Topic.topic_recency_decay(Some(last_seen), NOW_MS); + assert_eq!(decay, 1.0); + } + + #[test] + fn recency_decay_age_seven_days_is_half() { + let last_seen = NOW_MS - 7 * DAY_MS; + let decay = TreePolicy::Topic.topic_recency_decay(Some(last_seen), NOW_MS); + assert!( + (decay - 0.5).abs() < 1e-4, + "expected ~0.5 at 7 days, got {decay}" + ); + } + + #[test] + fn recency_decay_age_thirty_days_is_zero() { + let last_seen = NOW_MS - 30 * DAY_MS; + let decay = TreePolicy::Topic.topic_recency_decay(Some(last_seen), NOW_MS); + assert!(decay.abs() < 1e-4, "expected ~0.0 at 30 days, got {decay}"); + } + + #[test] + fn recency_decay_age_sixty_days_is_zero() { + let last_seen = NOW_MS - 60 * DAY_MS; + let decay = TreePolicy::Topic.topic_recency_decay(Some(last_seen), NOW_MS); + assert_eq!(decay, 0.0, "expected exactly 0.0 beyond 30 days"); + } + + // ── 4. Recency decay mid-range interpolation ────────────────────────────── + + #[test] + fn recency_decay_four_days_is_between_half_and_one() { + // 4 days falls in the 1–7 day band (1.0 → 0.5). + let last_seen = NOW_MS - 4 * DAY_MS; + let decay = TreePolicy::Topic.topic_recency_decay(Some(last_seen), NOW_MS); + assert!( + decay > 0.5 && decay < 1.0, + "expected decay in (0.5, 1.0) at 4 days, got {decay}" + ); + } + + // ── 5. Hotness: zero-signal entity ──────────────────────────────────────── + + #[test] + fn hotness_zero_signal_entity_is_zero() { + // mention_count=0 → ln(1)=0; sources=0; last_seen=None → recency=0; + // centrality=None → 0; query_hits=0 → 0. Total must be 0. + let stats = zero_stats(); + let h = TreePolicy::Topic.topic_hotness("entity:zero", &stats, NOW_MS); + assert_eq!(h, 0.0, "zero-signal entity should have hotness 0.0"); + } + + // ── 6. Hotness: high-signal entity exceeds creation threshold ───────────── + + #[test] + fn hotness_high_signal_exceeds_creation_threshold() { + let stats = EntityIndexStats { + mention_count_30d: 50, + distinct_sources: 5, + last_seen_ms: Some(NOW_MS - DAY_MS / 2), // half a day ago → recency = 1.0 + query_hits_30d: 10, + graph_centrality: Some(1.0), + }; + let h = TreePolicy::Topic.topic_hotness("entity:hot", &stats, NOW_MS); + let threshold = TreePolicy::Topic.topic_creation_threshold(); + assert!( + h > threshold, + "high-signal hotness ({h:.3}) should exceed creation threshold ({threshold})" + ); + } + + // ── 7. Query-hits boost is significant ──────────────────────────────────── + + #[test] + fn hotness_query_hits_boost_is_double() { + // Two otherwise identical entities; one has query_hits=5, the other 0. + // The difference must equal 2.0 * 5 = 10.0. + let base = EntityIndexStats { + mention_count_30d: 3, + distinct_sources: 1, + last_seen_ms: None, + query_hits_30d: 0, + graph_centrality: None, + }; + let with_queries = EntityIndexStats { + query_hits_30d: 5, + ..base.clone() + }; + + let h_base = TreePolicy::Topic.topic_hotness("entity:base", &base, NOW_MS); + let h_queries = TreePolicy::Topic.topic_hotness("entity:queries", &with_queries, NOW_MS); + + let expected_boost = 2.0 * 5.0_f32; + assert!( + (h_queries - h_base - expected_boost).abs() < 1e-4, + "query boost should be {expected_boost}, got {:.3}", + h_queries - h_base + ); + } + + // ── 8. Graph centrality contributes ────────────────────────────────────── + + #[test] + fn hotness_graph_centrality_contributes() { + let base = EntityIndexStats { + mention_count_30d: 2, + distinct_sources: 1, + last_seen_ms: None, + query_hits_30d: 0, + graph_centrality: None, + }; + let with_centrality = EntityIndexStats { + graph_centrality: Some(3.5), + ..base.clone() + }; + + let h_base = TreePolicy::Topic.topic_hotness("entity:central_base", &base, NOW_MS); + let h_central = TreePolicy::Topic.topic_hotness("entity:central", &with_centrality, NOW_MS); + + assert!( + (h_central - h_base - 3.5).abs() < 1e-4, + "centrality contribution should be 3.5, got {:.3}", + h_central - h_base + ); + } + + // ── 9. Ancient single mention decays toward zero ────────────────────────── + + #[test] + fn hotness_ancient_single_mention_is_near_zero() { + // 1 mention, 1 source, last seen 365 days ago → recency = 0. + // hotness = ln(2) + 0.5 * 1 + 0 + 0 + 0 ≈ 0.693 + 0.5 = 1.193 + // That should be well below the creation threshold (10.0). + let stats = EntityIndexStats { + mention_count_30d: 1, + distinct_sources: 1, + last_seen_ms: Some(NOW_MS - 365 * DAY_MS), + query_hits_30d: 0, + graph_centrality: None, + }; + let h = TreePolicy::Topic.topic_hotness("entity:ancient", &stats, NOW_MS); + let threshold = TreePolicy::Topic.topic_creation_threshold(); + assert!( + h < threshold, + "ancient single-mention hotness ({h:.3}) should be below creation threshold ({threshold})" + ); + // Recency component must be zero (age >> 30 days). + let recency = TreePolicy::Topic.topic_recency_decay(Some(NOW_MS - 365 * DAY_MS), NOW_MS); + assert_eq!(recency, 0.0, "recency for 365-day-old entity must be 0.0"); + } +} diff --git a/core/src/tree_source/file.rs b/core/src/tree_source/file.rs new file mode 100644 index 0000000..8646c83 --- /dev/null +++ b/core/src/tree_source/file.rs @@ -0,0 +1,219 @@ +//! Per-source `_source.md` registry mirror. +//! +//! Sits at `/raw//_source.md` next to the +//! per-kind raw subdirs (`emails/`, `chats/`, `documents/`, …). The file +//! is **frontmatter-only** — its YAML head is the registry record for +//! one source, the body is intentionally empty so Obsidian / `.base` +//! files can render it without distractions. +//! +//! Today this is a *mirror* of the `mem_tree_trees` row for the source's +//! tree (kind + scope + last_sealed_at). SQLite remains the source of +//! truth; the file is rewritten whenever the registry creates or +//! refreshes a tree so the on-disk view stays current. The contract is +//! one-way: nothing reads back from this file at runtime. +//! +//! Future direction: as more per-source state moves out of SQLite (the +//! sibling `tree/store.rs` rows that are naturally one-row-per +//! source), this file becomes the load-into-memory authority and the +//! SQLite columns get retired. We keep that migration small and explicit +//! by gating it behind callers; this module just owns the on-disk shape. +//! +//! Atomicity: writes go through the same tempfile-+-rename pattern the +//! sibling `content_store::raw` writer uses, so a crash mid-write leaves +//! either the previous file intact or no file at all — never a partial +//! one. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use crate::store::content::raw::raw_source_dir; +use crate::store::trees::types::Tree; +use crate::Config; + +/// Filename of the per-source registry mirror inside `raw//`. +pub const SOURCE_FILE_NAME: &str = "_source.md"; + +/// Resolve the absolute path of `_source.md` for `source_id` under the +/// configured content root. +pub fn source_file_path(config: &Config, source_id: &str) -> PathBuf { + let root = config.memory_tree_content_root(); + raw_source_dir(&root, source_id).join(SOURCE_FILE_NAME) +} + +/// Render the YAML frontmatter for a tree row. Body is empty — this is a +/// metadata-only file. Field order is fixed so re-renders for the same +/// row produce byte-identical output (idempotent rewrites, clean diffs). +fn render(tree: &Tree) -> String { + let mut out = String::with_capacity(256); + out.push_str("---\n"); + out.push_str(&format!("tree_id: {}\n", yaml_scalar(&tree.id))); + out.push_str(&format!("kind: {}\n", tree.kind.as_str())); + out.push_str(&format!("scope: {}\n", yaml_scalar(&tree.scope))); + out.push_str(&format!("status: {}\n", tree.status.as_str())); + out.push_str(&format!("max_level: {}\n", tree.max_level)); + out.push_str(&format!("created_at: {}\n", iso8601(tree.created_at))); + match tree.last_sealed_at { + Some(t) => out.push_str(&format!("last_sealed_at: {}\n", iso8601(t))), + None => out.push_str("last_sealed_at: null\n"), + } + match tree.root_id.as_ref() { + Some(id) => out.push_str(&format!("root_id: {}\n", yaml_scalar(id))), + None => out.push_str("root_id: null\n"), + } + out.push_str("---\n"); + out +} + +fn iso8601(t: DateTime) -> String { + t.to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} + +/// Quote a YAML scalar if it contains characters that would otherwise +/// break the parse (colons, leading whitespace, quote chars). The +/// scalars we emit (tree ids, scopes) are user-derived, so a defensive +/// quote keeps Obsidian's parser from misreading e.g. `gmail:foo` as a +/// nested mapping. +fn yaml_scalar(s: &str) -> String { + let needs_quote = s.is_empty() + || s.contains(':') + || s.contains('#') + || s.contains('"') + || s.contains('\'') + || s.starts_with(|c: char| c.is_whitespace()) + || s.ends_with(|c: char| c.is_whitespace()); + if !needs_quote { + return s.to_string(); + } + let escaped = s.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") +} + +/// Write (or rewrite) `_source.md` for `tree`. Idempotent: rewriting +/// with the same tree state produces the same bytes. Creates parent +/// directories as needed so callers don't have to. +pub fn write_source_file(config: &Config, tree: &Tree) -> Result { + let path = source_file_path(config, &tree.scope); + let parent = path + .parent() + .ok_or_else(|| anyhow::anyhow!("source file path has no parent: {}", path.display()))?; + fs::create_dir_all(parent) + .with_context(|| format!("create source file dir {}", parent.display()))?; + let bytes = render(tree); + write_atomic(&path, bytes.as_bytes()) + .with_context(|| format!("write source file {}", path.display()))?; + Ok(path) +} + +fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + let parent = path + .parent() + .ok_or_else(|| anyhow::anyhow!("path has no parent: {}", path.display()))?; + let tmp = parent.join(format!( + ".tmp_source_{}_{}.md", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + )); + let mut f = fs::File::create(&tmp).with_context(|| format!("create tmp {}", tmp.display()))?; + f.write_all(bytes) + .with_context(|| format!("write tmp {}", tmp.display()))?; + f.sync_all() + .with_context(|| format!("fsync tmp {}", tmp.display()))?; + drop(f); + fs::rename(&tmp, path) + .with_context(|| format!("rename {} -> {}", tmp.display(), path.display()))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::trees::types::{TreeKind, TreeStatus}; + use chrono::TimeZone; + use tempfile::TempDir; + + fn cfg() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + fn sample_tree(scope: &str) -> Tree { + Tree { + id: "source:abc".into(), + kind: TreeKind::Source, + scope: scope.into(), + ask: None, + root_id: None, + max_level: 0, + status: TreeStatus::Active, + created_at: Utc.timestamp_millis_opt(1_700_000_000_000).unwrap(), + last_sealed_at: None, + } + } + + #[test] + fn writes_frontmatter_only_file() { + let (_tmp, cfg) = cfg(); + let tree = sample_tree("gmail:acct-1"); + let path = write_source_file(&cfg, &tree).unwrap(); + assert!( + path.ends_with("raw/gmail-acct-1/_source.md"), + "{}", + path.display() + ); + let body = fs::read_to_string(&path).unwrap(); + // Bracketed by frontmatter delimiters with no body after. + assert!(body.starts_with("---\n")); + assert!(body.trim_end().ends_with("---")); + assert!(body.contains("tree_id: source:abc") || body.contains("tree_id: \"source:abc\"")); + assert!(body.contains("kind: source")); + assert!(body.contains("status: active")); + assert!(body.contains("last_sealed_at: null")); + } + + #[test] + fn rewrite_is_byte_identical_for_same_state() { + let (_tmp, cfg) = cfg(); + let tree = sample_tree("slack:#eng"); + let path = write_source_file(&cfg, &tree).unwrap(); + let first = fs::read(&path).unwrap(); + write_source_file(&cfg, &tree).unwrap(); + let second = fs::read(&path).unwrap(); + assert_eq!(first, second); + } + + #[test] + fn updates_last_sealed_at_on_rewrite() { + let (_tmp, cfg) = cfg(); + let mut tree = sample_tree("slack:#eng"); + write_source_file(&cfg, &tree).unwrap(); + tree.last_sealed_at = Some(Utc.timestamp_millis_opt(1_700_000_500_000).unwrap()); + tree.max_level = 3; + let path = write_source_file(&cfg, &tree).unwrap(); + let body = fs::read_to_string(&path).unwrap(); + assert!(body.contains("max_level: 3")); + assert!(body.contains("last_sealed_at: 2023-11-14"), "{body}"); + } + + #[test] + fn quotes_scalars_with_colons() { + let (_tmp, cfg) = cfg(); + let tree = sample_tree("gmail:user@example.com"); + let path = write_source_file(&cfg, &tree).unwrap(); + let body = fs::read_to_string(&path).unwrap(); + // scope contains ':' → must be quoted to round-trip through YAML. + assert!(body.contains("scope: \"gmail:user@example.com\""), "{body}"); + } +} diff --git a/core/src/tree_source/mod.rs b/core/src/tree_source/mod.rs new file mode 100644 index 0000000..d4d53a2 --- /dev/null +++ b/core/src/tree_source/mod.rs @@ -0,0 +1,15 @@ +//! Source tree instance — policy layer for per-ingest-source trees. +//! +//! This module owns the parts of the source-tree path that are not generic: +//! - [`mod@file`] — the `_source.md` on-disk mirror (one file per ingest source) +//! - [`registry`] — `get_or_create_source_tree`: wraps the generic +//! [`crate::tree::tree::registry::get_or_create_tree`] +//! and triggers the `_source.md` write as a source-specific side-effect. +//! +//! Generic tree mechanics (storage, buffer management, bucket-seal, +//! flush, id generation) live in [`crate::tree::tree`]. + +pub mod file; +pub mod registry; + +pub use registry::get_or_create_source_tree; diff --git a/core/src/tree_source/registry.rs b/core/src/tree_source/registry.rs new file mode 100644 index 0000000..1ae4f8e --- /dev/null +++ b/core/src/tree_source/registry.rs @@ -0,0 +1,78 @@ +//! Source-tree registry — thin wrapper around the generic +//! [`crate::tree::tree::registry::get_or_create_tree`] +//! that adds the source-specific `_source.md` on-disk mirror write after +//! every get-or-create call. + +use anyhow::Result; + +#[cfg(test)] +use tinymemory_api::host::test_support::TestHostConfig; + +use super::file; +use crate::store::trees::types::Tree; +use crate::tree::tree::TreeFactory; +use crate::Config; + +/// Look up the source tree for `scope`, or create a new one. +/// +/// Scope format convention (Phase 3a): use the ingested chunk's +/// `metadata.source_id` verbatim, so re-ingesting the same Slack channel +/// or Gmail account keeps appending to the same tree. +/// +/// After every successful get-or-create the `_source.md` on-disk mirror +/// for this source is (re)written. The write is best-effort — a failure +/// is logged but does not abort the call. +pub fn get_or_create_source_tree(config: &Config, scope: &str) -> Result { + log::debug!( + "[sources::registry] get_or_create_source_tree scope={}", + crate::util::redact::redact(scope) + ); + let tree = TreeFactory::source(scope).get_or_create(config)?; + if let Err(e) = file::write_source_file(config, &tree) { + log::warn!( + "[tree_source::registry] write_source_file failed scope={} err={e:#}", + crate::util::redact::redact(scope) + ); + } + Ok(tree) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::trees::types::TreeKind; + use tempfile::TempDir; + + fn test_config() -> (TempDir, TestHostConfig) { + crate::test_seams::init(); + let tmp = TempDir::new().unwrap(); + let mut cfg = TestHostConfig::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + (tmp, cfg) + } + + #[test] + fn get_or_create_is_idempotent_on_scope() { + let (_tmp, cfg) = test_config(); + let first = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + let second = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + assert_eq!(first.id, second.id); + assert_eq!(first.kind, TreeKind::Source); + } + + #[test] + fn different_scopes_yield_different_trees() { + let (_tmp, cfg) = test_config(); + let a = get_or_create_source_tree(&cfg, "slack:#eng").unwrap(); + let b = get_or_create_source_tree(&cfg, "gmail:user@example.com").unwrap(); + assert_ne!(a.id, b.id); + } + + #[test] + fn writes_source_file_on_create() { + let (_tmp, cfg) = test_config(); + let tree = get_or_create_source_tree(&cfg, "gmail:user@example.com").unwrap(); + let path = file::source_file_path(&cfg, &tree.scope); + assert!(path.exists(), "expected _source.md at {}", path.display()); + } +} diff --git a/core/src/util/README.md b/core/src/util/README.md new file mode 100644 index 0000000..eac048c --- /dev/null +++ b/core/src/util/README.md @@ -0,0 +1,12 @@ +# util/ + +Shared utility helpers used across the memory-tree subsystem. Kept pure-function and dependency-light so any module in `tree/` can pull them in without cycle risk. + +## Files + +- [`mod.rs`](mod.rs) — module banner; re-exports `redact`. +- [`redact.rs`](redact.rs) — log-time PII redaction. `redact(s)` hashes a string to 8 stable hex chars (safe to grep when the raw value is available externally). `redact_endpoint(url)` strips scheme, path, query, fragment, and credentials, keeping only `host[:port]`. + +## When to use + +Per CLAUDE.md: never log secrets or full PII. After the participant-bucketing change, source_ids and content_paths can embed full email addresses, so any log line that prints them must redact first. diff --git a/core/src/util/mod.rs b/core/src/util/mod.rs new file mode 100644 index 0000000..0c32b1c --- /dev/null +++ b/core/src/util/mod.rs @@ -0,0 +1,3 @@ +//! Shared utility helpers for the memory-tree subsystem. + +pub mod redact; diff --git a/core/src/util/redact.rs b/core/src/util/redact.rs new file mode 100644 index 0000000..a9b2ab4 --- /dev/null +++ b/core/src/util/redact.rs @@ -0,0 +1,136 @@ +//! PII redaction helpers for log output. +//! +//! Per project rule (CLAUDE.md): "Never log secrets or full PII." +//! After the participant-bucketing change introduced in the MD-content PR, +//! source_ids and content_paths can embed full email addresses, so any log +//! line that prints them needs to redact. + +use sha2::{Digest, Sha256}; + +/// Redact a string by hashing it to 8 hex chars. Stable across runs for the +/// same input — safe to grep for in logs when debugging with the raw value +/// available externally. +/// +/// Use for source_ids, entity_ids, content_paths and similar PII-bearing +/// strings in log output. +pub fn redact(s: &str) -> String { + let mut h = Sha256::new(); + h.update(s.as_bytes()); + let d = h.finalize(); + format!("{:08x}", u32::from_be_bytes([d[0], d[1], d[2], d[3]])) +} + +/// Redact a URL/endpoint by stripping path, query, fragment and credentials, +/// keeping only the host (and port if present). +/// +/// Examples: +/// - `"http://localhost:11434/api/chat"` → `"localhost:11434"` +/// - `"https://user:pass@example.com/foo?q=1"` → `"example.com"` +/// - `"ollama://host:1234"` → `"host:1234"` +/// +/// Does not pull in a URL-parsing crate; uses cheap string splitting which is +/// sufficient for the endpoint-config strings this codebase passes around. +pub fn redact_endpoint(url: &str) -> String { + // Strip scheme (everything before "://"). + let after_scheme = url.split_once("://").map(|(_, r)| r).unwrap_or(url); + // Take only the authority (everything up to the first '/', '?', or '#') so + // any '@' in the path / query (e.g. `?email=foo@bar`) doesn't get treated + // as a userinfo separator. + let authority = after_scheme + .split(['/', '?', '#']) + .next() + .unwrap_or(after_scheme); + // Within the authority, the LAST '@' separates userinfo from host:port. + // (RFC 3986: userinfo may itself contain '@' — split-on-first would + // truncate the host. Use rsplit so `user:p@ss@example.com` extracts + // `example.com` correctly.) + let host_port = authority + .rsplit_once('@') + .map(|(_, r)| r) + .unwrap_or(authority); + host_port.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── redact ─────────────────────────────────────────────────────────────── + + #[test] + fn redact_returns_eight_hex_chars() { + let r = redact("alice@example.com"); + assert_eq!(r.len(), 8, "must be 8 hex chars; got {r:?}"); + assert!(r.chars().all(|c| c.is_ascii_hexdigit()), "must be hex"); + } + + #[test] + fn redact_is_stable_across_calls() { + assert_eq!(redact("alice@example.com"), redact("alice@example.com")); + } + + #[test] + fn redact_is_different_for_different_inputs() { + assert_ne!(redact("alice@example.com"), redact("bob@example.com")); + } + + #[test] + fn redact_empty_string_does_not_panic() { + let r = redact(""); + assert_eq!(r.len(), 8); + } + + // ── redact_endpoint ───────────────────────────────────────────────────── + + #[test] + fn redact_endpoint_strips_path_and_query() { + assert_eq!( + redact_endpoint("http://localhost:11434/api/chat"), + "localhost:11434" + ); + } + + #[test] + fn redact_endpoint_strips_credentials() { + assert_eq!( + redact_endpoint("https://user:pass@example.com/foo"), + "example.com" + ); + } + + #[test] + fn redact_endpoint_no_scheme_passthrough() { + // No "://" present — treat the whole string as host/path; still strip path. + assert_eq!(redact_endpoint("localhost:11434/api"), "localhost:11434"); + } + + #[test] + fn redact_endpoint_just_host() { + assert_eq!(redact_endpoint("https://example.com"), "example.com"); + } + + #[test] + fn redact_endpoint_strips_fragment() { + assert_eq!(redact_endpoint("http://host:9090/path#frag"), "host:9090"); + } + + #[test] + fn redact_endpoint_strips_query() { + assert_eq!(redact_endpoint("http://host/path?q=1"), "host"); + } + + #[test] + fn redact_endpoint_empty_does_not_panic() { + let r = redact_endpoint(""); + // Empty input: no scheme, no host — returns empty string. + assert_eq!(r, ""); + } + + #[test] + fn redact_endpoint_ollama_style() { + assert_eq!( + redact_endpoint("http://127.0.0.1:11434/v1/chat/completions"), + "127.0.0.1:11434" + ); + } +} diff --git a/core/tests/fixtures/ingestion/README.md b/core/tests/fixtures/ingestion/README.md new file mode 100644 index 0000000..ec11013 --- /dev/null +++ b/core/tests/fixtures/ingestion/README.md @@ -0,0 +1,25 @@ +# Ingestion Fixtures + +These fixtures are plain-text source samples for memory ingestion tests. + +They are intentionally written as raw strings rather than strongly typed JSON so +future ingestion tests can exercise the same path used for real imported text. + +Current fixtures: + +- `gmail_thread_example.txt` + Gmail-like thread with headers, quoted replies, task ownership, dates, and + durable user/project facts. + +- `notion_page_example.txt` + Notion-like project page with sections, bullet lists, decisions, owners, + milestones, and operating notes. + +Suggested test usage: + +- Load fixture text as a string. +- Pass it through chunking and extraction. +- Assert that ingestion can recover: + - entities such as people, tools, projects, and dates + - relations such as ownership, dependencies, and responsibilities + - durable memory facts such as preferences, deadlines, and decisions diff --git a/core/tests/fixtures/ingestion/gmail_thread_example.txt b/core/tests/fixtures/ingestion/gmail_thread_example.txt new file mode 100644 index 0000000..70c44a1 --- /dev/null +++ b/core/tests/fixtures/ingestion/gmail_thread_example.txt @@ -0,0 +1,85 @@ +From: Sanil Jain +To: Asha Mehta , Ravi Kulkarni +Cc: OpenHuman Core +Subject: Re: Memory integration plan for OpenHuman desktop +Date: Tue, 12 Mar 2026 09:14:00 +0530 +Thread-Id: memory-integration-2026-03 + +Hi Asha and Ravi, + +Quick summary after today's sync: + +1. We should keep JSON-RPC as the transport for the desktop core. +2. The memory layer in the Rust core should use namespace as the main scope key. +3. We do not need user_id in the local storage contract for the current desktop runtime. +4. The frontend can adapt to richer result payloads as long as they still arrive inside JSON-RPC result. + +Current work items: +- Ravi owns the Rust memory API alignment for list, delete, query, and recall. +- Asha owns the Neocortex v2 ingestion experiment using the GLiNER relex model. +- Sanil will review response models so they follow the Neocortex API style. + +Important project facts: +- Project name: OpenHuman +- Subproject: memory-layer-completion +- Target milestone: March 22, 2026 +- Preferred embedding model for local experiments: text-embedding-3-small +- Preferred extraction mode to try first: sentence + +Known constraints: +- The desktop app is local-first. +- Core RPC currently binds to localhost only. +- We should avoid introducing user_id into every memory request unless we later support multi-user or remote runtimes. + +Action items: +- Ravi: draft typed request/response structs for memory.query_namespace and memory.recall_namespace by Friday. +- Asha: prepare two ingestion fixtures, one Gmail-like and one Notion-like, with enough structure to test entity and relation extraction. +- Sanil: decide whether memory.init becomes a no-op compatibility method or is removed from the frontend wrappers. + +One durable preference to remember: +I prefer keeping the memory core simple first and delaying graph traversal until after ingestion and recall are stable. + +Thanks, +Sanil + +--- + +From: Asha Mehta +To: Sanil Jain , Ravi Kulkarni +Subject: Re: Memory integration plan for OpenHuman desktop +Date: Tue, 12 Mar 2026 08:41:00 +0530 + +Agreed. + +For the Neocortex donor path, I reviewed the neocortex_v2 extractor again: +- It uses a single GLiNER relex model. +- It supports sentence-level and chunk-level extraction. +- It adds recipient and spatial relation heuristics. + +I think we should preserve those heuristics when we port the ingestion flow into OpenHuman. + +Also, please record this: +- Ravi prefers narrower worker ownership to avoid merge conflicts. +- I prefer evaluation fixtures that include dates, owners, and product decisions. + +Regards, +Asha + +--- + +From: Ravi Kulkarni +To: Sanil Jain , Asha Mehta +Subject: Re: Memory integration plan for OpenHuman desktop +Date: Tue, 12 Mar 2026 08:09:00 +0530 + +One more note before I start: + +- I will treat namespace as mandatory for memory query and recall. +- I will treat memory file APIs as optional until the core contract settles. +- I want the Gmail importer to preserve subject, sender, recipients, and sent_at metadata. + +Dependency note: +- The frontend wrapper work depends on finalizing the result shape from the Rust core. +- The ingestion evaluation can run in parallel once the storage mapping is clear. + +Ravi diff --git a/core/tests/fixtures/ingestion/notion_page_example.txt b/core/tests/fixtures/ingestion/notion_page_example.txt new file mode 100644 index 0000000..2439210 --- /dev/null +++ b/core/tests/fixtures/ingestion/notion_page_example.txt @@ -0,0 +1,132 @@ +# OpenHuman Memory Layer Roadmap + +Workspace: tinyhumans / engineering +Owner: Sanil Jain +Last edited: 2026-03-14 +Status: In Progress +Tags: memory, rust-core, ingestion, neocortex + +## Overview + +This page tracks the work needed to complete the OpenHuman memory layer in the Rust core. + +The current direction is: +- keep JSON-RPC as the transport +- use namespace as the storage and retrieval scope key +- avoid requiring user_id in local memory APIs +- adopt Neocortex-style typed request and response models inside JSON-RPC result + +## Core Decisions + +### Decision 1: Transport +We will keep JSON-RPC 2.0 as the transport for the desktop core. + +### Decision 2: Scope +Namespace is the primary logical partition for local memory. +Examples: +- conversations +- conscious +- skill-gmail +- skill-notion + +### Decision 3: Ingestion donor +We will use neocortex_v2 as the donor path for better memory extraction. +Important features to preserve: +- joint entity and relation extraction +- sentence-level extraction option +- relation constraints +- recipient relation synthesis +- spatial relation synthesis + +## Deliverables + +### Thread 0: Contract +Owner: Sanil Jain +Deliverables: +- final memory RPC names +- request and response model table +- decision on memory.init +- decision on file APIs + +### Thread 1: Core Memory Domain +Owner: Ravi Kulkarni +Deliverables: +- stable document storage semantics +- stable namespace list and document list behavior +- stable query and recall behavior +- clarified graph and KV scope + +### Thread 3: Ingestion +Owner: Asha Mehta +Deliverables: +- extraction adapter plan +- mapping into memory_docs, vector_chunks, and graph_namespace +- sample-data evaluation + +## Current Data Model Notes + +### Documents +Documents should preserve: +- document_id +- namespace +- title +- content +- metadata +- created_at +- updated_at + +### Graph facts +Graph storage should capture facts like: +- Ravi works_on memory-layer-completion +- Asha evaluates neocortex_v2 +- OpenHuman uses JSON-RPC +- memory-layer-completion depends_on API-contract + +### Durable preferences +Examples of durable user or team memory: +- Sanil prefers core-first delivery over UI-first delivery. +- Ravi prefers strict ownership boundaries for parallel agents. +- Asha prefers evaluation fixtures with realistic semi-structured text. + +## Milestones + +### Milestone A +Name: Core contract locked +Due date: 2026-03-18 +Success criteria: +- final RPC method names agreed +- JSON-RPC transport explicitly retained +- response envelope strategy documented + +### Milestone B +Name: Core memory operational +Due date: 2026-03-22 +Success criteria: +- list, delete, query, and recall work in Rust +- stable outputs exist for frontend adaptation + +### Milestone C +Name: Ingestion quality baseline +Due date: 2026-03-26 +Success criteria: +- Gmail-like and Notion-like fixtures ingest successfully +- extracted entities and relations are reviewed manually + +## Risks + +- The frontend currently expects raw values for some memory methods. +- neocortex_v2 preserves duplicate relation evidence, while OpenHuman may prefer aggregation. +- If we do not define request and response models early, parallel agents may diverge. + +## Testing Notes + +Use these sample source types for ingestion tests: +- Gmail thread as raw imported message text +- Notion page as raw exported document text + +Assertions should check for: +- person names +- project names +- ownership relations +- deadlines and dates +- decisions and preferences diff --git a/src/registry/class.rs b/src/registry/class.rs index 218e2f6..81f14b2 100644 --- a/src/registry/class.rs +++ b/src/registry/class.rs @@ -21,14 +21,24 @@ use serde::{Deserialize, Serialize}; pub enum DriverClassParseError { /// The raw class value is unsupported. Unknown { - /// The unrecognized input, retained for diagnostics that stay local. + /// The unrecognized input. raw: String, }, } impl fmt::Display for DriverClassParseError { + /// Renders the offending value. + /// + /// A config typo (`class = "embeded"`) is the only way to reach this, and + /// the message is what the operator sees. Without the raw value it says + /// only that *some* class was unrecognized, which does not point at the + /// line to fix — and this error carries it already. + /// + /// The value comes from the host's own config file, not from a driver or + /// the network, so echoing it discloses nothing the reader did not write. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("unknown driver class") + let Self::Unknown { raw } = self; + write!(f, "unknown driver class: {raw}") } } diff --git a/src/registry/mod.rs b/src/registry/mod.rs index 37b4714..005e97b 100644 --- a/src/registry/mod.rs +++ b/src/registry/mod.rs @@ -227,7 +227,11 @@ impl DriverRegistry { &format!("{} has no class line", labels.driver_entry), )?, Some(raw) => { - let class = DriverClass::parse(raw).map_err(|_| refuse("unknown driver class"))?; + // Render the parse error rather than discarding it: it carries the + // offending value, and a config typo is the only way to get + // here, so naming it is the difference between a refusal an + // operator can act on and one they cannot. + let class = DriverClass::parse(raw).map_err(|error| refuse(&error.to_string()))?; // A reserved id names a fixed implementation, so an explicit // `class` line may confirm it but never override it. if let Some(fixed) = self.reserved_class(id) { diff --git a/src/registry/test.rs b/src/registry/test.rs index d6db7f3..0a770e4 100644 --- a/src/registry/test.rs +++ b/src/registry/test.rs @@ -85,7 +85,7 @@ fn an_unparseable_class_is_refused_with_the_raw_value() { labels(), ) .expect_err("a misspelled class is refused"); - assert_eq!(refusal.reason, "unknown driver class"); + assert_eq!(refusal.reason, "unknown driver class: emebdded"); } /// The rule that keeps a bound engine truthfully labelled: a reserved id's diff --git a/vendor/tinyagents b/vendor/tinyagents new file mode 160000 index 0000000..27a3f39 --- /dev/null +++ b/vendor/tinyagents @@ -0,0 +1 @@ +Subproject commit 27a3f39dc6d7db676efe58d0f7b89752a8ab4746 diff --git a/vendor/tinycortex b/vendor/tinycortex index bf37602..be7b395 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit bf3760213fbae5e4cacf18c4a495930815e33feb +Subproject commit be7b395354271082953d2594765aded73975b54c