diff --git a/CHANGELOG.md b/CHANGELOG.md index fd097f1..7cda7d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `trachyte-db` IPC exposure: `index_search` / `index_insert_file` commands (per-call open of `/.trachyte/index.db`), `IpcError::InvalidQuery` for malformed FTS5 queries, `IpcError::Db`, typed `ipc/db.ts` wrapper +- Event-driven `Indexer` in core (upsert/delete/rename on vault events + startup sweep that converges missed events); `IndexDriver` seam + `MemoryIndexDriver` (test default) and `tauriIndexDriver` in the app; `extractIndexContent` frontmatter strip; `IndexedFileMeta`/`IndexEvent` types; write commands `index_upsert_file` / `index_delete_file` / `index_list_files` + typed `ipc/db.ts` wrappers + ### Changed - CI Hardening for `ci.yml` now checks inside app/desktop to confirm build ablility diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 7105c08..1147248 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -18,6 +18,9 @@ pub fn run() { trachyte_ipc::commands::doctor, trachyte_ipc::commands::index_search, trachyte_ipc::commands::index_insert_file, + trachyte_ipc::commands::index_upsert_file, + trachyte_ipc::commands::index_delete_file, + trachyte_ipc::commands::index_list_files, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/apps/desktop/src/ipc/db.ts b/apps/desktop/src/ipc/db.ts index 395fb65..cfe0772 100644 --- a/apps/desktop/src/ipc/db.ts +++ b/apps/desktop/src/ipc/db.ts @@ -1,4 +1,5 @@ import { invoke } from "@tauri-apps/api/core"; +import type { IndexedFileMeta } from "@trachyte/core"; export async function indexInsertFile( vaultPath: string, @@ -12,3 +13,20 @@ export async function indexInsertFile( export async function indexSearch(vaultPath: string, q: string): Promise { return invoke("index_search", { vaultPath, q }); } + +export async function indexUpsertFile( + vaultPath: string, + relPath: string, + content: string, + mtime: number, +): Promise { + return invoke("index_upsert_file", { vaultPath, relPath, content, mtime }); +} + +export async function indexDeleteFile(vaultPath: string, relPath: string): Promise { + return invoke("index_delete_file", { vaultPath, relPath }); +} + +export async function indexListFiles(vaultPath: string): Promise { + return invoke("index_list_files", { vaultPath }); +} diff --git a/apps/desktop/src/ipc/index-driver.ts b/apps/desktop/src/ipc/index-driver.ts new file mode 100644 index 0000000..19f70e9 --- /dev/null +++ b/apps/desktop/src/ipc/index-driver.ts @@ -0,0 +1,8 @@ +import type { IndexDriver } from "@trachyte/core"; +import { indexDeleteFile, indexListFiles, indexUpsertFile } from "./db.js"; + +export const tauriIndexDriver: IndexDriver = { + upsertFile: indexUpsertFile, + deleteFile: indexDeleteFile, + listFiles: indexListFiles, +}; diff --git a/crates/trachyte-db/src/lib.rs b/crates/trachyte-db/src/lib.rs index 14eb1ac..a0c6682 100644 --- a/crates/trachyte-db/src/lib.rs +++ b/crates/trachyte-db/src/lib.rs @@ -14,6 +14,19 @@ pub mod schema; pub use error::DbError; pub use hash::hash_content; +/// Metadata for a file row in the index. +#[derive(Debug, Clone, serde::Serialize)] +pub struct FileMeta { + /// Vault-relative path of the indexed file. + pub path: String, + /// Last-modified timestamp as supplied at insert time. + pub mtime: i64, + /// Content size in bytes. + pub size: i64, + /// BLAKE3 hex digest of the content. + pub hash: String, +} + /// A SQLite index database with WAL + FTS5 configured. pub struct Database { conn: Connection, @@ -53,6 +66,63 @@ impl Database { Ok(id) } + /// Insert a file into the index, or update it if the path already exists + /// Refreshes the FTS mirror so search reflects the latest content + pub fn upsert_file( + &self, + path: &str, + content: &str, + mtime: i64, + size: i64, + hash: &str, + ) -> Result { + self.conn.execute( + "INSERT INTO files (path, mtime, size, hash) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(path) DO UPDATE SET + mtime = excluded.mtime, size = excluded.size, hash = excluded.hash", + rusqlite::params![path, mtime, size, hash], + )?; + self.conn.execute( + "DELETE FROM content_fts WHERE path = ?1", + rusqlite::params![path], + )?; + fts::insert_content(&self.conn, path, content)?; + let id = self.conn.query_row( + "SELECT id FROM files WHERE path = ?1", + rusqlite::params![path], + |row| row.get(0), + )?; + Ok(id) + } + + /// Remove a file and its FTS mirror row from the index. + /// Headings, tags, and backlinks are removed by `ON DELETE CASCADE`. + pub fn delete_file(&self, path: &str) -> Result<(), DbError> { + self.conn.execute( + "DELETE FROM content_fts WHERE path = ?1", + rusqlite::params![path], + )?; + self.conn + .execute("DELETE FROM files WHERE path = ?1", rusqlite::params![path])?; + Ok(()) + } + + /// List all indexed files with their metadata, ordered by path. + pub fn list_meta(&self) -> Result, DbError> { + let mut stmt = self + .conn + .prepare("SELECT path, mtime, size, hash FROM files ORDER BY path")?; + let rows = stmt.query_map([], |row| { + Ok(FileMeta { + path: row.get(0)?, + mtime: row.get(1)?, + size: row.get(2)?, + hash: row.get(3)?, + }) + })?; + rows.collect::, _>>().map_err(DbError::from) + } + /// Full-text search over file contents; returns matching paths. pub fn search(&self, query: &str) -> Result, DbError> { fts::search(&self.conn, query) @@ -106,4 +176,48 @@ mod tests { assert!(dir.path().join("index.db-wal").exists()); } + + #[test] + fn upsert_updates_existing_row() { + let (_dir, db) = open_db(); + let id1 = db + .upsert_file("Notes/A.md", "first version", 1, 13, "hash-1") + .unwrap(); + let id2 = db + .upsert_file("Notes/A.md", "second version", 2, 15, "hash-2") + .unwrap(); + + assert_eq!(id1, id2, "upsert keeps the same row id"); + let hits = db.search("second").unwrap(); + assert_eq!(hits, vec!["Notes/A.md".to_string()]); + assert!( + db.search("first").unwrap().is_empty(), + "FTS row was refreshed" + ); + } + + #[test] + fn delete_removes_file_and_fts_row() { + let (_dir, db) = open_db(); + db.insert_file("Notes/A.md", "hello world", 0, 11, "deadbeef") + .unwrap(); + assert_eq!(db.search("hello").unwrap().len(), 1); + + db.delete_file("Notes/A.md").unwrap(); + + assert!(db.search("hello").unwrap().is_empty()); + assert!(db.list_meta().unwrap().is_empty()); + } + + #[test] + fn list_meta_returns_all_rows() { + let (_dir, db) = open_db(); + db.insert_file("Notes/A.md", "alpha", 1, 5, "h-a").unwrap(); + db.insert_file("Notes/B.md", "beta", 2, 4, "h-b").unwrap(); + + let metas = db.list_meta().unwrap(); + assert_eq!(metas.len(), 2); + assert_eq!(metas[0].path, "Notes/A.md"); + assert_eq!(metas[1].path, "Notes/B.md"); + } } diff --git a/crates/trachyte-ipc/src/commands.rs b/crates/trachyte-ipc/src/commands.rs index 3530bf7..f737c1a 100644 --- a/crates/trachyte-ipc/src/commands.rs +++ b/crates/trachyte-ipc/src/commands.rs @@ -190,6 +190,42 @@ fn db_to_ipc(e: trachyte_db::DbError) -> IpcError { } } +/// Insert / update a file in the vault index (`.trachyte/index.db`). +#[tauri::command] +#[specta::specta] +pub fn index_upsert_file( + vault_path: String, + rel_path: String, + content: String, + mtime: i64, +) -> Result { + let db = open_index(&vault_path)?; + db.upsert_file( + &rel_path, + &content, + mtime, + content.len() as i64, + &hash_of(&content), + ) + .map_err(db_to_ipc) +} + +/// Remove a file from the vault index. +#[tauri::command] +#[specta::specta] +pub fn index_delete_file(vault_path: String, rel_path: String) -> Result<(), IpcError> { + let db = open_index(&vault_path)?; + db.delete_file(&rel_path).map_err(db_to_ipc) +} + +/// List all indexed files with their metadata. +#[tauri::command] +#[specta::specta] +pub fn index_list_files(vault_path: String) -> Result, IpcError> { + let db = open_index(&vault_path)?; + db.list_meta().map_err(db_to_ipc) +} + /// Tests #[cfg(test)] mod tests { @@ -279,4 +315,43 @@ mod tests { let err = index_search(path, "\"".into()).unwrap_err(); assert!(matches!(err, IpcError::InvalidQuery(_))); } + + #[test] + fn index_upsert_updates_same_path() { + let dir = tempfile::tempdir().unwrap(); + let path = create_vault(dir.path()); + + let id1 = index_upsert_file(path.clone(), "Notes/A.md".into(), "first".into(), 0).unwrap(); + let id2 = index_upsert_file(path.clone(), "Notes/A.md".into(), "second".into(), 0).unwrap(); + assert_eq!(id1, id2); + + let hits = index_search(path.clone(), "second".into()).unwrap(); + assert_eq!(hits, vec!["Notes/A.md".to_string()]); + assert!(index_search(path, "first".into()).unwrap().is_empty()); + } + + #[test] + fn index_delete_removes_row() { + let dir = tempfile::tempdir().unwrap(); + let path = create_vault(dir.path()); + index_upsert_file(path.clone(), "Notes/A.md".into(), "hello world".into(), 0).unwrap(); + + index_delete_file(path.clone(), "Notes/A.md".into()).unwrap(); + + assert!(index_search(path, "world".into()).unwrap().is_empty()); + } + + #[test] + fn index_list_files_returns_meta() { + let dir = tempfile::tempdir().unwrap(); + let path = create_vault(dir.path()); + index_upsert_file(path.clone(), "Notes/A.md".into(), "alpha".into(), 1).unwrap(); + index_upsert_file(path.clone(), "Notes/B.md".into(), "beta".into(), 2).unwrap(); + + let metas = index_list_files(path).unwrap(); + assert_eq!(metas.len(), 2); + assert_eq!(metas[0].path, "Notes/A.md"); + assert_eq!(metas[0].mtime, 1); + assert_eq!(metas[0].size, 5); + } } diff --git a/eslint.config.js b/eslint.config.js index 34d71c6..ba70ff6 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -34,6 +34,7 @@ export default tseslint.config( rules: { ...reactHooks.configs.recommended.rules, ...jsxA11y.configs.recommended.rules, + "@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_" }], }, }, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f807fb1..e99436e 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -4,3 +4,4 @@ export * from "./settings/index.js"; export * from "./events/types.js"; export * from "./path.js"; export * from "./doctor/index.js"; +export * from "./index/index.js"; diff --git a/packages/core/src/index/__tests__/indexer.spec.ts b/packages/core/src/index/__tests__/indexer.spec.ts new file mode 100644 index 0000000..444aeec --- /dev/null +++ b/packages/core/src/index/__tests__/indexer.spec.ts @@ -0,0 +1,142 @@ +import { describe, it, expect } from "vitest"; +import { EventBus } from "../../vault/event-bus.js"; +import { MemoryAdapter } from "../../storage/memory.js"; +import { joinPath } from "../../path.js"; +import { Indexer } from "../indexer.js"; +import { MemoryIndexDriver } from "../driver.js"; + +const VAULT = "/vault"; + +function flush(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +function setup() { + const fs = new MemoryAdapter(); + const index = new MemoryIndexDriver(); + const bus = new EventBus(); + const indexer = new Indexer({ fs, index, events: bus, vaultPath: VAULT }); + return { fs, index, bus, indexer }; +} + +describe("Indexer — event-driven", () => { + it("indexes a file on file:created", async () => { + const { fs, index, bus, indexer } = setup(); + indexer.start(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "hello world"); + bus.emit({ type: "file:created", path: "Notes/A.md" }); + await flush(); + + expect(await index.listFiles(VAULT)).toHaveLength(1); + indexer.dispose(); + }); + + it("re-indexes updated content on file:changed", async () => { + const { fs, index, bus, indexer } = setup(); + indexer.start(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "v1"); + bus.emit({ type: "file:created", path: "Notes/A.md" }); + await flush(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "v2"); + bus.emit({ type: "file:changed", path: "Notes/A.md" }); + await flush(); + + expect(index.content("Notes/A.md")).toBe("v2"); + indexer.dispose(); + }); + + it("removes a row on file:deleted", async () => { + const { fs, index, bus, indexer } = setup(); + indexer.start(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "hello"); + bus.emit({ type: "file:created", path: "Notes/A.md" }); + await flush(); + bus.emit({ type: "file:deleted", path: "Notes/A.md" }); + await flush(); + + expect(await index.listFiles(VAULT)).toHaveLength(0); + indexer.dispose(); + }); + + it("moves the row on file:renamed", async () => { + const { fs, index, bus, indexer } = setup(); + indexer.start(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "hello"); + bus.emit({ type: "file:created", path: "Notes/A.md" }); + await flush(); + await fs.writeFile(joinPath(VAULT, "Notes/B.md"), "hello"); + bus.emit({ type: "file:renamed", from: "Notes/A.md", to: "Notes/B.md" }); + await flush(); + + const metas = await index.listFiles(VAULT); + expect(metas.map((m) => m.path)).toEqual(["Notes/B.md"]); + indexer.dispose(); + }); + + it("ignores non-md files and .trachyte entries", async () => { + const { fs, index, bus, indexer } = setup(); + indexer.start(); + await fs.writeFile(joinPath(VAULT, "Assets/pic.png"), "bin"); + await fs.writeFile(joinPath(VAULT, ".trachyte/settings.json"), "{}"); + bus.emit({ type: "file:created", path: "Assets/pic.png" }); + bus.emit({ type: "file:created", path: ".trachyte/settings.json" }); + await flush(); + + expect(await index.listFiles(VAULT)).toHaveLength(0); + indexer.dispose(); + }); + + it("strips frontmatter from indexed content", async () => { + const { fs, index, bus, indexer } = setup(); + indexer.start(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "---\ntitle: X\n---\n\nBody text"); + bus.emit({ type: "file:created", path: "Notes/A.md" }); + await flush(); + + expect(index.content("Notes/A.md")).toBe("\n\nBody text"); + indexer.dispose(); + }); +}); + +describe("Indexer — sweep", () => { + it("converges files added before start in one sweep", async () => { + const { fs, index, indexer } = setup(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "alpha"); + await fs.writeFile(joinPath(VAULT, "Daily/2026-08-07.md"), "beta"); + + indexer.start(); + await indexer.sweep(); + + const metas = await index.listFiles(VAULT); + expect(metas.map((m) => m.path).sort()).toEqual(["Daily/2026-08-07.md", "Notes/A.md"]); + indexer.dispose(); + }); + + it("removes rows whose file is gone from disk", async () => { + const { fs, index, indexer } = setup(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "alpha"); + indexer.start(); + await indexer.sweep(); + await fs.deleteFile(joinPath(VAULT, "Notes/A.md")); + await indexer.sweep(); + + expect(await index.listFiles(VAULT)).toHaveLength(0); + indexer.dispose(); + }); +}); + +describe("Indexer — lifecycle", () => { + it("start and dispose are idempotent (StrictMode-safe)", async () => { + const { fs, index, bus, indexer } = setup(); + indexer.start(); + indexer.start(); + await fs.writeFile(joinPath(VAULT, "Notes/A.md"), "x"); + bus.emit({ type: "file:created", path: "Notes/A.md" }); + await flush(); + + expect(await index.listFiles(VAULT)).toHaveLength(1); + + indexer.dispose(); + indexer.dispose(); + }); +}); diff --git a/packages/core/src/index/driver.ts b/packages/core/src/index/driver.ts new file mode 100644 index 0000000..aaf32eb --- /dev/null +++ b/packages/core/src/index/driver.ts @@ -0,0 +1,46 @@ +import type { IndexedFileMeta } from "./types.js"; + +export interface IndexDriver { + upsertFile(vaultPath: string, relPath: string, content: string, mtime: number): Promise; + deleteFile(vaultPath: string, relPath: string): Promise; + listFiles(vaultPath: string): Promise; +} + +export class MemoryIndexDriver implements IndexDriver { + private files = new Map(); + private meta = new Map(); + private ids = new Map(); + private nextId = 1; + + upsertFile(_vaultPath: string, relPath: string, content: string, mtime: number): Promise { + this.files.set(relPath, content); + let id = this.ids.get(relPath); + if (id === undefined) { + id = this.nextId++; + this.ids.set(relPath, id); + } + const existing = this.meta.get(relPath); + this.meta.set(relPath, { + path: relPath, + mtime, + size: content.length, + hash: existing?.hash ?? `hash-${id}`, + }); + return Promise.resolve(id); + } + + deleteFile(_vaultPath: string, relPath: string): Promise { + this.files.delete(relPath); + this.meta.delete(relPath); + this.ids.delete(relPath); + return Promise.resolve(); + } + + listFiles(_vaultPath: string): Promise { + return Promise.resolve([...this.meta.values()]); + } + + content(path: string): string | undefined { + return this.files.get(path); + } +} diff --git a/packages/core/src/index/index.ts b/packages/core/src/index/index.ts new file mode 100644 index 0000000..da5b393 --- /dev/null +++ b/packages/core/src/index/index.ts @@ -0,0 +1,4 @@ +export { Indexer, type IndexerDeps } from "./indexer.js"; +export { MemoryIndexDriver, type IndexDriver } from "./driver.js"; +export { extractIndexContent } from "./parser/md.js"; +export { type IndexedFileMeta, type IndexEvent } from "./types.js"; diff --git a/packages/core/src/index/indexer.ts b/packages/core/src/index/indexer.ts new file mode 100644 index 0000000..d414a78 --- /dev/null +++ b/packages/core/src/index/indexer.ts @@ -0,0 +1,124 @@ +import { joinPath } from "../path.js"; +import { type FSAdapter } from "../storage/adapter.js"; +import { type VaultEvent } from "../events/types.js"; +import { type IndexDriver } from "./driver.js"; +import { extractIndexContent } from "./parser/md.js"; +import { type IndexEvent } from "./types.js"; + +export interface IndexerDeps { + fs: FSAdapter; + index: IndexDriver; + events: { on(handler: (event: VaultEvent) => void): () => void }; + vaultPath: string; + onIndex?: (event: IndexEvent) => void; +} + +function isIndexablePath(relPath: string): boolean { + return relPath.endsWith(".md") && !relPath.startsWith(".trachyte"); +} + +/** + * Keeps the vault index in sync with files on disk. + * + * Reacts to vault events (`file:created`/`file:changed`/`file:deleted`/`file:renamed`) + * and runs a startup sweep to converge missed events. + */ +export class Indexer { + private unsubscribe: (() => void) | null = null; + + constructor(private readonly deps: IndexerDeps) {} + + start(): void { + if (this.unsubscribe !== null) return; + this.unsubscribe = this.deps.events.on((event) => void this.handle(event)); + void this.sweep(); + } + + dispose(): void { + this.unsubscribe?.(); + this.unsubscribe = null; + } + + async sweep(): Promise { + const diskMd = await this.walkVault(this.deps.vaultPath); + const diskSet = new Set(diskMd); + const indexed = await this.deps.index.listFiles(this.deps.vaultPath); + for (const meta of indexed) { + if (!diskSet.has(meta.path)) { + await this.deps.index.deleteFile(this.deps.vaultPath, meta.path); + } + } + for (const relPath of diskMd) { + await this.upsertFile(relPath); + } + } + + private async handle(event: VaultEvent): Promise { + try { + switch (event.type) { + case "file:created": + case "file:changed": + await this.upsertFile(event.path); + break; + case "file:deleted": + await this.deleteFile(event.path); + break; + case "file:renamed": + await this.deleteFile(event.from); + await this.upsertFile(event.to); + break; + case "vault:opened": + break; + } + } catch { + // Transient FS race (e.g. a file vanished between the event and the read) — + // the next sweep will reconcile the index. + } + } + + private async upsertFile(relPath: string): Promise { + if (!isIndexablePath(relPath)) return; + const content = await this.deps.fs.readFile(joinPath(this.deps.vaultPath, relPath)); + await this.deps.index.upsertFile( + this.deps.vaultPath, + relPath, + extractIndexContent(content), + Date.now(), + ); + this.deps.onIndex?.({ type: "index:upserted", path: relPath }); + } + + private async deleteFile(relPath: string): Promise { + if (!isIndexablePath(relPath)) return; + await this.deps.index.deleteFile(this.deps.vaultPath, relPath); + this.deps.onIndex?.({ type: "index:deleted", path: relPath }); + } + + private async walkVault(absDir: string): Promise { + let entries: string[]; + try { + entries = await this.deps.fs.listFiles(absDir); + } catch { + return []; // a file or unreadable dir — nothing to index here + } + + const result: string[] = []; + for (const entry of entries) { + const rel = entry.slice(this.deps.vaultPath.length + 1); + if (rel.startsWith(".trachyte")) continue; + let isDir = false; + try { + await this.deps.fs.listFiles(entry); + isDir = true; + } catch { + isDir = false; + } + if (isDir) { + result.push(...(await this.walkVault(entry))); + } else if (rel.endsWith(".md")) { + result.push(rel); + } + } + return result; + } +} diff --git a/packages/core/src/index/parser/md.ts b/packages/core/src/index/parser/md.ts new file mode 100644 index 0000000..6500aa4 --- /dev/null +++ b/packages/core/src/index/parser/md.ts @@ -0,0 +1,16 @@ +/** + * Extract the indexable text from a markdown file + * + * For v0.0.1 this only strips a leading YAML frontmatter block + * (`---\n…\n---`) so FTS doesn't index frontmatter keys. Heading/tag/link + * extraction is deferred to later PRs. + */ +export function extractIndexContent(markdown: string): string { + const trimmed = markdown.startsWith("\uFEFF") ? markdown.slice(1) : markdown; + if (!trimmed.startsWith("---")) return markdown; + + const end = trimmed.indexOf("\n---", 3); + if (end === -1) return markdown; + + return trimmed.slice(end + 4); +} diff --git a/packages/core/src/index/types.ts b/packages/core/src/index/types.ts new file mode 100644 index 0000000..fa3c368 --- /dev/null +++ b/packages/core/src/index/types.ts @@ -0,0 +1,9 @@ +export interface IndexedFileMeta { + path: string; + mtime: number; + size: number; + hash: string; +} + +export type IndexEvent = + { type: "index:upserted"; path: string } | { type: "index:deleted"; path: string };