From aacd586351944c0656de13f52ae95d9d48b5d9de Mon Sep 17 00:00:00 2001 From: Wadi Date: Wed, 4 Nov 2026 03:10:25 +0100 Subject: [PATCH] fix: keep Update working when one of two identical files goes Chunk IDs come from file content, so byte-identical files share them. ChunkStore.remove matched chunks by doc_id, collected both files' chunks and passed duplicate IDs to vectors.delete. Chroma raised DuplicateIDError, nothing was written, and every later Update failed the same way. Match chunks on the source path instead. Delete only the vectors no remaining chunk uses, and write the surviving chunk back over the rest so a citation never names the removed file. Chunk IDs stay shared on purpose: search merges results by chunk ID, which stops the same passage filling two answer slots. --- src/open_rag_engine/store.py | 25 ++++++++---- tests/test_ingest.py | 73 +++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 9 deletions(-) diff --git a/src/open_rag_engine/store.py b/src/open_rag_engine/store.py index b7bc1ff..f2dd985 100644 --- a/src/open_rag_engine/store.py +++ b/src/open_rag_engine/store.py @@ -13,7 +13,7 @@ from langchain_core.documents import Document from langchain_core.embeddings import Embeddings -from .documents import CHUNK_ID, DOC_ID +from .documents import CHUNK_ID, SOURCE @dataclass(frozen=True) @@ -55,22 +55,26 @@ def chunks(self) -> list[Document]: return [Document(**json.loads(line)) for line in handle if line.strip()] def add(self, source: str, doc_id: str, chunks: list[Document], index_fingerprint: str) -> None: - if chunks: - self.vectors.add_documents([_for_chroma(c) for c in chunks], ids=[c.metadata[CHUNK_ID] for c in chunks]) + self._write_vectors(chunks) self._write_chunks([*self.chunks(), *chunks]) manifest = self.manifest() self._write_manifest(Manifest(index_fingerprint, {**manifest.documents, source: doc_id})) def remove(self, source: str) -> None: + """Drop one file. Byte-identical files share chunk IDs, so Chroma holds one vector for them all: it stays until + the last of them goes, and is relabelled so a citation never names the file that was removed.""" manifest = self.manifest() - doc_id = manifest.documents.get(source) - if doc_id is None: + if source not in manifest.documents: return kept, dropped = [], [] for chunk in self.chunks(): - (dropped if chunk.metadata.get(DOC_ID) == doc_id else kept).append(chunk) - if dropped: - self.vectors.delete(ids=[c.metadata[CHUNK_ID] for c in dropped]) + (dropped if chunk.metadata.get(SOURCE) == source else kept).append(chunk) + remaining = {c.metadata[CHUNK_ID]: c for c in kept} + dropped_ids = {c.metadata[CHUNK_ID] for c in dropped} + unused = sorted(dropped_ids - remaining.keys()) + if unused: + self.vectors.delete(ids=unused) + self._write_vectors([remaining[chunk_id] for chunk_id in sorted(dropped_ids & remaining.keys())]) self._write_chunks(kept) self._write_manifest( Manifest(manifest.index_fingerprint, {k: v for k, v in manifest.documents.items() if k != source}) @@ -85,6 +89,11 @@ def reset(self) -> None: path.unlink(missing_ok=True) self.vectors = self._open_vectors() + def _write_vectors(self, chunks: list[Document]) -> None: + """Add these chunks to Chroma, replacing any vector that already has the same ID.""" + if chunks: + self.vectors.add_documents([_for_chroma(c) for c in chunks], ids=[c.metadata[CHUNK_ID] for c in chunks]) + def _write_chunks(self, chunks: list[Document]) -> None: lines = ( json.dumps({"page_content": c.page_content, "metadata": c.metadata}, ensure_ascii=False) for c in chunks diff --git a/tests/test_ingest.py b/tests/test_ingest.py index cd88780..4db353c 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -3,11 +3,14 @@ import pytest from open_rag_engine.chunking import make_splitter -from open_rag_engine.ingest import IndexMismatchError, ingest +from open_rag_engine.ingest import IndexMismatchError, IngestReport, ingest +from open_rag_engine.pipeline import RetrievalPipeline from open_rag_engine.store import ChunkStore from .conftest import HashEmbeddings +VALVE = "The pressure relief valve opens at six bar." + def _store(config) -> ChunkStore: return ChunkStore(config.data_dir, config.collection, HashEmbeddings()) @@ -88,3 +91,71 @@ def test_progress_is_reported_per_file(corpus: Path, config): on_progress=lambda done, total, source: seen.append((done, total)), ) assert seen == [(1, 5), (2, 5), (3, 5), (4, 5), (5, 5)] + + +def _update(folder: Path, config, store: ChunkStore) -> IngestReport: + """What the app's Update button does: index the folder and drop files that are gone.""" + return ingest([folder], store, make_splitter(config.chunking, None), config.index_fingerprint(), prune=True) + + +def _names(sources) -> set[str]: + return {Path(str(source)).name for source in sources} + + +def _vector_files(store: ChunkStore) -> set[str]: + return _names(meta["source"] for meta in store.vectors.get()["metadatas"]) + + +@pytest.fixture +def twins(corpus: Path) -> Path: + """Two byte-identical files, as when a document is saved twice. They share chunk IDs, so Chroma holds one copy.""" + (corpus / "valve.md").write_text(VALVE, encoding="utf-8") + (corpus / "valve copy.md").write_text(VALVE, encoding="utf-8") + return corpus + + +@pytest.mark.parametrize(("deleted", "kept"), [("valve.md", "valve copy.md"), ("valve copy.md", "valve.md")]) +def test_deleting_one_of_two_identical_files_keeps_the_other(twins: Path, config, deleted: str, kept: str): + store, _ = _ingest([twins], config) + (twins / deleted).unlink() + report = _update(twins, config, store) + assert _names(report.removed) == {deleted} + assert kept in _names(store.manifest().documents) and deleted not in _names(store.manifest().documents) + assert kept in _names(c.metadata["source"] for c in store.chunks()) + assert deleted not in _names(c.metadata["source"] for c in store.chunks()) + # A citation must never name the deleted file, whichever copy Chroma's shared vector was labelled with. + assert kept in _vector_files(store) and deleted not in _vector_files(store) + + +def test_update_keeps_working_after_removing_a_copy(twins: Path, config): + store, _ = _ingest([twins], config) + (twins / "valve copy.md").unlink() + _update(twins, config, store) + report = _update(twins, config, store) + assert len(report.unchanged) == 6 and not report.added and not report.removed + + +def test_changing_one_of_two_identical_files_keeps_the_other(twins: Path, config): + store, _ = _ingest([twins], config) + (twins / "valve copy.md").write_text("The relief valve now opens at eight bar.", encoding="utf-8") + report = _update(twins, config, store) + assert _names(report.updated) == {"valve copy.md"} + vectors = store.vectors.get() + files_by_text = dict(zip(vectors["documents"], vectors["metadatas"], strict=True)) + assert _names([files_by_text[VALVE]["source"]]) == {"valve.md"} + assert "The relief valve now opens at eight bar." in files_by_text + + +def test_deleting_both_identical_files_removes_their_vector(twins: Path, config): + store, _ = _ingest([twins], config) + (twins / "valve.md").unlink() + (twins / "valve copy.md").unlink() + _update(twins, config, store) + assert len(store.chunks()) == 5 and len(store.vectors.get()["ids"]) == 5 + assert VALVE not in store.vectors.get()["documents"] + + +def test_identical_files_take_one_place_in_search_results(twins: Path, config): + store, _ = _ingest([twins], config) + result = RetrievalPipeline(store, config.retrieval, top_n=5).run("pressure relief valve") + assert [p.doc.page_content for p in result.passages].count(VALVE) == 1