Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions src/open_rag_engine/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings

from .documents import CHUNK_ID, DOC_ID
from .documents import CHUNK_ID, SOURCE


@dataclass(frozen=True)
Expand Down Expand Up @@ -55,22 +55,26 @@ def chunks(self) -> list[Document]:
return [Document(**json.loads(line)) for line in handle if line.strip()]

def add(self, source: str, doc_id: str, chunks: list[Document], index_fingerprint: str) -> None:
if chunks:
self.vectors.add_documents([_for_chroma(c) for c in chunks], ids=[c.metadata[CHUNK_ID] for c in chunks])
self._write_vectors(chunks)
self._write_chunks([*self.chunks(), *chunks])
manifest = self.manifest()
self._write_manifest(Manifest(index_fingerprint, {**manifest.documents, source: doc_id}))

def remove(self, source: str) -> None:
"""Drop one file. Byte-identical files share chunk IDs, so Chroma holds one vector for them all: it stays until
the last of them goes, and is relabelled so a citation never names the file that was removed."""
manifest = self.manifest()
doc_id = manifest.documents.get(source)
if doc_id is None:
if source not in manifest.documents:
return
kept, dropped = [], []
for chunk in self.chunks():
(dropped if chunk.metadata.get(DOC_ID) == doc_id else kept).append(chunk)
if dropped:
self.vectors.delete(ids=[c.metadata[CHUNK_ID] for c in dropped])
(dropped if chunk.metadata.get(SOURCE) == source else kept).append(chunk)
remaining = {c.metadata[CHUNK_ID]: c for c in kept}
dropped_ids = {c.metadata[CHUNK_ID] for c in dropped}
unused = sorted(dropped_ids - remaining.keys())
if unused:
self.vectors.delete(ids=unused)
self._write_vectors([remaining[chunk_id] for chunk_id in sorted(dropped_ids & remaining.keys())])
self._write_chunks(kept)
self._write_manifest(
Manifest(manifest.index_fingerprint, {k: v for k, v in manifest.documents.items() if k != source})
Expand All @@ -85,6 +89,11 @@ def reset(self) -> None:
path.unlink(missing_ok=True)
self.vectors = self._open_vectors()

def _write_vectors(self, chunks: list[Document]) -> None:
"""Add these chunks to Chroma, replacing any vector that already has the same ID."""
if chunks:
self.vectors.add_documents([_for_chroma(c) for c in chunks], ids=[c.metadata[CHUNK_ID] for c in chunks])

def _write_chunks(self, chunks: list[Document]) -> None:
lines = (
json.dumps({"page_content": c.page_content, "metadata": c.metadata}, ensure_ascii=False) for c in chunks
Expand Down
73 changes: 72 additions & 1 deletion tests/test_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@
import pytest

from open_rag_engine.chunking import make_splitter
from open_rag_engine.ingest import IndexMismatchError, ingest
from open_rag_engine.ingest import IndexMismatchError, IngestReport, ingest
from open_rag_engine.pipeline import RetrievalPipeline
from open_rag_engine.store import ChunkStore

from .conftest import HashEmbeddings

VALVE = "The pressure relief valve opens at six bar."


def _store(config) -> ChunkStore:
return ChunkStore(config.data_dir, config.collection, HashEmbeddings())
Expand Down Expand Up @@ -88,3 +91,71 @@ def test_progress_is_reported_per_file(corpus: Path, config):
on_progress=lambda done, total, source: seen.append((done, total)),
)
assert seen == [(1, 5), (2, 5), (3, 5), (4, 5), (5, 5)]


def _update(folder: Path, config, store: ChunkStore) -> IngestReport:
"""What the app's Update button does: index the folder and drop files that are gone."""
return ingest([folder], store, make_splitter(config.chunking, None), config.index_fingerprint(), prune=True)


def _names(sources) -> set[str]:
return {Path(str(source)).name for source in sources}


def _vector_files(store: ChunkStore) -> set[str]:
return _names(meta["source"] for meta in store.vectors.get()["metadatas"])


@pytest.fixture
def twins(corpus: Path) -> Path:
"""Two byte-identical files, as when a document is saved twice. They share chunk IDs, so Chroma holds one copy."""
(corpus / "valve.md").write_text(VALVE, encoding="utf-8")
(corpus / "valve copy.md").write_text(VALVE, encoding="utf-8")
return corpus


@pytest.mark.parametrize(("deleted", "kept"), [("valve.md", "valve copy.md"), ("valve copy.md", "valve.md")])
def test_deleting_one_of_two_identical_files_keeps_the_other(twins: Path, config, deleted: str, kept: str):
store, _ = _ingest([twins], config)
(twins / deleted).unlink()
report = _update(twins, config, store)
assert _names(report.removed) == {deleted}
assert kept in _names(store.manifest().documents) and deleted not in _names(store.manifest().documents)
assert kept in _names(c.metadata["source"] for c in store.chunks())
assert deleted not in _names(c.metadata["source"] for c in store.chunks())
# A citation must never name the deleted file, whichever copy Chroma's shared vector was labelled with.
assert kept in _vector_files(store) and deleted not in _vector_files(store)


def test_update_keeps_working_after_removing_a_copy(twins: Path, config):
store, _ = _ingest([twins], config)
(twins / "valve copy.md").unlink()
_update(twins, config, store)
report = _update(twins, config, store)
assert len(report.unchanged) == 6 and not report.added and not report.removed


def test_changing_one_of_two_identical_files_keeps_the_other(twins: Path, config):
store, _ = _ingest([twins], config)
(twins / "valve copy.md").write_text("The relief valve now opens at eight bar.", encoding="utf-8")
report = _update(twins, config, store)
assert _names(report.updated) == {"valve copy.md"}
vectors = store.vectors.get()
files_by_text = dict(zip(vectors["documents"], vectors["metadatas"], strict=True))
assert _names([files_by_text[VALVE]["source"]]) == {"valve.md"}
assert "The relief valve now opens at eight bar." in files_by_text


def test_deleting_both_identical_files_removes_their_vector(twins: Path, config):
store, _ = _ingest([twins], config)
(twins / "valve.md").unlink()
(twins / "valve copy.md").unlink()
_update(twins, config, store)
assert len(store.chunks()) == 5 and len(store.vectors.get()["ids"]) == 5
assert VALVE not in store.vectors.get()["documents"]


def test_identical_files_take_one_place_in_search_results(twins: Path, config):
store, _ = _ingest([twins], config)
result = RetrievalPipeline(store, config.retrieval, top_n=5).run("pressure relief valve")
assert [p.doc.page_content for p in result.passages].count(VALVE) == 1
Loading