Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions ingest/bm25_indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@

from rank_bm25 import BM25Okapi

from server.application_paths import ApplicationPaths
from server.corpus import open_chunks_table, project_columns

logger = logging.getLogger(__name__)
Expand All @@ -93,15 +94,15 @@
# Paths and identity constants
# ---------------------------------------------------------------------------

REPO_ROOT = Path(__file__).resolve().parent.parent
_APPLICATION_PATHS = ApplicationPaths.resolve()

# The directory layout convention: ``<lancedb>/derived/bm25/v<N>/``.
# Confine the ``f"v{N}"`` literal to ``_bm25_version_dir`` below — every
# other reference goes through that helper.
BM25_DIR_NAME = "bm25"
BM25_INDEX_NAME = "bm25.pkl"
BM25_CHUNK_IDS_NAME = "chunk_ids.json"
BM25_STATS_PATH = REPO_ROOT / "var" / "arxmcp" / "ops" / "bm25-stats.jsonl"
BM25_STATS_PATH = _APPLICATION_PATHS.ops / "bm25-stats.jsonl"


def resolve_bm25_root(
Expand Down
60 changes: 55 additions & 5 deletions ingest/bulk_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,25 @@
active path), so it cannot accidentally pick up half-ingested
data.

**What staging does not isolate:** the per-paper intermediate trees
(``corpus/parsed``, ``corpus/chunks``, ``corpus/embeddings``,
``corpus/preamble``) are shared by every dataset's runs, on purpose.
They are a per-paper cache, not a dataset's record. The parsed and raw
trees are downloads, the same whichever dataset reads them. The chunk and
embedding trees are what this code makes of those downloads: the same
paper and code give the same files, each set is replaced through a
temporary file and ``os.replace`` (two runs writing one paper concurrently
each leave a whole set), and nothing reuses a set without checking it.
The embedder re-encodes any paper whose sidecar records another chunker,
embedder or input-policy version or other chunks, and the rows a run
writes come from the embed step of that same run, never from the tree
alone. An aborted run that leaves another code version's sets behind
therefore costs a re-embed, not a wrong row. A tree per dataset would
re-embed every paper once per notebook and per staging run. The ops
files are per target where it matters: callers pass ``ops_dir``,
``log_path`` and ``failures_path``, and a notebook's run keeps them in
the notebook's ``ops/``.

**Scope at v1 (per research synthesis D1):** This module ships
the SCAFFOLDING — orchestrator + CLI. The actual ingest of the
200K-paper corpus is a 1-2 day GPU run that requires a Bittorrent
Expand All @@ -41,9 +60,11 @@
``ops/parser-failures/bulk.jsonl``. Nougat PDF fallback is
deferred (synthesis D2).

**Single-writer constraint** (`ingest/store.py:44-55`): the loop
is sequential at the write boundary. No parallel ``write_chunks``
calls.
**One writer per dataset:** a run holds the target dataset's write
lock (:mod:`ingest.write_lock`) from its chunker-version check to its
index build, and the loop is sequential at the write boundary. A second
writer of the same dataset is refused at once, and the CLI exits with
:data:`ingest.write_lock.WRITE_LOCKED_EXIT_CODE`.

**Chunker-version guard:** a run refuses, before
its first write, a target whose arXiv rows another
Expand Down Expand Up @@ -83,6 +104,7 @@
refuse_stale_chunker_rows,
write_chunks,
)
from ingest.write_lock import dataset_write_lock, exits_when_write_locked
from server.application_paths import ApplicationPaths

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -425,8 +447,9 @@ def run_bulk_ingest(
"""Run the full bulk-ingest loop. Returns the aggregate summary.

The loop is **single-process sequential** at the write
boundary (``ingest.store.write_chunks`` is single-writer-per-
dataset). GPU batching happens inside ``embed_paper``. Naive
boundary, and the run holds the dataset's write lock throughout
(:class:`ingest.write_lock.DatasetWriteLockedError` if another
writer has it). GPU batching happens inside ``embed_paper``. Naive
re-runs are safe: the embedder's per-paper sidecar carries the
versions and each chunk's reuse identity, so already-processed
papers short-circuit at the embed step regardless (a sidecar
Expand All @@ -445,7 +468,33 @@ def run_bulk_ingest(
ar5iv_cache_dir=ar5iv_cache_dir,
parsed_dir=parsed_dir,
)
with dataset_write_lock(lancedb_staging_path):
return _run_locked(
paper_ids,
lancedb_staging_path=lancedb_staging_path,
ar5iv_cache_dir=ar5iv_cache_dir,
parsed_dir=parsed_dir,
failures_path=failures_path,
log_path=log_path,
ops_dir=ops_dir,
progress_interval=progress_interval,
limit=limit,
)


def _run_locked(
paper_ids: list[str],
*,
lancedb_staging_path: Path,
ar5iv_cache_dir: Path,
parsed_dir: Path,
failures_path: Path,
log_path: Path,
ops_dir: Path,
progress_interval: int,
limit: int | None,
) -> IngestSummary:
"""The run itself, under the dataset write lock."""
# Never re-chunk in place across a chunker version.
refuse_stale_chunker_rows(lancedb_staging_path)

Expand Down Expand Up @@ -544,6 +593,7 @@ def _run_dry(
# ---------------------------------------------------------------------------


@exits_when_write_locked
def _cli(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
description="arXMCP bulk ingest orchestrator (E11_S01).",
Expand Down
4 changes: 4 additions & 0 deletions ingest/derived_indexes.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
from dataclasses import dataclass
from pathlib import Path

from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock

logger = logging.getLogger(__name__)


Expand All @@ -29,6 +31,7 @@ class DerivedIndexSummary:
theorem_name_papers: int


@holds_dataset_write_lock("lancedb_path")
def build_derived_indexes(lancedb_path: str | Path) -> DerivedIndexSummary:
"""Build the BM25, definitions and theorem-name indexes of a dataset.

Expand Down Expand Up @@ -59,6 +62,7 @@ def build_derived_indexes(lancedb_path: str | Path) -> DerivedIndexSummary:
return summary


@exits_when_write_locked
def _cli(argv: list[str]) -> int:
import argparse

Expand Down
14 changes: 9 additions & 5 deletions ingest/embed_equations.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@
Same batch size (``EMBED_BATCH_DEFAULT = 32``), same dtype
(``float32``), same L2 normalization invariant.

**Concurrency.** Single-writer-per-table contract. SQLite WAL it's
not — LanceDB merge_insert against the same table from two
processes can interleave. Document and serialize externally.
**Concurrency.** LanceDB merge_insert against the same table from two
processes can interleave, so :func:`embed_pending_equations` runs under
the dataset write lock (:mod:`ingest.write_lock`).
"""

from __future__ import annotations
Expand All @@ -31,10 +31,12 @@
from ingest.embedder import EMBED_BATCH_DEFAULT, _encode_batch
from ingest.index_equations import open_or_create_equations_table
from ingest.schema import EQUATIONS_SCHEMA_V1
from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock
from server.application_paths import ApplicationPaths

logger = logging.getLogger(__name__)

REPO_ROOT = Path(__file__).resolve().parent.parent
_APPLICATION_PATHS = ApplicationPaths.resolve()


def _embedding_input(row: dict[str, Any]) -> str:
Expand All @@ -51,6 +53,7 @@ def _embedding_input(row: dict[str, Any]) -> str:
return pl or ctx


@holds_dataset_write_lock("lancedb_path")
def embed_pending_equations(
lancedb_path: str | Path,
batch_size: int = EMBED_BATCH_DEFAULT,
Expand Down Expand Up @@ -152,6 +155,7 @@ def embed_pending_equations(
# ---------------------------------------------------------------------------


@exits_when_write_locked
def _cli(argv: list[str]) -> int:
import argparse

Expand All @@ -160,7 +164,7 @@ def _cli(argv: list[str]) -> int:
)
parser.add_argument(
"--lancedb-path",
default=str(REPO_ROOT / "var" / "arxmcp" / "index" / "lancedb"),
default=str(_APPLICATION_PATHS.lancedb),
help="LanceDB dataset root (default: var/arxmcp/index/lancedb)",
)
parser.add_argument(
Expand Down
18 changes: 11 additions & 7 deletions ingest/extract_equations.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@
at 4000 characters (matches the handler-side ``latex_or_mathml``
Pydantic constraint). No sentence tokenization at v1.

**Concurrency.** Single-writer-per-paper contract; callers MUST
serialize concurrent invocations against the same LanceDB table. The
delete-then-insert pair (per-paper sweep) is two separate MVCC
commits; concurrent runs can interleave and orphan rows.
**Concurrency.** The delete-then-insert pair (per-paper sweep) is two
separate MVCC commits that concurrent runs could interleave, orphaning
rows, so :func:`extract_equations_for_paper` runs under the dataset
write lock (:mod:`ingest.write_lock`).
"""

from __future__ import annotations
Expand All @@ -63,11 +63,13 @@
from ingest.identifiers import is_valid_arxiv_paper_id
from ingest.index_equations import open_or_create_equations_table
from ingest.schema import EQUATIONS_SCHEMA_V1
from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock
from server.application_paths import ApplicationPaths

logger = logging.getLogger(__name__)

REPO_ROOT = Path(__file__).resolve().parent.parent
PARSED_DIR = REPO_ROOT / "var" / "arxmcp" / "corpus" / "parsed"
_APPLICATION_PATHS = ApplicationPaths.resolve()
PARSED_DIR = _APPLICATION_PATHS.corpus / "parsed"

#: Cap on the ``context_sentence`` column. Matches the handler-side
#: ``latex_or_mathml`` Pydantic max_length so corpus and query rows
Expand Down Expand Up @@ -349,6 +351,7 @@ def _table_has_any_rows_for_paper(tbl: Any, paper_id: str) -> bool:
return True


@holds_dataset_write_lock("lancedb_path")
def extract_equations_for_paper(
paper_id: str,
lancedb_path: str | Path,
Expand Down Expand Up @@ -422,6 +425,7 @@ def extract_equations_for_paper(
# ---------------------------------------------------------------------------


@exits_when_write_locked
def _cli(argv: list[str]) -> int:
import argparse

Expand All @@ -431,7 +435,7 @@ def _cli(argv: list[str]) -> int:
parser.add_argument("paper_id", help="arXiv paper id (e.g. 2401.00001)")
parser.add_argument(
"--lancedb-path",
default=str(REPO_ROOT / "var" / "arxmcp" / "index" / "lancedb"),
default=str(_APPLICATION_PATHS.lancedb),
help="LanceDB dataset root (default: var/arxmcp/index/lancedb)",
)
parser.add_argument(
Expand Down
5 changes: 3 additions & 2 deletions ingest/graph_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
import kuzu

from ingest.kuzudb_schema import apply_schema
from server.application_paths import ApplicationPaths
from tools.arxiv_fetch import (
DEFAULT_503_BACKOFF_SECONDS,
MAX_503_BACKOFF_SECONDS,
Expand Down Expand Up @@ -741,13 +742,13 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument(
"--checkpoint",
type=Path,
default=Path("var/arxmcp/ops/graph-ingest-checkpoint.json"),
default=ApplicationPaths.resolve().ops / "graph-ingest-checkpoint.json",
help="atomic-write checkpoint path (rewritten after each batch)",
)
parser.add_argument(
"--kuzudb",
type=Path,
default=Path("var/arxmcp/index/kuzu"),
default=ApplicationPaths.resolve().kuzu,
help="Kùzu database file (default: var/arxmcp/index/kuzu)",
)
parser.add_argument(
Expand Down
19 changes: 10 additions & 9 deletions ingest/index_definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,14 @@
The handler at :mod:`server.handlers.definitions` reads from the same
table; this module is the writer half of that contract.

**CONCURRENCY:** :func:`index_definitions_for_paper` assumes a
**single-writer-per-paper** contract — callers MUST serialize
concurrent invocations for the same ``paper_id``. The
**CONCURRENCY:** :func:`index_definitions_for_paper` and
:func:`index_definitions_for_corpus` run under the dataset write lock
(:mod:`ingest.write_lock`), so a second writer of the dataset is
refused. Without it the
delete-then-insert pair is two separate LanceDB MVCC commits, so two
concurrent runs for the same paper can interleave as
concurrent runs for the same paper could interleave as
``A.delete → B.delete → A.add → B.add`` (yielding duplicate rows) or
``A.add → B.delete → B.add`` (discarding A's writes). Different
papers may be indexed in parallel without coordination. The
production ingest driver landing in E11 must enforce this — either
via a per-paper file-system lock or a single-worker stage in the
pipeline DAG.
``A.add → B.delete → B.add`` (discarding A's writes).
"""

from __future__ import annotations
Expand All @@ -67,6 +64,7 @@
DEFINITIONS_TABLE_NAME,
LANCE_STORAGE_OPTIONS,
)
from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock

if TYPE_CHECKING:
from ingest.preamble_types import PreambleDoc
Expand Down Expand Up @@ -386,6 +384,7 @@ def _ensure_scalar_indexes(tbl: Any) -> dict[str, bool]:
return created


@holds_dataset_write_lock("lancedb_path")
def index_definitions_for_paper(
paper_id: str,
lancedb_path: str | Path,
Expand Down Expand Up @@ -441,6 +440,7 @@ def index_definitions_for_paper(



@holds_dataset_write_lock("lancedb_path")
def index_definitions_for_corpus(
lancedb_path: str | Path, paper_ids: list[str] | None = None,
) -> tuple[int, int]:
Expand All @@ -462,6 +462,7 @@ def index_definitions_for_corpus(
return total, len(paper_ids)


@exits_when_write_locked
def _cli(argv: list[str]) -> int:
import argparse

Expand Down
13 changes: 8 additions & 5 deletions ingest/index_equations.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
predicate filters rows that need processing.

**Concurrency.** Like the definitions indexer in
:mod:`ingest.index_definitions`, this writer assumes a
**single-writer contract** — concurrent invocations against the same
LanceDB table can interleave delete/insert pairs in ways that
duplicate or drop rows. The production ingest driver landing in E11
must serialize the equation-tree population step.
:mod:`ingest.index_definitions`, concurrent invocations against the same
LanceDB table could interleave delete/insert pairs in ways that
duplicate or drop rows, so :func:`index_equation_trees` runs under the
dataset write lock (:mod:`ingest.write_lock`) and a second writer is
refused.
"""

from __future__ import annotations
Expand All @@ -38,6 +38,7 @@
EQUATIONS_TABLE_NAME,
LANCE_STORAGE_OPTIONS,
)
from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock
from server.retrieval.equations import (
MATHML_TREE_NORMALIZER_VERSION,
parse_mathml_to_tree,
Expand Down Expand Up @@ -80,6 +81,7 @@ def open_or_create_equations_table(lancedb_path: str | Path) -> Any:
)


@holds_dataset_write_lock("lancedb_path")
def index_equation_trees(
lancedb_path: str | Path, *, rebuild: bool = False,
) -> dict[str, int]:
Expand Down Expand Up @@ -173,6 +175,7 @@ def index_equation_trees(



@exits_when_write_locked
def _cli(argv: list[str]) -> int:
import argparse

Expand Down
Loading
Loading