feat: paper fetch

.env.example

@@ -6,6 +6,13 @@ OPENROUTER_API_KEY=your_openrouter_api_key_here
 OPENAI_API_KEY=your_openai_api_key_here
 # OR See the Providers section in README.md for other supported providers
 
+# Optional: Scientific paper fetching
+# Asta and Semantic Scholar use separate API keys.
+# ASTA_API_KEY=your_asta_api_key_here
+# S2_API_KEY=your_semantic_scholar_api_key_here
+# Set a contact email to enable Unpaywall DOI lookups.
+# UNPAYWALL_EMAIL=you@your-institution.org
+
 # Optional: Weights & Biases / Weave experiment tracking
 # Get your API key from: https://wandb.ai/authorize
 # Uncomment all three WANDB_* variables to enable logging.

docs/configuration/environment.md

@@ -50,6 +50,31 @@ If you use `--use-provisioning-key`, set:
 OPENROUTER_API_KEY=sk-or-v1-xxxxxxxxxxxx
 ```
 
+## Paper Fetching
+
+The paper-fetch tool reads these optional variables. Export them in your shell
+or add them to `.env`; `agentomics-run` forwards them into Docker.
+
+| Variable | Purpose |
+|----------|---------|
+| `ASTA_API_KEY` | Authenticate Asta MCP search requests. This is separate from the Semantic Scholar key. |
+| `S2_API_KEY` | Authenticate Semantic Scholar batch requests for paper identifiers and PDF metadata. |
+| `UNPAYWALL_EMAIL` | Contact email sent with DOI lookups. Unpaywall is skipped if this is unset. |
+
+Asta and Semantic Scholar requests are attempted without keys when they are
+unset, but may encounter rate limits or service failures. Crossref and the
+repository resolvers do not require credentials in this implementation.
+
+```bash
+# Optional paper-fetch credentials
+# ASTA_API_KEY=your_asta_key
+# S2_API_KEY=your_semantic_scholar_key
+# UNPAYWALL_EMAIL=you@your-institution.org
+```
+
+See [Fetching Scientific Papers](../user-guide/running-agent.md#fetching-scientific-papers)
+for the retrieval flow and its limitations.
+
 ## Weights & Biases (W&B)
 
 Enable experiment tracking and logging:
@@ -124,6 +149,11 @@ OPENROUTER_API_KEY=sk-or-v1-xxxxxxxxxxxx
 # OPENAI_API_KEY=sk-xxxxxxxxxxxx
 # ANTHROPIC_API_KEY=sk-ant-xxxxxxxxxxxx
 
+# Paper fetching (optional)
+# ASTA_API_KEY=your_asta_key
+# S2_API_KEY=your_semantic_scholar_key
+# UNPAYWALL_EMAIL=you@your-institution.org
+
 # Weights & Biases (optional)
 WANDB_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxx
 WANDB_PROJECT_NAME=agentomics

docs/user-guide/outputs.md

@@ -24,7 +24,8 @@ outputs/<agent_id>/
 │   ├── shared/
 │   │   ├── config.json
 │   │   ├── dataset_metadata.json
-│   │   └── splits/
+│   │   ├── splits/
+│   │   └── fetched_papers/    # PDFs downloaded by paper_fetch, when used
 │   ├── iteration_0/
 │   ├── iteration_1/
 │   └── ...
@@ -91,6 +92,17 @@ run/iteration_N/
 └── ...                       # Other iteration artifacts
 ```
 
+## Fetched Papers
+
+The `paper_fetch` tool saves PDFs to
+`outputs/<agent_id>/run/shared/fetched_papers/` on the host. Each filename is the
+Semantic Scholar paper ID followed by `.pdf`. The returned `file_path` points to
+the file inside the run's container workspace.
+
+The folder is shared across iterations of the run. Existing files with a PDF
+signature are reused on later calls for the same paper ID. Only successfully
+saved or cached PDFs are returned to the agent; no HTML fallback is used.
+
 ## Reports
 
 `reports/best_iteration.md` and `reports/best_iteration.pdf` are copies of the

envs/environment.yaml

@@ -6,7 +6,9 @@ dependencies:
   - conda-pack==0.9.1
   - pandas==2.3.1
   - matplotlib==3.8.4
-  - pydantic-ai==1.53.0
+  - pydantic-ai==1.77.0
+  - markdownify==1.2.3
+  - ddgs==9.14.4
   - pip
   - pip:
     - weave==0.52.14 # Often breaks with new versions, update as necessary
@@ -21,3 +23,4 @@ dependencies:
     - openai==2.16.0
     - anthropic==0.79.0
     - pytest==9.1.1
+    - mcp==1.28.0

src/agentomics/agents/paper_fetch.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+from pydantic_ai import Agent, ModelRetry
+from pydantic_ai.models import Model
+from pydantic_ai.settings import ModelSettings
+from pydantic_ai.common_tools.web_fetch import web_fetch_tool
+
+from agentomics.tools.save_paper_pdf_tool import create_save_paper_pdf_tool
+from agentomics.utils.config import Config
+
+
+PAPER_FETCH_SYSTEM_PROMPT = """
+Your goal is to find scientific papers informing a machine learning modeling task.
+
+Work in three phases:
+1. Check the returned search results for relevance to the query. If the same paper appears several times, choose a single host to retrieve it from.
+2. Read. Use web_fetch on the web pages the search returned to confirm what each paper actually
+   is and to find its PDF link. Publisher landing pages state the real PDF URL; read it rather than
+   assembling one from the article id. Never report a paper you have not fetched.
+3. Retrieve. Call save_paper_pdf for each paper relevant to the query. It writes the PDF to disk; you do not
+   need to read the PDF contents yourself. If a URL returns a download page instead of a PDF, go back
+   to the landing page for the direct link rather than guessing.
+
+Report only papers whose PDF you successfully saved.
+"""
+
+class Paper(BaseModel):
+    title: str = Field(
+        description="The title of the paper."
+        )
+    pdf_url: str | None = Field(
+        default=None,
+        description="The URL the PDF was downloaded from.")
+    pdf_filename: str | None = Field(
+        default=None,
+        description="Filename of the saved PDF, as returned by save_paper_pdf. "
+        "Empty if the PDF could not be retrieved.")
+    relevance: str = Field(
+        description="Why this paper is relevant to the query, and what it contributes to the modeling task.")
+
+class PaperFetchOutput(BaseModel):
+    papers: list[Paper] = Field(
+        description="List of relevant fetched papers. Empty if no relevant papers were retrieved or saved.")
+
+def create_paper_fetch_agent(config: Config, model: Model) -> Agent[None, PaperFetchOutput]:
+    agent = Agent(
+        model=model,
+        system_prompt=PAPER_FETCH_SYSTEM_PROMPT,
+        tools=[
+            web_fetch_tool(timeout=config.web_fetch_timeout),
+            create_save_paper_pdf_tool(config),
+        ],
+        model_settings=ModelSettings(temperature=config.temperature),
+        output_type=PaperFetchOutput,
+        retries=config.max_validation_retries
+    )
+    attach_output_validator(agent, config)
+    return agent
+
+def attach_output_validator(agent: Agent[None, PaperFetchOutput], config: Config) -> None:
+    @agent.output_validator
+    async def _papers_stored_in_shared_dir(result: PaperFetchOutput) -> PaperFetchOutput:
+        papers_dir = config.fetched_papers_dir.resolve()
+
+        for paper in result.papers:
+            if paper.pdf_filename is None:
+                continue
+            pdf_path = papers_dir / Path(paper.pdf_filename).name
+            if not pdf_path.is_file():
+                raise ModelRetry(f"'{paper.pdf_filename}' is not in {papers_dir}. Save it with save_paper_pdf or clear pdf_filename.")
+        return result
+
+def build_prompt(config: Config, query: str, websearch_result: list[dict]) -> str:
+    return f"""Based on the query received as input:
+    {query}
+    and the results of web search for that query:
+    {websearch_result}
+    - Save the pdf of relevant papers at: {config.fetched_papers_dir}"""

src/agentomics/cli/run.py

@@ -108,6 +108,7 @@ def _build_container_arguments(arguments: argparse.Namespace) -> list[str]:
 def _docker_environment_arguments(agent_id: str) -> list[str]:
     docker_arguments = docker_environment_arguments(
         "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "OPENROUTER_API_KEY",
+        "ASTA_API_KEY", "S2_API_KEY", "UNPAYWALL_EMAIL",
         "PROVISIONING_OPENROUTER_API_KEY", "OLLAMA_BASE_URL",
         "WANDB_API_KEY", "WANDB_PROJECT_NAME", "WANDB_ENTITY",
         "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY",