From 0b700efddb9bbd178da3dac6c73aee1615bca202 Mon Sep 17 00:00:00 2001
From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com>
Date: Sun, 18 Oct 2026 17:49:14 +0800
Subject: [PATCH 1/7] feat: implement fields-study-flow FSF-01..07 roadmap
fixes
---
.cursor/mcp.json | 2 +-
.vscode/mcp.json | 2 +-
README.md | 19 +-
README.zh-CN.md | 19 +-
docs/MCP.md | 18 +-
fields_study_flow/cli.py | 19 +-
fields_study_flow/mcp_server.py | 19 +
fields_study_flow/mcp_tools.py | 53 +-
fields_study_flow/models.py | 11 +
fields_study_flow/prerequisites.py | 66 ++
fields_study_flow/ranking.py | 54 +-
fields_study_flow/roadmap.py | 32 +-
fields_study_flow/source-registry.yaml | 259 ++++++--
fields_study_flow/sources_adapters.py | 147 +++++
pyproject.toml | 2 +-
scripts/sync_capability_docs.py | 51 ++
source-registry.yaml | 259 ++++++--
tests/fixtures/ranking_cases.json | 812 +++++++++++++++++++++++++
tests/test_adapters.py | 96 +++
tests/test_mcp_protocol.py | 49 ++
tests/test_mcp_tools.py | 21 +-
tests/test_ranking.py | 52 +-
tests/test_roadmap.py | 37 ++
23 files changed, 1935 insertions(+), 164 deletions(-)
create mode 100644 fields_study_flow/prerequisites.py
create mode 100644 fields_study_flow/sources_adapters.py
create mode 100644 scripts/sync_capability_docs.py
create mode 100644 tests/fixtures/ranking_cases.json
create mode 100644 tests/test_adapters.py
create mode 100644 tests/test_mcp_protocol.py
diff --git a/.cursor/mcp.json b/.cursor/mcp.json
index d4174fb..2e9a67e 100644
--- a/.cursor/mcp.json
+++ b/.cursor/mcp.json
@@ -2,7 +2,7 @@
"mcpServers": {
"fields-study-flow": {
"command": "python",
- "args": ["-m", "fields_study_flow.mcp_server"]
+ "args": ["-m", "fields_study_flow.mcp_server", "--mcp"]
}
}
}
diff --git a/.vscode/mcp.json b/.vscode/mcp.json
index 3ccff3a..bcf3823 100644
--- a/.vscode/mcp.json
+++ b/.vscode/mcp.json
@@ -3,7 +3,7 @@
"fields-study-flow": {
"type": "stdio",
"command": "python",
- "args": ["-m", "fields_study_flow.mcp_server"]
+ "args": ["-m", "fields_study_flow.mcp_server", "--mcp"]
}
}
}
diff --git a/README.md b/README.md
index 474063e..1b7df34 100644
--- a/README.md
+++ b/README.md
@@ -177,7 +177,7 @@ fields_study_flow/
ranking.py # scoring, target-paper boost, canonical URL de-duplication
roadmap.py # roadmap schema and Markdown rendering
mcp_tools.py # tool functions for agents
- mcp_server.py # simple JSON-lines tool server
+ mcp_server.py # SDK MCP stdio (--mcp), legacy JSONL default
cli.py # command-line interface
```
@@ -204,12 +204,25 @@ Current test coverage includes language preference parsing, bilingual query gene
## Roadmap
-- Live official API adapters for GitHub, YouTube, OpenAlex, Semantic Scholar, Hugging Face, and Papers with Code.
+- Additional live API adapters for YouTube, OpenAlex, Semantic Scholar, Hugging Face, and Papers with Code.
- A richer AI/CS prerequisite graph with versioned topic nodes.
- Progress tracking and spaced-review exports.
- Browser-friendly roadmap preview.
-- Full MCP SDK packaging.
+- Wider MCP host GUI compatibility checks.
## License
MIT. See [LICENSE](LICENSE).
+
+
+## Current capabilities and limits
+
+`roadmap` remains offline by default. Add `--live --sources arxiv,github` to query public metadata APIs. MCP `searchResources` uses `filters={"offline": false}` for live queries; `type`, `language`, and `difficulty` filters apply to actual results. Offline catalog entries are suggestions, not live search matches. Other registered platforms are link/catalog only. Retrieval reports `offline`, `live`, `cache`, `mixed`, or `degraded` with reasons; network errors return no invented matches. API calls use a 5-second timeout, a 3-second minimum interval per service instance and a one-hour disk cache (`FIELDS_STUDY_FLOW_CACHE` overrides its directory). There are no paid APIs or authenticated fetches.
+
+`ingestUrl` is URL-only by default. `fetchMetadata=true` requests public GitHub repository or arXiv metadata through the same service, refusing credentials and redirects. Unsupported hosts remain URL-only with an explicit degraded reason. GitHub resource language is the interface/metadata language (English); programming language is recorded separately and is not a natural-language classifier.
+
+Standard MCP hosts must run `python -m fields_study_flow.mcp_server --mcp`. Both editor configurations use this command. The no-flag invocation and `--legacy-jsonl` retain the previous JSONL protocol, including the `git4study` alias. See [MCP transport](docs/MCP.md). The official SDK 2.x negotiates its supported protocol versions; local tests exercise initialize, eight tool schemas, calls, and errors. Editor GUI attachment remains an external check.
+
+Ranking exposes relevance, trust, structure, difficulty, outcome and language components plus unmatched goal terms. A small bilingual alias map and domain levels improve explanations; this is lexical ranking, not semantic search. Thirty fixed review cases live in `tests/fixtures/ranking_cases.json`. Low lexical relevance is labeled supplemental. Prerequisites use a curated mathematics/programming/ML/DL/Transformer/LLM graph, skip explicitly known topics and appear before resource study. Numerical hours determine relative week ranges; unknown durations and infeasible target dates are explicit. These estimates are planning aids, not completion guarantees. Target papers anchor core study rather than the first foundation item.
+
+Safety checks use parsed URL hosts/schemes and structured `metadata.access_policy` actions (`metadata`, `link`, `summarize`). Legacy notes retain compatibility checks for affirmative prohibited instructions; generated safety advice is not treated as an access operation.
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 8b4360a..322e8bc 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -180,7 +180,7 @@ fields_study_flow/
ranking.py # 评分、目标论文 boost、URL 规范化去重
roadmap.py # roadmap schema 与 Markdown 渲染
mcp_tools.py # agent 可调用工具函数
- mcp_server.py # JSON-lines 工具服务器
+ mcp_server.py # SDK MCP stdio（--mcp）与旧 JSONL
cli.py # 命令行入口
```
@@ -207,12 +207,25 @@ pytest -q
## 路线图
-- 接入 GitHub、YouTube、OpenAlex、Semantic Scholar、Hugging Face、Papers with Code 的官方 API adapter；
+- 继续接入 YouTube、OpenAlex、Semantic Scholar、Hugging Face、Papers with Code adapter；
- 扩展 AI/CS 前置知识图谱；
- 增加进度追踪和间隔复习导出；
- 增加浏览器版路线预览；
-- 打包正式 MCP SDK server。
+- 补充编辑器 GUI 的 MCP 连接验证。
## License
MIT. See [LICENSE](LICENSE).
+
+
+## 当前能力与限制
+
+CLI `roadmap` 默认离线；`--live --sources arxiv,github` 显式调用公开元数据 API。MCP `searchResources` 传 `filters={"offline": false}` 启用实时检索，`type`、`language`、`difficulty` 会真正过滤资源。其他 registry 平台目前只有链接/目录能力。返回 `offline/live/cache/mixed/degraded` 和降级原因；失败不会伪造查询结果。默认超时 5 秒、同一服务实例请求至少间隔 3 秒、磁盘缓存 1 小时，缓存目录可由 `FIELDS_STUDY_FLOW_CACHE` 设置。不使用付费 API 或认证内容。
+
+`ingestUrl` 默认只录入 URL；`fetchMetadata=true` 从公开 GitHub/arXiv API 获取元数据，拒绝凭据和重定向，其他来源明确降级。GitHub 的 `language=en` 表示元数据界面语言；编程语言另存 metadata，不保证仓库正文语言分类。
+
+标准 MCP 启动命令为 `python -m fields_study_flow.mcp_server --mcp`，两份编辑器配置已同步。无参数启动和 `--legacy-jsonl` 保留旧 JSONL，`git4study` 入口兼容。协议版本由官方 SDK 2.x 协商，详见 [MCP 文档](docs/MCP.md)。本地 SDK 测试涵盖握手、8 工具 schema、调用和错误；编辑器 GUI 实际连接仍需外部验收。
+
+排序输出相关性、可信度、结构、难度、成果偏好、语言等分项及缺失词项。小型中英别名表和领域等级避免全局降级；仍是词汇匹配，不是语义模型。30 条固定样例位于 `tests/fixtures/ranking_cases.json`。低相关性标为补充资料，不声称匹配。先修图覆盖数学/编程/ML/DL/Transformer/LLM，明确跳过已知主题；工时产生相对周排期，缺失时长标 unknown，期限不足会提示。目标论文放在核心阶段。工时是人工估算，不能保证实际完成时间。
+
+合规校验使用解析后的 URL 主机/协议和 `metadata.access_policy` 的结构化动作（metadata/link/summarize）；旧记录的肯定违规指令保留兼容校验，不把生成的否定安全说明当成访问操作。
diff --git a/docs/MCP.md b/docs/MCP.md
index d98fafb..de94dad 100644
--- a/docs/MCP.md
+++ b/docs/MCP.md
@@ -1,17 +1,9 @@
-# MCP Integration
+# MCP transport
-`fields_study_flow.mcp_tools` contains pure Python functions that can be wrapped by an MCP host.
+Install the package, then configure your host to run `python -m fields_study_flow.mcp_server --mcp`. The Cursor and VS Code examples select this mode explicitly. Use the Python interpreter from the environment where the package and its MCP SDK dependency are installed.
-For local experiments, `python -m fields_study_flow.mcp_server` runs a JSON-lines tool process. Each line should be:
+The server uses the official MCP Python SDK 2.x `MCPServer` stdio transport. Protocol versions and capabilities are negotiated by that SDK; this project does not implement additional protocol versions. Eight tools expose schemas derived from their existing typed signatures: assessKnowledge, discoverSources, searchResources, ingestUrl, rankResources, buildRoadmap, validateSources, exportPlan. Unknown tools and invalid arguments return SDK tool errors. Logs use stderr; stdout is reserved for protocol messages.
-```json
-{"tool":"searchResources","arguments":{"query":"Transformer derivation","languagePreference":"en-first"}}
-```
+For compatibility, `python -m fields_study_flow.mcp_server --legacy-jsonl` accepts one `{"tool":"assessKnowledge","arguments":{"goal":"python"}}` per line and returns `{"ok":true,"result":...}`. Running without a flag retains this legacy behavior, including `python -m git4study.mcp_server`. JSONL is not MCP. Use `--mcp` for standard hosts.
-The process returns:
-
-```json
-{"ok":true,"result":{}}
-```
-
-The JSON-lines server is intentionally small and deterministic. Production MCP packaging can map the same functions onto a full MCP SDK server without changing the tool behavior.
+Run `python -m pytest tests/test_mcp_protocol.py -q` for an actual SDK stdio initialization, tools/list, buildRoadmap call, error cases, legacy subprocess checks and configuration checks. Editor GUI attachment is a separate external check.
diff --git a/fields_study_flow/cli.py b/fields_study_flow/cli.py
index 743b11e..65e7004 100644
--- a/fields_study_flow/cli.py
+++ b/fields_study_flow/cli.py
@@ -45,7 +45,9 @@ def build_parser() -> argparse.ArgumentParser:
roadmap.add_argument("--weekly-hours", type=int)
roadmap.add_argument("--target-date")
roadmap.add_argument("--output-dir", default="fields-study-flow-output")
- roadmap.add_argument("--offline", action="store_true", help="Use the bundled deterministic resource catalog. This is the MVP default.")
+ retrieval = roadmap.add_mutually_exclusive_group()
+ retrieval.add_argument("--offline", action="store_true", help="Use the bundled deterministic resource catalog (default).")
+ retrieval.add_argument("--live", action="store_true", help="Search public arXiv and GitHub metadata.")
paper = subparsers.add_parser("paper", help="Generate a paper deep-reading roadmap.")
paper.add_argument("--url", required=True)
@@ -58,6 +60,7 @@ def build_parser() -> argparse.ArgumentParser:
ingest = subparsers.add_parser("ingest-url", help="Parse a user-provided resource URL at metadata level.")
ingest.add_argument("url")
ingest.add_argument("--source-hint")
+ ingest.add_argument("--fetch-metadata", action="store_true")
discover = subparsers.add_parser("discover-sources", help="List source adapters for a goal and language policy.")
discover.add_argument("--goal", required=True)
@@ -83,14 +86,14 @@ def _roadmap(args: argparse.Namespace) -> int:
target_date=args.target_date,
)
registry = SourceRegistry.default()
- if not args.offline:
- print("Live API connectors are not enabled in this MVP; using the bundled deterministic catalog.", file=sys.stderr)
- resources = offline_resources_for_goal(args.goal)
- selected_sources = _parse_sources(args.sources)
- if selected_sources:
- resources = _filter_resources_by_sources(resources, selected_sources, registry)
+ from fields_study_flow.sources_adapters import SearchService
+ selected_sources = sorted(_parse_sources(args.sources)) or None
+ retrieval = SearchService().search(args.goal, selected_sources, offline=not args.live)
+ resources = [Resource(**r) for r in retrieval['resources']]
ranked = rank_resources(resources, profile)
roadmap = build_roadmap(profile, ranked)
+ roadmap['retrieval'] = {k: v for k, v in retrieval.items() if k != 'resources'}
+ print(f"Retrieval: {retrieval['mode']}; {retrieval['degraded']}", file=sys.stderr)
write_outputs(Path(args.output_dir), profile, ranked, roadmap, registry.snapshot())
print(Path(args.output_dir).resolve())
return 0
@@ -116,7 +119,7 @@ def _paper(args: argparse.Namespace) -> int:
def _ingest_url(args: argparse.Namespace) -> int:
- print(json.dumps(ingestUrl(args.url, args.source_hint), ensure_ascii=False, indent=2))
+ print(json.dumps(ingestUrl(args.url, args.source_hint, fetchMetadata=args.fetch_metadata), ensure_ascii=False, indent=2))
return 0
diff --git a/fields_study_flow/mcp_server.py b/fields_study_flow/mcp_server.py
index acf4c18..789b42d 100644
--- a/fields_study_flow/mcp_server.py
+++ b/fields_study_flow/mcp_server.py
@@ -20,6 +20,25 @@
def main() -> int:
+ import argparse
+
+ parser = argparse.ArgumentParser(description="MCP stdio or legacy JSONL tools")
+ mode = parser.add_mutually_exclusive_group()
+ mode.add_argument("--mcp", action="store_true", help="Standard MCP SDK stdio transport")
+ mode.add_argument("--legacy-jsonl", action="store_true", help="Legacy JSONL (also the compatibility default)")
+ args = parser.parse_args()
+ if args.mcp:
+ from mcp.server.mcpserver import MCPServer
+
+ server = MCPServer("fields-study-flow", version="0.1.0", subscriptions=False)
+ for name, tool in TOOLS.items():
+ server.add_tool(tool, name=name)
+ server.run(transport="stdio")
+ return 0
+ return legacy_main()
+
+
+def legacy_main() -> int:
"""Small JSON-lines tool server for local agent integration examples.
Full MCP hosts can wrap these same tool functions. This process accepts one
diff --git a/fields_study_flow/mcp_tools.py b/fields_study_flow/mcp_tools.py
index 807762b..c1c752f 100644
--- a/fields_study_flow/mcp_tools.py
+++ b/fields_study_flow/mcp_tools.py
@@ -70,7 +70,7 @@ def discoverSources(
"output_language": normalize_output_language(outputLanguage),
"resource_language_preference": str(normalize_resource_language_preference(resourceLanguagePreference)),
"region": region,
- "sources": [source.to_dict() for source in sources],
+ "sources": [{**source.to_dict(), "adapter_status": "implemented" if source.id in {"arxiv", "github"} else "link-only"} for source in sources],
}
@@ -80,23 +80,31 @@ def searchResources(
filters: dict[str, Any] | None = None,
languagePreference: str = "balanced",
) -> dict[str, Any]:
- allowed_sources = set(sources or [])
- resources = offline_resources_for_goal(query)
- if allowed_sources:
- resources = [resource for resource in resources if resource.source in allowed_sources or resource.source.replace("-", "_") in allowed_sources]
+ from fields_study_flow.sources_adapters import SearchService
+ options = filters or {}
+ result = SearchService().search(query, sources, options, offline=options.get('offline', True))
+ resources = [Resource(**resource) for resource in result['resources']]
profile = LearnerProfile(
goal=query,
resource_language_preference=normalize_resource_language_preference(languagePreference),
)
resources = rank_resources(resources, profile)
return {
+ **result,
"queries": build_language_queries(query, languagePreference),
"filters": filters or {},
"resources": [resource.to_dict() for resource in resources],
}
-def ingestUrl(url: str, sourceHint: str | None = None) -> dict[str, Any]:
+def ingestUrl(url: str, sourceHint: str | None = None, fetchMetadata: bool = False) -> dict[str, Any]:
+ degraded = 'Metadata fetch not requested; URL-only import'
+ if fetchMetadata:
+ from fields_study_flow.sources_adapters import SearchService
+ try:
+ return SearchService().ingest(url)
+ except (OSError, ValueError, KeyError) as exc:
+ degraded = f'{type(exc).__name__}: {exc}'
source = sourceHint or _guess_source(url)
return Resource(
title=url.rstrip("/").split("/")[-1] or url,
@@ -108,6 +116,7 @@ def ingestUrl(url: str, sourceHint: str | None = None) -> dict[str, Any]:
concepts=[],
trust_score=0.45,
license_or_access_note="User-provided URL. Parse metadata only; do not bypass login or copy long content.",
+ metadata={"access_policy": {"action": "metadata", "bypass_login": False}, "retrieval_mode": "degraded" if fetchMetadata else "offline", "degraded": degraded},
).to_dict()
@@ -120,7 +129,7 @@ def rankResources(
profile = _profile_from_dict(learnerProfile)
if languagePreference:
profile.resource_language_preference = normalize_resource_language_preference(languagePreference)
- ranked = rank_resources([Resource(**resource) for resource in resources], profile)
+ ranked = rank_resources([Resource(**resource) for resource in resources], profile, target_outcome=targetOutcome)
return {"target_outcome": targetOutcome, "resources": [resource.to_dict() for resource in ranked]}
@@ -141,15 +150,29 @@ def validateSources(plan: dict[str, Any]) -> dict[str, Any]:
for resource in phase.get("resources", []):
url = resource.get("url", "")
access_note = resource.get("license_or_access_note", "")
- normalized_url = url.lower()
- normalized_note = access_note.lower()
- if not url.startswith("http"):
- issues.append(f"Resource has non-http URL: {resource.get('title')}")
- if any(term in normalized_url for term in DISALLOWED_URL_TERMS):
+ from urllib.parse import urlsplit
+ import re
+
+ parsed = urlsplit(url)
+ host = (parsed.hostname or '').lower()
+ if parsed.scheme.lower() not in {'http', 'https'} or not host or parsed.username or parsed.password:
+ issues.append(f"Resource has unsafe URL: {resource.get('title')}")
+ if any(term in host for term in DISALLOWED_URL_TERMS):
issues.append(f"Disallowed source detected: {url}")
- for term in DISALLOWED_ACCESS_NOTE_TERMS:
- if term in normalized_note:
- issues.append(f"Disallowed access instruction ({term}): {resource.get('title')}")
+ policy = resource.get('metadata', {}).get('access_policy')
+ if policy is not None:
+ if (policy.get('action', 'metadata') not in {'metadata', 'link', 'summarize'}
+ or policy.get('bypass_login') or policy.get('mirror') or policy.get('authorized') is False):
+ issues.append(f"Disallowed access policy: {resource.get('title')}")
+ else:
+ # Legacy records encoded operations in notes. Ignore explicitly negated clauses;
+ # structured records never infer operations from explanatory prose.
+ for clause in re.split(r'[.;]', access_note.lower()):
+ if re.search(r'\b(do not|don.t|never|must not)\b', clause):
+ continue
+ for term in DISALLOWED_ACCESS_NOTE_TERMS:
+ if term in clause:
+ issues.append(f"Disallowed access instruction ({term}): {resource.get('title')}")
if not access_note:
issues.append(f"Missing access note: {resource.get('title')}")
return {"valid": not issues, "issues": issues}
diff --git a/fields_study_flow/models.py b/fields_study_flow/models.py
index 83cd2d0..0d4aa7b 100644
--- a/fields_study_flow/models.py
+++ b/fields_study_flow/models.py
@@ -19,6 +19,15 @@ class LearnerProfile:
def __post_init__(self) -> None:
self.resource_language_preference = normalize_resource_language_preference(self.resource_language_preference)
+ import math
+ import re
+ from datetime import date
+ if self.weekly_hours is not None and (isinstance(self.weekly_hours, bool) or not isinstance(self.weekly_hours, (int, float)) or not math.isfinite(self.weekly_hours) or not 0 < self.weekly_hours <= 168):
+ raise ValueError('weekly_hours must be a finite number in (0, 168]')
+ if self.target_date is not None:
+ if not isinstance(self.target_date, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}', self.target_date):
+ raise ValueError('target_date must be YYYY-MM-DD')
+ date.fromisoformat(self.target_date)
def to_dict(self) -> dict[str, Any]:
data = asdict(self)
@@ -43,6 +52,8 @@ class Resource:
translation_note: str = ""
metadata: dict[str, Any] = field(default_factory=dict)
score: float = 0.0
+ score_components: dict[str, float] = field(default_factory=dict)
+ missing_concepts: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return asdict(self)
diff --git a/fields_study_flow/prerequisites.py b/fields_study_flow/prerequisites.py
new file mode 100644
index 0000000..f3cf243
--- /dev/null
+++ b/fields_study_flow/prerequisites.py
@@ -0,0 +1,66 @@
+"""Small curated prerequisite DAG and numerical scheduling estimates (not deadlines)."""
+import math
+import re
+from datetime import date
+from fields_study_flow.ranking import _tokenize
+
+GRAPH = {
+ 'math': ([], 8),
+ 'python': ([], 6),
+ 'machine learning': (['math', 'python'], 10),
+ 'deep learning': (['machine learning'], 10),
+ 'transformer': (['deep learning'], 8),
+ 'diffusion': (['deep learning'], 8),
+ 'llm': (['transformer'], 8),
+}
+
+
+def prerequisite_nodes(profile):
+ terms = _tokenize(profile.goal)
+ targets = [key for key in GRAPH if key in terms or set(key.split()) <= terms]
+ if 'yolo' in terms or 'cnn' in terms:
+ targets.append('deep learning')
+ known = set()
+ for topic in profile.known_topics:
+ known.update(_tokenize(topic))
+ known.add(topic.lower())
+ nodes, seen = [], set()
+ def visit(key):
+ if key in seen:
+ return
+ seen.add(key)
+ deps, hours = GRAPH[key]
+ for dep in deps:
+ visit(dep)
+ skipped = key in known or set(key.split()) <= known
+ nodes.append({'id': key, 'depends_on': deps, 'estimated_hours': hours, 'status': 'skipped' if skipped else 'required', 'reason': 'Already known according to learner profile' if skipped else 'Prerequisite for target concepts'})
+ for target in targets:
+ visit(target)
+ return nodes
+
+
+def schedule(profile, nodes, resources):
+ items, unknown, total = [], [], 0.0
+ for node in nodes:
+ if node['status'] == 'skipped':
+ continue
+ items.append({'id': node['id'], 'hours': node['estimated_hours'], 'kind': 'prerequisite'})
+ for resource in resources:
+ match = re.fullmatch(r'(\d+(?:\.\d+)?)(?:-(\d+(?:\.\d+)?))?h', resource.estimated_time)
+ hours = float(match.group(2) or match.group(1)) if match else None
+ if hours is None:
+ unknown.append(resource.url)
+ items.append({'id': resource.url, 'hours': hours, 'kind': 'resource'})
+ for item in items:
+ if item['hours'] is None:
+ item.update(start_week=None, end_week=None)
+ continue
+ item['start_week'] = math.floor(total / profile.weekly_hours) + 1 if profile.weekly_hours else None
+ total += item['hours']
+ item['end_week'] = max(1, math.ceil(total / profile.weekly_hours)) if profile.weekly_hours else None
+ weeks = math.ceil(total / profile.weekly_hours) if profile.weekly_hours else None
+ feasible = None
+ if profile.target_date and profile.weekly_hours:
+ available = max(0, (date.fromisoformat(profile.target_date) - date.today()).days) / 7 * profile.weekly_hours
+ feasible = False if total > available else None if unknown else True
+ return {'items': items, 'estimated_hours': total, 'unknown_estimates': unknown, 'total_weeks': weeks, 'target_date': profile.target_date, 'deadline_feasible': feasible, 'note': 'Estimates exclude unknown durations; prerequisite and resource study may overlap.'}
diff --git a/fields_study_flow/ranking.py b/fields_study_flow/ranking.py
index 89c56e1..2afdb6d 100644
--- a/fields_study_flow/ranking.py
+++ b/fields_study_flow/ranking.py
@@ -17,14 +17,34 @@
}
-def rank_resources(resources: list[Resource], profile: LearnerProfile) -> list[Resource]:
+ALIASES = {
+ 'transformer': ('变换器', 'transformer'),
+ 'diffusion': ('扩散', 'diffusion'),
+ 'linear algebra': ('线性代数', '线代', 'linear algebra'),
+ 'probability': ('概率', 'probability'),
+ 'machine learning': ('机器学习', 'machine learning'),
+ 'deep learning': ('深度学习', 'deep learning'),
+ 'python': ('编程', 'python'),
+ 'database': ('数据库', 'database'),
+ 'transaction': ('事务', 'transaction'),
+ 'attention': ('注意力', 'attention'),
+}
+DOMAINS = {'programming': {'python'}, 'math': {'linear algebra', 'probability', 'math'},
+ 'ml_dl': {'transformer', 'diffusion', 'machine learning', 'deep learning', 'yolo', 'cnn', 'ppo', 'trpo', 'attention'}}
+
+
+def rank_resources(resources: list[Resource], profile: LearnerProfile, target_outcome: str = 'understand') -> list[Resource]:
scored: list[Resource] = []
for resource in resources:
language_multiplier = language_weight(resource.language, profile.resource_language_preference)
if language_multiplier == 0:
continue
- resource.score = round(_base_score(resource, profile) * language_multiplier, 4)
- resource.why_recommended = resource.why_recommended or _recommendation_reason(resource, profile)
+ relevance = _concept_match_score(resource, profile)
+ outcome = 0.2 if (target_outcome in {'reproduce', 'implement', '复现'} and resource.type == 'repository') or (target_outcome in {'derive', '推导'} and resource.type == 'paper') else 0.0
+ resource.score_components = {'relevance': relevance, 'trust': max(0., min(1., resource.trust_score)), 'structure': _structure_score(resource), 'difficulty': _difficulty_fit(resource, profile), 'outcome': outcome, 'language': language_multiplier}
+ resource.missing_concepts = sorted(_tokenize(profile.goal) - _tokenize(resource.title + ' ' + ' '.join(resource.concepts)))
+ resource.score = round((_base_score(resource, profile) + outcome) * language_multiplier, 4)
+ resource.why_recommended = _recommendation_reason(resource, profile)
resource.translation_note = resource.translation_note or _translation_note(resource, profile.output_language)
scored.append(resource)
return _dedupe_by_url(sorted(scored, key=lambda item: item.score, reverse=True))
@@ -58,7 +78,7 @@ def _concept_match_score(resource: Resource, profile: LearnerProfile) -> float:
if not goal_terms:
return 0.4
overlap = len(goal_terms & all_terms) / max(1, len(goal_terms))
- return min(1.0, 0.25 + overlap)
+ return min(1.0, overlap)
def _structure_score(resource: Resource) -> float:
@@ -78,7 +98,9 @@ def _structure_score(resource: Resource) -> float:
def _difficulty_fit(resource: Resource, profile: LearnerProfile) -> float:
- level_values = set(profile.levels.values())
+ terms = _tokenize(profile.goal)
+ domain = max(DOMAINS, key=lambda key: len(terms & DOMAINS[key]))
+ level_values = {profile.levels.get(domain, 'familiar')}
if "beginner" in level_values or "初学" in level_values:
target = 0
elif "familiar" in level_values or "熟悉" in level_values:
@@ -93,7 +115,7 @@ def _difficulty_fit(resource: Resource, profile: LearnerProfile) -> float:
def _recommendation_reason(resource: Resource, profile: LearnerProfile) -> str:
- reason = "matches your goal"
+ reason = "matches your goal" if _concept_match_score(resource, profile) >= 0.1 else "No direct concept match; supplemental resource"
if resource.source == "github" and (resource.metadata.get("has_curriculum") or resource.metadata.get("has_notebooks")):
reason += " and includes structured code or notebooks"
elif resource.type == "paper":
@@ -115,6 +137,9 @@ def _translation_note(resource: Resource, output_language: str) -> str:
def _tokenize(value: str) -> set[str]:
normalized = value.lower().replace("/", " ").replace("-", " ")
+ for canonical, aliases in ALIASES.items():
+ for alias in aliases:
+ normalized = normalized.replace(alias, ' ' + canonical + ' ')
tokens = {token.strip(".,:;()[]{}") for token in normalized.split() if token.strip()}
for keyword in ("transformer", "diffusion", "yolo", "ppo", "trpo", "cnn", "python"):
if keyword in normalized:
@@ -150,4 +175,19 @@ def _canonical_url_key(url: str) -> str:
normalized_path = path.removesuffix(".git").lower()
return f"github:{normalized_path}"
- return f"{host}{path.lower()}"
+ from urllib.parse import parse_qsl, urlencode
+
+ query = [(k, v) for k, v in parse_qsl(parsed.query, keep_blank_values=True)
+ if not k.lower().startswith('utm_') and k.lower() not in {'gclid', 'fbclid', 'msclkid'}]
+ if host in {'youtube.com', 'm.youtube.com', 'youtu.be'}:
+ params = dict(query)
+ video = path.lstrip('/') if host == 'youtu.be' else params.get('v')
+ if video:
+ return f"youtube:video:{video}"
+ if params.get('list'):
+ return f"youtube:list:{params['list']}"
+ query = [(k, v) for k, v in query if k not in {'si', 'feature'}]
+ if host in {'bilibili.com', 'm.bilibili.com'}:
+ query = [(k, v) for k, v in query if k not in {'spm_id_from', 'share_source', 'share_medium', 'share_session_id'}]
+ suffix = urlencode(sorted(query))
+ return f"{host}{path}" + (f"?{suffix}" if suffix else '')
diff --git a/fields_study_flow/roadmap.py b/fields_study_flow/roadmap.py
index c737ed2..5176438 100644
--- a/fields_study_flow/roadmap.py
+++ b/fields_study_flow/roadmap.py
@@ -17,8 +17,27 @@
def build_roadmap(profile: LearnerProfile, resources: list[Resource]) -> dict[str, Any]:
- phases = _build_phases(profile, resources)
+ from fields_study_flow.prerequisites import prerequisite_nodes, schedule
+ nodes = prerequisite_nodes(profile)
+ # Resource ordering follows explicit dependencies, then difficulty; target papers anchor core study.
+ remaining = sorted(resources, key=lambda r: (bool(r.metadata.get('target_paper')), {'beginner': 0, 'intermediate': 1, 'advanced': 2}.get(r.difficulty, 1)))
+ ordered, seen = [], set(profile.known_topics)
+ while remaining:
+ ready = next((r for r in remaining if all(p in seen or not any(p in other.concepts for other in remaining) for p in r.prerequisites)), remaining[0])
+ remaining.remove(ready)
+ ordered.append(ready)
+ seen.update(ready.concepts)
+ phases = _build_phases(profile, ordered)
+ if any(r.metadata.get('target_paper') for r in ordered):
+ targets = [r.to_dict() for r in ordered if r.metadata.get('target_paper')]
+ for phase in phases:
+ phase['resources'] = [r for r in phase['resources'] if not r['metadata'].get('target_paper')]
+ if len(phases) == 1:
+ phases.append({'name': _phase_name(profile, 'Phase 2', '阶段 2', 'Core Understanding', '核心理解'), 'objective': 'Study the target paper after prerequisites.', 'resources': []})
+ phases[1]['resources'].extend(targets)
return {
+ 'prerequisites': nodes,
+ 'schedule': schedule(profile, nodes, ordered),
"title": _title(profile),
"profile": profile.to_dict(),
"outputs": OUTPUT_FILES.copy(),
@@ -89,7 +108,16 @@ def render_markdown(roadmap: dict[str, Any]) -> str:
]
)
lines.append("")
- lines.extend(["## Checkpoints", ""])
+ lines.extend(['## Prerequisites', ''])
+ for node in roadmap.get('prerequisites', []):
+ lines.append(f"- {node['id']}: {node['status']} ({node['reason']}); {node['estimated_hours']}h")
+ lines.extend(['', '## Schedule', ''])
+ schedule = roadmap.get('schedule', {})
+ lines.append(f"- Estimated weeks: {schedule.get('total_weeks', 'unknown')}; target date: {schedule.get('target_date') or 'not set'}")
+ lines.append(f"- Deadline feasible: {schedule.get('deadline_feasible')}; unknown durations: {len(schedule.get('unknown_estimates', []))}")
+ for item in schedule.get('items', []):
+ lines.append(f"- {item['id']}: {item['hours'] if item['hours'] is not None else 'unknown'}h; weeks {item['start_week']}–{item['end_week']}")
+ lines.extend(['', "## Checkpoints", ""])
for checkpoint in roadmap["checkpoints"]:
lines.append(f"- {checkpoint}")
lines.extend(["", "## Safety Policy", ""])
diff --git a/fields_study_flow/source-registry.yaml b/fields_study_flow/source-registry.yaml
index 8957575..26e24d4 100644
--- a/fields_study_flow/source-registry.yaml
+++ b/fields_study_flow/source-registry.yaml
@@ -5,200 +5,351 @@
"id": "github",
"name": "GitHub",
"category": "code-learning",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search repositories, topics, READMEs, notebooks, awesome lists, and paper implementations through official API or user-provided URLs.",
- "quality_signals": ["license", "readme_structure", "curriculum", "notebooks", "recent_activity", "issue_quality", "stars_as_weak_signal"],
+ "quality_signals": [
+ "license",
+ "readme_structure",
+ "curriculum",
+ "notebooks",
+ "recent_activity",
+ "issue_quality",
+ "stars_as_weak_signal"
+ ],
"restricted": false,
- "notes": "Treat stars and forks as weak popularity signals, not learning quality."
+ "notes": "Implemented: public metadata adapter, bounded timeout, disk cache, no authentication. Treat stars and forks as weak popularity signals, not learning quality."
},
{
"id": "arxiv",
"name": "arXiv",
"category": "academic",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search metadata and link to abstract/PDF pages for open preprints.",
- "quality_signals": ["paper_title", "abstract", "authors", "citations_via_cross_source", "recency"],
- "restricted": false
+ "quality_signals": [
+ "paper_title",
+ "abstract",
+ "authors",
+ "citations_via_cross_source",
+ "recency"
+ ],
+ "restricted": false,
+ "notes": "Implemented: public metadata adapter, bounded timeout, disk cache, no authentication. "
},
{
"id": "openalex",
"name": "OpenAlex",
"category": "academic",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search scholarly metadata, concepts, citations, venues, and open access status.",
- "quality_signals": ["citation_graph", "concepts", "open_access", "venue", "recency"],
- "restricted": false
+ "quality_signals": [
+ "citation_graph",
+ "concepts",
+ "open_access",
+ "venue",
+ "recency"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "semantic-scholar",
"name": "Semantic Scholar",
"category": "academic",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search paper metadata, recommendations, references, and citations within API terms.",
- "quality_signals": ["citations", "references", "recommendations", "fields_of_study"],
- "restricted": false
+ "quality_signals": [
+ "citations",
+ "references",
+ "recommendations",
+ "fields_of_study"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "unpaywall",
"name": "Unpaywall",
"category": "academic",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Resolve legal open-access full-text locations from DOI metadata.",
- "quality_signals": ["open_access_status", "host_type", "license"],
- "restricted": false
+ "quality_signals": [
+ "open_access_status",
+ "host_type",
+ "license"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "papers-with-code",
"name": "Papers with Code",
"category": "academic-code",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api-or-client",
"auth_required": false,
"allowed_use": "Discover papers, tasks, datasets, methods, leaderboards, and linked code.",
- "quality_signals": ["task_match", "dataset_match", "official_code", "leaderboard_presence"],
- "restricted": false
+ "quality_signals": [
+ "task_match",
+ "dataset_match",
+ "official_code",
+ "leaderboard_presence"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "youtube",
"name": "YouTube",
"category": "video",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "official-api",
"auth_required": true,
"allowed_use": "Search public video and playlist metadata with YouTube Data API. Do not download videos.",
- "quality_signals": ["channel_authority", "playlist_structure", "chapters", "captions_available", "duration"],
- "restricted": false
+ "quality_signals": [
+ "channel_authority",
+ "playlist_structure",
+ "chapters",
+ "captions_available",
+ "duration"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "bilibili",
"name": "Bilibili",
"category": "video-cn",
- "languages": ["zh-CN"],
+ "languages": [
+ "zh-CN"
+ ],
"access_mode": "official-or-url-only",
"auth_required": false,
"allowed_use": "Prefer official/open platform capabilities. If search is unavailable, parse user-provided BV/URL metadata only.",
- "quality_signals": ["uploader_authority", "series_structure", "duration", "danmaku_comment_quality_as_weak_signal"],
- "restricted": false
+ "quality_signals": [
+ "uploader_authority",
+ "series_structure",
+ "duration",
+ "danmaku_comment_quality_as_weak_signal"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "zhihu",
"name": "Zhihu",
"category": "community-cn",
- "languages": ["zh-CN"],
+ "languages": [
+ "zh-CN"
+ ],
"access_mode": "official-or-url-only",
"auth_required": false,
"allowed_use": "Use official/open platform or user-provided URLs. Summarize and link; do not bulk scrape or bypass login.",
- "quality_signals": ["author_authority", "answer_structure", "citations", "recency"],
- "restricted": false
+ "quality_signals": [
+ "author_authority",
+ "answer_structure",
+ "citations",
+ "recency"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "hugging-face",
"name": "Hugging Face",
"category": "practice",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search models, datasets, Spaces, model cards, dataset cards, and course pages.",
- "quality_signals": ["card_quality", "downloads", "likes_as_weak_signal", "license", "task_match"],
- "restricted": false
+ "quality_signals": [
+ "card_quality",
+ "downloads",
+ "likes_as_weak_signal",
+ "license",
+ "task_match"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "kaggle",
"name": "Kaggle",
"category": "practice",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api",
"auth_required": true,
"allowed_use": "Search datasets, notebooks, competitions, and practice projects with user-configured credentials.",
- "quality_signals": ["dataset_usability", "notebook_votes_as_weak_signal", "competition_relevance", "recency"],
- "restricted": false
+ "quality_signals": [
+ "dataset_usability",
+ "notebook_votes_as_weak_signal",
+ "competition_relevance",
+ "recency"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "mit-ocw",
"name": "MIT OpenCourseWare",
"category": "course",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "public-web-or-catalog",
"auth_required": false,
"allowed_use": "Recommend course pages and materials by link. Do not mirror course content.",
- "quality_signals": ["institution", "syllabus", "assignments", "lecture_notes"],
- "restricted": false
+ "quality_signals": [
+ "institution",
+ "syllabus",
+ "assignments",
+ "lecture_notes"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "google-mlcc",
"name": "Google Machine Learning Crash Course",
"category": "course",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "public-web",
"auth_required": false,
"allowed_use": "Recommend official course pages by link.",
- "quality_signals": ["official_docs", "exercises", "curriculum"],
- "restricted": false
+ "quality_signals": [
+ "official_docs",
+ "exercises",
+ "curriculum"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "fast-ai",
"name": "fast.ai",
"category": "course",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "public-web",
"auth_required": false,
"allowed_use": "Recommend official course pages and notebooks by link.",
- "quality_signals": ["course_sequence", "notebooks", "assignments", "community"],
- "restricted": false
+ "quality_signals": [
+ "course_sequence",
+ "notebooks",
+ "assignments",
+ "community"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "deep-learning-ai",
"name": "DeepLearning.AI",
"category": "course",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "link-only",
"auth_required": false,
"allowed_use": "Recommend public course landing pages and free resources by link.",
- "quality_signals": ["instructor_authority", "course_sequence", "official_curriculum"],
- "restricted": false
+ "quality_signals": [
+ "instructor_authority",
+ "course_sequence",
+ "official_curriculum"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "coursera",
"name": "Coursera",
"category": "course",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "link-only",
"auth_required": true,
"allowed_use": "Recommend public landing pages only; do not access paywalled content.",
- "quality_signals": ["institution", "syllabus", "reviews_as_weak_signal"],
- "restricted": true
+ "quality_signals": [
+ "institution",
+ "syllabus",
+ "reviews_as_weak_signal"
+ ],
+ "restricted": true,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "xuetangx",
"name": "学堂在线",
"category": "course-cn",
- "languages": ["zh-CN"],
+ "languages": [
+ "zh-CN"
+ ],
"access_mode": "link-only",
"auth_required": false,
"allowed_use": "Recommend public course pages by link.",
- "quality_signals": ["institution", "syllabus", "course_sequence"],
- "restricted": false
+ "quality_signals": [
+ "institution",
+ "syllabus",
+ "course_sequence"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "icourse163",
"name": "中国大学 MOOC",
"category": "course-cn",
- "languages": ["zh-CN"],
+ "languages": [
+ "zh-CN"
+ ],
"access_mode": "link-only",
"auth_required": false,
"allowed_use": "Recommend public course pages by link.",
- "quality_signals": ["institution", "syllabus", "course_sequence"],
- "restricted": false
+ "quality_signals": [
+ "institution",
+ "syllabus",
+ "course_sequence"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
}
]
}
diff --git a/fields_study_flow/sources_adapters.py b/fields_study_flow/sources_adapters.py
new file mode 100644
index 0000000..620b98b
--- /dev/null
+++ b/fields_study_flow/sources_adapters.py
@@ -0,0 +1,147 @@
+"""Public metadata adapters. No credentials, content scraping, or automatic redirects."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import quote, urlencode, urlsplit
+from urllib.request import Request, build_opener, HTTPRedirectHandler
+from urllib.error import HTTPError
+
+from fields_study_flow.models import Resource
+from fields_study_flow.offline_catalog import offline_resources_for_goal
+
+
+@dataclass
+class Response:
+ status: int
+ body: bytes
+ url: str
+
+
+class NoRedirect(HTTPRedirectHandler):
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ return None
+
+
+def public_get(url, timeout):
+ request = Request(url, headers={'User-Agent': 'fields-study-flow/0.1 (public metadata)', 'Accept': 'application/json, application/atom+xml'})
+ try:
+ with build_opener(NoRedirect).open(request, timeout=timeout) as response:
+ return Response(response.status, response.read(2_000_001), response.url)
+ except HTTPError as exc:
+ return Response(exc.code, b'', url)
+
+
+class SearchService:
+ def __init__(self, cache_dir=None, transport=public_get, timeout=5, ttl=3600, min_interval=3):
+ self.cache_dir = Path(cache_dir or os.environ.get('FIELDS_STUDY_FLOW_CACHE', Path.home() / '.cache' / 'fields-study-flow'))
+ self.transport, self.timeout, self.ttl, self.min_interval = transport, timeout, ttl, min_interval
+ self.last_request = 0.0
+
+ def _fetch(self, url):
+ key = hashlib.sha256(url.encode()).hexdigest()
+ path = self.cache_dir / (key + '.json')
+ try:
+ data = json.loads(path.read_text(encoding='utf-8'))
+ if time.time() - data['timestamp'] < self.ttl:
+ return data['resources'], 'cache'
+ except (OSError, ValueError, KeyError):
+ pass
+ delay = self.min_interval - (time.monotonic() - self.last_request)
+ if delay > 0:
+ time.sleep(delay)
+ self.last_request = time.monotonic()
+ response = self.transport(url, self.timeout)
+ if response.url != url:
+ raise ValueError('redirect rejected')
+ if response.status != 200:
+ raise ValueError(f'HTTP {response.status}; no automatic retry')
+ if len(response.body) > 2_000_000:
+ raise ValueError('metadata response too large')
+ resources = self._normalize(response.body, 'github' if 'api.github.com' in url else 'arxiv')
+ try:
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps({'timestamp': time.time(), 'resources': resources}), encoding='utf-8')
+ except OSError:
+ pass
+ return resources, 'live'
+
+ def _normalize(self, body, source):
+ now = datetime.now(timezone.utc).isoformat()
+ output = []
+ if source == 'github':
+ payload = json.loads(body)
+ items = payload.get('items', [payload])
+ for item in items:
+ name = item['full_name']
+ # Always construct canonical public URLs rather than trusting response links.
+ if len(name.split('/')) != 2 or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._/' for c in name):
+ raise ValueError('invalid repository name')
+ output.append(Resource(title=name, url='https://github.com/' + name, source='github', type='repository', language='en', concepts=item.get('topics', []), trust_score=0.7, metadata={'id': f"github:{item['id']}", 'retrieved_at': now, 'source': source, 'description': item.get('description'), 'code_language': item.get('language'), 'stars': item.get('stargazers_count', 0)}).to_dict())
+ else:
+ ns = {'a': 'http://www.w3.org/2005/Atom'}
+ root = ET.fromstring(body)
+ for entry in root.findall('a:entry', ns):
+ identifier = entry.findtext('a:id', '', ns).rsplit('/abs/', 1)[-1]
+ import re
+ if not re.fullmatch(r'(?:\d{4}\.\d{4,5}|[a-z-]+/\d{7})(?:v\d+)?', identifier):
+ raise ValueError('invalid arXiv id')
+ output.append(Resource(title=' '.join(entry.findtext('a:title', '', ns).split()), url='https://arxiv.org/abs/' + identifier, source='arxiv', type='paper', language='en', difficulty='advanced', trust_score=0.85, metadata={'id': 'arxiv:' + identifier, 'retrieved_at': now, 'source': source, 'summary': entry.findtext('a:summary', '', ns)}).to_dict())
+ return output
+
+ def search(self, query, sources=None, filters=None, offline=False):
+ filters = filters or {}
+ selected = list(dict.fromkeys(sources or ['arxiv', 'github']))
+ reasons, modes, resources = [], [], []
+ if offline:
+ resources = [r.to_dict() for r in offline_resources_for_goal(query)]
+ if sources:
+ from fields_study_flow.sources import SourceRegistry
+ registry = SourceRegistry.default()
+ allowed = set(sources)
+ for source in sources:
+ registered = registry.sources.get(source)
+ if registered and registered.category == 'course':
+ allowed.add('course')
+ resources = [r for r in resources if r['source'] in allowed]
+ modes.append('offline')
+ reasons.append('Explicit offline catalog; curated suggestions, not live query matches')
+ else:
+ for source in selected:
+ if source not in {'arxiv', 'github'}:
+ reasons.append(f'{source}: adapter unavailable')
+ continue
+ url = ('https://api.github.com/search/repositories?' + urlencode({'q': query, 'per_page': 10}) if source == 'github' else 'https://export.arxiv.org/api/query?' + urlencode({'search_query': 'all:' + query, 'max_results': 10}))
+ try:
+ found, mode = self._fetch(url)
+ resources.extend(found)
+ modes.append(mode)
+ except (OSError, ValueError, KeyError, TypeError, ET.ParseError) as exc:
+ reasons.append(f'{source}: {type(exc).__name__}: {exc}')
+ for field in ('type', 'language', 'difficulty'):
+ if filters.get(field):
+ resources = [r for r in resources if r[field] == filters[field]]
+ return {'resources': resources, 'mode': ('offline' if offline else 'degraded' if reasons else modes[0] if len(set(modes)) == 1 else 'mixed'), 'degraded': reasons, 'source_modes': modes}
+
+ def ingest(self, url):
+ parsed = urlsplit(url)
+ if parsed.scheme.lower() != 'https' or parsed.username or parsed.password or parsed.hostname not in {'github.com', 'arxiv.org'}:
+ raise ValueError('only credential-free public HTTPS GitHub/arXiv metadata URLs are supported')
+ parts = parsed.path.strip('/').split('/')
+ if parsed.hostname == 'github.com' and len(parts) == 2 and all(parts):
+ api = 'https://api.github.com/repos/' + '/'.join(quote(p, safe='') for p in parts)
+ elif parsed.hostname == 'arxiv.org' and len(parts) >= 2 and parts[0] in {'abs', 'pdf'}:
+ api = 'https://export.arxiv.org/api/query?' + urlencode({'id_list': '/'.join(parts[1:]).removesuffix('.pdf')})
+ else:
+ raise ValueError('unsupported public metadata URL')
+ resources, mode = self._fetch(api)
+ if not resources:
+ raise ValueError('no public metadata returned')
+ resources[0]['metadata']['retrieval_mode'] = mode
+ return resources[0]
diff --git a/pyproject.toml b/pyproject.toml
index ad4076b..c689183 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,7 +11,7 @@ requires-python = ">=3.10"
license = "MIT"
authors = [{ name = "fields-study-flow contributors" }]
keywords = ["ai", "learning-path", "mcp", "skills", "education", "research"]
-dependencies = ["PyYAML>=6.0"]
+dependencies = ["PyYAML>=6.0", "mcp>=2.3,<3"]
[project.optional-dependencies]
dev = ["pytest>=8.0"]
diff --git a/scripts/sync_capability_docs.py b/scripts/sync_capability_docs.py
new file mode 100644
index 0000000..a8b4981
--- /dev/null
+++ b/scripts/sync_capability_docs.py
@@ -0,0 +1,51 @@
+"""Refresh capability descriptions; run from repository root."""
+import json
+from pathlib import Path
+
+for file in [Path('source-registry.yaml'), Path('fields_study_flow/source-registry.yaml')]:
+ data = json.loads(file.read_text(encoding='utf-8'))
+ for source in data['sources']:
+ source['notes'] = ('Implemented: public metadata adapter, bounded timeout, disk cache, no authentication. ' if source['id'] in {'arxiv', 'github'} else 'Link/catalog only: no live adapter implemented. ') + source.get('notes', '')
+ file.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
+
+for name, replacements, extra in [
+ ('README.md', {
+ 'mcp_server.py # simple JSON-lines tool server': 'mcp_server.py # SDK MCP stdio (--mcp), legacy JSONL default',
+ '- Live official API adapters for GitHub, YouTube, OpenAlex, Semantic Scholar, Hugging Face, and Papers with Code.': '- Additional live API adapters for YouTube, OpenAlex, Semantic Scholar, Hugging Face, and Papers with Code.',
+ '- Full MCP SDK packaging.': '- Wider MCP host GUI compatibility checks.',
+ 'Current source categories include:': 'Registered source categories (only arXiv/GitHub have live adapters):',
+ }, '''
+## Current capabilities and limits
+
+`roadmap` remains offline by default. Add `--live --sources arxiv,github` to query public metadata APIs. MCP `searchResources` uses `filters={"offline": false}` for live queries; `type`, `language`, and `difficulty` filters apply to actual results. Offline catalog entries are suggestions, not live search matches. Other registered platforms are link/catalog only. Retrieval reports `offline`, `live`, `cache`, `mixed`, or `degraded` with reasons; network errors return no invented matches. API calls use a 5-second timeout, a 3-second minimum interval per service instance and a one-hour disk cache (`FIELDS_STUDY_FLOW_CACHE` overrides its directory). There are no paid APIs or authenticated fetches.
+
+`ingestUrl` is URL-only by default. `fetchMetadata=true` requests public GitHub repository or arXiv metadata through the same service, refusing credentials and redirects. Unsupported hosts remain URL-only with an explicit degraded reason. GitHub resource language is the interface/metadata language (English); programming language is recorded separately and is not a natural-language classifier.
+
+Standard MCP hosts must run `python -m fields_study_flow.mcp_server --mcp`. Both editor configurations use this command. The no-flag invocation and `--legacy-jsonl` retain the previous JSONL protocol, including the `git4study` alias. See [MCP transport](docs/MCP.md). The official SDK 2.x negotiates its supported protocol versions; local tests exercise initialize, eight tool schemas, calls, and errors. Editor GUI attachment remains an external check.
+
+Ranking exposes relevance, trust, structure, difficulty, outcome and language components plus unmatched goal terms. A small bilingual alias map and domain levels improve explanations; this is lexical ranking, not semantic search. Thirty fixed review cases live in `tests/fixtures/ranking_cases.json`. Low lexical relevance is labeled supplemental. Prerequisites use a curated mathematics/programming/ML/DL/Transformer/LLM graph, skip explicitly known topics and appear before resource study. Numerical hours determine relative week ranges; unknown durations and infeasible target dates are explicit. These estimates are planning aids, not completion guarantees. Target papers anchor core study rather than the first foundation item.
+
+Safety checks use parsed URL hosts/schemes and structured `metadata.access_policy` actions (`metadata`, `link`, `summarize`). Legacy notes retain compatibility checks for affirmative prohibited instructions; generated safety advice is not treated as an access operation.
+'''),
+ ('README.zh-CN.md', {
+ 'mcp_server.py # JSON-lines 工具服务器': 'mcp_server.py # SDK MCP stdio（--mcp）与旧 JSONL',
+ '- 接入 GitHub、YouTube、OpenAlex、Semantic Scholar、Hugging Face、Papers with Code 的官方 API adapter；': '- 继续接入 YouTube、OpenAlex、Semantic Scholar、Hugging Face、Papers with Code adapter；',
+ '- 打包正式 MCP SDK server。': '- 补充编辑器 GUI 的 MCP 连接验证。',
+ }, '''
+## 当前能力与限制
+
+CLI `roadmap` 默认离线；`--live --sources arxiv,github` 显式调用公开元数据 API。MCP `searchResources` 传 `filters={"offline": false}` 启用实时检索，`type`、`language`、`difficulty` 会真正过滤资源。其他 registry 平台目前只有链接/目录能力。返回 `offline/live/cache/mixed/degraded` 和降级原因；失败不会伪造查询结果。默认超时 5 秒、同一服务实例请求至少间隔 3 秒、磁盘缓存 1 小时，缓存目录可由 `FIELDS_STUDY_FLOW_CACHE` 设置。不使用付费 API 或认证内容。
+
+`ingestUrl` 默认只录入 URL；`fetchMetadata=true` 从公开 GitHub/arXiv API 获取元数据，拒绝凭据和重定向，其他来源明确降级。GitHub 的 `language=en` 表示元数据界面语言；编程语言另存 metadata，不保证仓库正文语言分类。
+
+标准 MCP 启动命令为 `python -m fields_study_flow.mcp_server --mcp`，两份编辑器配置已同步。无参数启动和 `--legacy-jsonl` 保留旧 JSONL，`git4study` 入口兼容。协议版本由官方 SDK 2.x 协商，详见 [MCP 文档](docs/MCP.md)。本地 SDK 测试涵盖握手、8 工具 schema、调用和错误；编辑器 GUI 实际连接仍需外部验收。
+
+排序输出相关性、可信度、结构、难度、成果偏好、语言等分项及缺失词项。小型中英别名表和领域等级避免全局降级；仍是词汇匹配，不是语义模型。30 条固定样例位于 `tests/fixtures/ranking_cases.json`。低相关性标为补充资料，不声称匹配。先修图覆盖数学/编程/ML/DL/Transformer/LLM，明确跳过已知主题；工时产生相对周排期，缺失时长标 unknown，期限不足会提示。目标论文放在核心阶段。工时是人工估算，不能保证实际完成时间。
+
+合规校验使用解析后的 URL 主机/协议和 `metadata.access_policy` 的结构化动作（metadata/link/summarize）；旧记录的肯定违规指令保留兼容校验，不把生成的否定安全说明当成访问操作。
+''')]:
+ path = Path(name)
+ text = path.read_text(encoding='utf-8')
+ for old, new in replacements.items():
+ text = text.replace(old, new)
+ path.write_text(text + '\n' + extra, encoding='utf-8')
diff --git a/source-registry.yaml b/source-registry.yaml
index 8957575..26e24d4 100644
--- a/source-registry.yaml
+++ b/source-registry.yaml
@@ -5,200 +5,351 @@
"id": "github",
"name": "GitHub",
"category": "code-learning",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search repositories, topics, READMEs, notebooks, awesome lists, and paper implementations through official API or user-provided URLs.",
- "quality_signals": ["license", "readme_structure", "curriculum", "notebooks", "recent_activity", "issue_quality", "stars_as_weak_signal"],
+ "quality_signals": [
+ "license",
+ "readme_structure",
+ "curriculum",
+ "notebooks",
+ "recent_activity",
+ "issue_quality",
+ "stars_as_weak_signal"
+ ],
"restricted": false,
- "notes": "Treat stars and forks as weak popularity signals, not learning quality."
+ "notes": "Implemented: public metadata adapter, bounded timeout, disk cache, no authentication. Treat stars and forks as weak popularity signals, not learning quality."
},
{
"id": "arxiv",
"name": "arXiv",
"category": "academic",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search metadata and link to abstract/PDF pages for open preprints.",
- "quality_signals": ["paper_title", "abstract", "authors", "citations_via_cross_source", "recency"],
- "restricted": false
+ "quality_signals": [
+ "paper_title",
+ "abstract",
+ "authors",
+ "citations_via_cross_source",
+ "recency"
+ ],
+ "restricted": false,
+ "notes": "Implemented: public metadata adapter, bounded timeout, disk cache, no authentication. "
},
{
"id": "openalex",
"name": "OpenAlex",
"category": "academic",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search scholarly metadata, concepts, citations, venues, and open access status.",
- "quality_signals": ["citation_graph", "concepts", "open_access", "venue", "recency"],
- "restricted": false
+ "quality_signals": [
+ "citation_graph",
+ "concepts",
+ "open_access",
+ "venue",
+ "recency"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "semantic-scholar",
"name": "Semantic Scholar",
"category": "academic",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search paper metadata, recommendations, references, and citations within API terms.",
- "quality_signals": ["citations", "references", "recommendations", "fields_of_study"],
- "restricted": false
+ "quality_signals": [
+ "citations",
+ "references",
+ "recommendations",
+ "fields_of_study"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "unpaywall",
"name": "Unpaywall",
"category": "academic",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Resolve legal open-access full-text locations from DOI metadata.",
- "quality_signals": ["open_access_status", "host_type", "license"],
- "restricted": false
+ "quality_signals": [
+ "open_access_status",
+ "host_type",
+ "license"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "papers-with-code",
"name": "Papers with Code",
"category": "academic-code",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api-or-client",
"auth_required": false,
"allowed_use": "Discover papers, tasks, datasets, methods, leaderboards, and linked code.",
- "quality_signals": ["task_match", "dataset_match", "official_code", "leaderboard_presence"],
- "restricted": false
+ "quality_signals": [
+ "task_match",
+ "dataset_match",
+ "official_code",
+ "leaderboard_presence"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "youtube",
"name": "YouTube",
"category": "video",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "official-api",
"auth_required": true,
"allowed_use": "Search public video and playlist metadata with YouTube Data API. Do not download videos.",
- "quality_signals": ["channel_authority", "playlist_structure", "chapters", "captions_available", "duration"],
- "restricted": false
+ "quality_signals": [
+ "channel_authority",
+ "playlist_structure",
+ "chapters",
+ "captions_available",
+ "duration"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "bilibili",
"name": "Bilibili",
"category": "video-cn",
- "languages": ["zh-CN"],
+ "languages": [
+ "zh-CN"
+ ],
"access_mode": "official-or-url-only",
"auth_required": false,
"allowed_use": "Prefer official/open platform capabilities. If search is unavailable, parse user-provided BV/URL metadata only.",
- "quality_signals": ["uploader_authority", "series_structure", "duration", "danmaku_comment_quality_as_weak_signal"],
- "restricted": false
+ "quality_signals": [
+ "uploader_authority",
+ "series_structure",
+ "duration",
+ "danmaku_comment_quality_as_weak_signal"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "zhihu",
"name": "Zhihu",
"category": "community-cn",
- "languages": ["zh-CN"],
+ "languages": [
+ "zh-CN"
+ ],
"access_mode": "official-or-url-only",
"auth_required": false,
"allowed_use": "Use official/open platform or user-provided URLs. Summarize and link; do not bulk scrape or bypass login.",
- "quality_signals": ["author_authority", "answer_structure", "citations", "recency"],
- "restricted": false
+ "quality_signals": [
+ "author_authority",
+ "answer_structure",
+ "citations",
+ "recency"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "hugging-face",
"name": "Hugging Face",
"category": "practice",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "official-api",
"auth_required": false,
"allowed_use": "Search models, datasets, Spaces, model cards, dataset cards, and course pages.",
- "quality_signals": ["card_quality", "downloads", "likes_as_weak_signal", "license", "task_match"],
- "restricted": false
+ "quality_signals": [
+ "card_quality",
+ "downloads",
+ "likes_as_weak_signal",
+ "license",
+ "task_match"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "kaggle",
"name": "Kaggle",
"category": "practice",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "official-api",
"auth_required": true,
"allowed_use": "Search datasets, notebooks, competitions, and practice projects with user-configured credentials.",
- "quality_signals": ["dataset_usability", "notebook_votes_as_weak_signal", "competition_relevance", "recency"],
- "restricted": false
+ "quality_signals": [
+ "dataset_usability",
+ "notebook_votes_as_weak_signal",
+ "competition_relevance",
+ "recency"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "mit-ocw",
"name": "MIT OpenCourseWare",
"category": "course",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "public-web-or-catalog",
"auth_required": false,
"allowed_use": "Recommend course pages and materials by link. Do not mirror course content.",
- "quality_signals": ["institution", "syllabus", "assignments", "lecture_notes"],
- "restricted": false
+ "quality_signals": [
+ "institution",
+ "syllabus",
+ "assignments",
+ "lecture_notes"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "google-mlcc",
"name": "Google Machine Learning Crash Course",
"category": "course",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "public-web",
"auth_required": false,
"allowed_use": "Recommend official course pages by link.",
- "quality_signals": ["official_docs", "exercises", "curriculum"],
- "restricted": false
+ "quality_signals": [
+ "official_docs",
+ "exercises",
+ "curriculum"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "fast-ai",
"name": "fast.ai",
"category": "course",
- "languages": ["en"],
+ "languages": [
+ "en"
+ ],
"access_mode": "public-web",
"auth_required": false,
"allowed_use": "Recommend official course pages and notebooks by link.",
- "quality_signals": ["course_sequence", "notebooks", "assignments", "community"],
- "restricted": false
+ "quality_signals": [
+ "course_sequence",
+ "notebooks",
+ "assignments",
+ "community"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "deep-learning-ai",
"name": "DeepLearning.AI",
"category": "course",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "link-only",
"auth_required": false,
"allowed_use": "Recommend public course landing pages and free resources by link.",
- "quality_signals": ["instructor_authority", "course_sequence", "official_curriculum"],
- "restricted": false
+ "quality_signals": [
+ "instructor_authority",
+ "course_sequence",
+ "official_curriculum"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "coursera",
"name": "Coursera",
"category": "course",
- "languages": ["en", "zh-CN"],
+ "languages": [
+ "en",
+ "zh-CN"
+ ],
"access_mode": "link-only",
"auth_required": true,
"allowed_use": "Recommend public landing pages only; do not access paywalled content.",
- "quality_signals": ["institution", "syllabus", "reviews_as_weak_signal"],
- "restricted": true
+ "quality_signals": [
+ "institution",
+ "syllabus",
+ "reviews_as_weak_signal"
+ ],
+ "restricted": true,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "xuetangx",
"name": "学堂在线",
"category": "course-cn",
- "languages": ["zh-CN"],
+ "languages": [
+ "zh-CN"
+ ],
"access_mode": "link-only",
"auth_required": false,
"allowed_use": "Recommend public course pages by link.",
- "quality_signals": ["institution", "syllabus", "course_sequence"],
- "restricted": false
+ "quality_signals": [
+ "institution",
+ "syllabus",
+ "course_sequence"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
},
{
"id": "icourse163",
"name": "中国大学 MOOC",
"category": "course-cn",
- "languages": ["zh-CN"],
+ "languages": [
+ "zh-CN"
+ ],
"access_mode": "link-only",
"auth_required": false,
"allowed_use": "Recommend public course pages by link.",
- "quality_signals": ["institution", "syllabus", "course_sequence"],
- "restricted": false
+ "quality_signals": [
+ "institution",
+ "syllabus",
+ "course_sequence"
+ ],
+ "restricted": false,
+ "notes": "Link/catalog only: no live adapter implemented. "
}
]
}
diff --git a/tests/fixtures/ranking_cases.json b/tests/fixtures/ranking_cases.json
new file mode 100644
index 0000000..a97727f
--- /dev/null
+++ b/tests/fixtures/ranking_cases.json
@@ -0,0 +1,812 @@
+[
+ {
+ "goal": "transformer",
+ "resources": [
+ {
+ "title": "transformer",
+ "url": "https://example.org/0",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "transformer"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "transformer",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "TRANSFORMER",
+ "resources": [
+ {
+ "title": "transformer",
+ "url": "https://example.org/1",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "transformer"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "transformer",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "diffusion",
+ "resources": [
+ {
+ "title": "diffusion",
+ "url": "https://example.org/2",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "diffusion"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "diffusion",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "DIFFUSION",
+ "resources": [
+ {
+ "title": "diffusion",
+ "url": "https://example.org/3",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "diffusion"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "diffusion",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "python",
+ "resources": [
+ {
+ "title": "python",
+ "url": "https://example.org/4",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "python"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "python",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "PYTHON",
+ "resources": [
+ {
+ "title": "python",
+ "url": "https://example.org/5",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "python"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "python",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "probability",
+ "resources": [
+ {
+ "title": "probability",
+ "url": "https://example.org/6",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "probability"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "probability",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "PROBABILITY",
+ "resources": [
+ {
+ "title": "probability",
+ "url": "https://example.org/7",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "probability"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "probability",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "database",
+ "resources": [
+ {
+ "title": "database",
+ "url": "https://example.org/8",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "database"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "database",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "DATABASE",
+ "resources": [
+ {
+ "title": "database",
+ "url": "https://example.org/9",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "database"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "database",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "transaction",
+ "resources": [
+ {
+ "title": "transaction",
+ "url": "https://example.org/10",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "transaction"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "transaction",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "TRANSACTION",
+ "resources": [
+ {
+ "title": "transaction",
+ "url": "https://example.org/11",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "transaction"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "transaction",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "attention",
+ "resources": [
+ {
+ "title": "attention",
+ "url": "https://example.org/12",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "attention"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "attention",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "ATTENTION",
+ "resources": [
+ {
+ "title": "attention",
+ "url": "https://example.org/13",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "attention"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "attention",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "cnn",
+ "resources": [
+ {
+ "title": "cnn",
+ "url": "https://example.org/14",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "cnn"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "cnn",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "CNN",
+ "resources": [
+ {
+ "title": "cnn",
+ "url": "https://example.org/15",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "cnn"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "cnn",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "yolo",
+ "resources": [
+ {
+ "title": "yolo",
+ "url": "https://example.org/16",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "yolo"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "yolo",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "YOLO",
+ "resources": [
+ {
+ "title": "yolo",
+ "url": "https://example.org/17",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "yolo"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "yolo",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "ppo",
+ "resources": [
+ {
+ "title": "ppo",
+ "url": "https://example.org/18",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "ppo"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "ppo",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "PPO",
+ "resources": [
+ {
+ "title": "ppo",
+ "url": "https://example.org/19",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "ppo"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "ppo",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "trpo",
+ "resources": [
+ {
+ "title": "trpo",
+ "url": "https://example.org/20",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "trpo"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "trpo",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "TRPO",
+ "resources": [
+ {
+ "title": "trpo",
+ "url": "https://example.org/21",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "trpo"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "trpo",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "linear algebra",
+ "resources": [
+ {
+ "title": "linear algebra",
+ "url": "https://example.org/22",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "linear algebra"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "linear algebra",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "LINEAR ALGEBRA",
+ "resources": [
+ {
+ "title": "linear algebra",
+ "url": "https://example.org/23",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "linear algebra"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "linear algebra",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "machine learning",
+ "resources": [
+ {
+ "title": "machine learning",
+ "url": "https://example.org/24",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "machine learning"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "machine learning",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "MACHINE LEARNING",
+ "resources": [
+ {
+ "title": "machine learning",
+ "url": "https://example.org/25",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "machine learning"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "machine learning",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "deep learning",
+ "resources": [
+ {
+ "title": "deep learning",
+ "url": "https://example.org/26",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "deep learning"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "deep learning",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "DEEP LEARNING",
+ "resources": [
+ {
+ "title": "deep learning",
+ "url": "https://example.org/27",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "deep learning"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "deep learning",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "optimization",
+ "resources": [
+ {
+ "title": "optimization",
+ "url": "https://example.org/28",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "optimization"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "optimization",
+ "unrelated css"
+ ]
+ },
+ {
+ "goal": "OPTIMIZATION",
+ "resources": [
+ {
+ "title": "optimization",
+ "url": "https://example.org/29",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "optimization"
+ ]
+ },
+ {
+ "title": "unrelated css",
+ "url": "https://example.org/css",
+ "source": "course",
+ "type": "article",
+ "concepts": [
+ "css"
+ ]
+ }
+ ],
+ "expected_titles": [
+ "optimization",
+ "unrelated css"
+ ]
+ }
+]
\ No newline at end of file
diff --git a/tests/test_adapters.py b/tests/test_adapters.py
new file mode 100644
index 0000000..8b84142
--- /dev/null
+++ b/tests/test_adapters.py
@@ -0,0 +1,96 @@
+import json
+import pytest
+from fields_study_flow.sources_adapters import SearchService, Response
+
+ATOM = b'''http://arxiv.org/abs/2501.12345v1Graph learninggraph networks'''
+GITHUB = json.dumps({'items': [{'id': 42, 'full_name': 'example/graph', 'html_url': 'https://github.com/example/graph', 'description': 'Graph learning', 'language': 'Python', 'topics': ['graph'], 'stargazers_count': 3}]}).encode()
+
+
+def test_adapters_normalize_cache_and_filters(tmp_path):
+ calls = []
+ def transport(url, timeout):
+ calls.append(url)
+ return Response(200, GITHUB if 'api.github.com' in url else ATOM, url)
+ service = SearchService(cache_dir=tmp_path, transport=transport, min_interval=0)
+ live = service.search('graph', sources=['arxiv', 'github'], filters={'language': 'en'})
+ assert live['mode'] == 'live' and len(live['resources']) == 2
+ for r in live['resources']:
+ assert r['metadata']['id'] and r['metadata']['retrieved_at']
+ assert r['source'] in ['arxiv', 'github'] and r['url'].startswith('https://')
+ assert r['language'] == 'en' and r['title']
+ cached = service.search('graph', sources=['arxiv', 'github'], filters={'language': 'en'})
+ assert cached['mode'] == 'cache' and len(calls) == 2
+ papers = service.search('graph', sources=['arxiv', 'github'], filters={'type': 'paper'})
+ assert len(papers['resources']) == 1
+ assert not service.search('graph', sources=['youtube'])['resources']
+
+
+@pytest.mark.parametrize('response,reason', [(Response(429,b'', 'https://api.github.com/search/repositories'),'429'), (Response(200,b'not json','https://api.github.com/search/repositories'),'JSON'), (Response(200,GITHUB,'https://evil.example/redirect'),'redirect')])
+def test_failures_are_explicit(tmp_path, response, reason):
+ def transport(url, timeout):
+ return Response(response.status, response.body, url if 'evil' not in response.url else response.url)
+ service = SearchService(cache_dir=tmp_path, transport=transport, min_interval=0)
+ result = service.search('graph', sources=['github'])
+ assert result['mode'] == 'degraded' and not result['resources']
+ assert reason.lower() in str(result['degraded']).lower()
+
+
+def test_timeout_offline_and_credentials(tmp_path):
+ def timeout(*args):
+ raise TimeoutError('fixture timeout')
+ service = SearchService(cache_dir=tmp_path, transport=timeout, min_interval=0)
+ result = service.search('unseen-topic', sources=['arxiv'])
+ assert result['mode'] == 'degraded' and 'timeout' in str(result['degraded'])
+ assert service.search('unseen-topic', offline=True)['mode'] == 'offline'
+ for url in ['https://user:secret@github.com/a/b', 'http://127.0.0.1/admin', 'https://github.com.evil.example/a/b']:
+ with pytest.raises(ValueError):
+ service.ingest(url)
+
+
+def test_ingest_public_api_metadata(tmp_path):
+ def transport(url, timeout):
+ return Response(200, json.dumps({'id':42,'full_name':'example/graph','html_url':'https://github.com/example/graph','description':'Graph learning'}).encode(), url)
+ service = SearchService(cache_dir=tmp_path, transport=transport, min_interval=0)
+ resource = service.ingest('https://github.com/example/graph')
+ assert resource['metadata']['id'] == 'github:42'
+ assert resource['metadata']['retrieved_at']
+
+
+def test_ingest_tool_can_request_public_metadata(monkeypatch):
+ from fields_study_flow.mcp_tools import ingestUrl
+ def fake(self, url):
+ if 'example.org' in url:
+ raise ValueError('unsupported host')
+ return {'title': 'Public fixture', 'url': url}
+ monkeypatch.setattr(SearchService, 'ingest', fake)
+ assert ingestUrl('https://github.com/example/graph', fetchMetadata=True)['title'] == 'Public fixture'
+ assert ingestUrl('https://example.org', fetchMetadata=True)['metadata']['retrieval_mode'] == 'degraded'
+
+
+def test_cli_ingest_public_metadata_flag(monkeypatch, capsys):
+ from fields_study_flow.cli import main
+ monkeypatch.setattr(SearchService, 'ingest', lambda self, url: {'title': 'fixture', 'url': url})
+ assert main(['ingest-url', 'https://github.com/example/graph', '--fetch-metadata']) == 0
+ assert json.loads(capsys.readouterr().out)['title'] == 'fixture'
+
+
+def test_cli_uses_shared_service(tmp_path, monkeypatch):
+ from fields_study_flow.cli import main
+ calls = []
+ def fake(self, query, sources=None, filters=None, offline=False):
+ calls.append((query, sources, offline))
+ return {'resources': [], 'mode': 'degraded', 'degraded': ['fixture timeout']}
+ monkeypatch.setattr(SearchService, 'search', fake)
+ assert main(['roadmap', '--goal', 'graph', '--sources', 'arxiv', '--live', '--output-dir', str(tmp_path)]) == 0
+ plan = json.loads((tmp_path / 'roadmap.json').read_text(encoding='utf-8'))
+ assert calls == [('graph', ['arxiv'], False)]
+ assert plan['retrieval']['degraded'] == ['fixture timeout']
+
+
+def test_mcp_search_filters_and_degraded_metadata():
+ from fields_study_flow.mcp_tools import searchResources, discoverSources
+ result = searchResources('diffusion', filters={'offline': True, 'type': 'paper'})
+ assert result['mode'] == 'offline'
+ assert all(r['type'] == 'paper' for r in result['resources'])
+ sources = discoverSources('graph')['sources']
+ assert all(s['adapter_status'] == ('implemented' if s['id'] in {'github', 'arxiv'} else 'link-only') for s in sources)
diff --git a/tests/test_mcp_protocol.py b/tests/test_mcp_protocol.py
new file mode 100644
index 0000000..cc98037
--- /dev/null
+++ b/tests/test_mcp_protocol.py
@@ -0,0 +1,49 @@
+import asyncio
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from mcp.client.session import ClientSession
+from mcp.client.stdio import StdioServerParameters, stdio_client
+
+
+import pytest
+
+
+@pytest.mark.parametrize('config_path', ['.cursor/mcp.json', '.vscode/mcp.json'])
+def test_sdk_stdio_round_trip(config_path):
+ async def run():
+ config = json.loads(Path(config_path).read_text())
+ server = config.get('mcpServers', config.get('servers'))['fields-study-flow']
+ params = StdioServerParameters(command=sys.executable, args=server['args'])
+ async with stdio_client(params) as streams:
+ async with ClientSession(*streams) as session:
+ initialized = await session.initialize()
+ assert initialized.server_info.name == 'fields-study-flow'
+ tools = await session.list_tools()
+ assert len(tools.tools) == 8
+ assert all(t.input_schema.get('type') == 'object' for t in tools.tools)
+ result = await session.call_tool('buildRoadmap', {'goal': 'transformer', 'profile': {}, 'rankedResources': []})
+ assert not result.is_error
+ assert 'phases' in result.content[0].text
+ for name, args in [('unknown', {}), ('assessKnowledge', {})]:
+ bad = await session.call_tool(name, args)
+ assert bad.is_error
+ asyncio.run(asyncio.wait_for(run(), timeout=20))
+
+
+def test_legacy_jsonl_is_explicit_and_compatible():
+ for module, flag in [('fields_study_flow.mcp_server', '--legacy-jsonl'), ('fields_study_flow.mcp_server', None), ('git4study.mcp_server', None)]:
+ args = [sys.executable, '-m', module] + ([flag] if flag else [])
+ result = subprocess.run(args, input=json.dumps({'tool': 'assessKnowledge', 'arguments': {'goal': 'python'}})+'\n', text=True, capture_output=True, timeout=15)
+ assert result.returncode == 0
+ assert json.loads(result.stdout)['result']['goal'] == 'python'
+
+
+def test_editor_configs_select_mcp():
+ for directory in ('.cursor', '.vscode'):
+ config = json.loads((Path(directory) / 'mcp.json').read_text())
+ servers = config.get('mcpServers', config.get('servers'))
+ server = servers['fields-study-flow']
+ assert server['args'] == ['-m', 'fields_study_flow.mcp_server', '--mcp']
diff --git a/tests/test_mcp_tools.py b/tests/test_mcp_tools.py
index 7bd66c2..2e3d87e 100644
--- a/tests/test_mcp_tools.py
+++ b/tests/test_mcp_tools.py
@@ -1,4 +1,23 @@
-from fields_study_flow.mcp_tools import searchResources, validateSources
+from fields_study_flow.mcp_tools import searchResources, validateSources, ingestUrl, buildRoadmap
+
+
+def test_imported_safe_notes_and_parsed_host_validation():
+ for url in ['HTTPS://ARXIV.ORG/abs/1706.03762?utm_source=a', 'https://example.org/?next=sci-hub']:
+ resource = ingestUrl(url)
+ plan = buildRoadmap('paper', {}, [resource])
+ assert validateSources(plan)['valid']
+ safe = {'title': 'safe', 'url': 'https://example.org', 'license_or_access_note': 'do not bypass login; do not copy long copyrighted passages; do not download video'}
+ assert validateSources({'phases': [{'resources': [safe]}]})['valid']
+
+
+def test_structured_access_policy_rejects_unsafe_operations():
+ for policy in [{'action': 'download_video'}, {'action': 'hotlink'}, {'bypass_login': True}, {'authorized': False}, {'mirror': True}]:
+ resource = ingestUrl('https://example.org/open')
+ resource['metadata']['access_policy'] = policy
+ assert not validateSources({'phases': [{'resources': [resource]}]})['valid']
+ for url in ['https://SCI-HUB.example/p?utm_source=x', 'ftp://example.org/a', 'https://user:password@example.org/a']:
+ resource = {'url': url, 'title': 'bad', 'license_or_access_note': 'link'}
+ assert not validateSources({'phases': [{'resources': [resource]}]})['valid']
def test_search_resources_applies_hard_language_filter_at_tool_boundary():
diff --git a/tests/test_ranking.py b/tests/test_ranking.py
index 2d0822a..d135723 100644
--- a/tests/test_ranking.py
+++ b/tests/test_ranking.py
@@ -1,6 +1,56 @@
from fields_study_flow.language import ResourceLanguagePreference
from fields_study_flow.models import LearnerProfile, Resource
-from fields_study_flow.ranking import rank_resources
+from fields_study_flow.ranking import rank_resources, _canonical_url_key
+
+
+def test_platform_identity_fixture():
+ urls = [
+ 'https://youtube.com/watch?v=ONE',
+ 'https://youtube.com/watch?v=TWO',
+ 'https://youtube.com/watch?v=ONE&si=track&utm_source=x',
+ 'https://youtu.be/ONE?si=track',
+ 'https://youtube.com/playlist?list=A',
+ 'https://youtube.com/playlist?list=B',
+ 'https://bilibili.com/video/BVAbC?utm_source=x',
+ 'https://bilibili.com/video/BVAbD',
+ 'https://example.org/Path?id=1',
+ 'https://example.org/Path?id=2',
+ 'https://example.org/path?id=1',
+ ]
+ keys = [_canonical_url_key(u) for u in urls]
+ assert keys[0] == keys[2] == keys[3]
+ assert len(set(keys)) == 9
+ assert _canonical_url_key('https://github.com/Owner/Repo.git') == _canonical_url_key('https://github.com/owner/repo')
+
+
+def test_relevance_domain_aliases_and_outcome():
+ from fields_study_flow.ranking import _difficulty_fit
+ unrelated = Resource(title='Web CSS', url='https://example.org/web', source='course', type='article', concepts=['css'])
+ ranked = rank_resources([unrelated], LearnerProfile(goal='数据库事务'))
+ assert 'matches your goal' not in ranked[0].why_recommended
+ assert ranked[0].score_components['relevance'] == 0
+ assert ranked[0].missing_concepts
+ resource = Resource(title='Transformer', url='https://example.org/t', source='course', type='article', concepts=['transformer'], difficulty='advanced')
+ a = LearnerProfile(goal='transformer', levels={'ml_dl': 'advanced', 'programming': 'beginner'})
+ b = LearnerProfile(goal='transformer', levels={'ml_dl': 'advanced'})
+ assert _difficulty_fit(resource, a) == _difficulty_fit(resource, b) == 1
+ assert rank_resources([resource], LearnerProfile(goal='变换器'))[0].score == rank_resources([resource], LearnerProfile(goal='transformer'))[0].score
+ paper = Resource(title='Transformer', url='https://example.org/p', source='arxiv', type='paper', concepts=['transformer'])
+ code = Resource(title='Transformer', url='https://example.org/c', source='github', type='repository', concepts=['transformer'])
+ assert rank_resources([paper, code], b, target_outcome='reproduce')[0].type == 'repository'
+ assert rank_resources([code, paper], b, target_outcome='derive')[0].type == 'paper'
+
+
+def test_fixed_thirty_examples_are_reviewable():
+ import json
+ from pathlib import Path
+ cases = json.loads((Path(__file__).parent / 'fixtures' / 'ranking_cases.json').read_text(encoding='utf-8'))
+ assert len(cases) == 30
+ for case in cases:
+ resources = [Resource(**r) for r in case['resources']]
+ ranked = rank_resources(resources, LearnerProfile(goal=case['goal']))
+ assert [r.title for r in ranked] == case['expected_titles']
+ assert all('relevance' in r.score_components for r in ranked)
def test_rank_resources_values_teaching_structure_over_stars_only():
diff --git a/tests/test_roadmap.py b/tests/test_roadmap.py
index 223ca9c..21d924c 100644
--- a/tests/test_roadmap.py
+++ b/tests/test_roadmap.py
@@ -3,6 +3,43 @@
from fields_study_flow.roadmap import build_roadmap
+def test_invalid_profile_time_rejected():
+ import pytest
+ for hours in [-1, 0, True, '8', float('nan'), 169]:
+ with pytest.raises(ValueError):
+ LearnerProfile(goal='x', weekly_hours=hours)
+ for date in ['tomorrow', '2026-02-30', '2026-1-1']:
+ with pytest.raises(ValueError):
+ LearnerProfile(goal='x', target_date=date)
+
+
+def test_prerequisites_schedule_known_and_anchor():
+ from fields_study_flow.roadmap import render_markdown
+ target = Resource(title='Target Transformer', url='https://arxiv.org/abs/1706.03762', source='arxiv', type='paper', concepts=['transformer'], estimated_time='6h', metadata={'target_paper': True})
+ slow = build_roadmap(LearnerProfile(goal='transformer', weekly_hours=1, known_topics=['python']), [target])
+ fast = build_roadmap(LearnerProfile(goal='transformer', weekly_hours=20, known_topics=['python']), [target])
+ nodes = slow['prerequisites']
+ ids = [n['id'] for n in nodes]
+ for node in nodes:
+ assert all(ids.index(dep) < ids.index(node['id']) for dep in node['depends_on'])
+ assert next(n for n in nodes if n['id'] == 'python')['status'] == 'skipped'
+ assert next(n for n in nodes if n['id'] == 'python')['reason']
+ assert slow['schedule']['total_weeks'] > fast['schedule']['total_weeks']
+ assert [s['id'] for s in slow['schedule']['items']] == [s['id'] for s in fast['schedule']['items']]
+ assert slow['schedule']['target_date'] is None
+ assert all(not r['metadata'].get('target_paper') for r in slow['phases'][0]['resources'])
+ assert 'Checkpoints' in render_markdown(slow)
+ assert 'Schedule' in render_markdown(slow)
+ unknown = build_roadmap(LearnerProfile(goal='x', weekly_hours=2), [Resource(title='x',url='https://example.org',source='course',type='article')])
+ assert unknown['schedule']['unknown_estimates']
+
+
+def test_deadline_shortfall_is_explicit():
+ from datetime import date
+ plan = build_roadmap(LearnerProfile(goal='transformer', weekly_hours=1, target_date=date.today().isoformat()), [])
+ assert plan['schedule']['deadline_feasible'] is False
+
+
def test_build_roadmap_contains_required_files_and_resource_fields():
profile = LearnerProfile(
goal="从线代概率到理解 diffusion models",
From 630a92162b9efe5230eb5cff72cdc360477845dc Mon Sep 17 00:00:00 2001
From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com>
Date: Sun, 18 Oct 2026 18:39:24 +0800
Subject: [PATCH 2/7] fix: validate source policies and metadata with shared
transport limits
---
fields_study_flow/concepts.py | 53 ++++++++++++++++
fields_study_flow/mcp_tools.py | 36 +++++++----
fields_study_flow/ranking.py | 33 +---------
fields_study_flow/sources_adapters.py | 83 +++++++++++++++++++++----
tests/test_review_parsing.py | 55 +++++++++++++++++
tests/test_review_policy.py | 49 +++++++++++++++
tests/test_review_ranking.py | 36 +++++++++++
tests/test_review_transport.py | 88 +++++++++++++++++++++++++++
8 files changed, 380 insertions(+), 53 deletions(-)
create mode 100644 fields_study_flow/concepts.py
create mode 100644 tests/test_review_parsing.py
create mode 100644 tests/test_review_policy.py
create mode 100644 tests/test_review_ranking.py
create mode 100644 tests/test_review_transport.py
diff --git a/fields_study_flow/concepts.py b/fields_study_flow/concepts.py
new file mode 100644
index 0000000..0ba02e5
--- /dev/null
+++ b/fields_study_flow/concepts.py
@@ -0,0 +1,53 @@
+"""Shared bilingual concept identities for ranking and prerequisite matching."""
+import re
+
+
+ALIASES = {
+ 'transformer': ('变换器', 'transformer'),
+ 'diffusion': ('扩散', 'diffusion'),
+ 'linear algebra': ('线性代数', '线代', 'linear algebra'),
+ 'probability': ('概率', 'probability'),
+ 'machine learning': ('机器学习', 'machine learning'),
+ 'deep learning': ('深度学习', 'deep learning'),
+ 'python': ('编程', 'python'),
+ 'database': ('数据库', 'database'),
+ 'transaction': ('事务', 'transaction'),
+ 'attention': ('注意力', 'attention'),
+}
+DOMAINS = {
+ 'programming': {'python'},
+ 'math': {'linear algebra', 'probability', 'math'},
+ 'ml_dl': {'transformer', 'diffusion', 'machine learning', 'deep learning', 'llm', 'yolo', 'cnn', 'ppo', 'trpo', 'attention'},
+}
+
+_ALIAS_TO_CONCEPT = {alias: canonical for canonical, aliases in ALIASES.items() for alias in aliases}
+_PATTERN = re.compile('|'.join(
+ r'(? set[str]:
+ normalized = value.lower().replace('/', ' ').replace('-', ' ')
+ concepts = set()
+
+ def consume(match):
+ concepts.add(_ALIAS_TO_CONCEPT[match.group()])
+ return ' '
+
+ remaining = _PATTERN.sub(consume, normalized)
+ concepts.update(re.findall(r'[\w]+', remaining))
+ return concepts
+
+
+def concept_id(value: str) -> str:
+ normalized = ' '.join(value.lower().replace('-', ' ').split())
+ return _ALIAS_TO_CONCEPT.get(normalized, normalized)
+
+
+def dominant_domain(value: str) -> str | None:
+ terms = tokenize(value)
+ counts = {domain: len(terms & concepts) for domain, concepts in DOMAINS.items()}
+ maximum = max(counts.values())
+ matches = [domain for domain, count in counts.items() if count == maximum]
+ return matches[0] if maximum and len(matches) == 1 else None
diff --git a/fields_study_flow/mcp_tools.py b/fields_study_flow/mcp_tools.py
index c1c752f..f460fc5 100644
--- a/fields_study_flow/mcp_tools.py
+++ b/fields_study_flow/mcp_tools.py
@@ -1,6 +1,7 @@
from __future__ import annotations
from pathlib import Path
+import re
from typing import Any
from fields_study_flow.language import (
@@ -144,6 +145,26 @@ def buildRoadmap(
return build_roadmap(learner, resources)
+def _affirmative_access_actions(note: str) -> list[str]:
+ actions = []
+ action_pattern = r'\b(?:' + '|'.join(re.escape(term) for term in DISALLOWED_ACCESS_NOTE_TERMS) + r')\b'
+ # Only immediate prohibitions and coordinated actions share negation.
+ # Punctuation, adversatives, or a new subject start a new instruction.
+ clauses = re.split(r'[.,;:!?\n]|\b(?:but|however|then)\b|\b(?:and|or)\s+(?=(?:you|we|they|one)\b)', note.lower())
+ for clause in clauses:
+ negated, previous_end = False, 0
+ for match in re.finditer(action_pattern, clause):
+ before, after = clause[previous_end:match.start()], clause[match.end():]
+ direct = re.search(r"\b(?:do not|don't|never|must not|cannot|no)\s+(?:(?:ever|use|run)\s+)*$", before)
+ coordinated = re.fullmatch(r'\s*(?:and|or)\s+(?:(?:use|run)\s+)?', before)
+ negated = bool(direct or (negated and coordinated))
+ prohibited = re.match(r'\s+(?:is|are)\s+(?:prohibited|forbidden|banned|not allowed)\b', after)
+ if not negated and not prohibited:
+ actions.append(match.group())
+ previous_end = match.end()
+ return actions
+
+
def validateSources(plan: dict[str, Any]) -> dict[str, Any]:
issues: list[str] = []
for phase in plan.get("phases", []):
@@ -151,13 +172,12 @@ def validateSources(plan: dict[str, Any]) -> dict[str, Any]:
url = resource.get("url", "")
access_note = resource.get("license_or_access_note", "")
from urllib.parse import urlsplit
- import re
parsed = urlsplit(url)
- host = (parsed.hostname or '').lower()
+ host = (parsed.hostname or '').lower().rstrip('.')
if parsed.scheme.lower() not in {'http', 'https'} or not host or parsed.username or parsed.password:
issues.append(f"Resource has unsafe URL: {resource.get('title')}")
- if any(term in host for term in DISALLOWED_URL_TERMS):
+ if set(host.split('.')) & set(DISALLOWED_URL_TERMS):
issues.append(f"Disallowed source detected: {url}")
policy = resource.get('metadata', {}).get('access_policy')
if policy is not None:
@@ -165,14 +185,8 @@ def validateSources(plan: dict[str, Any]) -> dict[str, Any]:
or policy.get('bypass_login') or policy.get('mirror') or policy.get('authorized') is False):
issues.append(f"Disallowed access policy: {resource.get('title')}")
else:
- # Legacy records encoded operations in notes. Ignore explicitly negated clauses;
- # structured records never infer operations from explanatory prose.
- for clause in re.split(r'[.;]', access_note.lower()):
- if re.search(r'\b(do not|don.t|never|must not)\b', clause):
- continue
- for term in DISALLOWED_ACCESS_NOTE_TERMS:
- if term in clause:
- issues.append(f"Disallowed access instruction ({term}): {resource.get('title')}")
+ for term in _affirmative_access_actions(access_note):
+ issues.append(f"Disallowed access instruction ({term}): {resource.get('title')}")
if not access_note:
issues.append(f"Missing access note: {resource.get('title')}")
return {"valid": not issues, "issues": issues}
diff --git a/fields_study_flow/ranking.py b/fields_study_flow/ranking.py
index 2afdb6d..ce93611 100644
--- a/fields_study_flow/ranking.py
+++ b/fields_study_flow/ranking.py
@@ -4,6 +4,7 @@
import re
from urllib.parse import urlparse
+from fields_study_flow.concepts import dominant_domain, tokenize as _tokenize
from fields_study_flow.language import language_weight
from fields_study_flow.models import LearnerProfile, Resource
@@ -17,22 +18,6 @@
}
-ALIASES = {
- 'transformer': ('变换器', 'transformer'),
- 'diffusion': ('扩散', 'diffusion'),
- 'linear algebra': ('线性代数', '线代', 'linear algebra'),
- 'probability': ('概率', 'probability'),
- 'machine learning': ('机器学习', 'machine learning'),
- 'deep learning': ('深度学习', 'deep learning'),
- 'python': ('编程', 'python'),
- 'database': ('数据库', 'database'),
- 'transaction': ('事务', 'transaction'),
- 'attention': ('注意力', 'attention'),
-}
-DOMAINS = {'programming': {'python'}, 'math': {'linear algebra', 'probability', 'math'},
- 'ml_dl': {'transformer', 'diffusion', 'machine learning', 'deep learning', 'yolo', 'cnn', 'ppo', 'trpo', 'attention'}}
-
-
def rank_resources(resources: list[Resource], profile: LearnerProfile, target_outcome: str = 'understand') -> list[Resource]:
scored: list[Resource] = []
for resource in resources:
@@ -98,8 +83,8 @@ def _structure_score(resource: Resource) -> float:
def _difficulty_fit(resource: Resource, profile: LearnerProfile) -> float:
- terms = _tokenize(profile.goal)
- domain = max(DOMAINS, key=lambda key: len(terms & DOMAINS[key]))
+ domain = (dominant_domain(' '.join(resource.concepts))
+ or dominant_domain(resource.title) or dominant_domain(profile.goal))
level_values = {profile.levels.get(domain, 'familiar')}
if "beginner" in level_values or "初学" in level_values:
target = 0
@@ -135,18 +120,6 @@ def _translation_note(resource: Resource, output_language: str) -> str:
return "No translation needed."
-def _tokenize(value: str) -> set[str]:
- normalized = value.lower().replace("/", " ").replace("-", " ")
- for canonical, aliases in ALIASES.items():
- for alias in aliases:
- normalized = normalized.replace(alias, ' ' + canonical + ' ')
- tokens = {token.strip(".,:;()[]{}") for token in normalized.split() if token.strip()}
- for keyword in ("transformer", "diffusion", "yolo", "ppo", "trpo", "cnn", "python"):
- if keyword in normalized:
- tokens.add(keyword)
- return tokens
-
-
def _dedupe_by_url(resources: list[Resource]) -> list[Resource]:
seen: set[str] = set()
output: list[Resource] = []
diff --git a/fields_study_flow/sources_adapters.py b/fields_study_flow/sources_adapters.py
index 620b98b..e0843d5 100644
--- a/fields_study_flow/sources_adapters.py
+++ b/fields_study_flow/sources_adapters.py
@@ -5,6 +5,8 @@
import json
import os
import time
+import tempfile
+from threading import Lock
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import datetime, timezone
@@ -38,26 +40,46 @@ def public_get(url, timeout):
return Response(exc.code, b'', url)
+# Shared by every service instance in this process, independently per API host.
+_SOURCE_LOCKS = {'api.github.com': Lock(), 'export.arxiv.org': Lock()}
+_LAST_REQUEST: dict[str, float] = {}
+
+
class SearchService:
def __init__(self, cache_dir=None, transport=public_get, timeout=5, ttl=3600, min_interval=3):
self.cache_dir = Path(cache_dir or os.environ.get('FIELDS_STUDY_FLOW_CACHE', Path.home() / '.cache' / 'fields-study-flow'))
self.transport, self.timeout, self.ttl, self.min_interval = transport, timeout, ttl, min_interval
- self.last_request = 0.0
def _fetch(self, url):
key = hashlib.sha256(url.encode()).hexdigest()
path = self.cache_dir / (key + '.json')
try:
data = json.loads(path.read_text(encoding='utf-8'))
+ if not isinstance(data, dict) or not isinstance(data.get('timestamp'), (int, float)):
+ raise ValueError('invalid cache envelope')
+ cached = data.get('resources')
+ if not isinstance(cached, list):
+ raise ValueError('invalid cached resources')
+ for item in cached:
+ if not isinstance(item, dict):
+ raise ValueError('cached resource must be an object')
+ resource = Resource(**item)
+ if (not all(isinstance(getattr(resource, field), str) for field in ('title', 'url', 'source', 'type', 'language', 'difficulty'))
+ or not isinstance(resource.concepts, list)
+ or not all(isinstance(c, str) for c in resource.concepts)
+ or not isinstance(resource.metadata, dict)):
+ raise ValueError('invalid cached resource structure')
if time.time() - data['timestamp'] < self.ttl:
- return data['resources'], 'cache'
- except (OSError, ValueError, KeyError):
+ return cached, 'cache'
+ except (OSError, ValueError, KeyError, TypeError):
pass
- delay = self.min_interval - (time.monotonic() - self.last_request)
- if delay > 0:
- time.sleep(delay)
- self.last_request = time.monotonic()
- response = self.transport(url, self.timeout)
+ host = urlsplit(url).hostname
+ with _SOURCE_LOCKS[host]:
+ delay = self.min_interval - (time.monotonic() - _LAST_REQUEST.get(host, float('-inf')))
+ if delay > 0:
+ time.sleep(delay)
+ _LAST_REQUEST[host] = time.monotonic()
+ response = self.transport(url, self.timeout)
if response.url != url:
raise ValueError('redirect rejected')
if response.status != 200:
@@ -65,29 +87,66 @@ def _fetch(self, url):
if len(response.body) > 2_000_000:
raise ValueError('metadata response too large')
resources = self._normalize(response.body, 'github' if 'api.github.com' in url else 'arxiv')
+ temporary = None
try:
self.cache_dir.mkdir(parents=True, exist_ok=True)
- path.write_text(json.dumps({'timestamp': time.time(), 'resources': resources}), encoding='utf-8')
+ with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=self.cache_dir, suffix='.tmp', delete=False) as handle:
+ temporary = Path(handle.name)
+ json.dump({'timestamp': time.time(), 'resources': resources}, handle)
+ handle.flush()
+ os.fsync(handle.fileno())
+ os.replace(temporary, path)
except OSError:
pass
+ finally:
+ if temporary is not None:
+ temporary.unlink(missing_ok=True)
return resources, 'live'
def _normalize(self, body, source):
+ try:
+ return self._parse_metadata(body, source)
+ except (ValueError, TypeError, KeyError, AttributeError, ET.ParseError) as exc:
+ kind = 'JSON' if source == 'github' else 'XML'
+ raise ValueError(f'{source} {kind} metadata: {exc}') from exc
+
+ def _parse_metadata(self, body, source):
now = datetime.now(timezone.utc).isoformat()
output = []
if source == 'github':
payload = json.loads(body)
+ if not isinstance(payload, dict):
+ raise ValueError('response must be an object')
items = payload.get('items', [payload])
+ if not isinstance(items, list):
+ raise ValueError('items must be an array')
for item in items:
- name = item['full_name']
+ if not isinstance(item, dict):
+ raise ValueError('repository must be an object')
+ name = item.get('full_name')
# Always construct canonical public URLs rather than trusting response links.
- if len(name.split('/')) != 2 or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._/' for c in name):
- raise ValueError('invalid repository name')
+ if (not isinstance(name, str) or len(name.split('/')) != 2 or not all(name.split('/'))
+ or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._/' for c in name)):
+ raise ValueError('invalid repository full_name')
+ if type(item.get('id')) is not int or item['id'] <= 0:
+ raise ValueError('repository id must be a positive integer')
+ topics = item.get('topics', [])
+ if not isinstance(topics, list) or not all(isinstance(topic, str) for topic in topics):
+ raise ValueError('topics must be an array of strings')
+ if type(item.get('stargazers_count', 0)) is not int or item.get('stargazers_count', 0) < 0:
+ raise ValueError('stargazers_count must be a nonnegative integer')
+ for field in ('description', 'language'):
+ if item.get(field) is not None and not isinstance(item[field], str):
+ raise ValueError(f'{field} must be text or null')
output.append(Resource(title=name, url='https://github.com/' + name, source='github', type='repository', language='en', concepts=item.get('topics', []), trust_score=0.7, metadata={'id': f"github:{item['id']}", 'retrieved_at': now, 'source': source, 'description': item.get('description'), 'code_language': item.get('language'), 'stars': item.get('stargazers_count', 0)}).to_dict())
else:
ns = {'a': 'http://www.w3.org/2005/Atom'}
root = ET.fromstring(body)
+ if root.tag != '{http://www.w3.org/2005/Atom}feed':
+ raise ValueError('expected an Atom feed')
for entry in root.findall('a:entry', ns):
+ if not entry.findtext('a:title', '', ns).strip():
+ raise ValueError('entry title is required')
identifier = entry.findtext('a:id', '', ns).rsplit('/abs/', 1)[-1]
import re
if not re.fullmatch(r'(?:\d{4}\.\d{4,5}|[a-z-]+/\d{7})(?:v\d+)?', identifier):
diff --git a/tests/test_review_parsing.py b/tests/test_review_parsing.py
new file mode 100644
index 0000000..c0b8596
--- /dev/null
+++ b/tests/test_review_parsing.py
@@ -0,0 +1,55 @@
+from functools import partial
+import json
+
+import pytest
+
+from fields_study_flow import sources_adapters as adapters
+from fields_study_flow.mcp_tools import ingestUrl, searchResources
+from fields_study_flow.sources_adapters import Response, SearchService
+
+
+@pytest.mark.parametrize('source,body,reason', [
+ ('arxiv', b'', 'Atom feed'),
+ ('arxiv', b'http://arxiv.org/abs/2501.12345', 'title'),
+ ('github', b'[]', 'object'),
+ ('github', b'{"items": {}}', 'items'),
+ ('github', b'{"items": [[]]}', 'object'),
+ ('github', b'{"id":42,"full_name":null}', 'full_name'),
+ ('github', b'{"id":42,"full_name":"a/b","topics":"graph"}', 'topics'),
+ ('github', b'{"id":42,"full_name":"a/b","stargazers_count":{}}', 'stargazers_count'),
+ ('github', b'{"full_name":"a/b"}', 'id'),
+])
+@pytest.mark.parametrize('operation', ['search', 'ingest'])
+def test_invalid_metadata_degrades_at_both_tool_boundaries(tmp_path, monkeypatch, source, body, reason, operation):
+ def transport(url, timeout):
+ return Response(200, body, url)
+
+ monkeypatch.setattr(adapters, 'SearchService', partial(
+ SearchService, cache_dir=tmp_path, transport=transport, min_interval=0))
+ if operation == 'search':
+ result = searchResources('graph', [source], {'offline': False})
+ assert result['mode'] == 'degraded'
+ assert result['resources'] == []
+ explanation = str(result['degraded'])
+ else:
+ url = 'https://github.com/a/b' if source == 'github' else 'https://arxiv.org/abs/2501.12345'
+ result = ingestUrl(url, fetchMetadata=True)
+ assert result['metadata']['retrieval_mode'] == 'degraded'
+ assert result['type'] == 'link'
+ explanation = result['metadata']['degraded']
+ assert reason.lower() in explanation.lower()
+ assert not list(tmp_path.glob('*.json'))
+
+
+@pytest.mark.parametrize('payload', [[], {'timestamp': 99999999999, 'resources': [None]}])
+def test_malformed_cache_is_refetched(tmp_path, payload):
+ def transport(url, timeout):
+ return Response(200, b'{"id":42,"full_name":"a/b"}', url)
+
+ service = SearchService(cache_dir=tmp_path, transport=transport, min_interval=0)
+ service.search('graph', ['github'])
+ next(tmp_path.glob('*.json')).write_text(json.dumps(payload))
+ result = service.search('graph', ['github'])
+ assert result['mode'] == 'live'
+ assert result['resources'][0]['title'] == 'a/b'
diff --git a/tests/test_review_policy.py b/tests/test_review_policy.py
new file mode 100644
index 0000000..2ab4edf
--- /dev/null
+++ b/tests/test_review_policy.py
@@ -0,0 +1,49 @@
+import pytest
+
+from fields_study_flow.mcp_tools import validateSources
+
+
+def validate(note, host='example.org'):
+ resource = {'title': 'fixture', 'url': f'https://{host}/paper',
+ 'license_or_access_note': note}
+ return validateSources({'phases': [{'resources': [resource]}]})
+
+
+@pytest.mark.parametrize('note', [
+ 'yt-dlp is prohibited',
+ 'youtube-dl is not allowed',
+ 'Do not copy full text or download video',
+ 'Never bypass login and copy full text',
+ 'Do not download video; yt-dlp is forbidden',
+ 'Do not use yt-dlp or youtube-dl',
+ 'Do not copy full text and use yt-dlp',
+ 'No download video',
+])
+def test_prohibition_notes_are_allowed(note):
+ assert validate(note)['valid']
+
+
+@pytest.mark.parametrize('note', [
+ 'Do not copy full text, download video',
+ 'Do not copy full text but download video',
+ 'yt-dlp is prohibited; download video',
+ 'Do not bypass login. Use youtube-dl',
+ 'download video is not prohibited',
+ 'No login is required to download video',
+ 'No restrictions: use yt-dlp',
+ 'Do not copy full text and you may download video',
+ 'Do not copy full text! Download video',
+])
+def test_affirmative_action_outside_negation_is_rejected(note):
+ assert not validate(note)['valid']
+
+
+@pytest.mark.parametrize('host,valid', [
+ ('libgenerator.example', True),
+ ('notlibgen.example', True),
+ ('sci-hub.example', False),
+ ('www.libgen.example', False),
+ ('LIBGEN.example.', False),
+])
+def test_prohibited_hosts_match_domain_boundaries(host, valid):
+ assert validate('Link only', host)['valid'] is valid
diff --git a/tests/test_review_ranking.py b/tests/test_review_ranking.py
new file mode 100644
index 0000000..599e82f
--- /dev/null
+++ b/tests/test_review_ranking.py
@@ -0,0 +1,36 @@
+import pytest
+
+from fields_study_flow.models import LearnerProfile, Resource
+from fields_study_flow.ranking import rank_resources
+
+
+def difficulty(goal, concepts, levels):
+ resource = Resource(title='Study material', url='https://example.org/study',
+ source='course', type='course', difficulty='advanced', concepts=concepts)
+ return rank_resources([resource], LearnerProfile(goal=goal, levels=levels))[0].score_components['difficulty']
+
+
+@pytest.mark.parametrize('concept,domain', [
+ ('linear algebra', 'math'), ('线性代数', 'math'),
+ ('machine learning', 'ml_dl'), ('深度学习', 'ml_dl'),
+])
+def test_multiword_concepts_use_their_domain_grade(concept, domain):
+ assert difficulty(concept, [concept], {domain: 'advanced', 'programming': 'beginner'}) == 1.0
+
+
+@pytest.mark.parametrize('concept', ['linear algebra', 'machine learning', 'unmapped topic'])
+def test_unrelated_beginner_level_does_not_reduce_fit(concept):
+ assert difficulty(concept, [concept], {}) == pytest.approx(0.65)
+ assert difficulty(concept, [concept], {'programming': 'beginner'}) == pytest.approx(0.65)
+
+
+def test_resource_domain_takes_precedence_over_overall_goal():
+ assert difficulty('transformer', ['linear algebra'], {'math': 'advanced', 'ml_dl': 'beginner'}) == 1.0
+
+
+def test_multiword_concept_is_not_matched_by_a_single_word():
+ resource = Resource(title='Linear regression', concepts=['linear regression'],
+ url='https://example.org/linear', source='course', type='course')
+ ranked = rank_resources([resource], LearnerProfile(goal='linear algebra'))[0]
+ assert ranked.score_components['relevance'] == 0
+ assert ranked.missing_concepts == ['linear algebra']
diff --git a/tests/test_review_transport.py b/tests/test_review_transport.py
new file mode 100644
index 0000000..b447b1c
--- /dev/null
+++ b/tests/test_review_transport.py
@@ -0,0 +1,88 @@
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+import json
+from pathlib import Path
+from threading import Barrier, Event, Lock
+import time
+
+from fields_study_flow import sources_adapters as adapters
+from fields_study_flow.mcp_tools import ingestUrl, searchResources
+from fields_study_flow.sources_adapters import Response, SearchService
+
+
+REPO = {'id': 42, 'full_name': 'example/graph', 'topics': ['graph']}
+
+
+def test_mcp_calls_share_source_rate_limit_across_instances(tmp_path, monkeypatch):
+ timestamps = []
+
+ def transport(url, timeout):
+ timestamps.append(time.monotonic())
+ return Response(200, json.dumps(REPO).encode(), url)
+
+ monkeypatch.setattr(adapters, 'SearchService', partial(
+ SearchService, cache_dir=tmp_path, transport=transport, min_interval=0.04))
+ assert searchResources('one', ['github'], {'offline': False})['mode'] == 'live'
+ assert searchResources('two', ['github'], {'offline': False})['mode'] == 'live'
+ assert ingestUrl('https://github.com/example/graph', fetchMetadata=True)['metadata']['retrieval_mode'] == 'live'
+ assert len(timestamps) == 3
+ assert all(b - a >= 0.035 for a, b in zip(timestamps, timestamps[1:]))
+
+
+def test_concurrent_instances_share_source_rate_limit(tmp_path):
+ timestamps, lock, start = [], Lock(), Barrier(4)
+
+ def transport(url, timeout):
+ with lock:
+ timestamps.append(time.monotonic())
+ return Response(200, json.dumps(REPO).encode(), url)
+
+ def search(index):
+ service = SearchService(cache_dir=tmp_path / str(index), transport=transport, min_interval=0.04)
+ start.wait(timeout=5)
+ return service.search(str(index), ['github'])
+
+ with ThreadPoolExecutor(max_workers=4) as pool:
+ results = list(pool.map(search, range(4)))
+ assert all(result['mode'] == 'live' for result in results)
+ assert len(timestamps) == 4
+ assert all(b - a >= 0.035 for a, b in zip(timestamps, timestamps[1:]))
+
+
+def test_cache_replacement_keeps_complete_json_during_concurrent_writes(tmp_path, monkeypatch):
+ def transport(url, timeout):
+ return Response(200, json.dumps(REPO).encode(), url)
+
+ initial = SearchService(cache_dir=tmp_path, transport=transport, min_interval=0)
+ initial.search('graph', ['github'])
+ cache_path = next(tmp_path.glob('*.json'))
+ assert json.loads(cache_path.read_text())['resources'][0]['title'] == 'example/graph'
+ started, resume = Event(), Event()
+ original_open = Path.open
+
+ def paused_open(path, mode='r', *args, **kwargs):
+ handle = original_open(path, mode, *args, **kwargs)
+ if path == cache_path and 'w' in mode:
+ started.set()
+ assert resume.wait(timeout=5)
+ return handle
+
+ monkeypatch.setattr(Path, 'open', paused_open)
+
+ def refresh():
+ return SearchService(cache_dir=tmp_path, transport=transport, min_interval=0, ttl=0).search('graph', ['github'])
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ futures = [pool.submit(refresh) for _ in range(2)]
+ # With atomic writes there is no direct truncation; writers finish.
+ # A direct write is paused after truncation to make the race deterministic.
+ try:
+ started.wait(timeout=0.15)
+ visible = cache_path.read_text()
+ finally:
+ resume.set()
+ results = [future.result(timeout=5) for future in futures]
+ assert json.loads(visible)['resources'][0]['title'] == 'example/graph'
+ assert all(result['mode'] == 'live' for result in results)
+ assert json.loads(cache_path.read_text())['resources'][0]['title'] == 'example/graph'
+ assert sorted(p.name for p in tmp_path.iterdir()) == [cache_path.name]
From fffa3a9712bdd6c9cf83a606864d4fb0998f30b0 Mon Sep 17 00:00:00 2001
From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com>
Date: Sun, 18 Oct 2026 18:39:30 +0800
Subject: [PATCH 3/7] fix: preserve roadmap dependencies and propagate unknown
schedules
---
fields_study_flow/cli.py | 16 ++---
fields_study_flow/mcp_server.py | 18 +++++-
fields_study_flow/models.py | 19 ++++++
fields_study_flow/prerequisites.py | 74 +++++++++++++++++------
fields_study_flow/roadmap.py | 32 ++++------
tests/test_review_levels.py | 40 +++++++++++++
tests/test_review_mcp.py | 63 ++++++++++++++++++++
tests/test_review_roadmap.py | 95 ++++++++++++++++++++++++++++++
8 files changed, 312 insertions(+), 45 deletions(-)
create mode 100644 tests/test_review_levels.py
create mode 100644 tests/test_review_mcp.py
create mode 100644 tests/test_review_roadmap.py
diff --git a/fields_study_flow/cli.py b/fields_study_flow/cli.py
index 65e7004..f966df4 100644
--- a/fields_study_flow/cli.py
+++ b/fields_study_flow/cli.py
@@ -17,10 +17,11 @@
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
- if args.command == "roadmap":
- return _roadmap(args)
- if args.command == "paper":
- return _paper(args)
+ if args.command in {"roadmap", "paper"}:
+ try:
+ return _roadmap(args) if args.command == "roadmap" else _paper(args)
+ except ValueError as exc:
+ parser.error(str(exc))
if args.command == "ingest-url":
return _ingest_url(args)
if args.command == "discover-sources":
@@ -176,9 +177,10 @@ def _parse_sources(value: str) -> set[str]:
def _parse_levels(values: list[str]) -> dict[str, str]:
levels: dict[str, str] = {}
for value in values:
- if "=" in value:
- key, level = value.split("=", 1)
- levels[key.strip()] = level.strip()
+ if "=" not in value or not value.split("=", 1)[0].strip():
+ raise ValueError('--level must be domain=beginner|familiar|advanced')
+ key, level = value.split("=", 1)
+ levels[key.strip()] = level.strip()
return levels
diff --git a/fields_study_flow/mcp_server.py b/fields_study_flow/mcp_server.py
index 789b42d..fa9e0dc 100644
--- a/fields_study_flow/mcp_server.py
+++ b/fields_study_flow/mcp_server.py
@@ -19,6 +19,22 @@
}
+def _mcp_tool(tool: Callable[..., Any]) -> Callable[..., Any]:
+ from functools import wraps
+ from mcp.server.mcpserver.exceptions import ToolError
+
+ @wraps(tool)
+ def run(*args, **kwargs):
+ try:
+ return tool(*args, **kwargs)
+ except ValueError as exc:
+ # Expected input/dependency errors must reach the client; the SDK
+ # intentionally hides unexpected exception messages by default.
+ raise ToolError(str(exc)) from exc
+
+ return run
+
+
def main() -> int:
import argparse
@@ -32,7 +48,7 @@ def main() -> int:
server = MCPServer("fields-study-flow", version="0.1.0", subscriptions=False)
for name, tool in TOOLS.items():
- server.add_tool(tool, name=name)
+ server.add_tool(_mcp_tool(tool), name=name)
server.run(transport="stdio")
return 0
return legacy_main()
diff --git a/fields_study_flow/models.py b/fields_study_flow/models.py
index 0d4aa7b..7927f51 100644
--- a/fields_study_flow/models.py
+++ b/fields_study_flow/models.py
@@ -1,11 +1,21 @@
from __future__ import annotations
from dataclasses import asdict, dataclass, field
+from enum import Enum
from typing import Any
from fields_study_flow.language import ResourceLanguagePreference, normalize_resource_language_preference
+class KnowledgeLevel(str, Enum):
+ BEGINNER = 'beginner'
+ FAMILIAR = 'familiar'
+ ADVANCED = 'advanced'
+
+
+LEVEL_ALIASES = {'初学': 'beginner', '熟悉': 'familiar', '老手': 'advanced'}
+
+
@dataclass(slots=True)
class LearnerProfile:
goal: str
@@ -19,6 +29,15 @@ class LearnerProfile:
def __post_init__(self) -> None:
self.resource_language_preference = normalize_resource_language_preference(self.resource_language_preference)
+ if not isinstance(self.levels, dict):
+ raise ValueError('levels must map domains to beginner, familiar, or advanced')
+ for domain, level in self.levels.items():
+ try:
+ if not isinstance(level, str):
+ raise ValueError('level must be text')
+ KnowledgeLevel(LEVEL_ALIASES.get(level, level))
+ except ValueError:
+ raise ValueError(f'Invalid level {level!r} for {domain!r}; expected beginner, familiar, or advanced (aliases: 初学, 熟悉, 老手)') from None
import math
import re
from datetime import date
diff --git a/fields_study_flow/prerequisites.py b/fields_study_flow/prerequisites.py
index f3cf243..d5eeb66 100644
--- a/fields_study_flow/prerequisites.py
+++ b/fields_study_flow/prerequisites.py
@@ -2,7 +2,7 @@
import math
import re
from datetime import date
-from fields_study_flow.ranking import _tokenize
+from fields_study_flow.concepts import concept_id, tokenize
GRAPH = {
'math': ([], 8),
@@ -15,30 +15,68 @@
}
-def prerequisite_nodes(profile):
- terms = _tokenize(profile.goal)
- targets = [key for key in GRAPH if key in terms or set(key.split()) <= terms]
+def prerequisite_nodes(profile, resources=()):
+ terms = tokenize(profile.goal)
+ # A paper URL alone carries no topic. Use the supplied resource concepts,
+ # including curated target-paper tags, without fetching paper contents.
+ for resource in resources:
+ for concept in resource.concepts + resource.prerequisites:
+ terms.update(tokenize(concept))
+ targets = [key for key in GRAPH if key in terms]
if 'yolo' in terms or 'cnn' in terms:
targets.append('deep learning')
- known = set()
- for topic in profile.known_topics:
- known.update(_tokenize(topic))
- known.add(topic.lower())
- nodes, seen = [], set()
+ known = {concept_id(topic) for topic in profile.known_topics}
+ nodes, seen, visiting = [], set(), []
+
def visit(key):
+ if key in visiting:
+ raise ValueError('Prerequisite cycle: ' + ' -> '.join(visiting + [key]))
+ if key not in GRAPH:
+ raise ValueError(f'unknown prerequisite: {key}')
if key in seen:
return
- seen.add(key)
+ visiting.append(key)
deps, hours = GRAPH[key]
for dep in deps:
visit(dep)
- skipped = key in known or set(key.split()) <= known
+ visiting.pop()
+ seen.add(key)
+ skipped = key in known
nodes.append({'id': key, 'depends_on': deps, 'estimated_hours': hours, 'status': 'skipped' if skipped else 'required', 'reason': 'Already known according to learner profile' if skipped else 'Prerequisite for target concepts'})
for target in targets:
visit(target)
return nodes
+def order_resources(profile, nodes, resources):
+ """Stable concept-level topological order, shared by phases and schedule."""
+ remaining = sorted(resources, key=lambda resource: (
+ bool(resource.metadata.get('target_paper')),
+ {'beginner': 0, 'intermediate': 1, 'advanced': 2}.get(resource.difficulty, 1),
+ ))
+ learned = {concept_id(topic) for topic in profile.known_topics}
+ planned = {node['id'] for node in nodes}
+ provided = {concept_id(concept) for resource in resources for concept in resource.concepts}
+ # Prefer concrete resource providers over a generic curated estimate.
+ available = learned | (planned - provided)
+ unknown = [f'{resource.title}: {dependency}' for resource in resources
+ for dependency in resource.prerequisites
+ if concept_id(dependency) not in available | provided]
+ if unknown:
+ raise ValueError('unknown prerequisite(s): ' + '; '.join(unknown))
+ ordered = []
+ while remaining:
+ ready = next((resource for resource in remaining
+ if all(concept_id(dep) in available for dep in resource.prerequisites)), None)
+ if ready is None:
+ blocked = '; '.join(f'{resource.title}: {", ".join(resource.prerequisites)}' for resource in remaining)
+ raise ValueError('Resource dependency cycle: ' + blocked)
+ remaining.remove(ready)
+ ordered.append(ready)
+ available.update(concept_id(concept) for concept in ready.concepts)
+ return ordered
+
+
def schedule(profile, nodes, resources):
items, unknown, total = [], [], 0.0
for node in nodes:
@@ -51,16 +89,16 @@ def schedule(profile, nodes, resources):
if hours is None:
unknown.append(resource.url)
items.append({'id': resource.url, 'hours': hours, 'kind': 'resource'})
+ uncertain = False
for item in items:
if item['hours'] is None:
- item.update(start_week=None, end_week=None)
- continue
- item['start_week'] = math.floor(total / profile.weekly_hours) + 1 if profile.weekly_hours else None
- total += item['hours']
- item['end_week'] = max(1, math.ceil(total / profile.weekly_hours)) if profile.weekly_hours else None
- weeks = math.ceil(total / profile.weekly_hours) if profile.weekly_hours else None
+ uncertain = True
+ item['start_week'] = math.floor(total / profile.weekly_hours) + 1 if profile.weekly_hours and not uncertain else None
+ total += item['hours'] or 0
+ item['end_week'] = max(1, math.ceil(total / profile.weekly_hours)) if profile.weekly_hours and not uncertain else None
+ weeks = math.ceil(total / profile.weekly_hours) if profile.weekly_hours and not unknown else None
feasible = None
if profile.target_date and profile.weekly_hours:
available = max(0, (date.fromisoformat(profile.target_date) - date.today()).days) / 7 * profile.weekly_hours
feasible = False if total > available else None if unknown else True
- return {'items': items, 'estimated_hours': total, 'unknown_estimates': unknown, 'total_weeks': weeks, 'target_date': profile.target_date, 'deadline_feasible': feasible, 'note': 'Estimates exclude unknown durations; prerequisite and resource study may overlap.'}
+ return {'items': items, 'estimated_hours': None if unknown else total, 'known_estimated_hours': total, 'unknown_estimates': unknown, 'total_weeks': weeks, 'target_date': profile.target_date, 'deadline_feasible': feasible, 'note': 'Sequential study estimates; weeks after an unknown duration remain unknown. Known hours are a subtotal. Prerequisite and resource study may overlap.'}
diff --git a/fields_study_flow/roadmap.py b/fields_study_flow/roadmap.py
index 5176438..2b34b23 100644
--- a/fields_study_flow/roadmap.py
+++ b/fields_study_flow/roadmap.py
@@ -17,24 +17,10 @@
def build_roadmap(profile: LearnerProfile, resources: list[Resource]) -> dict[str, Any]:
- from fields_study_flow.prerequisites import prerequisite_nodes, schedule
- nodes = prerequisite_nodes(profile)
- # Resource ordering follows explicit dependencies, then difficulty; target papers anchor core study.
- remaining = sorted(resources, key=lambda r: (bool(r.metadata.get('target_paper')), {'beginner': 0, 'intermediate': 1, 'advanced': 2}.get(r.difficulty, 1)))
- ordered, seen = [], set(profile.known_topics)
- while remaining:
- ready = next((r for r in remaining if all(p in seen or not any(p in other.concepts for other in remaining) for p in r.prerequisites)), remaining[0])
- remaining.remove(ready)
- ordered.append(ready)
- seen.update(ready.concepts)
+ from fields_study_flow.prerequisites import order_resources, prerequisite_nodes, schedule
+ nodes = prerequisite_nodes(profile, resources)
+ ordered = order_resources(profile, nodes, resources)
phases = _build_phases(profile, ordered)
- if any(r.metadata.get('target_paper') for r in ordered):
- targets = [r.to_dict() for r in ordered if r.metadata.get('target_paper')]
- for phase in phases:
- phase['resources'] = [r for r in phase['resources'] if not r['metadata'].get('target_paper')]
- if len(phases) == 1:
- phases.append({'name': _phase_name(profile, 'Phase 2', '阶段 2', 'Core Understanding', '核心理解'), 'objective': 'Study the target paper after prerequisites.', 'resources': []})
- phases[1]['resources'].extend(targets)
return {
'prerequisites': nodes,
'schedule': schedule(profile, nodes, ordered),
@@ -142,9 +128,17 @@ def _build_phases(profile: LearnerProfile, resources: list[Resource]) -> list[di
("Practice and Reproduction", "实践复现", "Turn understanding into code, notes, or proof steps.", "把理解转成代码、笔记或推导步骤。"),
]
phases: list[dict[str, Any]] = []
- chunk_size = max(1, (len(resources) + len(buckets) - 1) // len(buckets))
+ target_index = next((i for i, resource in enumerate(resources) if resource.metadata.get('target_paper')), None)
+ if target_index is None:
+ chunk_size = max(1, (len(resources) + len(buckets) - 1) // len(buckets))
+ chunks = [resources[i * chunk_size:(i + 1) * chunk_size] for i in range(len(buckets))]
+ else:
+ # Change phase boundaries, never the dependency order. A target paper
+ # follows every foundation resource and precedes its dependent practice.
+ core_end = max(i for i, resource in enumerate(resources) if resource.metadata.get('target_paper')) + 1
+ chunks = [resources[:target_index], resources[target_index:core_end], resources[core_end:]]
for index, (en_name, zh_name, en_obj, zh_obj) in enumerate(buckets):
- chunk = resources[index * chunk_size : (index + 1) * chunk_size]
+ chunk = chunks[index]
if not chunk and index > 0:
continue
phases.append(
diff --git a/tests/test_review_levels.py b/tests/test_review_levels.py
new file mode 100644
index 0000000..a9d9286
--- /dev/null
+++ b/tests/test_review_levels.py
@@ -0,0 +1,40 @@
+import subprocess
+import sys
+
+import pytest
+
+from fields_study_flow.mcp_tools import assessKnowledge
+from fields_study_flow.models import LearnerProfile
+
+
+@pytest.mark.parametrize('level', ['begginer', 'expert', '', None, 2, ['beginner']])
+def test_invalid_domain_grade_is_rejected_with_allowed_values(level):
+ with pytest.raises(ValueError) as error:
+ LearnerProfile(goal='python', levels={'programming': level})
+ assert 'programming' in str(error.value)
+ assert 'beginner' in str(error.value)
+ assert 'familiar' in str(error.value)
+ assert 'advanced' in str(error.value)
+
+
+@pytest.mark.parametrize('level', ['beginner', 'familiar', 'advanced', '初学', '熟悉', '老手'])
+def test_documented_domain_grades_and_existing_aliases_are_accepted(level):
+ assert LearnerProfile(goal='python', levels={'programming': level}).levels['programming'] == level
+
+
+def test_mcp_assessment_rejects_grade_typo():
+ with pytest.raises(ValueError, match='begginer'):
+ assessKnowledge('python', confidenceLevels={'programming': 'begginer'})
+
+
+@pytest.mark.parametrize('value', ['math=begginer', 'math'])
+def test_cli_invalid_level_is_a_readable_usage_error(tmp_path, value):
+ result = subprocess.run([
+ sys.executable, '-m', 'fields_study_flow.cli', 'roadmap', '--goal',
+ 'linear algebra', '--level', value, '--output-dir', str(tmp_path / 'out'),
+ ], capture_output=True, text=True, timeout=15)
+ assert result.returncode == 2
+ assert 'error:' in result.stderr
+ assert 'beginner' in result.stderr
+ assert 'Traceback' not in result.stderr
+ assert not (tmp_path / 'out').exists()
diff --git a/tests/test_review_mcp.py b/tests/test_review_mcp.py
new file mode 100644
index 0000000..0705608
--- /dev/null
+++ b/tests/test_review_mcp.py
@@ -0,0 +1,63 @@
+import asyncio
+import json
+import sys
+
+from jsonschema import Draft202012Validator
+from mcp.client.session import ClientSession
+from mcp.client.stdio import StdioServerParameters, stdio_client
+
+
+TOOL_NAMES = {'assessKnowledge', 'discoverSources', 'searchResources', 'ingestUrl',
+ 'rankResources', 'buildRoadmap', 'validateSources', 'exportPlan'}
+
+
+def test_sdk_all_eight_tools_schema_calls_and_errors(tmp_path):
+ async def run():
+ params = StdioServerParameters(command=sys.executable, args=['-m', 'fields_study_flow.mcp_server', '--mcp'])
+ async with stdio_client(params) as streams:
+ async with ClientSession(*streams) as session:
+ initialized = await session.initialize()
+ assert initialized.server_info.name == 'fields-study-flow'
+ listing = await session.list_tools()
+ assert {tool.name for tool in listing.tools} == TOOL_NAMES
+ schemas = {tool.name: tool.input_schema for tool in listing.tools}
+ for schema in schemas.values():
+ assert schema['type'] == 'object'
+ Draft202012Validator.check_schema(schema)
+
+ async def call(name, arguments):
+ Draft202012Validator(schemas[name]).validate(arguments)
+ result = await session.call_tool(name, arguments)
+ assert not result.is_error, result
+ return json.loads(result.content[0].text)
+
+ profile = await call('assessKnowledge', {'goal': 'transformer', 'confidenceLevels': {'ml_dl': 'familiar'}})
+ assert profile['levels']['ml_dl'] == 'familiar'
+ sources = await call('discoverSources', {'goal': 'transformer'})
+ assert {'arxiv', 'github'} <= {source['id'] for source in sources['sources']}
+ search = await call('searchResources', {'query': 'transformer', 'filters': {'offline': True}})
+ assert search['mode'] == 'offline' and search['resources']
+ imported = await call('ingestUrl', {'url': 'https://arxiv.org/abs/1706.03762'})
+ assert imported['metadata']['retrieval_mode'] == 'offline'
+ ranked = await call('rankResources', {'resources': search['resources'], 'learnerProfile': profile})
+ assert ranked['resources'][0]['score_components']
+ plan = await call('buildRoadmap', {'goal': 'transformer', 'profile': profile, 'rankedResources': ranked['resources']})
+ assert plan['prerequisites']
+ checked = await call('validateSources', {'plan': plan})
+ assert checked['valid']
+ exported = await call('exportPlan', {'plan': plan, 'outputDir': str(tmp_path)})
+ assert exported['roadmap_json']
+ assert json.loads((tmp_path / 'roadmap.json').read_text(encoding='utf-8')) == plan
+
+ cycle = dict(imported, title='cycle', concepts=['self'], prerequisites=['self'])
+ for name, arguments, message in [
+ ('unknown', {}, 'unknown'),
+ ('assessKnowledge', {}, 'goal'),
+ ('assessKnowledge', {'goal': 'python', 'confidenceLevels': {'programming': 'begginer'}}, 'begginer'),
+ ('buildRoadmap', {'goal': 'custom', 'profile': {}, 'rankedResources': [cycle]}, 'cycle'),
+ ]:
+ result = await session.call_tool(name, arguments)
+ assert result.is_error
+ assert message in result.content[0].text.lower()
+
+ asyncio.run(asyncio.wait_for(run(), timeout=30))
diff --git a/tests/test_review_roadmap.py b/tests/test_review_roadmap.py
new file mode 100644
index 0000000..5d1f7e9
--- /dev/null
+++ b/tests/test_review_roadmap.py
@@ -0,0 +1,95 @@
+import json
+import subprocess
+import sys
+
+import pytest
+
+from fields_study_flow.models import LearnerProfile, Resource
+from fields_study_flow.roadmap import build_roadmap
+
+
+def material(name, *, concepts=None, prerequisites=None, hours='2h', target=False):
+ return Resource(title=name, url=f'https://example.org/{name}', source='course',
+ type='paper' if target else 'course', difficulty='advanced',
+ concepts=concepts or [name], prerequisites=prerequisites or [],
+ estimated_time=hours, metadata={'target_paper': target})
+
+
+def test_phases_and_schedule_share_dependency_order_even_for_target_paper():
+ resources = [material(f'step-{i}', prerequisites=[f'step-{i-1}'] if i else [])
+ for i in range(8)]
+ target = material('target', prerequisites=['step-7'], target=True)
+ plan = build_roadmap(LearnerProfile(goal='custom', weekly_hours=2), [target, *reversed(resources)])
+ expected = [f'https://example.org/step-{i}' for i in range(8)] + ['https://example.org/target']
+ assert [r['url'] for phase in plan['phases'] for r in phase['resources']] == expected
+ assert [item['id'] for item in plan['schedule']['items']] == expected
+ assert [item['start_week'] for item in plan['schedule']['items']] == [1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+
+@pytest.mark.parametrize('resources,reason', [
+ ([material('a', prerequisites=['b']), material('b', prerequisites=['a'])], 'cycle'),
+ ([material('a', prerequisites=['a'])], 'cycle'),
+ ([material('a', prerequisites=['unmapped prerequisite'])], 'unknown prerequisite'),
+])
+def test_unresolvable_dependencies_are_reported(resources, reason):
+ with pytest.raises(ValueError, match=reason) as error:
+ build_roadmap(LearnerProfile(goal='custom'), resources)
+ assert 'a' in str(error.value)
+
+
+def test_dependency_matching_uses_bilingual_concept_identity():
+ target = material('target', prerequisites=['线性代数'], target=True)
+ foundation = material('foundation', concepts=['linear algebra'])
+ plan = build_roadmap(LearnerProfile(goal='custom'), [target, foundation])
+ assert [r['title'] for phase in plan['phases'] for r in phase['resources']] == ['foundation', 'target']
+
+
+def test_explicitly_known_prerequisite_needs_no_resource():
+ target = material('target', prerequisites=['线性代数'], target=True)
+ plan = build_roadmap(LearnerProfile(goal='custom', known_topics=['linear algebra']), [target])
+ assert [r['title'] for phase in plan['phases'] for r in phase['resources']] == ['target']
+
+
+def test_target_resource_concepts_seed_prerequisites_without_goal_keyword():
+ target = material('target', concepts=['变换器'], target=True)
+ plan = build_roadmap(LearnerProfile(goal='Read this paper', known_topics=['机器学习']), [target])
+ assert [node['id'] for node in plan['prerequisites']] == ['math', 'python', 'machine learning', 'deep learning', 'transformer']
+ assert next(node for node in plan['prerequisites'] if node['id'] == 'machine learning')['status'] == 'skipped'
+
+
+def test_readme_paper_command_includes_transformer_prerequisites(tmp_path):
+ result = subprocess.run([
+ sys.executable, '-m', 'fields_study_flow.cli', 'paper', '--url',
+ 'https://arxiv.org/abs/1706.03762', '--with-videos', '--output-language',
+ 'bilingual', '--resource-language', 'en-first', '--output-dir', str(tmp_path),
+ ], capture_output=True, text=True, timeout=15)
+ assert result.returncode == 0, result.stderr
+ plan = json.loads((tmp_path / 'roadmap.json').read_text(encoding='utf-8'))
+ assert [node['id'] for node in plan['prerequisites']] == ['math', 'python', 'machine learning', 'deep learning', 'transformer']
+ assert all(not r['metadata'].get('target_paper') for r in plan['phases'][0]['resources'])
+
+
+def test_unknown_duration_propagates_without_erasing_known_hours():
+ resources = [material('foundation', hours='unknown'),
+ material('core', prerequisites=['foundation'], hours='4h'),
+ material('practice', prerequisites=['core'], hours='2h')]
+ plan = build_roadmap(LearnerProfile(goal='custom', weekly_hours=2, target_date='2099-01-01'), resources)
+ schedule = plan['schedule']
+ assert [item['hours'] for item in schedule['items']] == [None, 4, 2]
+ assert all(item['start_week'] is None and item['end_week'] is None for item in schedule['items'])
+ assert schedule['estimated_hours'] is None
+ assert schedule['known_estimated_hours'] == 6
+ assert schedule['total_weeks'] is None
+ assert schedule['deadline_feasible'] is None
+ assert schedule['unknown_estimates'] == ['https://example.org/foundation']
+
+
+def test_known_prefix_keeps_its_weeks_before_unknown_duration():
+ plan = build_roadmap(LearnerProfile(goal='custom', weekly_hours=2), [
+ material('intro'), material('foundation', prerequisites=['intro'], hours='unknown'),
+ material('core', prerequisites=['foundation']),
+ ])
+ items = plan['schedule']['items']
+ assert (items[0]['start_week'], items[0]['end_week']) == (1, 1)
+ assert items[2]['start_week'] is None
+ assert items[2]['hours'] == 2
From 240187b3fb86ff59aebad09a61240e57a732ca6d Mon Sep 17 00:00:00 2001
From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com>
Date: Sun, 18 Oct 2026 18:39:36 +0800
Subject: [PATCH 4/7] fix: pin reviewed dependencies and document actual
prerequisite scope
---
README.md | 10 ++++----
README.zh-CN.md | 12 ++++++----
constraints/windows-py314.txt | 43 +++++++++++++++++++++++++++++++++++
docs/REPRODUCIBILITY.md | 32 ++++++++++++++++++++++++++
4 files changed, 88 insertions(+), 9 deletions(-)
create mode 100644 constraints/windows-py314.txt
create mode 100644 docs/REPRODUCIBILITY.md
diff --git a/README.md b/README.md
index 1b7df34..3d15630 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@ Most learning-path tools stop at a plausible list of links. fields-study-flow ke
| Area | What it does |
| --- | --- |
| Personalized interview | Captures goal type, known topics, skill levels, time budget, output language, and resource language preference. |
-| AI/CS taxonomy | Seeds routes with math, programming, ML/DL, LLM, RL, systems, and paper-reading prerequisites. |
+| Curated prerequisites | Covers math, Python, ML/DL, Transformer, diffusion, and LLM; YOLO/CNN map to DL. RL, systems, and general paper-reading prerequisite graphs are not implemented. |
| Multi-source discovery | Models GitHub, arXiv, OpenAlex, Semantic Scholar, Unpaywall, Papers with Code, YouTube, Bilibili, Zhihu, Hugging Face, Kaggle, MIT OCW, fast.ai, Google MLCC, and more. |
| Language-aware ranking | Supports `zh-first`, `en-first`, `balanced`, `zh-only`, and `en-only`. |
| Paper deep reading | Makes the target paper first-class, then adds prerequisites, intuition resources, and reproduction checkpoints. |
@@ -200,7 +200,9 @@ python -m pip install -e .
pytest -q
```
-Current test coverage includes language preference parsing, bilingual query generation, registry policy filtering, GitHub-style ranking signals, URL de-duplication, MCP tool boundaries, safety validation, roadmap schema, and CLI smoke tests.
+For a reproducible Windows CPython 3.14.7 environment, use [the pinned constraints and build/install steps](docs/REPRODUCIBILITY.md). The constraints include transitive runtime, test, and build dependencies; unconstrained installs can resolve different versions.
+
+Current test coverage includes language preference parsing, bilingual query generation, registry policy filtering, GitHub-style ranking signals, URL de-duplication, MCP tool boundaries, safety validation, concurrent rate limiting and atomic cache writes, dependency ordering and unknown schedules, roadmap schema, and CLI smoke tests.
## Roadmap
@@ -217,12 +219,12 @@ MIT. See [LICENSE](LICENSE).
## Current capabilities and limits
-`roadmap` remains offline by default. Add `--live --sources arxiv,github` to query public metadata APIs. MCP `searchResources` uses `filters={"offline": false}` for live queries; `type`, `language`, and `difficulty` filters apply to actual results. Offline catalog entries are suggestions, not live search matches. Other registered platforms are link/catalog only. Retrieval reports `offline`, `live`, `cache`, `mixed`, or `degraded` with reasons; network errors return no invented matches. API calls use a 5-second timeout, a 3-second minimum interval per service instance and a one-hour disk cache (`FIELDS_STUDY_FLOW_CACHE` overrides its directory). There are no paid APIs or authenticated fetches.
+`roadmap` remains offline by default. Add `--live --sources arxiv,github` to query public metadata APIs. MCP `searchResources` uses `filters={"offline": false}` for live queries; `type`, `language`, and `difficulty` filters apply to actual results. Offline catalog entries are suggestions, not live search matches. Other registered platforms are link/catalog only. Retrieval reports `offline`, `live`, `cache`, `mixed`, or `degraded` with reasons; network errors return no invented matches. API calls use a 5-second timeout, a 3-second minimum interval shared across service instances per API host in one process, and a one-hour disk cache with atomic replacement (`FIELDS_STUDY_FLOW_CACHE` overrides its directory). There are no paid APIs or authenticated fetches.
`ingestUrl` is URL-only by default. `fetchMetadata=true` requests public GitHub repository or arXiv metadata through the same service, refusing credentials and redirects. Unsupported hosts remain URL-only with an explicit degraded reason. GitHub resource language is the interface/metadata language (English); programming language is recorded separately and is not a natural-language classifier.
Standard MCP hosts must run `python -m fields_study_flow.mcp_server --mcp`. Both editor configurations use this command. The no-flag invocation and `--legacy-jsonl` retain the previous JSONL protocol, including the `git4study` alias. See [MCP transport](docs/MCP.md). The official SDK 2.x negotiates its supported protocol versions; local tests exercise initialize, eight tool schemas, calls, and errors. Editor GUI attachment remains an external check.
-Ranking exposes relevance, trust, structure, difficulty, outcome and language components plus unmatched goal terms. A small bilingual alias map and domain levels improve explanations; this is lexical ranking, not semantic search. Thirty fixed review cases live in `tests/fixtures/ranking_cases.json`. Low lexical relevance is labeled supplemental. Prerequisites use a curated mathematics/programming/ML/DL/Transformer/LLM graph, skip explicitly known topics and appear before resource study. Numerical hours determine relative week ranges; unknown durations and infeasible target dates are explicit. These estimates are planning aids, not completion guarantees. Target papers anchor core study rather than the first foundation item.
+Ranking exposes relevance, trust, structure, difficulty, outcome and language components plus unmatched goal terms. A small bilingual alias map and domain levels improve explanations; this is lexical ranking, not semantic search. Thirty fixed review cases live in `tests/fixtures/ranking_cases.json`. Low lexical relevance is labeled supplemental. Prerequisites use the small curated graph described above, seeded by the goal and supplied resource concepts (including the README's known Transformer paper), and skip explicitly known topics. Arbitrary paper URLs do not imply extracted paper concepts. Phases and sequential schedules share one resource dependency order; cycles and unknown prerequisite references raise explicit errors. Target papers start core study after their dependencies. Unknown durations leave that item and all later weeks undetermined; `known_estimated_hours` retains the known subtotal while total hours/weeks remain unknown. These curated estimates may overlap between prerequisites and resource study and are planning aids, not completion guarantees. Learner levels accept `beginner`, `familiar`, `advanced` and the existing Chinese aliases `初学`, `熟悉`, `老手`; invalid levels report allowed values.
Safety checks use parsed URL hosts/schemes and structured `metadata.access_policy` actions (`metadata`, `link`, `summarize`). Legacy notes retain compatibility checks for affirmative prohibited instructions; generated safety advice is not treated as an access operation.
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 322e8bc..b9b7579 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -27,10 +27,10 @@ fields-study-flow 可以把“读懂这篇论文”、“学习 Transformer”
| 能力 | 说明 |
| --- | --- |
| 个性化访谈 | 收集目标类型、已知知识、各领域水平、时间预算、路线语言和资料语言偏好。 |
-| AI/CS 技能树 | 内置数学、编程、ML/DL、LLM、RL、系统、论文阅读等前置知识结构。 |
+| 人工维护的先修图 | 覆盖数学、Python、ML/DL、Transformer、diffusion、LLM；YOLO/CNN 映射到 DL。尚未实现 RL、系统和通用论文阅读的先修图。 |
| 多源资源发现 | 覆盖 GitHub、arXiv、OpenAlex、Semantic Scholar、Unpaywall、Papers with Code、YouTube、Bilibili、知乎、Hugging Face、Kaggle、MIT OCW、fast.ai、Google MLCC 等。 |
| 语言感知排序 | 支持 `zh-first`、`en-first`、`balanced`、`zh-only`、`en-only`。 |
-| 论文深读路线 | 把目标论文作为第一资源，再补充前置知识、直觉视频、代码实现和复现检查点。 |
+| 论文深读路线 | 把目标论文放入依赖之后的核心阶段，使用已有概念标签补充先修，并提供复现检查点。 |
| 安全边界 | 拒绝盗版来源、绕登录、下载视频、长篇复制版权内容等不合规路径。 |
| Agent 友好输出 | 生成 Markdown 与 JSON，方便后续 agent 继续执行、复盘或扩展。 |
@@ -203,7 +203,9 @@ python -m pip install -e .
pytest -q
```
-当前测试覆盖语言偏好解析、双语 query 生成、registry policy、GitHub 风格排序信号、URL 去重、MCP 工具边界、安全校验、roadmap schema 和 CLI smoke tests。
+Windows CPython 3.14.7 的可复现环境见[固定版本 constraints 与构建/安装说明](docs/REPRODUCIBILITY.md)。约束包含运行、测试和构建的传递依赖；无约束安装可能解析到其他版本。
+
+当前测试覆盖语言偏好解析、双语 query 生成、registry policy、GitHub 风格排序信号、URL 去重、MCP 工具边界、安全校验、并发限流与原子缓存写入、依赖顺序与未知排期、roadmap schema 和 CLI smoke tests。
## 路线图
@@ -220,12 +222,12 @@ MIT. See [LICENSE](LICENSE).
## 当前能力与限制
-CLI `roadmap` 默认离线；`--live --sources arxiv,github` 显式调用公开元数据 API。MCP `searchResources` 传 `filters={"offline": false}` 启用实时检索，`type`、`language`、`difficulty` 会真正过滤资源。其他 registry 平台目前只有链接/目录能力。返回 `offline/live/cache/mixed/degraded` 和降级原因；失败不会伪造查询结果。默认超时 5 秒、同一服务实例请求至少间隔 3 秒、磁盘缓存 1 小时，缓存目录可由 `FIELDS_STUDY_FLOW_CACHE` 设置。不使用付费 API 或认证内容。
+CLI `roadmap` 默认离线；`--live --sources arxiv,github` 显式调用公开元数据 API。MCP `searchResources` 传 `filters={"offline": false}` 启用实时检索，`type`、`language`、`difficulty` 会真正过滤资源。其他 registry 平台目前只有链接/目录能力。返回 `offline/live/cache/mixed/degraded` 和降级原因；失败不会伪造查询结果。默认超时 5 秒、同一进程内各服务实例按 API 主机共享至少 3 秒的请求间隔、磁盘缓存 1 小时并原子替换，缓存目录可由 `FIELDS_STUDY_FLOW_CACHE` 设置。不使用付费 API 或认证内容。
`ingestUrl` 默认只录入 URL；`fetchMetadata=true` 从公开 GitHub/arXiv API 获取元数据，拒绝凭据和重定向，其他来源明确降级。GitHub 的 `language=en` 表示元数据界面语言；编程语言另存 metadata，不保证仓库正文语言分类。
标准 MCP 启动命令为 `python -m fields_study_flow.mcp_server --mcp`，两份编辑器配置已同步。无参数启动和 `--legacy-jsonl` 保留旧 JSONL，`git4study` 入口兼容。协议版本由官方 SDK 2.x 协商，详见 [MCP 文档](docs/MCP.md)。本地 SDK 测试涵盖握手、8 工具 schema、调用和错误；编辑器 GUI 实际连接仍需外部验收。
-排序输出相关性、可信度、结构、难度、成果偏好、语言等分项及缺失词项。小型中英别名表和领域等级避免全局降级；仍是词汇匹配，不是语义模型。30 条固定样例位于 `tests/fixtures/ranking_cases.json`。低相关性标为补充资料，不声称匹配。先修图覆盖数学/编程/ML/DL/Transformer/LLM，明确跳过已知主题；工时产生相对周排期，缺失时长标 unknown，期限不足会提示。目标论文放在核心阶段。工时是人工估算，不能保证实际完成时间。
+排序输出相关性、可信度、结构、难度、成果偏好、语言等分项及缺失词项。小型中英别名表和领域等级避免全局降级；仍是词汇匹配，不是语义模型。30 条固定样例位于 `tests/fixtures/ranking_cases.json`。低相关性标为补充资料，不声称匹配。先修图使用上文的小型人工图，由目标和已提供的资源概念（包括 README 的已知 Transformer 论文）触发，并跳过明确已知主题；任意论文 URL 不代表已提取论文概念。阶段和串行周计划共用资源依赖顺序，环和未知先修引用会明确报错。目标论文在依赖之后进入核心阶段。未知时长使该项及后续周次未定，`known_estimated_hours` 保留已知小计，总工时与总周数保持 unknown。先修与资源学习工时可能重叠，人工估算不保证实际完成时间。学习等级接受 `beginner/familiar/advanced` 和已有中文别名 `初学/熟悉/老手`，错误等级会提示允许值。
合规校验使用解析后的 URL 主机/协议和 `metadata.access_policy` 的结构化动作（metadata/link/summarize）；旧记录的肯定违规指令保留兼容校验，不把生成的否定安全说明当成访问操作。
diff --git a/constraints/windows-py314.txt b/constraints/windows-py314.txt
new file mode 100644
index 0000000..44889d1
--- /dev/null
+++ b/constraints/windows-py314.txt
@@ -0,0 +1,43 @@
+# Resolved baseline: CPython 3.14.7, Windows x64. See docs/REPRODUCIBILITY.md.
+# Includes runtime, test, and build tools; the project itself is installed separately.
+annotated-types==0.8.0
+anyio==4.15.1
+attrs==26.1.0
+build==1.6.1
+cffi==2.1.1
+click==8.5.0
+colorama==0.4.6
+cryptography==50.0.2
+h11==0.16.0
+httpcore2==2.13.1
+httpx2==2.13.1
+idna==3.20
+iniconfig==2.3.1
+jsonschema==4.26.0
+jsonschema-specifications==2025.9.1
+mcp==2.3.0
+mcp-types==2.3.0
+opentelemetry-api==1.45.1
+packaging==26.3
+pip==26.2.1
+pluggy==1.7.0
+pycparser==3.11
+pydantic==2.14.1
+pydantic-core==2.50.1
+Pygments==2.21.0
+PyJWT==2.15.1
+pyproject-hooks==1.3.3
+pytest==9.1.1
+python-multipart==0.0.32
+pywin32==312 ; sys_platform == 'win32'
+PyYAML==6.0.3
+referencing==0.37.0
+rpds-py==2026.9.1
+setuptools==84.0.0
+sse-starlette==3.5.0
+starlette==1.8.0
+truststore==0.10.5
+typing-extensions==4.16.0
+typing-inspection==0.4.4
+uvicorn==0.54.0
+wheel==0.48.0
diff --git a/docs/REPRODUCIBILITY.md b/docs/REPRODUCIBILITY.md
new file mode 100644
index 0000000..3f02ccc
--- /dev/null
+++ b/docs/REPRODUCIBILITY.md
@@ -0,0 +1,32 @@
+# Reproducing the reviewed environment
+
+`constraints/windows-py314.txt` pins the resolved runtime dependencies, their transitive dependencies, pytest, and build tools for **CPython 3.14.7 on Windows x64**. `pyproject.toml` retains compatible dependency ranges for library consumers. A normal unconstrained `pip install .` does not reproduce this baseline.
+
+From a repository checkout in PowerShell:
+
+```powershell
+py -3.14 -m venv .venv
+$python = '.\.venv\Scripts\python.exe'
+& $python -m pip install --constraint constraints/windows-py314.txt pip
+& $python -m pip install --constraint constraints/windows-py314.txt build setuptools wheel
+& $python -m pip install --constraint constraints/windows-py314.txt --no-build-isolation -e '.[dev]'
+& $python -m pytest -q
+& $python -m pip check
+& $python -m build --wheel --no-isolation
+```
+
+`--no-isolation` / `--no-build-isolation` ensure the build uses the explicitly installed, pinned tools. Without these flags, a separate build environment can resolve different versions of setuptools.
+
+To verify the wheel outside the checkout, create another venv, use its Python, and install the wheel with an absolute path to the constraints file:
+
+```powershell
+& $wheelPython -m pip install --constraint $constraintsPath $wheelPath
+& $wheelPython -m pip check
+& $wheelCli discover-sources --goal transformer
+```
+
+`$wheelPython`, `$wheelCli`, `$constraintsPath`, and `$wheelPath` refer to that venv's Python/CLI and the absolute constraint/wheel paths. Run the CLI from a temporary directory so an import from the source checkout cannot hide a packaging omission.
+
+These are version constraints, not an artifact hash lock: package availability still depends on the configured index, and wheels are not claimed to be byte-identical. Other Python versions or operating systems require their own resolved baseline and test run. No cross-platform installation claim follows from the package's declared Python minimum.
+
+When updating dependencies, resolve in a fresh venv, update the pins including transitive dependencies, run the complete tests and MCP SDK stdio round trip, then build and install the wheel in a separate venv. Record the actual Python version, installed packages, and results before replacing this baseline.
From 0d41a7a1e05e410cc5915d98f430ee811f8880fd Mon Sep 17 00:00:00 2001
From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com>
Date: Sun, 18 Oct 2026 19:34:48 +0800
Subject: [PATCH 5/7] fix: distinguish access actions from scoped prohibitions
---
fields_study_flow/mcp_tools.py | 101 ++++++++++++++++++++++------
tests/test_review_policy_scope.py | 107 ++++++++++++++++++++++++++++++
2 files changed, 189 insertions(+), 19 deletions(-)
create mode 100644 tests/test_review_policy_scope.py
diff --git a/fields_study_flow/mcp_tools.py b/fields_study_flow/mcp_tools.py
index f460fc5..9df3211 100644
--- a/fields_study_flow/mcp_tools.py
+++ b/fields_study_flow/mcp_tools.py
@@ -26,14 +26,31 @@
"anna-archive",
)
-DISALLOWED_ACCESS_NOTE_TERMS = (
- "download video",
- "yt-dlp",
- "youtube-dl",
- "bypass login",
- "copy full text",
- "copy long copyrighted",
- "scrape restricted",
+# Match actions and their objects, including inflections and passive instructions.
+# These are deliberately local patterns: a prohibition elsewhere in a note must
+# not hide an affirmative instruction.
+_ACCESS_DETERMINER = r'(?:(?:the|a|an|this|that|these|those|any|all|some|your|their)\s+)*'
+_ACCESS_OBJECTS = (
+ (r'download(?:s|ed|ing)?', r'videos?(?:\s+content)?', 'downloaded'),
+ (r'bypass(?:es|ed|ing)?', r'login', 'bypassed'),
+ (r'(?:copy|copies|copied|copying)', r'(?:full\s+text|long\s+copyrighted(?:\s+(?:content|passages|excerpts))?)', 'copied'),
+ (r'scrap(?:e|es|ed|ing)', r'restricted(?:\s+(?:pages|content))?', 'scraped'),
+)
+_ACCESS_ACTION = re.compile(
+ r'\b(?:(?P' + '|'.join(
+ verb + r'\s+' + _ACCESS_DETERMINER + obj for verb, obj, _ in _ACCESS_OBJECTS
+ ) + r')|(?Pyt-dlp|youtube-dl)|(?P' + '|'.join(
+ obj + r'\s+(?:(?:must|should|may|can|cannot|will|shall|is|are|be|been|being|not|never|to)\s+)+' + participle
+ for _, obj, participle in _ACCESS_OBJECTS
+ ) + r'))\b'
+)
+_ACCESS_NEGATION = re.compile(
+ r"\b(?:not|never|cannot|no|don't|doesn't|mustn't|shouldn't|can't|won't)\s+"
+ r'(?:(?:ever|use|run|to|be|allowed|permitted|attempt|try)\s+)*$'
+)
+_ACCESS_PREVENTION = re.compile(r'\b(?:avoid\s+|(?:forbidden|prohibited|banned)\s+(?:to|from)\s+)$')
+_ACCESS_PROHIBITION = re.compile(
+ r'\s+(?:is|are)\s+(?:strictly\s+)?(?:prohibited|forbidden|banned|not\s+(?:allowed|permitted))\b'
)
@@ -147,19 +164,65 @@ def buildRoadmap(
def _affirmative_access_actions(note: str) -> list[str]:
actions = []
- action_pattern = r'\b(?:' + '|'.join(re.escape(term) for term in DISALLOWED_ACCESS_NOTE_TERMS) + r')\b'
- # Only immediate prohibitions and coordinated actions share negation.
- # Punctuation, adversatives, or a new subject start a new instruction.
- clauses = re.split(r'[.,;:!?\n]|\b(?:but|however|then)\b|\b(?:and|or)\s+(?=(?:you|we|they|one)\b)', note.lower())
+ # A fresh subject, modal, or imperative ends inherited negation, even when
+ # its subject is omitted ("do not copy ... and must download ...").
+ clauses = re.split(
+ r'[.;:!?\n]|\b(?:but|however|then)\b|\b(?:and|or)\s+'
+ r'(?=(?:you|we|they|he|she|it|one|the|users?|agents?|must|should|may|can|will|shall|do|please)\b)',
+ note.lower().replace('’', "'"),
+ )
for clause in clauses:
+ matches = list(_ACCESS_ACTION.finditer(clause))
+ # Commas share scope only in an action list with an explicit final
+ # conjunction. A bare comma still starts an independent instruction.
+ list_links = set()
+ for index in range(1, len(matches)):
+ gap = clause[matches[index - 1].end():matches[index].start()]
+ if not re.fullmatch(r'\s*,?\s*\b(?:and|or)\s+', gap):
+ continue
+ list_links.add(index)
+ for earlier in range(index - 1, 0, -1):
+ gap = clause[matches[earlier - 1].end():matches[earlier].start()]
+ if not re.fullmatch(r'\s*,\s*', gap):
+ break
+ list_links.add(earlier)
negated, previous_end = False, 0
- for match in re.finditer(action_pattern, clause):
- before, after = clause[previous_end:match.start()], clause[match.end():]
- direct = re.search(r"\b(?:do not|don't|never|must not|cannot|no)\s+(?:(?:ever|use|run)\s+)*$", before)
- coordinated = re.fullmatch(r'\s*(?:and|or)\s+(?:(?:use|run)\s+)?', before)
- negated = bool(direct or (negated and coordinated))
- prohibited = re.match(r'\s+(?:is|are)\s+(?:prohibited|forbidden|banned|not allowed)\b', after)
- if not negated and not prohibited:
+ prohibited = set()
+ for index, match in enumerate(matches):
+ if not _ACCESS_PROHIBITION.match(clause[match.end():]):
+ continue
+ prohibited.add(index)
+ # A trailing predicate can govern a list of gerunds or tool names.
+ # Do not extend it over an imperative or a separate finite clause.
+ for earlier in range(index - 1, -1, -1):
+ left, right = matches[earlier:earlier + 2]
+ same_form = (left.lastgroup == right.lastgroup == 'tool' or
+ (left.group().split()[0].endswith('ing') and
+ right.group().split()[0].endswith('ing')))
+ if not same_form or earlier + 1 not in list_links:
+ break
+ prohibited.add(earlier)
+ for index, match in enumerate(matches):
+ before = clause[previous_end:match.start()]
+ direct = _ACCESS_NEGATION.search(before)
+ prevention = _ACCESS_PREVENTION.search(before)
+ if prevention and not _ACCESS_NEGATION.search(before[:prevention.start()]):
+ direct = True
+ # A passive clause or gerund with its own predicate has its own
+ # subject; it cannot inherit the previous action's prohibition.
+ own_subject = (match.lastgroup == 'passive' or
+ re.match(r'\s+(?:is|are|must|should|may|can|will)\b', clause[match.end():]))
+ coordinated = (not own_subject and
+ (index in list_links or (',' not in before and
+ re.search(r'\b(?:and|or)\s+(?:(?:use|run)\s+)?$', before))))
+ # An infinitive purpose stays inside the negated action:
+ # "do not use yt-dlp / bypass login to download videos".
+ purpose = (index > 0 and not own_subject and
+ re.fullmatch(r'\s+to\s+', before))
+ passive_negative = (match.lastgroup == 'passive' and
+ re.search(r'\b(?:not|never|cannot)\b', match.group()))
+ negated = bool(direct or passive_negative or (negated and (coordinated or purpose)))
+ if not negated and index not in prohibited:
actions.append(match.group())
previous_end = match.end()
return actions
diff --git a/tests/test_review_policy_scope.py b/tests/test_review_policy_scope.py
new file mode 100644
index 0000000..171414b
--- /dev/null
+++ b/tests/test_review_policy_scope.py
@@ -0,0 +1,107 @@
+import pytest
+
+from fields_study_flow.mcp_tools import buildRoadmap, validateSources
+
+
+def validate(note):
+ resource = {'title': 'public course', 'url': 'https://example.org/course',
+ 'license_or_access_note': note}
+ return validateSources({'phases': [{'resources': [resource]}]})
+
+
+@pytest.mark.parametrize('note', [
+ 'Download videos',
+ 'Download the video now',
+ 'DOWNLOAD VIDEOS',
+ 'Please download these videos',
+ 'Bypass login to fetch the paper',
+ 'Download video content and bypass login',
+ 'Do not copy full text; download videos',
+ 'Do not download video content or you may bypass login',
+ 'Do not download video content but bypass login',
+ 'Never copy full text and then download videos',
+ 'Do not copy full text and do download videos',
+ 'Do not copy full text and please download videos',
+ 'Do not copy full text and must download videos',
+ 'Do not download videos; bypass login',
+ 'Videos must be downloaded',
+ 'The video is to be downloaded now',
+ 'Videos must not be downloaded, but bypass login',
+ 'Download videos and bypassing login is prohibited',
+ 'Do not download videos; login must be bypassed',
+ 'Downloading videos is not prohibited',
+ 'Downloading videos is required and bypassing login is prohibited',
+ 'Do not refuse to download videos',
+ 'Do not download videos and we will bypass login',
+ 'No login is required to download the videos',
+ 'Do not copy full text, download videos',
+ 'Videos must not be downloaded and login must be bypassed',
+ 'Do not download videos or full text must be copied',
+ 'Do not download videos and bypassing login is required',
+ 'Do not download videos and students must bypass login',
+ 'Do not download videos and login is to be bypassed',
+ 'Do not copy full text, download videos and you must bypass login',
+ 'Downloading videos is required, copying full text and bypassing login are prohibited',
+ 'Do not copy full text, read the abstract and download videos',
+ 'Do not download video content, read the abstract and bypass login',
+ 'Do not download videos, use yt-dlp',
+ 'You are not prohibited from downloading videos',
+ 'Do not refuse to bypass login',
+ 'Do not avoid downloading videos',
+ 'You are not forbidden to download videos',
+])
+def test_affirmative_video_and_bypass_clauses_are_rejected(note):
+ result = validate(note)
+ assert not result['valid'], note
+ assert any('Disallowed access instruction' in issue for issue in result['issues'])
+
+
+@pytest.mark.parametrize('note', [
+ 'Do not download video content or bypass login',
+ 'You must not download videos',
+ 'Never bypass login or download videos',
+ 'DO NOT DOWNLOAD VIDEOS',
+ 'Do not download the video now',
+ "Don't download videos or bypass login",
+ 'You should not download video content or bypass login',
+ 'Do not download video content and copy full text',
+ 'Do not bypass login to fetch the paper',
+ 'Videos must not be downloaded',
+ 'The video is not to be downloaded',
+ 'Downloading videos is prohibited',
+ 'Downloading videos or bypassing login is prohibited',
+ 'Download videos is prohibited',
+ 'Video content must never be downloaded',
+ 'Login must not be bypassed',
+ "Don't use yt-dlp to download videos",
+ 'Never download videos, and never bypass login.',
+ 'Do not download videos. Use public paper links.',
+ 'We do not download video content or bypass login',
+ 'Downloading videos and copying full text are not allowed',
+ 'Do not download videos or use yt-dlp',
+ 'Do not copy full text from the paper or download videos',
+ 'You must not bypass login or download the video',
+ 'Do not download video content, or bypass login',
+ 'Never bypass login, and download videos',
+ 'Do not bypass login to download videos',
+ 'No downloading videos or bypassing login',
+ 'Do not bypass login, scrape restricted pages, or download videos.',
+ 'Never download videos, copy full text and bypass login.',
+ 'Downloading videos, copying full text, and bypassing login are prohibited.',
+ 'You are not allowed to download videos',
+ 'You are forbidden to download the video',
+ 'You are prohibited from downloading videos',
+ 'Avoid downloading videos or bypassing login',
+ 'Do not attempt to download video content or bypass login',
+])
+def test_prohibitions_keep_action_and_clause_scope(note):
+ result = validate(note)
+ assert result['valid'], result['issues']
+
+
+def test_generated_plan_accepts_coordinated_safety_note():
+ resource = {'title': 'public course', 'url': 'https://example.org/course',
+ 'source': 'course', 'type': 'course', 'estimated_time': '2h',
+ 'license_or_access_note': 'Do not download video content or bypass login'}
+ plan = buildRoadmap('study', {}, [resource])
+ assert validateSources(plan) == {'valid': True, 'issues': []}
From 27a34f8da6fbced264c7e9493d20de67938b6592 Mon Sep 17 00:00:00 2001
From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com>
Date: Sun, 18 Oct 2026 19:35:01 +0800
Subject: [PATCH 6/7] fix: resolve resource dependencies covered by planned
foundations
---
fields_study_flow/prerequisites.py | 11 +++-
tests/test_review_mcp_scope.py | 63 ++++++++++++++++++
tests/test_review_prerequisite_coverage.py | 74 ++++++++++++++++++++++
3 files changed, 147 insertions(+), 1 deletion(-)
create mode 100644 tests/test_review_mcp_scope.py
create mode 100644 tests/test_review_prerequisite_coverage.py
diff --git a/fields_study_flow/prerequisites.py b/fields_study_flow/prerequisites.py
index d5eeb66..3fce818 100644
--- a/fields_study_flow/prerequisites.py
+++ b/fields_study_flow/prerequisites.py
@@ -55,7 +55,7 @@ def order_resources(profile, nodes, resources):
{'beginner': 0, 'intermediate': 1, 'advanced': 2}.get(resource.difficulty, 1),
))
learned = {concept_id(topic) for topic in profile.known_topics}
- planned = {node['id'] for node in nodes}
+ planned = {concept_id(node['id']) for node in nodes}
provided = {concept_id(concept) for resource in resources for concept in resource.concepts}
# Prefer concrete resource providers over a generic curated estimate.
available = learned | (planned - provided)
@@ -69,6 +69,15 @@ def order_resources(profile, nodes, resources):
ready = next((resource for resource in remaining
if all(concept_id(dep) in available for dep in resource.prerequisites)), None)
if ready is None:
+ # Concrete providers are preferred, but may themselves require the
+ # concept they teach. Curated foundations are already scheduled
+ # before every resource, so use that independent coverage before
+ # declaring a cycle. Never invent coverage for custom concepts.
+ covered = {concept_id(dep) for resource in remaining for dep in resource.prerequisites}
+ covered &= planned - available
+ if covered:
+ available.update(covered)
+ continue
blocked = '; '.join(f'{resource.title}: {", ".join(resource.prerequisites)}' for resource in remaining)
raise ValueError('Resource dependency cycle: ' + blocked)
remaining.remove(ready)
diff --git a/tests/test_review_mcp_scope.py b/tests/test_review_mcp_scope.py
new file mode 100644
index 0000000..7b32c14
--- /dev/null
+++ b/tests/test_review_mcp_scope.py
@@ -0,0 +1,63 @@
+import asyncio
+import json
+import sys
+
+from mcp.client.session import ClientSession
+from mcp.client.stdio import StdioServerParameters, stdio_client
+
+
+def test_sdk_stdio_access_action_scope():
+ async def run():
+ params = StdioServerParameters(command=sys.executable, args=['-m', 'fields_study_flow.mcp_server', '--mcp'])
+ async with stdio_client(params) as streams:
+ async with ClientSession(*streams) as session:
+ await session.initialize()
+ for note, valid in [
+ ('Download videos', False),
+ ('Download the video now', False),
+ ('Do not download video content or bypass login', True),
+ ('You must not download videos', True),
+ ('Bypass login to fetch the paper', False),
+ ('Never bypass login or download videos', True),
+ ('Videos must not be downloaded and login must be bypassed', False),
+ ('Do not bypass login, scrape restricted pages, or download videos.', True),
+ ]:
+ resource = {'title': 'course', 'url': 'https://example.org/course',
+ 'license_or_access_note': note}
+ result = await session.call_tool('validateSources', {'plan': {'phases': [{'resources': [resource]}]}})
+ assert not result.is_error, result
+ assert json.loads(result.content[0].text)['valid'] is valid, note
+ asyncio.run(asyncio.wait_for(run(), timeout=20))
+
+
+def test_sdk_stdio_planned_prerequisite_coverage():
+ async def run():
+ params = StdioServerParameters(command=sys.executable, args=['-m', 'fields_study_flow.mcp_server', '--mcp'])
+ async with stdio_client(params) as streams:
+ async with ClientSession(*streams) as session:
+ await session.initialize()
+ resource = {'title': 'advanced-python', 'url': 'https://example.org/python',
+ 'source': 'course', 'type': 'course', 'difficulty': 'advanced',
+ 'concepts': ['python'], 'prerequisites': ['python'], 'estimated_time': '2h',
+ 'license_or_access_note': 'Do not download video content or bypass login'}
+ arguments = {'goal': 'Python', 'profile': {'weekly_hours': 2}, 'rankedResources': [resource]}
+ result = await session.call_tool('buildRoadmap', arguments)
+ assert not result.is_error, result
+ plan = json.loads(result.content[0].text)
+ assert [(item['id'], item['hours'], item['start_week']) for item in plan['schedule']['items']] == [
+ ('python', 6, 1), ('https://example.org/python', 2, 4),
+ ]
+ checked = await session.call_tool('validateSources', {'plan': plan})
+ assert not checked.is_error
+ assert json.loads(checked.content[0].text) == {'valid': True, 'issues': []}
+
+ for concepts, dependencies, error in [
+ (['uncovered'], ['uncovered'], 'cycle'),
+ (['python'], ['missing-basics'], 'unknown prerequisite'),
+ ]:
+ invalid = dict(resource, concepts=concepts, prerequisites=dependencies)
+ result = await session.call_tool('buildRoadmap', dict(arguments, rankedResources=[invalid]))
+ assert result.is_error
+ assert error in result.content[0].text.lower()
+ assert 'advanced-python' in result.content[0].text
+ asyncio.run(asyncio.wait_for(run(), timeout=20))
diff --git a/tests/test_review_prerequisite_coverage.py b/tests/test_review_prerequisite_coverage.py
new file mode 100644
index 0000000..1519b43
--- /dev/null
+++ b/tests/test_review_prerequisite_coverage.py
@@ -0,0 +1,74 @@
+import pytest
+
+from fields_study_flow.models import LearnerProfile, Resource
+from fields_study_flow.roadmap import build_roadmap
+
+
+def course(title, concepts, prerequisites=(), difficulty='advanced'):
+ return Resource(title=title, url=f'https://example.org/{title}', source='course',
+ type='course', concepts=list(concepts), prerequisites=list(prerequisites),
+ estimated_time='2h', difficulty=difficulty)
+
+
+@pytest.mark.parametrize('concept,dependency', [('python', 'python'), ('编程', 'Python')])
+def test_planned_python_foundation_satisfies_advanced_self_dependency(concept, dependency):
+ advanced = course('advanced-python', [concept], [dependency])
+ plan = build_roadmap(LearnerProfile(goal='advanced Python', weekly_hours=2), [advanced])
+ assert plan['prerequisites'] == [{
+ 'id': 'python', 'depends_on': [], 'estimated_hours': 6, 'status': 'required',
+ 'reason': 'Prerequisite for target concepts',
+ }]
+ assert plan['schedule']['items'] == [
+ {'id': 'python', 'hours': 6, 'kind': 'prerequisite', 'start_week': 1, 'end_week': 3},
+ {'id': advanced.url, 'hours': 2, 'kind': 'resource', 'start_week': 4, 'end_week': 4},
+ ]
+ assert [r['title'] for phase in plan['phases'] for r in phase['resources']] == ['advanced-python']
+ assert plan['schedule']['total_weeks'] == 4
+
+
+def test_concrete_foundation_precedes_advanced_course_with_shared_concept():
+ # Equal difficulty and advanced-first input ensure ordering follows coverage.
+ advanced = course('advanced-python', ['python'], ['python'])
+ foundation = course('python-intro', ['python'])
+ plan = build_roadmap(LearnerProfile(goal='Python'), [advanced, foundation])
+ assert [r['title'] for phase in plan['phases'] for r in phase['resources']] == ['python-intro', 'advanced-python']
+ assert [item['id'] for item in plan['schedule']['items']] == ['python', foundation.url, advanced.url]
+
+
+def test_custom_foundation_covers_advanced_course_with_shared_concept():
+ advanced = course('advanced-custom', ['custom'], ['custom'])
+ foundation = course('custom-intro', ['custom'])
+ plan = build_roadmap(LearnerProfile(goal='study'), [advanced, foundation])
+ assert not plan['prerequisites']
+ assert [item['id'] for item in plan['schedule']['items']] == [foundation.url, advanced.url]
+
+
+def test_known_python_satisfies_self_dependency_without_replanning():
+ advanced = course('advanced-python', ['python'], ['编程'])
+ plan = build_roadmap(LearnerProfile(goal='Python', known_topics=['Python']), [advanced])
+ assert plan['prerequisites'][0]['status'] == 'skipped'
+ assert [item['id'] for item in plan['schedule']['items']] == [advanced.url]
+
+
+def test_planned_coverage_unblocks_resource_chain_before_cycle_detection():
+ # The Python estimate is an independent provider, so this apparent resource
+ # cycle is resolvable: Python foundation -> practice -> advanced Python.
+ advanced = course('advanced-python', ['python'], ['practice'])
+ practice = course('practice', ['practice'], ['python'])
+ plan = build_roadmap(LearnerProfile(goal='Python'), [advanced, practice])
+ assert [item['id'] for item in plan['schedule']['items']] == ['python', practice.url, advanced.url]
+ assert [r['title'] for phase in plan['phases'] for r in phase['resources']] == ['practice', 'advanced-python']
+
+
+@pytest.mark.parametrize('resources,expected', [
+ ([course('advanced-custom', ['custom'], ['custom'])], ['cycle', 'advanced-custom', 'custom']),
+ ([course('course-a', ['a'], ['b']), course('course-b', ['b'], ['a'])], ['cycle', 'course-a', 'course-b', 'a', 'b']),
+ ([course('advanced-python', ['python', 'a'], ['python', 'b']), course('course-b', ['b'], ['a'])],
+ ['cycle', 'advanced-python', 'course-b', 'a', 'b']),
+ ([course('advanced-python', ['python'], ['python', 'missing-basics'])], ['unknown prerequisite', 'advanced-python', 'missing-basics']),
+])
+def test_uncovered_dependencies_still_report_readable_errors(resources, expected):
+ with pytest.raises(ValueError) as error:
+ build_roadmap(LearnerProfile(goal='study'), resources)
+ for term in expected:
+ assert term in str(error.value)
From e97ee827ef50261bb1e152684f6111963a00cb51 Mon Sep 17 00:00:00 2001
From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com>
Date: Sun, 18 Oct 2026 20:01:09 +0800
Subject: [PATCH 7/7] fix: bound legacy policy obligations and prohibition
scope
---
README.md | 6 +-
README.zh-CN.md | 6 +-
fields_study_flow/mcp_tools.py | 23 +++--
tests/test_review_mcp_policy_grammar.py | 36 +++++++
tests/test_review_policy_grammar.py | 119 ++++++++++++++++++++++++
5 files changed, 180 insertions(+), 10 deletions(-)
create mode 100644 tests/test_review_mcp_policy_grammar.py
create mode 100644 tests/test_review_policy_grammar.py
diff --git a/README.md b/README.md
index 3d15630..7b8843a 100644
--- a/README.md
+++ b/README.md
@@ -227,4 +227,8 @@ Standard MCP hosts must run `python -m fields_study_flow.mcp_server --mcp`. Both
Ranking exposes relevance, trust, structure, difficulty, outcome and language components plus unmatched goal terms. A small bilingual alias map and domain levels improve explanations; this is lexical ranking, not semantic search. Thirty fixed review cases live in `tests/fixtures/ranking_cases.json`. Low lexical relevance is labeled supplemental. Prerequisites use the small curated graph described above, seeded by the goal and supplied resource concepts (including the README's known Transformer paper), and skip explicitly known topics. Arbitrary paper URLs do not imply extracted paper concepts. Phases and sequential schedules share one resource dependency order; cycles and unknown prerequisite references raise explicit errors. Target papers start core study after their dependencies. Unknown durations leave that item and all later weeks undetermined; `known_estimated_hours` retains the known subtotal while total hours/weeks remain unknown. These curated estimates may overlap between prerequisites and resource study and are planning aids, not completion guarantees. Learner levels accept `beginner`, `familiar`, `advanced` and the existing Chinese aliases `初学`, `熟悉`, `老手`; invalid levels report allowed values.
-Safety checks use parsed URL hosts/schemes and structured `metadata.access_policy` actions (`metadata`, `link`, `summarize`). Legacy notes retain compatibility checks for affirmative prohibited instructions; generated safety advice is not treated as an access operation.
+Safety checks use parsed URL hosts/schemes. When present, structured `metadata.access_policy` is authoritative for access actions: `metadata`, `link`, and `summarize` are allowed, while unsupported actions, login bypass, mirroring, or `authorized: false` are rejected. Legacy prose is not cross-checked against a present structured policy; URL and missing-note checks still apply independently.
+
+Without structured policy, English access notes receive **finite compatibility rules**, not arbitrary English semantic parsing. The rules recognize the listed download-video, bypass-login, copy-full-text/copyrighted-content, restricted-scraping actions and downloader tool names, including inflections, determiners, and limited passive auxiliaries (`must`, `need(s) to`, `has/have to`, `is/are required to`). Local prohibitions include `do not`, `never`, and trailing `is/are prohibited` or `not/never allowed/permitted`; coordinated action lists can share prohibition scope. Punctuation, `but/however/then/instead`, explicit new subjects, and recognized finite/imperative markers reset inherited scope, including markers before intervening ordinary actions. Thus `Videos need to be downloaded` and `Do not copy full text and instead watch and download video content` are rejected, while `Downloading videos is never permitted` is accepted.
+
+These lexical rules do not resolve arbitrary synonyms, indirect requests, quotation, conditionals, pronouns, nested clauses, all double negations, or other languages. Even some unlisted English constructions can be missed or misclassified; a legacy `valid` result is not proof that arbitrary prose is safe. Supply explicit structured policy for machine-generated access decisions. The implementation does not fetch, download, or execute instructions from these notes.
diff --git a/README.zh-CN.md b/README.zh-CN.md
index b9b7579..03cc53f 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -230,4 +230,8 @@ CLI `roadmap` 默认离线；`--live --sources arxiv,github` 显式调用公开
排序输出相关性、可信度、结构、难度、成果偏好、语言等分项及缺失词项。小型中英别名表和领域等级避免全局降级；仍是词汇匹配，不是语义模型。30 条固定样例位于 `tests/fixtures/ranking_cases.json`。低相关性标为补充资料，不声称匹配。先修图使用上文的小型人工图，由目标和已提供的资源概念（包括 README 的已知 Transformer 论文）触发，并跳过明确已知主题；任意论文 URL 不代表已提取论文概念。阶段和串行周计划共用资源依赖顺序，环和未知先修引用会明确报错。目标论文在依赖之后进入核心阶段。未知时长使该项及后续周次未定，`known_estimated_hours` 保留已知小计，总工时与总周数保持 unknown。先修与资源学习工时可能重叠，人工估算不保证实际完成时间。学习等级接受 `beginner/familiar/advanced` 和已有中文别名 `初学/熟悉/老手`，错误等级会提示允许值。
-合规校验使用解析后的 URL 主机/协议和 `metadata.access_policy` 的结构化动作（metadata/link/summarize）；旧记录的肯定违规指令保留兼容校验，不把生成的否定安全说明当成访问操作。
+合规校验使用解析后的 URL 主机/协议。存在 `metadata.access_policy` 时，以结构化 policy 为访问动作的权威：允许 `metadata/link/summarize`，拒绝其他动作、绕登录、镜像或 `authorized: false`；此时不再对旧文案做动作交叉检查。URL 与访问说明缺失检查仍独立执行。
+
+没有结构化 policy 时，英文自由文案仅使用**有限兼容规则**，不支持任意英文语义解析。规则识别既定的视频下载、绕登录、全文/长篇版权复制、受限页面抓取动作及下载工具名，覆盖词形、限定词和有限被动助动表达（`must`、`need(s) to`、`has/have to`、`is/are required to`）。局部明确禁止包括 `do not`、`never`、尾部 `is/are prohibited` 或 `not/never allowed/permitted`；并列动作列表可以共享禁止范围。标点、`but/however/then/instead`、显式新主语及识别到的限定谓语/祈使标记终止继承的否定，标记与敏感动作之间插入普通动作也如此。因此 `Videos need to be downloaded` 和 `Do not copy full text and instead watch and download video content` 被拒绝，`Downloading videos is never permitted` 被允许。
+
+这些词法规则不解析任意同义词、间接请求、引语、条件句、代词、嵌套从句、所有双重否定或其他语言。未列出的英文句式仍可能漏判或误判；legacy 返回 `valid` 不代表任意文案安全。机器生成的访问决策应提供明确的结构化 policy。校验器不会抓取、下载或执行文案中的指令。
diff --git a/fields_study_flow/mcp_tools.py b/fields_study_flow/mcp_tools.py
index 9df3211..b041a63 100644
--- a/fields_study_flow/mcp_tools.py
+++ b/fields_study_flow/mcp_tools.py
@@ -26,10 +26,12 @@
"anna-archive",
)
-# Match actions and their objects, including inflections and passive instructions.
-# These are deliberately local patterns: a prohibition elsewhere in a note must
-# not hide an affirmative instruction.
+# Bounded legacy-English compatibility, not a general semantic policy parser.
+# Match each action/object in either voice, then resolve only local prohibition
+# and coordination. Structured metadata.access_policy remains authoritative.
_ACCESS_DETERMINER = r'(?:(?:the|a|an|this|that|these|those|any|all|some|your|their)\s+)*'
+_ACCESS_PREDICATE = r'(?:must|should|may|can|cannot|will|shall|is|are|needs?|has|have)'
+_ACCESS_NEW_INSTRUCTION = re.compile(r'\b(?:' + _ACCESS_PREDICATE + r'|do|please)\b')
_ACCESS_OBJECTS = (
(r'download(?:s|ed|ing)?', r'videos?(?:\s+content)?', 'downloaded'),
(r'bypass(?:es|ed|ing)?', r'login', 'bypassed'),
@@ -40,7 +42,7 @@
r'\b(?:(?P' + '|'.join(
verb + r'\s+' + _ACCESS_DETERMINER + obj for verb, obj, _ in _ACCESS_OBJECTS
) + r')|(?Pyt-dlp|youtube-dl)|(?P' + '|'.join(
- obj + r'\s+(?:(?:must|should|may|can|cannot|will|shall|is|are|be|been|being|not|never|to)\s+)+' + participle
+ obj + r'\s+(?:(?:' + _ACCESS_PREDICATE + r'|be|been|being|not|never|to|required)\s+)+' + participle
for _, obj, participle in _ACCESS_OBJECTS
) + r'))\b'
)
@@ -50,7 +52,7 @@
)
_ACCESS_PREVENTION = re.compile(r'\b(?:avoid\s+|(?:forbidden|prohibited|banned)\s+(?:to|from)\s+)$')
_ACCESS_PROHIBITION = re.compile(
- r'\s+(?:is|are)\s+(?:strictly\s+)?(?:prohibited|forbidden|banned|not\s+(?:allowed|permitted))\b'
+ r'\s+(?:is|are)\s+(?:strictly\s+)?(?:prohibited|forbidden|banned|(?:not|never)\s+(?:allowed|permitted))\b'
)
@@ -167,7 +169,7 @@ def _affirmative_access_actions(note: str) -> list[str]:
# A fresh subject, modal, or imperative ends inherited negation, even when
# its subject is omitted ("do not copy ... and must download ...").
clauses = re.split(
- r'[.;:!?\n]|\b(?:but|however|then)\b|\b(?:and|or)\s+'
+ r'[.;:!?\n]|\b(?:but|however|then|instead)\b|\b(?:and|or)\s+'
r'(?=(?:you|we|they|he|she|it|one|the|users?|agents?|must|should|may|can|will|shall|do|please)\b)',
note.lower().replace('’', "'"),
)
@@ -211,8 +213,13 @@ def _affirmative_access_actions(note: str) -> list[str]:
# A passive clause or gerund with its own predicate has its own
# subject; it cannot inherit the previous action's prohibition.
own_subject = (match.lastgroup == 'passive' or
- re.match(r'\s+(?:is|are|must|should|may|can|will)\b', clause[match.end():]))
- coordinated = (not own_subject and
+ re.match(r'\s+' + _ACCESS_PREDICATE + r'\b', clause[match.end():]))
+ # A finite predicate or imperative marker anywhere in the gap
+ # starts a new instruction, even with intervening ordinary actions
+ # ("and students must watch and download videos"). Do not infer
+ # arbitrary English subjects or verbs; only these explicit markers.
+ new_instruction = _ACCESS_NEW_INSTRUCTION.search(before)
+ coordinated = (not own_subject and not new_instruction and
(index in list_links or (',' not in before and
re.search(r'\b(?:and|or)\s+(?:(?:use|run)\s+)?$', before))))
# An infinitive purpose stays inside the negated action:
diff --git a/tests/test_review_mcp_policy_grammar.py b/tests/test_review_mcp_policy_grammar.py
new file mode 100644
index 0000000..ed43aa8
--- /dev/null
+++ b/tests/test_review_mcp_policy_grammar.py
@@ -0,0 +1,36 @@
+import asyncio
+import json
+import sys
+
+from mcp.client.session import ClientSession
+from mcp.client.stdio import StdioServerParameters, stdio_client
+
+
+def test_sdk_stdio_policy_obligations_prohibitions_and_boundaries():
+ async def run():
+ params = StdioServerParameters(command=sys.executable, args=['-m', 'fields_study_flow.mcp_server', '--mcp'])
+ async with stdio_client(params) as streams:
+ async with ClientSession(*streams) as session:
+ await session.initialize()
+ for note, policy, valid in [
+ ('Videos need to be downloaded', None, False),
+ ('Downloading videos is never permitted', None, True),
+ ('Do not copy full text and instead watch and download video content', None, False),
+ ('The video has to be downloaded', None, False),
+ ('Downloading videos and bypassing login are never allowed', None, True),
+ ('Do not copy full text and viewers have to watch and download videos', None, False),
+ ('Do not copy full text and instead never download video content', None, True),
+ ('Download videos and bypassing login is never permitted', None, False),
+ ('Videos need to be downloaded', {'action': 'metadata'}, True),
+ ('Downloading videos is never permitted', {'action': 'download_video'}, False),
+ ]:
+ resource = {'title': 'public course', 'url': 'https://example.org/course',
+ 'license_or_access_note': note}
+ if policy is not None:
+ resource['metadata'] = {'access_policy': policy}
+ result = await session.call_tool('validateSources', {'plan': {'phases': [{'resources': [resource]}]}})
+ assert not result.is_error, result
+ checked = json.loads(result.content[0].text)
+ assert checked['valid'] is valid, (note, policy, checked)
+ assert bool(checked['issues']) is not valid
+ asyncio.run(asyncio.wait_for(run(), timeout=20))
diff --git a/tests/test_review_policy_grammar.py b/tests/test_review_policy_grammar.py
new file mode 100644
index 0000000..8ba5228
--- /dev/null
+++ b/tests/test_review_policy_grammar.py
@@ -0,0 +1,119 @@
+import pytest
+
+from fields_study_flow.mcp_tools import validateSources
+
+
+def validate(note, policy=None, url='https://example.org/course'):
+ resource = {'title': 'public course', 'url': url, 'license_or_access_note': note}
+ if policy is not None:
+ resource['metadata'] = {'access_policy': policy}
+ return validateSources({'phases': [{'resources': [resource]}]})
+
+
+@pytest.mark.parametrize('note', [
+ 'Videos need to be downloaded',
+ 'The video needs to be downloaded',
+ 'Video content needs to be downloaded',
+ 'These videos have to be downloaded',
+ 'The video has to be downloaded',
+ 'Videos are required to be downloaded',
+ 'You need to download videos',
+ 'Please download the videos',
+ 'Login needs to be bypassed',
+ 'Full text has to be copied',
+ 'Restricted pages need to be scraped',
+ 'Do not copy full text and videos need to be downloaded',
+ 'Do not download videos or login has to be bypassed',
+])
+def test_active_and_passive_obligations_are_rejected(note):
+ result = validate(note)
+ assert not result['valid'], note
+ assert any('Disallowed access instruction' in issue for issue in result['issues'])
+
+
+@pytest.mark.parametrize('note', [
+ 'Downloading videos is never permitted',
+ 'Downloading the video is never allowed',
+ 'Downloading video content is not permitted',
+ 'Bypassing login is never permitted',
+ 'Copying full text is never allowed',
+ 'Scraping restricted pages is never permitted',
+ 'Downloading videos and bypassing login are never permitted',
+ 'Downloading videos, copying full text, and bypassing login are never allowed',
+ 'yt-dlp and youtube-dl are never permitted',
+ 'You are never permitted to download videos',
+])
+def test_negated_permission_prohibits_only_its_action_list(note):
+ assert validate(note) == {'valid': True, 'issues': []}
+
+
+@pytest.mark.parametrize('note', [
+ 'Do not copy full text and instead watch and download video content',
+ 'Never copy full text; instead watch and download videos',
+ 'Do not bypass login and instead read and copy full text',
+ 'Do not copy full text and instead inspect and scrape restricted pages',
+ 'Do not copy full text and instead watch and use yt-dlp',
+ 'Do not download videos but read and bypass login',
+ 'Do not copy full text and then watch and download videos',
+ 'Do not copy full text and please watch and download videos',
+ 'Do not copy full text and students must watch and download videos',
+ 'Do not copy full text and students need to watch and download videos',
+ 'Do not copy full text and do watch and download videos',
+ 'Do not copy full text and viewers have to watch and download videos',
+])
+def test_new_instruction_does_not_inherit_an_earlier_prohibition(note):
+ result = validate(note)
+ assert not result['valid'], note
+ assert any('Disallowed access instruction' in issue for issue in result['issues'])
+
+
+@pytest.mark.parametrize('note', [
+ 'Do not copy full text and instead never download video content',
+ 'Do not bypass login; instead do not download videos',
+ 'Never copy full text; instead downloading videos is never permitted',
+ 'Do not bypass login; videos must never be downloaded',
+])
+def test_new_clause_can_introduce_its_own_prohibition(note):
+ assert validate(note) == {'valid': True, 'issues': []}
+
+
+@pytest.mark.parametrize('note', [
+ 'Downloading videos is never prohibited',
+ 'Downloading videos is not forbidden',
+ 'Downloading videos is permitted',
+ 'Downloading videos is not never permitted',
+ 'Downloading videos is not allowed; login needs to be bypassed',
+ 'Downloading videos is never permitted; download videos',
+ 'Download videos and bypassing login is never permitted',
+ 'Downloading videos is required and bypassing login is never permitted',
+])
+def test_negated_bans_and_unrelated_prohibitions_do_not_allow_actions(note):
+ assert not validate(note)['valid'], note
+
+
+@pytest.mark.parametrize('policy,note,valid', [
+ ({'action': 'metadata'}, 'Videos need to be downloaded', True),
+ ({'action': 'link'}, 'Download videos', True),
+ ({'action': 'summarize'}, 'Do not copy full text and instead watch and download video content', True),
+ ({'action': 'download_video'}, 'Downloading videos is never permitted', False),
+ ({'action': 'metadata', 'bypass_login': True}, 'Never bypass login', False),
+ ({'action': 'link', 'mirror': True}, 'Link only', False),
+ ({'action': 'summarize', 'authorized': False}, 'Link only', False),
+])
+def test_structured_policy_is_authoritative_over_legacy_prose(policy, note, valid):
+ result = validate(note, policy)
+ assert result['valid'] is valid
+ if not valid:
+ assert any('Disallowed access policy' in issue for issue in result['issues'])
+ assert not any('Disallowed access instruction' in issue for issue in result['issues'])
+
+
+@pytest.mark.parametrize('url,note,issue', [
+ ('https://sci-hub.example/paper', 'Link only', 'Disallowed source'),
+ ('ftp://example.org/course', 'Link only', 'unsafe URL'),
+ ('https://example.org/course', '', 'Missing access note'),
+])
+def test_structured_policy_preserves_independent_url_and_note_checks(url, note, issue):
+ result = validate(note, {'action': 'link'}, url)
+ assert not result['valid']
+ assert any(issue in message for message in result['issues'])