From 6e14b6eb58a98307cd85edaa12e3b38b0ad6bf80 Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Mon, 10 Aug 2026 12:01:28 +0800 Subject: [PATCH 1/2] ci: split optional dependency profiles --- ...210\206CI\344\276\235\350\265\226.plan.md" | 36 ++++++++++ .github/workflows/model-tests.yml | 67 +++++++++++++++++++ .github/workflows/tests.yml | 21 ++++-- README.md | 14 +++- app/services/first_limit_alpha_service.py | 5 +- requirements-base.txt | 13 ++++ requirements-ci.txt | 3 + requirements-kronos.txt | 5 ++ requirements-mcp.txt | 2 + requirements-training.txt | 2 + requirements.txt | 22 ++---- tests/test_dependency_profiles.py | 62 +++++++++++++++++ tests/test_first_limit_alpha.py | 8 ++- 13 files changed, 235 insertions(+), 25 deletions(-) create mode 100644 ".cursor/plans/20260810-11_\346\213\206\345\210\206CI\344\276\235\350\265\226.plan.md" create mode 100644 .github/workflows/model-tests.yml create mode 100644 requirements-base.txt create mode 100644 requirements-ci.txt create mode 100644 requirements-kronos.txt create mode 100644 requirements-mcp.txt create mode 100644 requirements-training.txt create mode 100644 tests/test_dependency_profiles.py diff --git "a/.cursor/plans/20260810-11_\346\213\206\345\210\206CI\344\276\235\350\265\226.plan.md" "b/.cursor/plans/20260810-11_\346\213\206\345\210\206CI\344\276\235\350\265\226.plan.md" new file mode 100644 index 0000000..4f6a87c --- /dev/null +++ "b/.cursor/plans/20260810-11_\346\213\206\345\210\206CI\344\276\235\350\265\226.plan.md" @@ -0,0 +1,36 @@ +# 拆分 CI 依赖 + +## 背景 / 目标 + +当前 Python Tests 对所有改动都安装完整 Torch / Hugging Face / MCP 依赖，而完整测试执行仅约 14 秒。目标是在保持 `requirements.txt` 全功能安装合同和模型相关变更覆盖的前提下，缩短普通 PR 的依赖安装时间并消除重复触发。 + +## 任务分解（checklist） + +- [ ] 拆分基础运行、Kronos 模型、MCP 与 CI 测试依赖。 +- [ ] 保留 `requirements.txt` 作为完整功能安装入口。 +- [ ] 普通 Python Tests 仅安装 CI 依赖，并启用 pip 缓存。 +- [ ] 限制常规触发为 main push 与面向 main 的 PR，避免 feature push + PR 重复运行。 +- [ ] 新增模型路径专用 workflow，安装模型依赖并验证导入。 +- [ ] 删除当前仓库无引用的 scikit-learn / lightgbm 依赖。 +- [ ] 更新 README 安装档位说明。 +- [ ] 在隔离环境验证基础安装不包含模型/MCP 包且 API 可启动、完整测试通过。 +- [ ] 提交、推送、创建 PR，等待 CI 通过后合并并关闭 Issue #5。 + +## 验收标准 + +- `requirements.txt` 递归安装基础、Kronos 与 MCP 全部功能依赖。 +- `requirements-ci.txt` 不包含 torch、Hugging Face、MCP、scikit-learn 或 lightgbm。 +- 普通 CI 在缺少可选包的环境中完成 API 启动与完整 pytest。 +- 模型代码或依赖变化会触发独立 model-import 检查。 +- workflow YAML 可解析，GitHub Actions 绿灯。 + +## 风险与回滚 + +- 风险：隐藏的顶层导入导致基础 CI 无法启动；用全新虚拟环境实际安装并启动 API 验证。 +- 风险：模型代码失去覆盖；以 paths 触发的专用 workflow 保留依赖和导入检查。 +- 回滚：恢复单一 requirements 与原 workflow 安装命令，不涉及数据库和生产数据。 + +## 关键决策记录 + +- 不把模型依赖从完整安装入口移除，只让普通 CI 使用轻量档位。 +- benchmark 不纳入普通 pytest；模型 workflow 只做确定性 import/构造检查，不下载远端权重。 diff --git a/.github/workflows/model-tests.yml b/.github/workflows/model-tests.yml new file mode 100644 index 0000000..f39f77c --- /dev/null +++ b/.github/workflows/model-tests.yml @@ -0,0 +1,67 @@ +name: Kronos Model Imports + +on: + push: + branches: [main] + paths: + - 'app/services/kronos_model/**' + - 'app/services/kronos_predict_service.py' + - 'app/services/first_limit_alpha_service.py' + - 'strategy/first_limit_alpha/train_sequence.py' + - 'tests/test_first_limit_alpha.py' + - 'requirements*.txt' + - '.github/workflows/model-tests.yml' + pull_request: + branches: [main] + paths: + - 'app/services/kronos_model/**' + - 'app/services/kronos_predict_service.py' + - 'app/services/first_limit_alpha_service.py' + - 'strategy/first_limit_alpha/train_sequence.py' + - 'tests/test_first_limit_alpha.py' + - 'requirements*.txt' + - '.github/workflows/model-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: kronos-imports-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + import-model-runtime: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: | + requirements*.txt + + - name: Install model runtime + run: | + python -m pip install --upgrade pip + pip install -r requirements-base.txt -r requirements-kronos.txt + pip check + + - name: Import Kronos runtime + run: | + python - <<'PY' + from app.services.kronos_model import Kronos, KronosPredictor, KronosTokenizer + + assert Kronos is not None + assert KronosPredictor is not None + assert KronosTokenizer is not None + print("Kronos runtime imports: ok") + PY + + - name: Test Torch sequence training + run: python -m pytest tests/test_first_limit_alpha.py -q -k sequence diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a40a5ee..b869a1f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,13 +2,21 @@ name: Python Tests on: push: - branches: ["**"] + branches: [main] pull_request: - branches: ["**"] + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: python-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout @@ -18,12 +26,15 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' + cache: pip + cache-dependency-path: | + requirements*.txt - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest + pip install -r requirements-ci.txt + pip check - name: Start API for regression tests env: diff --git a/README.md b/README.md index cb96b9c..9f4e1af 100644 --- a/README.md +++ b/README.md @@ -513,9 +513,22 @@ Alpha/ ### 安装依赖 ```bash +# 完整安装：基础 API / 数据源 + Kronos + MCP + 训练后端 pip3 install -r requirements.txt ``` +也可以按部署角色选择更小的依赖档位： + +| 安装档位 | 命令 | 用途 | +|----------|------|------| +| 基础运行 | `pip3 install -r requirements-base.txt` | FastAPI、行情数据、策略与本地缓存；Kronos / MCP 保持惰性可选 | +| Kronos | `pip3 install -r requirements-base.txt -r requirements-kronos.txt` | 增加 Torch、Hugging Face 与 Kronos 推理能力 | +| MCP | `pip3 install -r requirements-base.txt -r requirements-mcp.txt` | 增加独立 MCP Server | +| 策略训练 | `pip3 install -r requirements-base.txt -r requirements-training.txt` | 增加 FirstLimit Alpha 首选的 LightGBM 训练后端；未安装时自动使用 RandomForest | +| 完整功能 | `pip3 install -r requirements.txt` | 安装上述全部能力 | + +CI 使用 `requirements-ci.txt` 执行基础单元/API 回归；只有模型代码或模型依赖变化时，才运行独立的 Kronos 导入与 Torch 序列训练测试。模型权重仍在第一次真实预测时按需下载，不进入仓库或 CI 缓存。 + ### 启动服务 ```bash @@ -553,7 +566,6 @@ pip3 install -r requirements.txt 日志文件：默认 `logs/server-18890.log`，随 `PORT` 变化。 - ### 行情缓存维护 历史解析缺陷或错误补数可能污染 `data/market_kline.db`。维护命令默认只做只读预检；只有显式传入 `--apply` 才会先通过 SQLite backup API 创建并验证完整备份，再用单一事务清理早于 2000 年或无法解析的交易日期及其关联同步状态。 diff --git a/app/services/first_limit_alpha_service.py b/app/services/first_limit_alpha_service.py index 9452a07..eed06a2 100644 --- a/app/services/first_limit_alpha_service.py +++ b/app/services/first_limit_alpha_service.py @@ -25,7 +25,6 @@ TrainingConfig, ) from strategy.first_limit_alpha.feature_store import build_version_dir, latest_child, read_dataframe, read_json, write_json -from strategy.first_limit_alpha.train_sequence import FirstLimitSequenceTrainer DEFAULT_GRAPHIC_CONFIG: dict[str, Any] = { "threshold_candidate": 6.0, @@ -168,6 +167,10 @@ def train_baseline(self, training_cfg: TrainingConfig | None = None) -> dict[str return result def train_sequence(self, sequence_cfg: SequenceConfig | None = None) -> dict[str, Any]: + # Keep the core API usable without the optional Torch profile. Sequence + # training imports its backend only when this endpoint is invoked. + from strategy.first_limit_alpha.train_sequence import FirstLimitSequenceTrainer + samples = self._load_latest_samples() out_dir = build_version_dir(self.root, "sequence_models", prefix="gru") baseline_meta: dict[str, Any] = {} diff --git a/requirements-base.txt b/requirements-base.txt new file mode 100644 index 0000000..4e64c40 --- /dev/null +++ b/requirements-base.txt @@ -0,0 +1,13 @@ +# Core API and market-data runtime. +akshare>=1.16.0 +tushare>=1.4.0 +fastapi>=0.115.0 +uvicorn[standard]>=0.30.0 +pandas>=2.2.0 +pydantic>=2.8.0 +requests>=2.31.0 +httpx>=0.27.0 +tqdm>=4.60.0 +numpy>=1.24.0 +joblib>=1.4.0 +scikit-learn>=1.5.0 diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 0000000..dac93fd --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,3 @@ +# Deterministic unit/API regression environment; optional model/MCP imports stay lazy. +-r requirements-base.txt +pytest>=8.0.0 diff --git a/requirements-kronos.txt b/requirements-kronos.txt new file mode 100644 index 0000000..ec763f0 --- /dev/null +++ b/requirements-kronos.txt @@ -0,0 +1,5 @@ +# Optional Kronos model runtime. Model weights are downloaded lazily. +torch>=2.0.0 +einops>=0.8.1 +huggingface_hub>=0.33.0 +safetensors>=0.6.0 diff --git a/requirements-mcp.txt b/requirements-mcp.txt new file mode 100644 index 0000000..5ab660c --- /dev/null +++ b/requirements-mcp.txt @@ -0,0 +1,2 @@ +# Optional standalone MCP server runtime. +mcp>=1.27.0 diff --git a/requirements-training.txt b/requirements-training.txt new file mode 100644 index 0000000..35a7a05 --- /dev/null +++ b/requirements-training.txt @@ -0,0 +1,2 @@ +# Optional preferred backend for FirstLimit Alpha training. +lightgbm>=4.3.0 diff --git a/requirements.txt b/requirements.txt index a06085c..f32de13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,5 @@ -akshare>=1.16.0 -tushare>=1.4.0 -fastapi>=0.115.0 -uvicorn[standard]>=0.30.0 -pandas>=2.2.0 -pydantic>=2.8.0 -requests>=2.31.0 -httpx>=0.27.0 -torch>=2.0.0 -einops>=0.8.1 -huggingface_hub>=0.33.0 -safetensors>=0.6.0 -tqdm>=4.60.0 -numpy>=1.24.0 -mcp>=1.27.0 -scikit-learn>=1.5.0 -lightgbm>=4.3.0 +# Full Alpha installation: API/data runtime + Kronos + MCP + training backends. +-r requirements-base.txt +-r requirements-kronos.txt +-r requirements-mcp.txt +-r requirements-training.txt diff --git a/tests/test_dependency_profiles.py b/tests/test_dependency_profiles.py new file mode 100644 index 0000000..74b84d0 --- /dev/null +++ b/tests/test_dependency_profiles.py @@ -0,0 +1,62 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def _requirement_lines(name: str) -> set[str]: + return { + line.strip() + for line in (ROOT / name).read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + + +def test_full_install_includes_all_supported_profiles(): + assert _requirement_lines("requirements.txt") == { + "-r requirements-base.txt", + "-r requirements-kronos.txt", + "-r requirements-mcp.txt", + "-r requirements-training.txt", + } + + +def test_ci_profile_excludes_optional_heavy_dependencies(): + ci_lines = _requirement_lines("requirements-ci.txt") + base_lines = _requirement_lines("requirements-base.txt") + combined = "\n".join(sorted(ci_lines | base_lines)).lower() + + assert "-r requirements-base.txt" in ci_lines + assert "pytest>=8.0.0" in ci_lines + for package in ( + "torch", + "einops", + "huggingface_hub", + "safetensors", + "mcp", + "lightgbm", + ): + assert package not in combined + + +def test_model_workflow_tracks_model_paths_and_dependencies(): + workflow = (ROOT / ".github/workflows/model-tests.yml").read_text( + encoding="utf-8" + ) + + assert "app/services/kronos_model/**" in workflow + assert "app/services/kronos_predict_service.py" in workflow + assert "strategy/first_limit_alpha/train_sequence.py" in workflow + assert "requirements*.txt" in workflow + assert "requirements-kronos.txt" in workflow + assert "from app.services.kronos_model import" in workflow + assert "-k sequence" in workflow + + +def test_core_service_lazy_loads_torch_sequence_backend(): + service = (ROOT / "app/services/first_limit_alpha_service.py").read_text( + encoding="utf-8" + ) + + import_line = "from strategy.first_limit_alpha.train_sequence import FirstLimitSequenceTrainer" + assert service.index(import_line) > service.index("def train_sequence(") diff --git a/tests/test_first_limit_alpha.py b/tests/test_first_limit_alpha.py index 55bc57b..72119cc 100644 --- a/tests/test_first_limit_alpha.py +++ b/tests/test_first_limit_alpha.py @@ -3,6 +3,7 @@ from pathlib import Path import pandas as pd +import pytest from app.services.first_limit_alpha_service import FirstLimitAlphaService from app.services.kline_store import KlineSQLiteStore @@ -10,7 +11,6 @@ from strategy.first_limit_alpha.features import FirstLimitFeatureBuilder from strategy.first_limit_alpha.modeling import FirstLimitBaselineTrainer from strategy.first_limit_alpha.schema import LabelConfig, SampleBuildConfig, SequenceConfig, TrainingConfig -from strategy.first_limit_alpha.train_sequence import FirstLimitSequenceTrainer def _symbol_rows(base_close: float, event_mode: str, start: str = "2026-01-01") -> list[dict]: @@ -181,6 +181,9 @@ def test_baseline_trainer_excludes_future_columns(tmp_path: Path): def test_sequence_trainer_small_sample_returns_graceful_result(tmp_path: Path): + pytest.importorskip("torch") + from strategy.first_limit_alpha.train_sequence import FirstLimitSequenceTrainer + store = KlineSQLiteStore(str(tmp_path / "market_kline.db")) store.upsert_symbol_klines("600001", _symbol_rows(10.0, "strong"), "2026-04-22T11:00:00+08:00") samples = pd.DataFrame( @@ -200,6 +203,9 @@ def test_sequence_trainer_small_sample_returns_graceful_result(tmp_path: Path): def test_sequence_trainer_outputs_metrics_and_predictions(tmp_path: Path): + pytest.importorskip("torch") + from strategy.first_limit_alpha.train_sequence import FirstLimitSequenceTrainer + store = KlineSQLiteStore(str(tmp_path / "market_kline.db")) name_map = {} for idx in range(40): From 41952f0e6d69a8c20bb0e8baa03c7da60c0e38d9 Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Mon, 10 Aug 2026 12:05:40 +0800 Subject: [PATCH 2/2] ci: install pytest for model checks --- .github/workflows/model-tests.yml | 2 +- tests/test_dependency_profiles.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/model-tests.yml b/.github/workflows/model-tests.yml index f39f77c..564a389 100644 --- a/.github/workflows/model-tests.yml +++ b/.github/workflows/model-tests.yml @@ -49,7 +49,7 @@ jobs: - name: Install model runtime run: | python -m pip install --upgrade pip - pip install -r requirements-base.txt -r requirements-kronos.txt + pip install -r requirements-ci.txt -r requirements-kronos.txt pip check - name: Import Kronos runtime diff --git a/tests/test_dependency_profiles.py b/tests/test_dependency_profiles.py index 74b84d0..cf97ac5 100644 --- a/tests/test_dependency_profiles.py +++ b/tests/test_dependency_profiles.py @@ -48,6 +48,7 @@ def test_model_workflow_tracks_model_paths_and_dependencies(): assert "app/services/kronos_predict_service.py" in workflow assert "strategy/first_limit_alpha/train_sequence.py" in workflow assert "requirements*.txt" in workflow + assert "requirements-ci.txt" in workflow assert "requirements-kronos.txt" in workflow assert "from app.services.kronos_model import" in workflow assert "-k sequence" in workflow