From a27bcdc862a6e9d56c4400949fc9366721dd3d88 Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 21:15:36 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E5=85=83=E6=95=B0=E6=8D=AE=E6=A0=A1=E9=AA=8C=E5=8D=95=E5=85=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-130923-6b2088 Task-Id: VideoTranscriptAPI-20261008-16 --- .../progress/B-progress.md | 20 +++++ .../api/routes/uploads.py | 84 +++++++++++++++++++ tests/unit/test_upload_routes.py | 60 +++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 docs/sessions/261008-local-upload/progress/B-progress.md create mode 100644 src/video_transcript_api/api/routes/uploads.py create mode 100644 tests/unit/test_upload_routes.py diff --git a/docs/sessions/261008-local-upload/progress/B-progress.md b/docs/sessions/261008-local-upload/progress/B-progress.md new file mode 100644 index 00000000..a7c00bd4 --- /dev/null +++ b/docs/sessions/261008-local-upload/progress/B-progress.md @@ -0,0 +1,20 @@ +# B 卡进度 + +## 当前阶段 + +implementing；完成第一份独立可运行单元：bounded metadata decoder 与Unicode/字段约束测试通过。尚未实现 HTTP 接收、队列/SQLite 原子交接、worker readiness、实际integration验证。 + +## 本段结论（≤3句） + +- metadata 使用有界 UTF-8 base64url JSON；显示文件名规范化为 basename，不作为路径；处理选项沿用既有 normalize_processing_options。 +- `tests/unit/test_upload_routes.py` 通过，初始失败为值差异 AssertionError，修复后全绿。 +- 尚无验证失败归因或生产环境结论。 + +## 关键决策与否决方案 + +- 不消费客户端文件hash；元数据只声明正字节数，正文由服务端流式计数/hash。 +- source_url 只接受无userinfo的 http/https 文本，不抓取。 + +## 下一步唯一动作 + +扩展 CacheManager 持久 admission/root/media mapping 与 runtime 限额/预留，再接上真正 HTTP stream 和 dispatcher 测试。 diff --git a/src/video_transcript_api/api/routes/uploads.py b/src/video_transcript_api/api/routes/uploads.py new file mode 100644 index 00000000..154d6da4 --- /dev/null +++ b/src/video_transcript_api/api/routes/uploads.py @@ -0,0 +1,84 @@ +"""Bounded raw-byte local upload API.""" + +import base64 +import binascii +import json +import re +from pathlib import PurePosixPath +from urllib.parse import urlsplit + +from ..processing_options import normalize_processing_options + +_METADATA_MAX_BYTES = 4096 +_BASE64URL = re.compile(r"^[A-Za-z0-9_-]+={0,2}$") +_ALLOWED_METADATA = { + "filename", "byte_size", "title", "source_url", "retention", "processing_options" +} + + +def decode_upload_metadata(value: str) -> dict: + """Decode and validate the bounded UTF-8 JSON metadata header.""" + if not isinstance(value, str) or not value or len(value) > 6000 or not _BASE64URL.fullmatch(value): + raise ValueError("X-Upload-Metadata must be bounded base64url") + try: + encoded = value.rstrip("=") + raw = base64.b64decode( + encoded + "=" * (-len(encoded) % 4), altchars=b"-_", validate=True + ) + if len(raw) > _METADATA_MAX_BYTES: + raise ValueError("upload metadata exceeds 4096 bytes") + data = json.loads(raw.decode("utf-8")) + except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("X-Upload-Metadata must encode UTF-8 JSON") from exc + if not isinstance(data, dict) or set(data) != _ALLOWED_METADATA: + raise ValueError("upload metadata fields are incomplete or unsupported") + + filename = data["filename"] + if not isinstance(filename, str) or not filename.strip(): + raise ValueError("filename must be non-empty text") + if len(filename.encode("utf-8")) > 255: + raise ValueError("filename exceeds 255 UTF-8 bytes") + if any(ord(char) < 32 or ord(char) == 127 for char in filename): + raise ValueError("filename contains control characters") + display_name = PurePosixPath(filename.replace("\\", "/")).name + if not display_name or display_name in {".", ".."}: + raise ValueError("filename has no display basename") + + byte_size = data["byte_size"] + if not isinstance(byte_size, int) or isinstance(byte_size, bool) or byte_size <= 0: + raise ValueError("byte_size must be a positive integer") + title = data["title"] + if title is not None and ( + not isinstance(title, str) or len(title) > 200 + or any(ord(char) < 32 and char not in "\t\n\r" for char in title) + ): + raise ValueError("title must be at most 200 characters") + source_url = data["source_url"] + if source_url is not None: + if not isinstance(source_url, str) or len(source_url.encode("utf-8")) > 2048: + raise ValueError("source_url exceeds 2048 UTF-8 bytes") + parsed = urlsplit(source_url) + if ( + parsed.scheme.lower() not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or any(ord(char) < 32 or ord(char) == 127 for char in source_url) + ): + raise ValueError("source_url must be http(s) without userinfo") + retention = data["retention"] + if retention not in {"30d", "never"}: + raise ValueError("retention must be '30d' or 'never'") + options = data["processing_options"] + if not isinstance(options, dict): + raise ValueError("processing_options must be an object") + try: + normalized_options = normalize_processing_options(options) + except (TypeError, ValueError) as exc: + raise ValueError(f"invalid processing_options: {exc}") from exc + + return { + **data, + "filename": display_name, + "processing_options": normalized_options, + } diff --git a/tests/unit/test_upload_routes.py b/tests/unit/test_upload_routes.py new file mode 100644 index 00000000..9cd14ccb --- /dev/null +++ b/tests/unit/test_upload_routes.py @@ -0,0 +1,60 @@ +import base64 +import json + +import pytest + +from video_transcript_api.api.routes.uploads import decode_upload_metadata + + +def encode_metadata(payload): + raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def test_decode_upload_metadata_preserves_unicode_and_normalizes_options(): + payload = { + "filename": "../会议 录音.mp4", + "byte_size": 17, + "title": "讨论：咖啡与模型", + "source_url": "https://example.com/watch?id=1", + "retention": "30d", + "processing_options": {"calibrate": False, "summarize": True}, + } + + metadata = decode_upload_metadata(encode_metadata(payload)) + + assert metadata == { + **payload, + "filename": "会议 录音.mp4", + "processing_options": { + "calibrate": False, + "summarize": True, + "infer_speaker_names": True, + "chapters": True, + }, + } + + +@pytest.mark.parametrize( + "mutation", + [ + {"title": "x" * 201}, + {"filename": "é" * 128}, + {"source_url": "https://user:secret@example.com/a"}, + {"source_url": "javascript:alert(1)"}, + {"byte_size": 0}, + ], +) +def test_decode_upload_metadata_rejects_invalid_bounded_fields(mutation): + payload = { + "filename": "clip.mp4", + "byte_size": 17, + "title": None, + "source_url": None, + "retention": "30d", + "processing_options": {}, + **mutation, + } + + with pytest.raises(ValueError): + decode_upload_metadata(encode_metadata(payload)) From 9ed74d9b5997aea4acbd3722b2cd6db537571faa Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 22:18:08 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=B5=81=E5=BC=8F=E5=8F=97=E7=90=86=E4=B8=8E?= =?UTF-8?q?=E6=8C=81=E4=B9=85=E9=98=9F=E5=88=97=E4=BA=A4=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-130923-6b2088 Task-Id: VideoTranscriptAPI-20261008-16 --- config/config.example.jsonc | 8 +- docs/sessions/261008-local-upload/design.md | 8 +- .../progress/B-progress.md | 19 + goals/local-upload/M2-intake.md | 16 +- src/video_transcript_api/api/app.py | 18 +- src/video_transcript_api/api/context.py | 57 ++ .../api/routes/uploads.py | 313 ++++++++- .../api/services/transcription.py | 94 ++- .../cache/cache_manager.py | 176 ++++- tests/README.md | 8 +- tests/integration/test_upload_intake.py | 619 ++++++++++++++++++ tests/unit/test_upload_dispatch.py | 149 +++++ tests/unit/test_upload_routes.py | 118 ++++ 13 files changed, 1569 insertions(+), 34 deletions(-) create mode 100644 tests/integration/test_upload_intake.py create mode 100644 tests/unit/test_upload_dispatch.py diff --git a/config/config.example.jsonc b/config/config.example.jsonc index f2cd6f26..18ba6b30 100644 --- a/config/config.example.jsonc +++ b/config/config.example.jsonc @@ -62,7 +62,13 @@ "cache_retention_days": 360, // 缓存保留天数（每日定时清理超期缓存记录及文件），0 表示永久保留 "task_status_retention_days": 180, // task_status 表终态（success/failed）任务记录保留天数（每日定时清理），0 表示永久保留。注意：/view 链接依赖 task_status 行解析，本值不应短于 cache_retention_days，短于时按缓存保留期执行（钳制并记 warning）；cache_retention_days 为 0（缓存永久保留）时本清理会被跳过 "audit_log_retention_days": 180, // 审计日志保留天数（每日定时清理 audit.db 超期记录），0 表示永久保留 - "max_download_size_mb": 4096 // 单文件下载大小上限（MB），0 表示不限制，防止超长视频填满磁盘 + "max_download_size_mb": 4096, // 单文件下载大小上限（MB），0 表示不限制，防止超长视频填满磁盘 + "upload_limits": { // 未经容量实测的上传限制留 null；缺少任一正有限值时上传能力关闭 + "max_file_mib": null, + "max_media_hours": null, + "receive_concurrency": null, + "upload_temp_budget_mib": null + } }, // ============================================================ diff --git a/docs/sessions/261008-local-upload/design.md b/docs/sessions/261008-local-upload/design.md index 88462ac7..416b884a 100644 --- a/docs/sessions/261008-local-upload/design.md +++ b/docs/sessions/261008-local-upload/design.md @@ -4,7 +4,7 @@ 依据 2026-10-08 已批准的 v2（D16–D19、R1–R8）与最新授权“拆卡完整落地我们的设计方案”。本文件只持久化已批准的当前设计与跨卡 HTTP 合同；不是完整上传项目或生产启用许可。 -A 卡交付：`cache.db` 内独立上传身份/期限/撤销记录；上传根任务与旧 `task_status.view_token` 分离；Resolver 只从上传记录解析 `upload_` capability；缓存/任务清理只在分享有效期间保护必要成果与根记录。`VTA_UPLOADS_ENABLED` 缺失或不是 `true` 时，上传分享读取拒绝，且写一条可 grep 的 `UPLOAD_DISABLED` 日志。开关关闭不改变清理保护。当前没有上传 HTTP 接收端、真实本地 ASR worker、上传 UI、生产额度或恢复证明。 +A 卡交付：`cache.db` 内独立上传身份/期限/撤销记录；上传根任务与旧 `task_status.view_token` 分离；Resolver 只从上传记录解析 `upload_` capability；缓存/任务清理只在分享有效期间保护必要成果与根记录。`VTA_UPLOADS_ENABLED` 缺失或不是 `true` 时，上传分享读取拒绝，且写一条可 grep 的 `UPLOAD_DISABLED` 日志。开关关闭不改变清理保护。B 已接入 raw-stream HTTP、owner receipt、持久 admission、dispatcher 与既有本地媒体 ASR 路径；默认开关关闭，生产额度、容量实测、恢复证明及生产启用均仍未完成/授权。 主身份仍是现有 Bearer 用户。正文链接公开读取；提交归属由 owner 持有。每次正式上传独立成果，不跨上传去重；同一 owner 与同一幂等 key 是同一意图。上传开关不写入数据卷配置；没有完整恢复域证明，部署保持关闭。不得据代码测试宣称已授权生产部署。 @@ -14,7 +14,7 @@ A 卡交付：`cache.db` 内独立上传身份/期限/撤销记录；上传根 上传 root task 仍用现有 `task_status` 做进度和终态真相。由于旧列 `view_token TEXT NOT NULL`，上传根行写 `view_token=''`，`platform='local_upload'`；公开 token 只存 `local_uploads.view_token`。空 token 与 `upload_` token 都不准旧 `get_task_by_view_token` 或 audit 归属 legacy fallback 解析。写入撤销后不能复活；DB trigger 和 `COALESCE` 写入锁住 `revoked_at` 单调性。 -接收 key 格式为 `-`。首次登记只接受当前时刻之前 24 小时内的 key，未来最多容许 5 分钟时钟偏差；超过 24 小时的已有 owner/key 仍可查原记录，记录退休/清除后同 key 不得重新受理。此窗口是幂等记录生命周期，不是分享期限。当前 A 不做记录 GC；后续维护只能在确认查询/历史消费者及唯一记录窗口后回收。 +接收 key 格式为 `-`。新正式受理截止于 key 创建时间后 24 小时：到达截止点（now >= created_at + 24h）即拒绝；key 最多容许 5 分钟未来时钟偏差。超过 24 小时的已有 owner/key 仍可查原记录；过期 `receiving` 草稿及其暂存字节可退休/清理，key 时间戳仍保证其不能重新受理。此窗口是幂等记录生命周期，不是分享期限。`receiving` 不是已接受任务，不得返回 202；相同 key 重放只回原阶段回执、不读取 body 或重新入队，未知回执只查询，不自动换 key/重传。 `30d` 从 root 首次 `success` 或 `failed` 终态的 SQLite `completed_at` 固定加 30 天；未到终态时 `expires_at=NULL` 不等于长期，尚未终态的 root 只呈现处理中。`never` 才以 `expires_at=NULL` 表示不自动到期。子任务、重复终态写和重启不续期。撤销时间只写一次；关闭分享不取消已接受处理，也不专属擦除正文。 @@ -39,7 +39,7 @@ Resolver 每次新请求都查独立上传记录并核对开关、撤销和截 ### 响应 - `GET /api/uploads/capabilities`（Bearer）：`{enabled:boolean, default_retention:'30d', retention_options:['30d','never'], limits:{max_file_mib,max_media_hours,receive_concurrency,upload_temp_budget_mib}}`。四项额度必须正且有限、来自经过真实消费环境验证的配置；当前生产值未知/未测，缺一项即 `enabled=false`。不得用示例值声称已测。 -- `POST /api/uploads`：只有数据库正式接受后才返回 202；同意图返回已有 receipt，变化 metadata 返回 409。FastAPI 错误必须明确 status/detail，不得假 202。 +- `POST /api/uploads`：只有 root、accepted 状态及媒体映射在 SQLite 正式提交、且单次队列 handoff 成功后才返回 202；相同意图返回原回执（包括仍为 receiving 的阶段回执），变化 metadata 返回 409。receiving/failed/未知回执均不伪装为 accepted/202；队列或提交失败不能留下可用分享 capability。FastAPI 错误必须明确 status/detail。 - `GET /api/uploads/by-idempotency-key/{key}`（Bearer/owner）：已存在 receipt 或 404；不能借查询创建任务。 - receipt：`upload_id`、`state`（接收阶段，不复制 `task_status`）、`task_id`（未正式受理时 null）、`view_token`（正式受理后独立 `upload_` token，否则 null）、`retention`、`expires_at`（终态前与 `never` 为 null）、`share_active`、`error_code`（无错误为 null）。处理进度继续由既有 `GET /api/task/{task_id}` 承载。 - `DELETE /api/uploads/{upload_id}/share`（Bearer/owner）：`{upload_id, share_active:false, revoked_at}`。重复关闭返回原截止，不得复活。关闭/到期后所有新 public read 和新 reprocess 拒绝；已经授权的传输和已接受处理可继续。 @@ -56,7 +56,7 @@ Resolver 每次新请求都查独立上传记录并核对开关、撤销和截 | 有效产物不被 cache/task/audit 清理删除；开关 off 不影响保护；关闭后恢复原清理；媒体仍临时 | `cleanup_old_cache/cleanup_task_status`；现有 `task_exists` 与 `TempFileManager` | local policy cleanup/controlled ordering tests、`tests/cache/test_task_status_cleanup.py`、`tests/cache/test_cache_cleanup.py` | | URL 读、去重、history 与 task 行为不改变 | 原有 URL routes/CacheManager | `tests/unit/test_view_token_resolver.py`、`test_api_routes.py`、`test_history_routes.py` 及 cache cleanup 套件 | -当前真实开发验证入口：`uv sync --frozen`；重复五次 `uv run --frozen pytest -q tests/unit/test_local_upload_policy.py tests/unit/test_view_token_resolver.py tests/cache/test_task_status_cleanup.py`；完整 gate `make test`。对环境变量依赖的测试必须在实际消费 Resolver 的环境运行。部署/systemd 恢复及容量验证仍是上线前置条件；本卡不触及生产 unit、数据卷或服务。 +当前真实开发验证入口：A 的窄测与 `make test`；B 真实 HTTP/SQLite/dispatcher 入口为 `uv run --frozen pytest -q tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py`。B 测试覆盖真实 Bearer、raw body 实际字节与服务端 SHA-256、容量/队列/SQL/断开、重复与丢回执、owner 撤销及 stale dispatcher。所有生产额度仍为 null/未知，`VTA_UPLOADS_ENABLED` 缺失时 intake 与 share 都关闭；部署/systemd 恢复及容量验证仍是上线前置条件，本卡不触及生产 unit、数据卷或服务。 ## 非目标与依赖 diff --git a/docs/sessions/261008-local-upload/progress/B-progress.md b/docs/sessions/261008-local-upload/progress/B-progress.md index a7c00bd4..08ffffd6 100644 --- a/docs/sessions/261008-local-upload/progress/B-progress.md +++ b/docs/sessions/261008-local-upload/progress/B-progress.md @@ -18,3 +18,22 @@ implementing；完成第一份独立可运行单元：bounded metadata decoder ## 下一步唯一动作 扩展 CacheManager 持久 admission/root/media mapping 与 runtime 限额/预留，再接上真正 HTTP stream 和 dispatcher 测试。 + +## Milestone 2：流式HTTP与持久队列交接 + +### 本段结论（≤3句） + +- 已实现 GET capabilities、raw `application/octet-stream` POST、owner receipt/DELETE、服务端流式 byte count/SHA-256、metadata边界、unique owner/key、24h截止与过期receiving清理。 +- SQLite root/accepted/mapping一次事务提交后只调用一次有界队列 `put_nowait`；worker 在 PROCESSING/executor.submit 前核验持久受理与 root/media/path 映射，local_upload 走现有 ASR 路径且不经过 URL downloader。 +- 真实 ASGI+临时SQLite/dispatcher窄测：`tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py` 全绿（32 passed）；尚未跑全仓 make test、裸shell/systemd验证或 CI。 + +### 关键决策与否决方案 + +- upload limits 只从现有 `storage.upload_limits` 消费，示例四值均 null；`VTA_UPLOADS_ENABLED` 独立于数据卷配置，只有env、正有限额度及实际存活的 process_task_queue 同时满足才报告 enabled。 +- body接收目录在开始读取前持久记录并由 TempFileManager active 集保护；未移交时由接收者清理，移交后worker清理；worker future完成释放原始输入的磁盘/inflight预留。 +- `receiving` key在创建+24h截止前保留唯一；正式接受在截止点及以后显式410，过期receiving草稿和未移交暂存路径由maintenance退休；不自动重试/换key。 +- 首次QueueFull在读body前拒绝；若数据库已提交后`put_nowait`仍失败，事务补偿撤回root/token映射、保留失败receipt并清理文件，不能返回202。 + +## 下一步唯一动作 + +完成文档/测试说明与全仓门禁，核对每个不变量的producer/consumer锁定测试，再更新本进度并提交推送下一里程碑。 diff --git a/goals/local-upload/M2-intake.md b/goals/local-upload/M2-intake.md index b4958da4..610f7f24 100644 --- a/goals/local-upload/M2-intake.md +++ b/goals/local-upload/M2-intake.md @@ -2,8 +2,8 @@ lane: local-upload id: M2 slug: intake -status: 未开始 -owner: delegate implementer dlg-20261008-094534-0fd910 +status: 已实现待独立审查与CI +owner: delegate implementer dlg-20261008-130923-6b2088 order: 2 priority: 高 depends_on: [local-upload/M1] @@ -14,10 +14,10 @@ merged_pr: null - **预期产出**：Bearer 认证的能力查询、raw octet-stream 接收、有限 metadata header、真实额度准入、owner/key receipt 与诚实 HTTP 状态。 - **当前范围**：B 卡；不把 A 的 SQLite store 方法误报为已存在 HTTP 接收端。 -- **关键决策**：沿用 `docs/sessions/261008-local-upload/design.md` 的 wire 合同；key 窗口24小时、未来容差5分钟；同意图不消费新 body、不重入队，变化 metadata 409。 +- **关键决策**：沿用 `docs/sessions/261008-local-upload/design.md` 的 wire 合同；新正式受理截止为 key 创建时间+24h（到截止时拒绝），未来容差5分钟；`receiving` 不等于 accepted/202，同意图只回原receipt、不消费新 body、不重入队，变化 metadata 409。 - **推进前必须拿到的证据**： - - [ ] 真实 FastAPI/ASGI 与临时 cache.db 上由 client 发 raw 文件字节及所有 headers；断言 handler 实收字节、metadata 解码、commit receipt 与真实 task/queue producer 参数。跨进程 payload 保留 producer fixture。 - - [ ] 无开关、关闭开关、任一额度缺失/无效时实际返回 disabled 并拒绝接收；四项额度全为正有限数才测 enabled。生产测量前不填真实容量结论。 - - [ ] 同 key 重放时 consumer 证明没有读取新 body、没有第二次 queue 入队；查询只读回原 receipt；丢回执不自动生成新 key/重传。 - - [ ] 在 CI、无会话身份裸 shell 和获准隔离 systemd unit 分别验证真实消费环境变量与接收行为。systemd/生产改动另需部署授权。 -- **完成条件**：202 只在持久正式接受后返回；未知/失败不假成功；queue/SQLite 交接窗口由真实入口测试锁死，不加 retry/fallback/outbox。 + - [x] 真实 FastAPI/ASGI 与临时 cache.db 上由 client 发 raw 文件字节及 headers；锁定实际落盘字节、服务端 SHA-256、Unicode metadata、root row 与 dispatcher 实际消费路径。producer fixture 保留在 `tests/integration/test_upload_intake.py`。 + - [x] 缺/错 Bearer、默认关闭、额度未知、非 octet-stream、声明/实收超限、receiver/inflight/free-space 满载均在 body/file/ASR 边界前显式拒绝。 + - [x] 同 key 重放/变化 metadata、相同字节新 key、真实 ASGI 断开、队列满、SQLite commit fault、queue put fault、窗口跨期与丢失202回执重查均有 producer→consumer 测试；same-intent 不读 body/不重入队。 + - [ ] 最终CI及系统级部署 unit 的环境传递仍由 lead 验收；未改真实 systemd/service 或生产环境。本地隔离裸shell验证待最终阶段执行。 +- **完成条件**：本地代码与测试已覆盖 202/失败边界；正式合并、CI 与真实部署环境变量/容量证明由 lead 后续托管，生产仍关闭。 diff --git a/src/video_transcript_api/api/app.py b/src/video_transcript_api/api/app.py index ee6ae6da..7b7af59a 100644 --- a/src/video_transcript_api/api/app.py +++ b/src/video_transcript_api/api/app.py @@ -29,7 +29,7 @@ run_with_runtime, unbind_runtime, ) -from .routes import audit, health, tasks, users, views +from .routes import audit, health, tasks, uploads, users, views from .services.transcription import process_llm_queue, process_task_queue from .services.terminal_status import run_terminal_notification_dispatcher @@ -160,6 +160,15 @@ async def _periodic_maintenance(config: dict) -> None: # 出现在里面，不会被波及（见 CacheManager.recover_orphaned_tasks # 的详细说明）。 runtime = get_runtime() + retired_upload_intents = await runtime.run_maintenance( + get_cache_manager().cleanup_expired_local_upload_receiving, + storage.get("temp_dir", ""), + now=now, + ) + if retired_upload_intents: + logger.info( + f"定期清理：退休 {retired_upload_intents} 条过期上传接收记录" + ) if getattr(runtime, "recovery_pending", False): restrict_to_task_ids = getattr(runtime, "startup_recovery_task_ids", None) recovered = await runtime.run_maintenance( @@ -363,6 +372,7 @@ async def log_external_access(request: Request, call_next): app.include_router(health.router) app.include_router(tasks.router) + app.include_router(uploads.router) app.include_router(audit.router) app.include_router(users.router) app.include_router(views.router) @@ -376,6 +386,12 @@ async def startup_event(): old_files_count = temp_manager.clean_up_old_files() if old_files_count > 0: logger.info(f"启动时清理了 {old_files_count} 个旧临时文件") + retired_upload_intents = await app.state.runtime.run_maintenance( + get_cache_manager().cleanup_expired_local_upload_receiving, + temp_manager.base_dir, + ) + if retired_upload_intents: + logger.info(f"启动时退休了 {retired_upload_intents} 条过期上传接收记录") init_all_notifiers() diff --git a/src/video_transcript_api/api/context.py b/src/video_transcript_api/api/context.py index bb6b6fbd..a4fff4b0 100644 --- a/src/video_transcript_api/api/context.py +++ b/src/video_transcript_api/api/context.py @@ -10,6 +10,7 @@ import contextvars import datetime import hashlib +import math import os import queue import re @@ -177,6 +178,7 @@ def validate_config(config: Any) -> dict: "task_status_retention_days", "audit_log_retention_days", "max_download_size_mb", + "upload_limits", }, "storage", ) @@ -209,6 +211,34 @@ def validate_config(config: Any) -> dict: raise ConfigError( "storage.max_download_size_mb must be a non-negative integer" ) + upload_limits = storage.get("upload_limits") + if upload_limits is not None: + upload_limits = _require_dict(storage, "upload_limits") + _reject_unknown( + upload_limits, + {"max_file_mib", "max_media_hours", "receive_concurrency", "upload_temp_budget_mib"}, + "storage.upload_limits", + ) + for field in ("max_file_mib", "max_media_hours", "upload_temp_budget_mib"): + value = upload_limits.get(field) + if value is not None and ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(value) + or value <= 0 + ): + raise ConfigError( + f"storage.upload_limits.{field} must be a positive finite number or null" + ) + concurrency = upload_limits.get("receive_concurrency") + if concurrency is not None and ( + not isinstance(concurrency, int) + or isinstance(concurrency, bool) + or concurrency <= 0 + ): + raise ConfigError( + "storage.upload_limits.receive_concurrency must be a positive integer or null" + ) # llm is required, not optional-like the backend_sections below: unlike # ASR backends (which stay dormant when disabled/absent), RuntimeContext @@ -957,6 +987,12 @@ def __init__(self, config: dict): # 的数据结构或独立模块。 self.terminal_write_pending: set[str] = set() self._terminal_write_pending_guard = threading.Lock() + upload_limits = self.config["storage"].get("upload_limits") or {} + self.upload_receiver_slots = threading.BoundedSemaphore( + upload_limits.get("receive_concurrency") or 1 + ) + self._upload_budget_guard = threading.Lock() + self._upload_reserved_bytes: dict[str, int] = {} def start(self) -> None: if self.started: @@ -1004,6 +1040,26 @@ def start(self) -> None: ) self.started = True + def reserve_upload_temp( + self, task_id: str, byte_count: int, budget_bytes: int, temp_dir: str + ) -> bool: + import shutil + + with self._upload_budget_guard: + current = sum(self._upload_reserved_bytes.values()) + if current + byte_count > budget_bytes: + return False + free_bytes = shutil.disk_usage(temp_dir).free + safety_margin = max(1024 * 1024, budget_bytes // 20) + if free_bytes < byte_count + safety_margin: + return False + self._upload_reserved_bytes[task_id] = byte_count + return True + + def release_upload_temp(self, task_id: str) -> None: + with self._upload_budget_guard: + self._upload_reserved_bytes.pop(task_id, None) + def track_future( self, future: concurrent.futures.Future, *, kind: str = "transcription", task_id: str | None = None, @@ -1076,6 +1132,7 @@ def discard(done_future): finally: if task_id is not None: self.inflight_registry.release(kind, task_id) + self.release_upload_temp(task_id) if kind == "llm": self.llm_submit_semaphore.release() with self._worker_futures_condition: diff --git a/src/video_transcript_api/api/routes/uploads.py b/src/video_transcript_api/api/routes/uploads.py index 154d6da4..5ab36593 100644 --- a/src/video_transcript_api/api/routes/uploads.py +++ b/src/video_transcript_api/api/routes/uploads.py @@ -1,19 +1,39 @@ """Bounded raw-byte local upload API.""" +import asyncio import base64 import binascii +import datetime +import hashlib import json +import math +import os import re +import uuid from pathlib import PurePosixPath from urllib.parse import urlsplit +from fastapi import APIRouter, Depends, Header, HTTPException, Request +from fastapi.responses import JSONResponse + +from ..context import ( + get_cache_manager, + get_config, + get_inflight_registry, + get_task_queue, + get_temp_manager, +) from ..processing_options import normalize_processing_options +from ..services.transcription import verify_token +from ...utils.logging import logger +router = APIRouter(prefix="/api/uploads", tags=["uploads"]) _METADATA_MAX_BYTES = 4096 _BASE64URL = re.compile(r"^[A-Za-z0-9_-]+={0,2}$") _ALLOWED_METADATA = { "filename", "byte_size", "title", "source_url", "retention", "processing_options" } +_MIB = 1024 * 1024 def decode_upload_metadata(value: str) -> dict: @@ -38,7 +58,7 @@ def decode_upload_metadata(value: str) -> dict: raise ValueError("filename must be non-empty text") if len(filename.encode("utf-8")) > 255: raise ValueError("filename exceeds 255 UTF-8 bytes") - if any(ord(char) < 32 or ord(char) == 127 for char in filename): + if any(ord(char) < 32 or 127 <= ord(char) <= 159 for char in filename): raise ValueError("filename contains control characters") display_name = PurePosixPath(filename.replace("\\", "/")).name if not display_name or display_name in {".", ".."}: @@ -50,7 +70,7 @@ def decode_upload_metadata(value: str) -> dict: title = data["title"] if title is not None and ( not isinstance(title, str) or len(title) > 200 - or any(ord(char) < 32 and char not in "\t\n\r" for char in title) + or any(ord(char) < 32 or 127 <= ord(char) <= 159 for char in title) ): raise ValueError("title must be at most 200 characters") source_url = data["source_url"] @@ -63,7 +83,7 @@ def decode_upload_metadata(value: str) -> dict: or not parsed.hostname or parsed.username is not None or parsed.password is not None - or any(ord(char) < 32 or ord(char) == 127 for char in source_url) + or any(ord(char) < 32 or 127 <= ord(char) <= 159 for char in source_url) ): raise ValueError("source_url must be http(s) without userinfo") retention = data["retention"] @@ -82,3 +102,290 @@ def decode_upload_metadata(value: str) -> dict: "filename": display_name, "processing_options": normalized_options, } + + +def _upload_limits(config: dict) -> dict: + configured = config.get("storage", {}).get("upload_limits") + fields = ("max_file_mib", "max_media_hours", "receive_concurrency", "upload_temp_budget_mib") + if not isinstance(configured, dict): + return {field: None for field in fields} + limits = {field: configured.get(field) for field in fields} + for field in ("max_file_mib", "max_media_hours", "upload_temp_budget_mib"): + value = limits[field] + if ( + not isinstance(value, (int, float)) or isinstance(value, bool) + or not math.isfinite(value) or value <= 0 + ): + return {field: None for field in fields} + concurrency = limits["receive_concurrency"] + if not isinstance(concurrency, int) or isinstance(concurrency, bool) or concurrency <= 0: + return {field: None for field in fields} + return limits + + +def _worker_ready(request: Request) -> bool: + runtime = getattr(request.app.state, "runtime", None) + processor = getattr(request.app.state, "queue_processor", None) + return bool( + runtime is not None + and getattr(runtime, "started", False) + and processor is not None + and not processor.done() + and getattr(processor.get_coro(), "__qualname__", "") == "process_task_queue" + and hasattr(runtime, "reserve_upload_temp") + ) + + +def _uploads_enabled(request: Request, config: dict, limits: dict) -> bool: + return ( + os.environ.get("VTA_UPLOADS_ENABLED") == "true" + and all(value is not None for value in limits.values()) + and _worker_ready(request) + ) + + +def _receipt(upload: dict) -> dict: + accepted = upload.get("state") == "accepted" and not upload.get("error_code") + state = upload.get("state") if not upload.get("error_code") else "failed" + return { + "upload_id": upload["upload_id"], + "state": state, + "task_id": upload.get("root_task_id") if accepted else None, + "view_token": upload.get("view_token") if accepted else None, + "retention": upload["retention"], + "expires_at": upload.get("expires_at"), + "share_active": bool( + accepted + and upload.get("revoked_at") is None + and ( + upload["retention"] == "never" + and upload.get("expires_at") is None + or upload["retention"] == "30d" + and ( + upload.get("expires_at") is None + and upload.get("root_status") not in {"success", "failed"} + or upload.get("expires_at") is not None + and upload["expires_at"] > datetime.datetime.now( + datetime.timezone.utc + ).strftime("%Y-%m-%d %H:%M:%S") + ) + ) + ), + "error_code": upload.get("error_code"), + } + + +def _reject(status_code: int, reason: str, detail: str) -> HTTPException: + logger.warning("UPLOAD_INTAKE_REJECTED reason={} status={}", reason, status_code) + return HTTPException(status_code=status_code, detail=detail) + + +@router.get("/capabilities") +async def upload_capabilities( + request: Request, + user_info: dict = Depends(verify_token), +): + config = get_config() + limits = _upload_limits(config) + return { + "enabled": _uploads_enabled(request, config, limits), + "default_retention": "30d", + "retention_options": ["30d", "never"], + "limits": limits, + } + + +@router.post( + "", + openapi_extra={ + "requestBody": { + "required": True, + "content": { + "application/octet-stream": { + "schema": {"type": "string", "format": "binary"} + } + }, + } + }, +) +async def receive_upload( + request: Request, + idempotency_key: str | None = Header(None, alias="Idempotency-Key"), + metadata_header: str | None = Header(None, alias="X-Upload-Metadata"), + user_info: dict = Depends(verify_token), +): + # No body parameter: FastAPI resolves auth and these bounded headers before streaming. + config = get_config() + limits = _upload_limits(config) + if not _uploads_enabled(request, config, limits): + raise _reject(503, "disabled_or_not_ready", "上传接收未启用或worker尚未就绪") + content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/octet-stream": + raise _reject(415, "content_type", "Content-Type必须为application/octet-stream") + if not idempotency_key: + raise _reject(400, "idempotency_key", "缺少Idempotency-Key") + try: + metadata = decode_upload_metadata(metadata_header or "") + except ValueError as exc: + raise _reject(400, "metadata", str(exc)) from exc + max_bytes = int(limits["max_file_mib"] * _MIB) + if metadata["byte_size"] > max_bytes: + raise _reject(413, "declared_size", "上传文件超过允许大小") + + cache_manager = get_cache_manager() + try: + intent = cache_manager.register_local_upload( + owner_user_id=user_info.get("user_id"), + idempotency_key=idempotency_key, + retention=metadata["retention"], + intent_metadata=metadata, + ) + except ValueError as exc: + message = str(exc) + status = 409 if "metadata conflicts" in message else 410 if "24-hour" in message else 400 + raise _reject(status, "idempotency", message) from exc + if not intent["_created"]: + return JSONResponse(_receipt(cache_manager.get_local_upload_by_id(intent["upload_id"])), status_code=200) + + upload_id = intent["upload_id"] + task_id = cache_manager.generate_task_id() + runtime = request.app.state.runtime + receiver_acquired = runtime.upload_receiver_slots.acquire(blocking=False) + inflight = get_inflight_registry() + inflight_acquired = False + budget_acquired = False + body_started = False + transferred = False + failure_code = None + temp_manager = get_temp_manager() + task_dir = None + media_path = None + if not receiver_acquired: + failure_code = "receiver_limit" + cache_manager.set_local_upload_error(upload_id, failure_code) + raise _reject(503, failure_code, "上传接收并发已满") + try: + if not inflight.try_register("transcription", task_id): + failure_code = "inflight_limit" + cache_manager.set_local_upload_error(upload_id, failure_code) + raise _reject(503, failure_code, "转录队列已满") + inflight_acquired = True + declared_bytes = metadata["byte_size"] + # Keep one source-sized processing footprint reserved until the existing worker finishes. + reservation_bytes = declared_bytes * 2 + budget_bytes = int(limits["upload_temp_budget_mib"] * _MIB) + if not runtime.reserve_upload_temp( + task_id, reservation_bytes, budget_bytes, str(temp_manager.base_dir) + ): + failure_code = "upload_temp_budget" + cache_manager.set_local_upload_error(upload_id, failure_code) + raise _reject(507, failure_code, "上传临时存储预算或可用磁盘空间不足") + budget_acquired = True + task_dir = temp_manager.create_task_dir(task_id) + temp_manager.mark_active(task_id) + media_path = task_dir / "upload-source.bin" + cache_manager.set_local_upload_receiving_path(upload_id, str(media_path)) + task_queue = get_task_queue() + if task_queue.full(): + failure_code = "queue_full" + cache_manager.set_local_upload_error(upload_id, failure_code) + raise _reject(503, failure_code, "转录队列已满，上传未正式受理") + body_started = True + observed_bytes = 0 + digest = hashlib.sha256() + with os.fdopen( + os.open(media_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb" + ) as media_file: + async for chunk in request.stream(): + if not chunk: + continue + next_size = observed_bytes + len(chunk) + if next_size > max_bytes or next_size > declared_bytes: + failure_code = "observed_size_limit" + raise _reject(413, failure_code, "实际上传字节超过声明或允许大小") + digest.update(chunk) + media_file.write(chunk) + observed_bytes = next_size + if observed_bytes != declared_bytes: + failure_code = "declared_size_mismatch" + raise _reject(400, failure_code, "实际上传字节与byte_size不一致") + media_file.flush() + os.fsync(media_file.fileno()) + + if task_queue.full(): + failure_code = "queue_full" + cache_manager.set_local_upload_error(upload_id, failure_code) + raise _reject(503, failure_code, "转录队列已满，上传未正式受理") + media_id = f"upload_{uuid.uuid4().hex}" + try: + accepted = cache_manager.accept_local_upload( + upload_id, + media_id=media_id, + title=metadata["title"] or metadata["filename"], + processing_options=metadata["processing_options"], + task_id=task_id, + filename=metadata["filename"], + source_url=metadata["source_url"], + request_metadata=metadata, + media_path=str(media_path), + byte_size=observed_bytes, + sha256=digest.hexdigest(), + ) + except ValueError as exc: + if "24-hour" in str(exc): + failure_code = "acceptance_window_expired" + raise _reject(410, failure_code, "上传幂等key已超过24小时受理截止时间") from exc + raise + task_queue.put_nowait({"id": task_id, "url": metadata["source_url"] or "", "platform": "local_upload"}) + transferred = True + return JSONResponse(_receipt(cache_manager.get_local_upload_by_id(accepted["upload_id"])), status_code=202) + except asyncio.QueueFull as exc: + failure_code = "queue_full" + cache_manager.cancel_local_upload_acceptance(upload_id, task_id, failure_code) + raise _reject(503, failure_code, "转录队列已满，上传未正式受理") from exc + finally: + if body_started and not transferred: + failure_code = failure_code or "receive_incomplete" + cache_manager.set_local_upload_error(upload_id, failure_code) + logger.warning("UPLOAD_INTAKE_REJECTED reason={} status=failed", failure_code) + if not transferred: + if task_dir is not None: + temp_manager.clean_up_task(task_id) + if task_dir.exists(): + raise OSError(f"failed to remove untransferred upload directory: {task_dir}") + if budget_acquired: + runtime.release_upload_temp(task_id) + if inflight_acquired: + inflight.release("transcription", task_id) + if receiver_acquired: + runtime.upload_receiver_slots.release() + + +@router.get("/by-idempotency-key/{key}") +async def get_upload_receipt( + key: str, + user_info: dict = Depends(verify_token), +): + upload = get_cache_manager().get_local_upload_by_owner_key(user_info.get("user_id"), key) + if upload is None: + raise HTTPException(status_code=404, detail="上传回执不存在") + return _receipt(upload) + + +@router.delete("/{upload_id}/share") +async def stop_upload_share( + upload_id: str, + user_info: dict = Depends(verify_token), +): + cache_manager = get_cache_manager() + upload = cache_manager.get_local_upload_by_id(upload_id) + if upload is None or upload["owner_user_id"] != user_info.get("user_id"): + raise HTTPException(status_code=404, detail="上传记录不存在") + if upload["state"] != "accepted": + raise HTTPException(status_code=409, detail="尚未正式受理的上传没有分享可关闭") + revoked = cache_manager.revoke_local_upload(upload_id) + return { + "upload_id": upload_id, + "share_active": False, + "revoked_at": revoked["revoked_at"], + } diff --git a/src/video_transcript_api/api/services/transcription.py b/src/video_transcript_api/api/services/transcription.py index 9cb80b41..e70be79c 100644 --- a/src/video_transcript_api/api/services/transcription.py +++ b/src/video_transcript_api/api/services/transcription.py @@ -1,7 +1,9 @@ import asyncio import datetime +import json import os import threading +from pathlib import Path import time from typing import Optional, Dict, Any @@ -576,6 +578,45 @@ async def process_task_queue(): processing_options = normalize_processing_options(task.get("processing_options")) try: + local_upload = None + local_media_path = None + local_media_id = None + if task.get("platform") == "local_upload": + local_upload = cache_manager.get_admitted_local_upload_by_task(task_id) + temp_manager = get_temp_manager() + task_dir = temp_manager.base_dir / f"task_{task_id}" + candidate = Path(local_upload["media_path"]) if local_upload else None + resolved_task_dir = task_dir.resolve() + valid_mapping = bool( + local_upload + and candidate + and candidate.name == "upload-source.bin" + and candidate.is_file() + and candidate.resolve(strict=True).is_relative_to(resolved_task_dir) + ) + if not valid_mapping: + logger.error( + "local upload stale admission rejected before ASR: task_id={}", task_id + ) + cache_manager.update_task_status( + task_id, + TaskStatus.FAILED, + platform="local_upload", + error_message="Local upload durable admission or media mapping is missing", + suppress_terminal_notification=True, + ) + get_runtime().inflight_registry.release("transcription", task_id) + get_runtime().release_upload_temp(task_id) + temp_manager.clean_up_task(task_id) + if task_dir.exists(): + raise OSError(f"failed to remove stale upload media directory: {task_dir}") + continue + local_media_path = str(candidate.resolve(strict=True)) + local_media_id = local_upload["media_id"] + metadata = json.loads(local_upload["request_metadata"]) + metadata_override = {"title": metadata.get("title") or local_upload["filename"]} + url = local_upload["source_url"] or "" + cache_manager.update_task_status(task_id, TaskStatus.PROCESSING, download_url=download_url) runtime = get_runtime() @@ -592,6 +633,8 @@ def run_and_finalize( notification_channel=notification_channel, notification_webhooks=dict(notification_webhooks), processing_options=dict(processing_options), + local_media_path=local_media_path, + local_media_id=local_media_id, ): try: process_transcription( @@ -606,6 +649,8 @@ def run_and_finalize( processing_options=processing_options, preparsed_url=preparsed_url, url_parse_attempted=url_parse_attempted, + local_media_path=local_media_path, + local_media_id=local_media_id, ) logger.info(f"任务完成: {task_id}") except Exception as exc: @@ -653,8 +698,15 @@ def run_and_finalize( # 引入新的失败面。 try: get_runtime().inflight_registry.release("transcription", task_id) + get_runtime().release_upload_temp(task_id) except Exception: logger.exception(f"释放在途任务登记表名额失败: {task_id}") + if task.get("platform") == "local_upload": + temp_manager = get_temp_manager() + task_dir = temp_manager.base_dir / f"task_{task_id}" + temp_manager.clean_up_task(task_id) + if task_dir.exists(): + raise OSError(f"failed to remove unsubmitted upload media directory: {task_dir}") # G1 修复（CI review 第 2 轮 major）：此前这里认为 # "update_task_status 自身的异常也不能向上传播"——重新核实后 # 发现这个理由不成立：下面的 task_queue.task_done() 是外层 @@ -927,7 +979,7 @@ def process_transcription( task_id, url, use_speaker_recognition=False, wechat_webhook=None, download_url=None, metadata_override=None, notification_channel=None, notification_webhooks=None, processing_options=None, preparsed_url=None, - url_parse_attempted=False, + url_parse_attempted=False, local_media_path=None, local_media_id=None, ): """ 处理视频转录 @@ -1112,7 +1164,12 @@ def _fail_task_and_notify( logger.info(f"[URL解析] 开始解析 URL: {check_url[:100]}") with tracker.track("url_parse"): - if preparsed_url is not None: + if local_media_path is not None: + platform = "local_upload" + video_id = local_media_id + parse_url = url + logger.info("[local-upload] using durable server-owned media id={}", video_id) + elif preparsed_url is not None: platform = preparsed_url.platform video_id = preparsed_url.video_id parse_url = preparsed_url.normalized_url or url @@ -1155,7 +1212,7 @@ def _fail_task_and_notify( is_generic_downloader = platform == 'generic' with tracker.track("cache_check"): - if video_id and platform and not is_generic_downloader: + if video_id and platform and not is_generic_downloader and platform != "local_upload": logger.info( f"[缓存检测] 检查缓存: platform={platform}, video_id={video_id}, " f"use_speaker_recognition={use_speaker_recognition}" @@ -1690,7 +1747,7 @@ def _fail_task_and_notify( from urllib.parse import urlparse url_scheme = urlparse(url).scheme.lower() - if url_scheme not in ("http", "https"): + if local_media_path is not None or url_scheme not in ("http", "https"): logger.info( f"[元数据获取] URL scheme 非 http/https（{url_scheme or '空'}），" f"跳过元数据探测，直接使用 metadata_override 兜底: {parse_url}" @@ -1769,7 +1826,9 @@ def _fail_task_and_notify( # 下载器准备 from ...downloaders.generic import GenericDownloader download_downloader = None - if has_separate_download_url: + if local_media_path is not None: + download_downloader = None + elif has_separate_download_url: download_downloader = GenericDownloader() elif metadata_downloader: download_downloader = metadata_downloader @@ -2097,7 +2156,9 @@ def _finalize_presentation_fields( # ========== 原有逻辑（非 YouTube API Server 路径）========== # 已在前面完成元数据解析与下载器准备 original_downloader = None - if not download_url: + if local_media_path is not None: + original_downloader = None + elif not download_url: original_downloader = metadata_downloader or create_downloader(url) else: logger.info("已提供 download_url，使用解析的元数据，跳过传统下载器的 get_video_info") @@ -2242,7 +2303,10 @@ def _finalize_presentation_fields( # 下载文件 local_file = None actual_downloader = None - if has_separate_download_url: + if local_media_path is not None: + local_file = local_media_path + logger.info("[local-upload] ASR input is the owned upload file") + elif has_separate_download_url: actual_downloader = download_downloader actual_download_url = download_url or url logger.info(f"使用 GenericDownloader 下载文件: {actual_download_url}") @@ -2325,6 +2389,22 @@ def _finalize_presentation_fields( return _fail_task_and_notify( str(admission_exc), title=video_title, author_name=author, ) + if local_media_path is not None: + max_hours = get_config().get("storage", {}).get("upload_limits", {}).get( + "max_media_hours" + ) + if probed_media_duration is None: + return _fail_task_and_notify( + "上传媒体时长未知，拒绝进入转录（media_duration_unknown）", + title=video_title, + author_name=author, + ) + if max_hours is None or probed_media_duration > max_hours * 3600: + return _fail_task_and_notify( + "上传媒体超过允许时长，拒绝进入转录（media_duration_limit）", + title=video_title, + author_name=author, + ) # 时长载体（issue #155）：优先取本次准入探测解析出的容器时长。 # generic / recorder:// 走自实现的 download_file，从不调用 diff --git a/src/video_transcript_api/cache/cache_manager.py b/src/video_transcript_api/cache/cache_manager.py index 6abbed7c..dfc7ec0b 100644 --- a/src/video_transcript_api/cache/cache_manager.py +++ b/src/video_transcript_api/cache/cache_manager.py @@ -358,6 +358,12 @@ def _init_database(self): expires_at TEXT, revoked_at TEXT, error_code TEXT, + filename TEXT, + source_url TEXT, + request_metadata TEXT NOT NULL DEFAULT '{}', + media_path TEXT, + byte_size INTEGER, + sha256 TEXT, UNIQUE(owner_user_id, idempotency_key), FOREIGN KEY(root_task_id) REFERENCES task_status(task_id) ) @@ -370,6 +376,22 @@ def _init_database(self): 'CREATE INDEX IF NOT EXISTS idx_local_upload_expiry ' 'ON local_uploads(retention, expires_at, revoked_at)' ) + upload_columns = { + row[1] for row in cursor.execute("PRAGMA table_info(local_uploads)").fetchall() + } + for column, declaration in ( + ("filename", "TEXT"), + ("source_url", "TEXT"), + ("request_metadata", "TEXT NOT NULL DEFAULT '{}'"), + ("media_path", "TEXT"), + ("byte_size", "INTEGER"), + ("sha256", "TEXT"), + ): + if column not in upload_columns: + cursor.execute( + f"ALTER TABLE local_uploads ADD COLUMN {column} {declaration}" + ) + cursor.execute(''' CREATE TRIGGER IF NOT EXISTS local_upload_revocation_write_once BEFORE UPDATE OF revoked_at ON local_uploads @@ -627,7 +649,7 @@ def _validate_upload_key_window( now = now.astimezone(datetime.timezone.utc) if created_at > now + cls._UPLOAD_KEY_FUTURE_SKEW: raise ValueError("idempotency key exceeds allowed clock skew") - if now - created_at > cls._UPLOAD_IDEMPOTENCY_WINDOW: + if now - created_at >= cls._UPLOAD_IDEMPOTENCY_WINDOW: raise ValueError("idempotency key is outside the 24-hour acceptance window") return created_at @@ -670,6 +692,7 @@ def register_local_upload( record = dict(existing) if record["metadata_fingerprint"] != metadata_fingerprint: raise ValueError("idempotency key metadata conflicts with recorded intent") + record["_created"] = False return record created_at = self._validate_upload_key_window(idempotency_key, now) upload_id = f"upload_{uuid.uuid4().hex}" @@ -689,7 +712,73 @@ def register_local_upload( ), ) cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) - return dict(cursor.fetchone()) + record = dict(cursor.fetchone()) + record["_created"] = True + return record + + def set_local_upload_receiving_path(self, upload_id: str, media_path: str) -> None: + with self._get_cursor() as cursor: + cursor.execute( + """UPDATE local_uploads SET media_path = ? + WHERE upload_id = ? AND state = 'receiving' AND root_task_id IS NULL""", + (media_path, upload_id), + ) + if cursor.rowcount != 1: + raise ValueError("local upload is not in receiving state") + + def cleanup_expired_local_upload_receiving( + self, temp_dir: str, *, now: Optional[datetime.datetime] = None + ) -> int: + """Retire receiving-only idempotency records after their key window.""" + now = now or datetime.datetime.now(datetime.timezone.utc) + if now.tzinfo is None: + raise ValueError("now must be timezone-aware") + now = now.astimezone(datetime.timezone.utc) + with self._get_cursor() as cursor: + cursor.execute( + """SELECT upload_id, idempotency_key, media_path + FROM local_uploads WHERE state = 'receiving'""" + ) + candidates = [dict(row) for row in cursor.fetchall()] + + expired = [ + row for row in candidates + if now >= self._upload_key_timestamp(row["idempotency_key"]) + + self._UPLOAD_IDEMPOTENCY_WINDOW + ] + base_dir = Path(temp_dir).resolve() + for row in expired: + if row["media_path"]: + media_path = Path(row["media_path"]) + resolved_path = media_path.resolve() + if not resolved_path.is_relative_to(base_dir): + raise ValueError("receiving upload media path escaped configured temp directory") + if media_path.exists(): + if not media_path.is_file(): + raise ValueError("receiving upload media mapping is not a file") + media_path.unlink() + + deleted = 0 + for row in expired: + with self._get_cursor() as cursor: + cursor.execute("BEGIN IMMEDIATE") + cursor.execute( + "SELECT state, idempotency_key FROM local_uploads WHERE upload_id = ?", + (row["upload_id"],), + ) + current = cursor.fetchone() + if ( + current is not None + and current["state"] == "receiving" + and now >= self._upload_key_timestamp(current["idempotency_key"]) + + self._UPLOAD_IDEMPOTENCY_WINDOW + ): + cursor.execute( + "DELETE FROM local_uploads WHERE upload_id = ? AND state = 'receiving'", + (row["upload_id"],), + ) + deleted += cursor.rowcount + return deleted def accept_local_upload( self, @@ -699,9 +788,15 @@ def accept_local_upload( title: Optional[str] = None, processing_options: Optional[Dict[str, Any]] = None, task_id: Optional[str] = None, + filename: Optional[str] = None, + source_url: Optional[str] = None, + request_metadata: Optional[Dict[str, Any]] = None, + media_path: Optional[str] = None, + byte_size: Optional[int] = None, + sha256: Optional[str] = None, now: Optional[datetime.datetime] = None, ) -> Dict[str, Any]: - """Create a root task with a blank legacy token and persist its separate share token.""" + """Atomically create the root and durable accepted admission for one upload.""" if not media_id: raise ValueError("media_id must be non-empty") now = now or datetime.datetime.now(datetime.timezone.utc) @@ -723,23 +818,84 @@ def accept_local_upload( """INSERT INTO task_status (task_id, view_token, url, platform, media_id, status, title, processing_options, submitted_by) - VALUES (?, '', '', 'local_upload', ?, 'queued', ?, ?, ?)""", - (task_id, media_id, title, options_json, upload["owner_user_id"]), + VALUES (?, '', ?, 'local_upload', ?, 'queued', ?, ?, ?)""", + (task_id, source_url or "", media_id, title, options_json, upload["owner_user_id"]), ) cursor.execute( """UPDATE local_uploads - SET state = 'accepted', root_task_id = ?, media_id = ?, view_token = ? + SET state = 'accepted', root_task_id = ?, media_id = ?, view_token = ?, + filename = ?, source_url = ?, request_metadata = ?, media_path = ?, + byte_size = ?, sha256 = ? WHERE upload_id = ? AND state = 'receiving' AND revoked_at IS NULL""", - (task_id, media_id, view_token, upload_id), + ( + task_id, media_id, view_token, filename, source_url, + json.dumps(request_metadata or {}, ensure_ascii=False, sort_keys=True), + media_path, byte_size, sha256, upload_id, + ), ) if cursor.rowcount != 1: raise RuntimeError("local upload acceptance lost its receiving state") cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) return dict(cursor.fetchone()) + def get_admitted_local_upload_by_task( + self, task_id: str + ) -> Optional[Dict[str, Any]]: + """Return only a queued root backed by a durable accepted upload mapping.""" + with self._get_cursor() as cursor: + cursor.execute( + """SELECT u.* FROM local_uploads u + JOIN task_status t ON t.task_id = u.root_task_id + WHERE u.root_task_id = ? AND u.state = 'accepted' + AND u.error_code IS NULL AND t.task_id = u.root_task_id + AND t.platform = 'local_upload' AND t.media_id = u.media_id + AND t.submitted_by = u.owner_user_id AND t.view_token = '' + AND t.status = 'queued'""", + (task_id,), + ) + row = cursor.fetchone() + return dict(row) if row is not None else None + + def set_local_upload_error(self, upload_id: str, error_code: str) -> None: + with self._get_cursor() as cursor: + cursor.execute( + "UPDATE local_uploads SET error_code = ? WHERE upload_id = ?", + (error_code, upload_id), + ) + if cursor.rowcount != 1: + raise ValueError("local upload does not exist") + + def cancel_local_upload_acceptance( + self, upload_id: str, task_id: str, error_code: str + ) -> None: + """Undo the durable draft if the one-shot in-memory queue handoff fails.""" + with self._get_cursor() as cursor: + cursor.execute("BEGIN IMMEDIATE") + cursor.execute( + """UPDATE local_uploads + SET state = 'receiving', root_task_id = NULL, media_id = NULL, + view_token = NULL, expires_at = NULL, media_path = NULL, + byte_size = NULL, sha256 = NULL, error_code = ? + WHERE upload_id = ? AND root_task_id = ? AND state = 'accepted'""", + (error_code, upload_id, task_id), + ) + if cursor.rowcount != 1: + raise RuntimeError("cannot cancel missing local upload acceptance") + cursor.execute( + "DELETE FROM task_status WHERE task_id = ? AND status = 'queued'", + (task_id,), + ) + if cursor.rowcount != 1: + raise RuntimeError("cannot remove unqueued local upload root") + def get_local_upload_by_id(self, upload_id: str) -> Optional[Dict[str, Any]]: with self._get_cursor() as cursor: - cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) + cursor.execute( + """SELECT u.*, t.status AS root_status FROM local_uploads u + LEFT JOIN task_status t ON t.task_id = u.root_task_id + WHERE u.upload_id = ?""", + (upload_id,), + ) row = cursor.fetchone() return dict(row) if row is not None else None @@ -748,7 +904,9 @@ def get_local_upload_by_owner_key( ) -> Optional[Dict[str, Any]]: with self._get_cursor() as cursor: cursor.execute( - "SELECT * FROM local_uploads WHERE owner_user_id = ? AND idempotency_key = ?", + """SELECT u.*, t.status AS root_status FROM local_uploads u + LEFT JOIN task_status t ON t.task_id = u.root_task_id + WHERE u.owner_user_id = ? AND u.idempotency_key = ?""", (owner_user_id, idempotency_key), ) row = cursor.fetchone() diff --git a/tests/README.md b/tests/README.md index e65c40b0..23dde4b6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -46,7 +46,13 @@ uv run pytest tests/deployment 默认门禁会阻断指向非 loopback 地址的 AF_INET/AF_INET6 出站连接；DNS 解析、 loopback 与 Unix 域套接字放行，真实外网测试仍应放在 `tests/manual/`。 -本地上传安全基线：`tests/unit/test_local_upload_policy.py` 用临时 SQLite、真实 `ViewTokenResolver` 与 cache/task cleanup 消费者验证 owner/key 幂等、blank legacy token 隔离、`upload_` token、固定终态期限、write-once 撤销、开关关闭后的拒读与有效成果保护；另启动真实 Python 子进程消费同一 SQLite，验证缺省关闭与显式环境开关。`tests/cache/test_cache_cleanup.py` 的缓存 fixture 仅写 `tmp_path`，并从缓存 producer 返回的实际文件路径删除，不触碰仓库 `test_cache_dir/`。 +本地上传安全基线：`tests/unit/test_local_upload_policy.py` 用临时 SQLite、真实 `ViewTokenResolver` 与 cache/task cleanup 消费者验证 owner/key 幂等、blank legacy token 隔离、`upload_` token、固定终态期限、write-once 撤销、开关关闭后的拒读与有效成果保护；另启动真实 Python 子进程消费同一 SQLite，验证缺省关闭与显式环境开关。B 接收回归运行： + +```bash +uv run --frozen pytest -q tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py +``` + +其中 `test_upload_intake.py` 由真实 Bearer `UserManager`、FastAPI/ASGI raw request、临时 cache.db 与真实 `process_task_queue` consumer 构成；断言落盘 bytes 与 SHA-256、Unicode wire metadata、同 key零body/零重复入队、distinct key独立任务、actual receiver/inflight/free-space/bytes限制、queue/SQLite失败、ASGI断开、丢回执查询原任务、owner stop write-once。ASR调用被替换为捕获真实 dispatcher 参数的本地消费者，生产 ASR/通知未调用；独立 `test_upload_dispatch.py` 证明 stale admission 不进PROCESSING/executor/ASR，并证明 local media未知/无音轨/超时长在ASR前拒绝且不走URL downloader。示例配置的四项 upload limits 为 null；这些测试不能被用作生产容量测量或部署授权。`tests/cache/test_cache_cleanup.py` 的缓存 fixture 仅写 `tmp_path`，并从缓存 producer 返回的实际文件路径删除，不触碰仓库 `test_cache_dir/`。 任务通知数量回归：`tests/unit/test_notification_budget.py` 从真实转录入口运行下载、平台字幕、YouTube API 字幕、部分/全部缓存命中与两种失败路径，断言成功路径携带完整总结和分享回执、失败不带回执；同文件还以 25 MiB 下载 fixture 锁定 generic 下载不发进度通知。`tests/unit/test_terminal_notification_outbox.py` 覆盖单行 outbox、缺失快照可见降级与标记 sent、两条成功消息共用快照及失败路径。`tests/unit/test_notification_e2e_delivery.py` 使用真实 `process_transcription → _TaskNotifier → NotificationRouter → WeComChannel/FeishuChannel → wecom-notifier FIFO → HTTP JSON`（只替换最终外部 POST），并覆盖即时与 outbox 补发入口；检查 SQLite 实际快照字节、双渠道最终 payload 中完整总结先于分享回执、精确 view URL 与网页首段。`tests/unit/test_completion_share.py` 锁定 Markdown→网页同源 HTML→首个 p 可见文本与缺总结状态。相关设计及 N1–N4 更新见 `docs/sessions/261008-completion-share/design.md`。 diff --git a/tests/integration/test_upload_intake.py b/tests/integration/test_upload_intake.py new file mode 100644 index 00000000..ef74df7c --- /dev/null +++ b/tests/integration/test_upload_intake.py @@ -0,0 +1,619 @@ +import asyncio +import base64 +import datetime +import hashlib +import json +import queue +import shutil +import sqlite3 +import threading +import uuid +from contextlib import asynccontextmanager, contextmanager +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from starlette.requests import ClientDisconnect + +from video_transcript_api.api.context import ( + ConfigError, + RuntimeContext, + bind_runtime, + unbind_runtime, + validate_config, +) +from video_transcript_api.api.routes import uploads +from video_transcript_api.api.services import transcription +from video_transcript_api.cache.cache_manager import CacheManager +from video_transcript_api.utils.accounts.user_manager import UserManager +from video_transcript_api.utils.tempfile_manager import TempFileManager + + +# Representative raw File/Blob producer payload, sent as application/octet-stream. +RAW_PRODUCER_BYTES = b"\x00VTA upload producer fixture\nmeeting audio bytes\x00" +PRODUCER_METADATA = { + "filename": "C:\\fakepath\\会议 录音.mp4", + "byte_size": len(RAW_PRODUCER_BYTES), + "title": "讨论：咖啡与模型", + "source_url": "https://display.example/recording?id=42", + "retention": "30d", + "processing_options": {"calibrate": False, "summarize": True}, +} +OWNER_TOKEN = "integration-owner-token" +OTHER_TOKEN = "integration-other-token" + + +def _key(): + epoch_ms = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000) + return f"{epoch_ms}-{uuid.uuid4()}" + + +def _metadata_header(payload): + raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +class BodyReadCounter: + def __init__(self, app, counters): + self.app = app + self.counters = counters + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or scope["method"] != "POST" or scope["path"] != "/api/uploads": + await self.app(scope, receive, send) + return + body_id = dict(scope["headers"]).get(b"x-body-id", b"").decode("ascii") + + async def counted_receive(): + message = await receive() + if message["type"] == "http.request" and message.get("body"): + self.counters[body_id] = self.counters.get(body_id, 0) + 1 + return message + + await self.app(scope, counted_receive, send) + + +def _config(tmp_path): + return { + "api": {"host": "127.0.0.1", "port": 8000, "auth_token": "integration-config-token"}, + "concurrent": {"max_workers": 1, "queue_size": 2, "llm_max_workers": 1}, + "storage": { + "cache_dir": str(tmp_path / "cache"), + "workspace_dir": str(tmp_path / "workspace"), + "temp_dir": str(tmp_path / "temp"), + "audit_db": str(tmp_path / "audit.db"), + "upload_limits": { + "max_file_mib": 2, + "max_media_hours": 1, + "receive_concurrency": 1, + "upload_temp_budget_mib": 8, + }, + }, + "web": {"base_url": "http://localhost:8000"}, + "llm": { + "api_key": "test-llm-key", + "base_url": "http://127.0.0.1:1/v1", + "calibrate_model": "test-calibrate-model", + "summary_model": "test-summary-model", + }, + "log": {"file": str(tmp_path / "app.log")}, + } + + +@pytest.fixture() +def upload_client(tmp_path, monkeypatch): + config = _config(tmp_path) + runtime = RuntimeContext(config) + runtime.cache_manager = CacheManager(config["storage"]["cache_dir"]) + runtime.temp_manager = TempFileManager(config["storage"]["temp_dir"]) + runtime.task_queue = asyncio.Queue(maxsize=config["concurrent"]["queue_size"]) + runtime.executor = ThreadPoolExecutor(max_workers=1) + runtime.started = True + users_file = tmp_path / "users.json" + users_file.write_text(json.dumps({"users": { + OWNER_TOKEN: {"user_id": "alice", "name": "Alice"}, + OTHER_TOKEN: {"user_id": "bob", "name": "Bob"}, + }}), encoding="utf-8") + monkeypatch.setattr( + transcription, "user_manager", + UserManager(users_config_path=str(users_file), fallback_config=config), + ) + worker_calls = [] + worker_results = queue.Queue() + worker_lock = threading.Lock() + + def fake_process_transcription(*args, **kwargs): + path = kwargs["local_media_path"] + result = { + "task_id": args[0], "url": args[1], + "local_media_id": kwargs["local_media_id"], + "bytes": Path(path).read_bytes(), "path": path, + } + with worker_lock: + worker_calls.append(result) + worker_results.put(result) + + monkeypatch.setattr(transcription, "process_transcription", fake_process_transcription) + counters = {} + app = FastAPI() + app.include_router(uploads.router) + app.add_middleware(BodyReadCounter, counters=counters) + + @app.middleware("http") + async def bind_upload_runtime(request, call_next): + token = bind_runtime(runtime) + try: + return await call_next(request) + finally: + unbind_runtime(token) + + @asynccontextmanager + async def lifespan(app): + token = bind_runtime(runtime) + app.state.runtime = runtime + app.state.queue_processor = asyncio.create_task(transcription.process_task_queue()) + try: + yield + finally: + app.state.queue_processor.cancel() + with pytest.raises(asyncio.CancelledError): + await app.state.queue_processor + runtime.executor.shutdown(wait=True) + unbind_runtime(token) + + app.router.lifespan_context = lifespan + with TestClient(app, raise_server_exceptions=False) as client: + yield client, runtime, counters, worker_calls, worker_results + + +def _post(client, key, payload, raw_bytes, body_id, token=OWNER_TOKEN, **headers): + request_headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/octet-stream", + "Idempotency-Key": key, + "X-Upload-Metadata": _metadata_header(payload), + "X-Body-Id": body_id, + } + request_headers.update(headers) + return client.post("/api/uploads", content=raw_bytes, headers=request_headers) + + +def _wait_for_worker_release(runtime): + with runtime._worker_futures_condition: + assert runtime._worker_futures_condition.wait_for( + lambda: not runtime.worker_futures, timeout=2 + ) + assert runtime._upload_reserved_bytes == {} + assert runtime.inflight_registry.size("transcription") == 0 + + +def test_real_http_stream_sqlite_dispatch_idempotency_and_owner_stop(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, worker_results = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + post_schema = client.get("/openapi.json").json()["paths"]["/api/uploads"]["post"] + assert "application/octet-stream" in post_schema["requestBody"]["content"] + header_parameters = {item["name"] for item in post_schema["parameters"]} + assert {"Idempotency-Key", "X-Upload-Metadata"} <= header_parameters + capabilities = client.get( + "/api/uploads/capabilities", headers={"Authorization": f"Bearer {OWNER_TOKEN}"} + ) + assert capabilities.status_code == 200 + assert capabilities.json() == { + "enabled": True, "default_retention": "30d", + "retention_options": ["30d", "never"], + "limits": { + "max_file_mib": 2, "max_media_hours": 1, + "receive_concurrency": 1, "upload_temp_budget_mib": 8, + }, + } + + assert _post(client, _key(), PRODUCER_METADATA, RAW_PRODUCER_BYTES, "bad-auth", token="wrong").status_code == 401 + assert body_reads.get("bad-auth", 0) == 0 + wrong_type = _post( + client, _key(), PRODUCER_METADATA, RAW_PRODUCER_BYTES, "wrong-type", + **{"Content-Type": "multipart/form-data"}, + ) + assert wrong_type.status_code == 415 + assert body_reads.get("wrong-type", 0) == 0 + invalid_metadata = client.post( + "/api/uploads", + content=RAW_PRODUCER_BYTES, + headers={ + "Authorization": f"Bearer {OWNER_TOKEN}", + "Content-Type": "application/octet-stream", + "Idempotency-Key": _key(), + "X-Upload-Metadata": "not_base64url", + "X-Body-Id": "bad-metadata", + }, + ) + assert invalid_metadata.status_code == 400 + assert body_reads.get("bad-metadata", 0) == 0 + + key = _key() + response = _post(client, key, PRODUCER_METADATA, RAW_PRODUCER_BYTES, "producer") + assert response.status_code == 202 + receipt = response.json() + assert receipt["state"] == "accepted" + assert receipt["task_id"] + assert receipt["view_token"].startswith("upload_") + assert body_reads.get("producer", 0) > 0 + stored = runtime.cache_manager.get_local_upload_by_owner_key("alice", key) + assert Path(stored["media_path"]).read_bytes() == RAW_PRODUCER_BYTES + assert Path(stored["media_path"]).stat().st_mode & 0o777 == 0o600 + assert stored["byte_size"] == len(RAW_PRODUCER_BYTES) + assert stored["sha256"] == hashlib.sha256(RAW_PRODUCER_BYTES).hexdigest() + assert stored["filename"] == "会议 录音.mp4" + root = runtime.cache_manager.get_task_by_id(receipt["task_id"]) + assert root["view_token"] == "" + assert root["platform"] == "local_upload" + assert root["media_id"] == stored["media_id"] + assert Path(stored["media_path"]).parent == Path(runtime.temp_manager.base_dir) / f"task_{receipt['task_id']}" + dispatched = worker_results.get(timeout=2) + assert dispatched["task_id"] == receipt["task_id"] + assert dispatched["local_media_id"] == stored["media_id"] + assert dispatched["bytes"] == RAW_PRODUCER_BYTES + assert "upload-source.bin" in dispatched["path"] + _wait_for_worker_release(runtime) + + replay = _post(client, key, PRODUCER_METADATA, b"R" * len(RAW_PRODUCER_BYTES), "replay") + assert replay.status_code == 200 + assert replay.json()["upload_id"] == receipt["upload_id"] + assert body_reads.get("replay", 0) == 0 + conflict = _post( + client, key, {**PRODUCER_METADATA, "title": "changed"}, + RAW_PRODUCER_BYTES, "conflict", + ) + assert conflict.status_code == 409 + assert body_reads.get("conflict", 0) == 0 + assert len(worker_calls) == 1 + + independent = _post(client, _key(), PRODUCER_METADATA, RAW_PRODUCER_BYTES, "independent") + assert independent.status_code == 202 + independent_receipt = independent.json() + assert independent_receipt["upload_id"] != receipt["upload_id"] + assert independent_receipt["task_id"] != receipt["task_id"] + assert worker_results.get(timeout=2)["task_id"] == independent_receipt["task_id"] + _wait_for_worker_release(runtime) + assert len(worker_calls) == 2 + + owner_receipt = client.get( + f"/api/uploads/by-idempotency-key/{key}", + headers={"Authorization": f"Bearer {OWNER_TOKEN}"}, + ) + assert owner_receipt.status_code == 200 + assert owner_receipt.json()["upload_id"] == receipt["upload_id"] + assert client.get( + f"/api/uploads/by-idempotency-key/{key}", + headers={"Authorization": f"Bearer {OTHER_TOKEN}"}, + ).status_code == 404 + stopped = client.delete( + f"/api/uploads/{receipt['upload_id']}/share", + headers={"Authorization": f"Bearer {OWNER_TOKEN}"}, + ) + assert stopped.status_code == 200 + assert stopped.json()["share_active"] is False + repeated = client.delete( + f"/api/uploads/{receipt['upload_id']}/share", + headers={"Authorization": f"Bearer {OWNER_TOKEN}"}, + ) + assert repeated.json()["revoked_at"] == stopped.json()["revoked_at"] + assert client.delete( + f"/api/uploads/{independent_receipt['upload_id']}/share", + headers={"Authorization": f"Bearer {OTHER_TOKEN}"}, + ).status_code == 404 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("max_file_mib", 0), + ("max_media_hours", float("inf")), + ("upload_temp_budget_mib", float("nan")), + ("receive_concurrency", True), + ("receive_concurrency", 1.5), + ], +) +def test_upload_limit_config_rejects_nonpositive_nonfinite_or_noninteger(tmp_path, field, value): + config = _config(tmp_path) + config["storage"]["upload_limits"][field] = value + with pytest.raises(ConfigError, match="storage.upload_limits"): + validate_config(config) + + +def test_disabled_and_missing_limits_reject_before_body(upload_client, monkeypatch): + client, runtime, body_reads, _, _ = upload_client + monkeypatch.delenv("VTA_UPLOADS_ENABLED", raising=False) + assert client.get( + "/api/uploads/capabilities", headers={"Authorization": f"Bearer {OWNER_TOKEN}"} + ).json()["enabled"] is False + denied = _post(client, _key(), PRODUCER_METADATA, RAW_PRODUCER_BYTES, "disabled") + assert denied.status_code == 503 + assert body_reads.get("disabled", 0) == 0 + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + runtime.config["storage"]["upload_limits"]["upload_temp_budget_mib"] = None + assert client.get( + "/api/uploads/capabilities", headers={"Authorization": f"Bearer {OWNER_TOKEN}"} + ).json()["enabled"] is False + denied = _post(client, _key(), PRODUCER_METADATA, RAW_PRODUCER_BYTES, "missing-limit") + assert denied.status_code == 503 + assert body_reads.get("missing-limit", 0) == 0 + + +def test_receiver_and_inflight_limits_reject_without_body_consumption(upload_client, monkeypatch): + client, runtime, body_reads, _, _ = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + assert runtime.upload_receiver_slots.acquire(blocking=False) + receiver_full = _post(client, _key(), PRODUCER_METADATA, RAW_PRODUCER_BYTES, "receiver-full") + assert receiver_full.status_code == 503 + assert body_reads.get("receiver-full", 0) == 0 + runtime.upload_receiver_slots.release() + registry = runtime.inflight_registry + assert registry.try_register("transcription", "occupied-one") + assert registry.try_register("transcription", "occupied-two") + inflight_full = _post(client, _key(), PRODUCER_METADATA, RAW_PRODUCER_BYTES, "inflight-full") + assert inflight_full.status_code == 503 + assert body_reads.get("inflight-full", 0) == 0 + registry.release("transcription", "occupied-one") + registry.release("transcription", "occupied-two") + assert registry.size("transcription") == 0 + assert not list(Path(runtime.temp_manager.base_dir).glob("task_*/upload-source.bin")) + + +def test_actual_free_space_shortage_rejects_before_body_or_file(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, _ = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + monkeypatch.setattr(shutil, "disk_usage", lambda _path: SimpleNamespace(free=1)) + key = _key() + response = _post(client, key, PRODUCER_METADATA, RAW_PRODUCER_BYTES, "disk-shortage") + assert response.status_code == 507 + assert body_reads.get("disk-shortage", 0) == 0 + record = runtime.cache_manager.get_local_upload_by_owner_key("alice", key) + assert record["error_code"] == "upload_temp_budget" + assert record["root_task_id"] is None + assert runtime.inflight_registry.size("transcription") == 0 + assert runtime._upload_reserved_bytes == {} + assert worker_calls == [] + assert not list(Path(runtime.temp_manager.base_dir).glob("task_*/upload-source.bin")) + + +def test_declared_and_observed_limits_check_bytes_not_content_length(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, _ = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + declared = {**PRODUCER_METADATA, "byte_size": 2 * 1024 * 1024 + 1} + response = _post(client, _key(), declared, RAW_PRODUCER_BYTES, "declared-too-large") + assert response.status_code == 413 + assert body_reads.get("declared-too-large", 0) == 0 + actual_bytes = b"B" * (2 * 1024 * 1024 + 1) + observed = _post( + client, _key(), {**PRODUCER_METADATA, "byte_size": 2 * 1024 * 1024}, + actual_bytes, "observed-too-large", **{"Content-Length": "1"}, + ) + assert observed.status_code == 413 + assert body_reads.get("observed-too-large", 0) > 0 + assert runtime.task_queue.qsize() == 0 + assert worker_calls == [] + assert not list(Path(runtime.temp_manager.base_dir).glob("task_*/upload-source.bin")) + assert runtime._upload_reserved_bytes == {} + + +def test_client_disconnect_cleans_only_untransferred_partial_media(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, _ = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + key = _key() + scope = { + "type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", "method": "POST", "scheme": "http", + "path": "/api/uploads", "raw_path": b"/api/uploads", "query_string": b"", + "root_path": "", "client": ("testclient", 50000), "server": ("testserver", 80), + "headers": [ + (b"authorization", f"Bearer {OWNER_TOKEN}".encode()), + (b"content-type", b"application/octet-stream"), + (b"idempotency-key", key.encode()), + (b"x-upload-metadata", _metadata_header(PRODUCER_METADATA).encode()), + (b"x-body-id", b"client-disconnect"), + ], + } + receive_index = 0 + + async def receive(): + nonlocal receive_index + receive_index += 1 + if receive_index == 1: + return {"type": "http.request", "body": RAW_PRODUCER_BYTES[:8], "more_body": True} + return {"type": "http.disconnect"} + + async def send(_message): + return None + + with pytest.raises(ClientDisconnect): + client.portal.call(client.app, scope, receive, send) + record = runtime.cache_manager.get_local_upload_by_owner_key("alice", key) + assert record["state"] == "receiving" + assert record["error_code"] == "receive_incomplete" + assert record["root_task_id"] is None + assert body_reads.get("client-disconnect", 0) == 1 + assert runtime.task_queue.qsize() == 0 + assert worker_calls == [] + assert not list(Path(runtime.temp_manager.base_dir).glob("task_*/upload-source.bin")) + assert runtime._upload_reserved_bytes == {} + + +def test_queue_full_has_no_success_receipt_or_dispatch(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, _ = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + full_queue = asyncio.Queue(maxsize=1) + full_queue.put_nowait({"occupies": "the only queue slot"}) + monkeypatch.setattr(uploads, "get_task_queue", lambda: full_queue) + key = _key() + response = _post(client, key, PRODUCER_METADATA, RAW_PRODUCER_BYTES, "queue-full") + assert response.status_code == 503 + assert full_queue.qsize() == 1 + assert body_reads.get("queue-full", 0) == 0 + record = runtime.cache_manager.get_local_upload_by_owner_key("alice", key) + assert record["error_code"] == "queue_full" + assert record["state"] == "receiving" + assert record["root_task_id"] is None + assert worker_calls == [] + assert not list(Path(runtime.temp_manager.base_dir).glob("task_*/upload-source.bin")) + assert runtime._upload_reserved_bytes == {} + + +def test_sql_commit_failure_rolls_back_root_and_never_queues(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, _ = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + cache = runtime.cache_manager + + @contextmanager + def fail_root_commit(): + connection = cache._get_connection() + cursor = connection.cursor() + inserted_root = False + + class CursorProxy: + def execute(self, sql, parameters=()): + nonlocal inserted_root + if "INSERT INTO task_status" in sql: + inserted_root = True + return cursor.execute(sql, parameters) + + def __getattr__(self, name): + return getattr(cursor, name) + + try: + yield CursorProxy() + if inserted_root: + raise sqlite3.OperationalError("injected commit failure") + connection.commit() + except Exception: + connection.rollback() + raise + finally: + cursor.close() + + monkeypatch.setattr(cache, "_get_cursor", fail_root_commit) + key = _key() + response = _post(client, key, PRODUCER_METADATA, RAW_PRODUCER_BYTES, "commit-failure") + assert response.status_code == 500 + assert body_reads.get("commit-failure", 0) > 0 + assert runtime.task_queue.qsize() == 0 + record = cache.get_local_upload_by_owner_key("alice", key) + assert record["state"] == "receiving" + assert record["root_task_id"] is None + with cache._get_cursor() as cursor: + cursor.execute("SELECT COUNT(*) FROM task_status") + assert cursor.fetchone()[0] == 0 + assert worker_calls == [] + assert runtime._upload_reserved_bytes == {} + assert not list(Path(runtime.temp_manager.base_dir).glob("task_*/upload-source.bin")) + + +def test_acceptance_window_expiry_after_receive_is_explicit_410(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, _ = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + validate = CacheManager._validate_upload_key_window.__func__ + calls = 0 + + def expire_at_accept(cls, key, now): + nonlocal calls + calls += 1 + if calls == 2: + now += datetime.timedelta(hours=24) + return validate(cls, key, now) + + monkeypatch.setattr(CacheManager, "_validate_upload_key_window", classmethod(expire_at_accept)) + key = _key() + response = _post(client, key, PRODUCER_METADATA, RAW_PRODUCER_BYTES, "window-expired") + assert calls == 2 + assert response.status_code == 410 + assert body_reads.get("window-expired", 0) > 0 + record = runtime.cache_manager.get_local_upload_by_owner_key("alice", key) + assert record["state"] == "receiving" + assert record["error_code"] == "acceptance_window_expired" + assert record["root_task_id"] is None + assert runtime.task_queue.qsize() == 0 + assert worker_calls == [] + assert not list(Path(runtime.temp_manager.base_dir).glob("task_*/upload-source.bin")) + + +def test_queue_put_failure_rolls_back_formal_receipt(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, _ = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + + class FailingQueue(asyncio.Queue): + def full(self): + return False + + def put_nowait(self, item): + raise asyncio.QueueFull + + failing_queue = FailingQueue(maxsize=1) + monkeypatch.setattr(uploads, "get_task_queue", lambda: failing_queue) + key = _key() + response = _post(client, key, PRODUCER_METADATA, RAW_PRODUCER_BYTES, "put-failure") + assert response.status_code == 503 + assert body_reads.get("put-failure", 0) > 0 + record = runtime.cache_manager.get_local_upload_by_owner_key("alice", key) + assert record["state"] == "receiving" + assert record["error_code"] == "queue_full" + assert record["root_task_id"] is None + assert record["view_token"] is None + with runtime.cache_manager._get_cursor() as cursor: + cursor.execute("SELECT COUNT(*) FROM task_status") + assert cursor.fetchone()[0] == 0 + assert failing_queue.qsize() == 0 + assert worker_calls == [] + assert runtime._upload_reserved_bytes == {} + assert not list(Path(runtime.temp_manager.base_dir).glob("task_*/upload-source.bin")) + + +def test_lost_202_receipt_retry_queries_original_without_requeue(upload_client, monkeypatch): + client, runtime, body_reads, worker_calls, worker_results = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + key = _key() + scope = { + "type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", "method": "POST", "scheme": "http", + "path": "/api/uploads", "raw_path": b"/api/uploads", "query_string": b"", + "root_path": "", "client": ("testclient", 50000), "server": ("testserver", 80), + "headers": [ + (b"authorization", f"Bearer {OWNER_TOKEN}".encode()), + (b"content-type", b"application/octet-stream"), + (b"idempotency-key", key.encode()), + (b"x-upload-metadata", _metadata_header(PRODUCER_METADATA).encode()), + (b"x-body-id", b"lost-receipt"), + ], + } + sent_body = False + server_status = [] + + async def receive(): + nonlocal sent_body + if not sent_body: + sent_body = True + return {"type": "http.request", "body": RAW_PRODUCER_BYTES, "more_body": False} + return {"type": "http.disconnect"} + + async def discard_response(message): + if message["type"] == "http.response.start": + server_status.append(message["status"]) + # Simulate a lost network response: the server sends it, producer receives none. + + client.portal.call(client.app, scope, receive, discard_response) + assert server_status == [202] + assert body_reads.get("lost-receipt", 0) > 0 + original = runtime.cache_manager.get_local_upload_by_owner_key("alice", key) + assert original["state"] == "accepted" + task_id = original["root_task_id"] + assert worker_results.get(timeout=2)["task_id"] == task_id + _wait_for_worker_release(runtime) + + retry = _post(client, key, PRODUCER_METADATA, b"Z" * len(RAW_PRODUCER_BYTES), "receipt-retry") + assert retry.status_code == 200 + assert retry.json()["task_id"] == task_id + assert retry.json()["upload_id"] == original["upload_id"] + assert body_reads.get("receipt-retry", 0) == 0 + assert len(worker_calls) == 1 + assert runtime.task_queue.qsize() == 0 diff --git a/tests/unit/test_upload_dispatch.py b/tests/unit/test_upload_dispatch.py new file mode 100644 index 00000000..e91f136b --- /dev/null +++ b/tests/unit/test_upload_dispatch.py @@ -0,0 +1,149 @@ +import asyncio +import datetime +import uuid +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from video_transcript_api.api.services import transcription +from video_transcript_api.cache.cache_manager import CacheManager +from video_transcript_api.errors import InvalidMediaError +from video_transcript_api.utils.tempfile_manager import TempFileManager + + +def _key(): + now = datetime.datetime.now(datetime.timezone.utc) + return f"{int(now.timestamp() * 1000)}-{uuid.uuid4()}" + + +def test_dispatcher_fails_stale_local_mapping_before_processing_or_executor_submit( + tmp_path, monkeypatch +): + cache = CacheManager(str(tmp_path / "cache")) + intent = cache.register_local_upload( + owner_user_id="alice", idempotency_key=_key(), retention="30d" + ) + task_id = cache.generate_task_id() + upload = cache.accept_local_upload( + intent["upload_id"], + media_id=f"upload_{uuid.uuid4().hex}", + task_id=task_id, + filename="clip.mp4", + request_metadata={"filename": "clip.mp4"}, + media_path=str(tmp_path / "missing-upload.bin"), + byte_size=12, + sha256="f" * 64, + ) + assert upload["root_task_id"] == task_id + assert cache.get_admitted_local_upload_by_task(task_id)["media_path"].endswith( + "missing-upload.bin" + ) + + queue = asyncio.Queue() + queue.put_nowait({"id": task_id, "url": "https://example.invalid/display-only", "platform": "local_upload"}) + temp_manager = TempFileManager(str(tmp_path / "temp")) + task_dir = temp_manager.create_task_dir(task_id) + runtime = MagicMock() + runtime.inflight_registry.release = MagicMock() + runtime.release_upload_temp = MagicMock() + executor = MagicMock() + process_calls = [] + original_update = cache.update_task_status + failed = asyncio.Event() + + def update_and_signal(*args, **kwargs): + result = original_update(*args, **kwargs) + if args[1] == "failed": + failed.set() + return result + + monkeypatch.setattr(transcription, "cache_manager", cache) + monkeypatch.setattr(transcription, "task_queue", queue) + monkeypatch.setattr(transcription, "executor", executor) + monkeypatch.setattr(transcription, "get_runtime", lambda: runtime) + monkeypatch.setattr(transcription, "get_temp_manager", lambda: temp_manager) + monkeypatch.setattr(cache, "update_task_status", update_and_signal) + monkeypatch.setattr( + transcription, + "process_transcription", + lambda *args, **kwargs: process_calls.append((args, kwargs)), + ) + + async def run_dispatcher_once(): + dispatcher = asyncio.create_task(transcription.process_task_queue()) + await asyncio.wait_for(failed.wait(), timeout=2) + dispatcher.cancel() + try: + await dispatcher + except asyncio.CancelledError: + pass + + asyncio.run(run_dispatcher_once()) + + root = cache.get_task_by_id(task_id) + assert root["status"] == "failed" + assert process_calls == [] + executor.submit.assert_not_called() + runtime.inflight_registry.release.assert_called_once_with("transcription", task_id) + runtime.release_upload_temp.assert_called_once_with(task_id) + assert not task_dir.exists() + assert queue._unfinished_tasks == 0 + + +@pytest.mark.parametrize( + ("probe_result", "probe_error", "expected_code"), + [ + (None, None, "media_duration_unknown"), + (None, InvalidMediaError("no_audio_track"), "no_audio_track"), + (3601.0, None, "media_duration_limit"), + ], +) +def test_local_media_admission_rejects_before_asr_and_never_uses_url_downloader( + tmp_path, monkeypatch, probe_result, probe_error, expected_code +): + task_id = f"task_{uuid.uuid4().hex}" + temp_manager = TempFileManager(str(tmp_path / "temp")) + task_dir = temp_manager.create_task_dir(task_id) + media_path = task_dir / "upload-source.bin" + media_path.write_bytes(b"real local media fixture") + transcriber = MagicMock() + terminal_calls = [] + + def probe(path): + assert path == str(media_path) + if probe_error is not None: + raise probe_error + return probe_result + + monkeypatch.setattr(transcription, "get_temp_manager", lambda: temp_manager) + monkeypatch.setattr(transcription, "_resolve_view_url", lambda *args: None) + monkeypatch.setattr(transcription, "get_notification_router", MagicMock) + monkeypatch.setattr(transcription, "create_downloader", lambda *args: pytest.fail("local upload called URL downloader")) + monkeypatch.setattr(transcription, "_ensure_audio_track", probe) + monkeypatch.setattr(transcription, "Transcriber", lambda: transcriber) + monkeypatch.setattr(transcription.slow_alert, "track_transcription", lambda *args, **kwargs: None) + monkeypatch.setattr( + transcription, + "finalize_terminal_status_and_notify", + lambda *args, **kwargs: terminal_calls.append(kwargs["error_message"]) or True, + ) + monkeypatch.setattr( + transcription, + "get_config", + lambda: {"storage": {"upload_limits": {"max_media_hours": 1}}}, + ) + + result = transcription.process_transcription( + task_id, + "https://display-only.example/source", + metadata_override={"title": "本地媒体"}, + local_media_path=str(media_path), + local_media_id=f"upload_{uuid.uuid4().hex}", + ) + + assert result["status"] == "failed" + assert expected_code in result["message"] or expected_code in result.get("error", "") + assert terminal_calls + transcriber.transcribe.assert_not_called() + assert not media_path.exists() diff --git a/tests/unit/test_upload_routes.py b/tests/unit/test_upload_routes.py index 9cd14ccb..42b2c5e1 100644 --- a/tests/unit/test_upload_routes.py +++ b/tests/unit/test_upload_routes.py @@ -1,9 +1,14 @@ import base64 +import datetime import json +import sqlite3 +import uuid import pytest +from video_transcript_api.api.routes import uploads from video_transcript_api.api.routes.uploads import decode_upload_metadata +from video_transcript_api.cache.cache_manager import CacheManager def encode_metadata(payload): @@ -11,6 +16,18 @@ def encode_metadata(payload): return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") +def test_rejection_log_has_grepable_upload_intake_event(monkeypatch): + observed = [] + monkeypatch.setattr(uploads.logger, "warning", lambda *args: observed.append(args)) + + exception = uploads._reject(413, "declared_size", "too large") + + assert exception.status_code == 413 + assert observed == [ + ("UPLOAD_INTAKE_REJECTED reason={} status={}", "declared_size", 413) + ] + + def test_decode_upload_metadata_preserves_unicode_and_normalizes_options(): payload = { "filename": "../会议 录音.mp4", @@ -39,6 +56,8 @@ def test_decode_upload_metadata_preserves_unicode_and_normalizes_options(): "mutation", [ {"title": "x" * 201}, + {"title": "bad\nname"}, + {"filename": "clip\u0000.mp4"}, {"filename": "é" * 128}, {"source_url": "https://user:secret@example.com/a"}, {"source_url": "javascript:alert(1)"}, @@ -58,3 +77,102 @@ def test_decode_upload_metadata_rejects_invalid_bounded_fields(mutation): with pytest.raises(ValueError): decode_upload_metadata(encode_metadata(payload)) + + +def test_receiving_cleanup_retires_only_expired_keys_and_removes_staging_file(tmp_path): + now = datetime.datetime(2026, 10, 8, 12, 0, tzinfo=datetime.timezone.utc) + old_creation = now - datetime.timedelta(hours=23) + old_key = f"{int(old_creation.timestamp() * 1000)}-{uuid.uuid4()}" + current_key = f"{int(now.timestamp() * 1000)}-{uuid.uuid4()}" + manager = CacheManager(str(tmp_path / "cache")) + temp_dir = tmp_path / "temp" + old_path = temp_dir / "task_old" / "partial.bin" + current_path = temp_dir / "task_current" / "partial.bin" + old_path.parent.mkdir(parents=True) + current_path.parent.mkdir(parents=True) + old_path.write_bytes(b"partial old body") + current_path.write_bytes(b"still receiving") + + old_record = manager.register_local_upload( + owner_user_id="alice", idempotency_key=old_key, retention="30d", now=now + ) + current_record = manager.register_local_upload( + owner_user_id="alice", idempotency_key=current_key, retention="30d", now=now + ) + manager.set_local_upload_receiving_path(old_record["upload_id"], str(old_path)) + manager.set_local_upload_receiving_path(current_record["upload_id"], str(current_path)) + + retired = manager.cleanup_expired_local_upload_receiving( + str(temp_dir), now=now + datetime.timedelta(hours=2) + ) + + assert retired == 1 + assert not old_path.exists() + assert current_path.read_bytes() == b"still receiving" + assert manager.get_local_upload_by_owner_key("alice", old_key) is None + assert manager.get_local_upload_by_owner_key("alice", current_key)["upload_id"] == current_record["upload_id"] + with pytest.raises(ValueError, match="24-hour"): + manager.register_local_upload( + owner_user_id="alice", + idempotency_key=old_key, + retention="30d", + now=now + datetime.timedelta(hours=25), + ) + + +def test_cache_manager_adds_upload_mapping_columns_to_existing_a_schema(tmp_path): + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + database = cache_dir / "cache.db" + connection = sqlite3.connect(database) + connection.execute( + """CREATE TABLE local_uploads ( + upload_id TEXT PRIMARY KEY, + owner_user_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + idempotency_created_at TEXT NOT NULL, + metadata_fingerprint TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('receiving', 'accepted')), + retention TEXT NOT NULL CHECK(retention IN ('30d', 'never')), + root_task_id TEXT UNIQUE, + media_id TEXT UNIQUE, + view_token TEXT UNIQUE, + created_at TEXT NOT NULL, + expires_at TEXT, + revoked_at TEXT, + error_code TEXT, + UNIQUE(owner_user_id, idempotency_key) + )""" + ) + connection.commit() + connection.close() + + CacheManager(str(cache_dir)) + + connection = sqlite3.connect(database) + columns = {row[1] for row in connection.execute("PRAGMA table_info(local_uploads)")} + connection.close() + assert { + "filename", "source_url", "request_metadata", "media_path", "byte_size", "sha256" + } <= columns + + +def test_acceptance_deadline_is_exactly_key_creation_plus_24_hours(tmp_path): + now = datetime.datetime(2026, 10, 8, 12, 0, tzinfo=datetime.timezone.utc) + created = now - datetime.timedelta(hours=23) + key = f"{int(created.timestamp() * 1000)}-{uuid.uuid4()}" + manager = CacheManager(str(tmp_path / "cache")) + intent = manager.register_local_upload( + owner_user_id="alice", idempotency_key=key, retention="30d", now=now + ) + + with pytest.raises(ValueError, match="24-hour"): + manager.accept_local_upload( + intent["upload_id"], + media_id="upload-test-media", + now=created + datetime.timedelta(hours=24), + ) + assert manager.get_local_upload_by_id(intent["upload_id"])["state"] == "receiving" + with manager._get_cursor() as cursor: + cursor.execute("SELECT COUNT(*) FROM task_status") + assert cursor.fetchone()[0] == 0 From 840facda8e0cb251a1197cf3647c5accd5c496ff Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 22:27:24 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0B=E5=8D=A1=E9=AA=8C=E8=AF=81=E4=B8=8E?= =?UTF-8?q?=E4=BA=A4=E4=BB=98=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-130923-6b2088 Task-Id: VideoTranscriptAPI-20261008-16 --- .../261008-local-upload/progress/B-progress.md | 15 ++++++++++++++- goals/local-upload/M2-intake.md | 3 ++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/sessions/261008-local-upload/progress/B-progress.md b/docs/sessions/261008-local-upload/progress/B-progress.md index 08ffffd6..327e9258 100644 --- a/docs/sessions/261008-local-upload/progress/B-progress.md +++ b/docs/sessions/261008-local-upload/progress/B-progress.md @@ -34,6 +34,19 @@ implementing；完成第一份独立可运行单元：bounded metadata decoder - `receiving` key在创建+24h截止前保留唯一；正式接受在截止点及以后显式410，过期receiving草稿和未移交暂存路径由maintenance退休；不自动重试/换key。 - 首次QueueFull在读body前拒绝；若数据库已提交后`put_nowait`仍失败，事务补偿撤回root/token映射、保留失败receipt并清理文件，不能返回202。 +## Milestone 3：全仓验证与交付 + +### 本段结论（≤3句） + +- `make test` 完整门禁 exit 0；输出到 `/tmp/vta-upload-B-make-test.log`，pytest 到 100%，3 个既有用例 skip，无失败。 +- 同一未设置 `VTA_UPLOADS_ENABLED` 的消费测试分别在 `env -i` 裸shell与临时 systemd user unit `vta-upload-B-env-verify-1791469452-1251284.service` 运行成功；未修改真实 unit/生产配置。 +- commit `9ed74d9b5997aea4acbd3722b2cd6db537571faa` 已 push，draft PR #202：`https://github.com/zlxlabs/VideoTranscriptAPI/pull/202`；本地树clean。CI / 独立主审由lead托管，尚未ready/merge。 + +### 关键决策与未验证项 + +- 上传默认保持关闭，示例四项生产额度均为null；本地测试额度只存在临时 fixture，不作为生产容量结论。 +- 真实生产 systemd unit、ASR服务容量、恢复域和部署仍未验证；本卡未触碰真实配置、数据、通知或生产ASR。 + ## 下一步唯一动作 -完成文档/测试说明与全仓门禁，核对每个不变量的producer/consumer锁定测试，再更新本进度并提交推送下一里程碑。 +等待 Pi lead 对 draft PR #202 独立验收并托管CI/主审；执行器不标ready、不合并、不部署。 diff --git a/goals/local-upload/M2-intake.md b/goals/local-upload/M2-intake.md index 610f7f24..91a17456 100644 --- a/goals/local-upload/M2-intake.md +++ b/goals/local-upload/M2-intake.md @@ -19,5 +19,6 @@ merged_pr: null - [x] 真实 FastAPI/ASGI 与临时 cache.db 上由 client 发 raw 文件字节及 headers；锁定实际落盘字节、服务端 SHA-256、Unicode metadata、root row 与 dispatcher 实际消费路径。producer fixture 保留在 `tests/integration/test_upload_intake.py`。 - [x] 缺/错 Bearer、默认关闭、额度未知、非 octet-stream、声明/实收超限、receiver/inflight/free-space 满载均在 body/file/ASR 边界前显式拒绝。 - [x] 同 key 重放/变化 metadata、相同字节新 key、真实 ASGI 断开、队列满、SQLite commit fault、queue put fault、窗口跨期与丢失202回执重查均有 producer→consumer 测试；same-intent 不读 body/不重入队。 - - [ ] 最终CI及系统级部署 unit 的环境传递仍由 lead 验收；未改真实 systemd/service 或生产环境。本地隔离裸shell验证待最终阶段执行。 + - [x] 未设置VTA开关的真实consumer测试在`env -i`裸shell与隔离的临时systemd user unit分别运行成功；未改真实systemd/service或生产环境。 + - [ ] Hosted CI及实际生产unit环境传递仍由lead验收；本地隔离验证不等于生产部署/容量证明。 - **完成条件**：本地代码与测试已覆盖 202/失败边界；正式合并、CI 与真实部署环境变量/容量证明由 lead 后续托管，生产仍关闭。 From 5b1f3526fa958deffd704ba43cbc13be47db95df Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 23:38:42 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E9=98=9F=E5=88=97=E4=BA=A4=E6=8E=A5=E4=B8=8E=E6=8E=A5=E6=94=B6?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E9=80=80=E4=BC=91=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-151543-50422d Task-Id: VideoTranscriptAPI-20261008-16 --- docs/sessions/261008-local-upload/design.md | 4 +- .../progress/B-progress.md | 28 ++- goals/local-upload/M2-intake.md | 5 +- .../api/routes/uploads.py | 3 +- .../cache/cache_manager.py | 61 +++--- tests/README.md | 2 +- tests/integration/test_upload_intake.py | 184 +++++++++++++++++- 7 files changed, 243 insertions(+), 44 deletions(-) diff --git a/docs/sessions/261008-local-upload/design.md b/docs/sessions/261008-local-upload/design.md index 416b884a..3c754d1e 100644 --- a/docs/sessions/261008-local-upload/design.md +++ b/docs/sessions/261008-local-upload/design.md @@ -39,7 +39,7 @@ Resolver 每次新请求都查独立上传记录并核对开关、撤销和截 ### 响应 - `GET /api/uploads/capabilities`（Bearer）：`{enabled:boolean, default_retention:'30d', retention_options:['30d','never'], limits:{max_file_mib,max_media_hours,receive_concurrency,upload_temp_budget_mib}}`。四项额度必须正且有限、来自经过真实消费环境验证的配置；当前生产值未知/未测，缺一项即 `enabled=false`。不得用示例值声称已测。 -- `POST /api/uploads`：只有 root、accepted 状态及媒体映射在 SQLite 正式提交、且单次队列 handoff 成功后才返回 202；相同意图返回原回执（包括仍为 receiving 的阶段回执），变化 metadata 返回 409。receiving/failed/未知回执均不伪装为 accepted/202；队列或提交失败不能留下可用分享 capability。FastAPI 错误必须明确 status/detail。 +- `POST /api/uploads`：root、accepted 状态和媒体映射在 `BEGIN IMMEDIATE` 事务中写入，并在同一无 await SQL 临界步骤内仅调用一次有界队列 `put_nowait`；事务提交成功后才正式受理并返回 202。SQLite 与进程内队列并非共同事务：队列 put 成功而 SQLite commit 失败时允许留下stale队列项，但dispatcher必须先核验持久admission/root/media映射、在 PROCESSING/executor.submit/ASR 前拒绝它。QueueFull自然回滚未受理SQL，不依赖事后撤回已accepted root。相同意图返回原回执（包括仍为 receiving 的阶段回执），变化 metadata 返回 409；receiving/failed/未知回执均不伪装为 accepted/202。FastAPI 错误必须明确 status/detail。 - `GET /api/uploads/by-idempotency-key/{key}`（Bearer/owner）：已存在 receipt 或 404；不能借查询创建任务。 - receipt：`upload_id`、`state`（接收阶段，不复制 `task_status`）、`task_id`（未正式受理时 null）、`view_token`（正式受理后独立 `upload_` token，否则 null）、`retention`、`expires_at`（终态前与 `never` 为 null）、`share_active`、`error_code`（无错误为 null）。处理进度继续由既有 `GET /api/task/{task_id}` 承载。 - `DELETE /api/uploads/{upload_id}/share`（Bearer/owner）：`{upload_id, share_active:false, revoked_at}`。重复关闭返回原截止，不得复活。关闭/到期后所有新 public read 和新 reprocess 拒绝；已经授权的传输和已接受处理可继续。 @@ -56,7 +56,7 @@ Resolver 每次新请求都查独立上传记录并核对开关、撤销和截 | 有效产物不被 cache/task/audit 清理删除；开关 off 不影响保护；关闭后恢复原清理；媒体仍临时 | `cleanup_old_cache/cleanup_task_status`；现有 `task_exists` 与 `TempFileManager` | local policy cleanup/controlled ordering tests、`tests/cache/test_task_status_cleanup.py`、`tests/cache/test_cache_cleanup.py` | | URL 读、去重、history 与 task 行为不改变 | 原有 URL routes/CacheManager | `tests/unit/test_view_token_resolver.py`、`test_api_routes.py`、`test_history_routes.py` 及 cache cleanup 套件 | -当前真实开发验证入口：A 的窄测与 `make test`；B 真实 HTTP/SQLite/dispatcher 入口为 `uv run --frozen pytest -q tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py`。B 测试覆盖真实 Bearer、raw body 实际字节与服务端 SHA-256、容量/队列/SQL/断开、重复与丢回执、owner 撤销及 stale dispatcher。所有生产额度仍为 null/未知，`VTA_UPLOADS_ENABLED` 缺失时 intake 与 share 都关闭；部署/systemd 恢复及容量验证仍是上线前置条件，本卡不触及生产 unit、数据卷或服务。 +当前真实开发验证入口：A 的窄测与 `make test`；B 真实 HTTP/SQLite/dispatcher 入口为 `uv run --frozen pytest -q tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py`。B 测试覆盖真实 Bearer、raw body 实际字节与服务端 SHA-256、容量、queue put 与 SQL commit 双顺序、cleanup/accept 两顺序、真实 stale dispatcher、断开、重复/丢回执与撤销。所有生产额度仍为 null/未知，`VTA_UPLOADS_ENABLED` 缺失时 intake 与 share 都关闭；部署/systemd 恢复及容量验证仍是上线前置条件，本卡不触及生产 unit、数据卷或服务。 ## 非目标与依赖 diff --git a/docs/sessions/261008-local-upload/progress/B-progress.md b/docs/sessions/261008-local-upload/progress/B-progress.md index 327e9258..b9afab2f 100644 --- a/docs/sessions/261008-local-upload/progress/B-progress.md +++ b/docs/sessions/261008-local-upload/progress/B-progress.md @@ -24,7 +24,7 @@ implementing；完成第一份独立可运行单元：bounded metadata decoder ### 本段结论（≤3句） - 已实现 GET capabilities、raw `application/octet-stream` POST、owner receipt/DELETE、服务端流式 byte count/SHA-256、metadata边界、unique owner/key、24h截止与过期receiving清理。 -- SQLite root/accepted/mapping一次事务提交后只调用一次有界队列 `put_nowait`；worker 在 PROCESSING/executor.submit 前核验持久受理与 root/media/path 映射，local_upload 走现有 ASR 路径且不经过 URL downloader。 +- H0实现曾在SQLite root/accepted/mapping commit之后才调用队列 `put_nowait`；H1续补将同步 `put_nowait` 移入同一事务提交前，dispatcher仍在 PROCESSING/executor.submit 前核验持久受理与 root/media/path 映射，local_upload 走现有ASR路径且不经过URL downloader。 - 真实 ASGI+临时SQLite/dispatcher窄测：`tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py` 全绿（32 passed）；尚未跑全仓 make test、裸shell/systemd验证或 CI。 ### 关键决策与否决方案 @@ -32,7 +32,7 @@ implementing；完成第一份独立可运行单元：bounded metadata decoder - upload limits 只从现有 `storage.upload_limits` 消费，示例四值均 null；`VTA_UPLOADS_ENABLED` 独立于数据卷配置，只有env、正有限额度及实际存活的 process_task_queue 同时满足才报告 enabled。 - body接收目录在开始读取前持久记录并由 TempFileManager active 集保护；未移交时由接收者清理，移交后worker清理；worker future完成释放原始输入的磁盘/inflight预留。 - `receiving` key在创建+24h截止前保留唯一；正式接受在截止点及以后显式410，过期receiving草稿和未移交暂存路径由maintenance退休；不自动重试/换key。 -- 首次QueueFull在读body前拒绝；若数据库已提交后`put_nowait`仍失败，事务补偿撤回root/token映射、保留失败receipt并清理文件，不能返回202。 +- 队列预满时在读body前拒绝；H0的事后撤回补偿仅属旧实现，H1已删除。QueueFull由事务回滚留在receiving/失败回执；commit失败时已入队的stale item由dispatcher durable gate丢弃。 ## Milestone 3：全仓验证与交付 @@ -47,6 +47,26 @@ implementing；完成第一份独立可运行单元：bounded metadata decoder - 上传默认保持关闭，示例四项生产额度均为null；本地测试额度只存在临时 fixture，不作为生产容量结论。 - 真实生产 systemd unit、ASR服务容量、恢复域和部署仍未验证；本卡未触碰真实配置、数据、通知或生产ASR。 -## 下一步唯一动作 +## Milestone H1：补齐 R3 与接收文件所有权顺序 + +### 当前阶段 + +implementing；H0 `840facda8e0cb251a1197cf3647c5accd5c496ff` 后续审查指出两项原合同缺口。先在固定H0上固化四个真实AssertionError红测，之后完成最小顺序修复；H1窄测与A/URL回归已绿，本段准备作为H1小提交推送。 + +### 本段结论（≤3句） + +- H0真实HTTP/SQLite/queue handoff负对照观察到commit前状态已是accepted/root queued；H1在H0上四个候选producer/consumer顺序测试实际失败均为AssertionError，修复后四项通过。 +- 当前H1将一次同步`put_nowait`从route移入`CacheManager.accept_local_upload`事务回调；commit失败会留下stale queue payload，由dispatcher在PROCESSING/submit/ASR前拒绝。事后cancel acceptance方法和callsite已删除。 +- receiving cleanup现在先于unlink在`BEGIN IMMEDIATE`中重读、核验owner-key时间/仍receiving/路径未变并删除记录，commit后才unlink；两个controlled order测试分别锁定accepted文件保留和retire先赢。 + +### 关键决策与验证 + +- 新参数`enqueue`只服务HTTP队列跨SQLite原子发布边界；callback同步调用且无await，既有A store-level direct acceptance仍可不带queue callback。没有新增服务层/状态/锁/重试/fallback。 +- 真实测试位于`tests/integration/test_upload_intake.py`：队列put hook用独立SQLite连接观察未提交receiving/root-none和实际payload；commit fault时实际queue收到payload，真实dispatcher drain后没有PROCESSING/executor submit/ASR；cleanup/accept两顺序均使用真实临时SQLite与文件字节，Pause仅位于测试fixture。 +- 红验命令：`uv run --frozen pytest -q tests/integration/test_upload_intake.py::test_http_queue_handoff_precedes_durable_acceptance_commit tests/integration/test_upload_intake.py::test_sql_commit_failure_leaves_stale_queue_item_for_dispatcher_to_drop tests/integration/test_upload_intake.py::test_receiving_cleanup_rechecks_after_accept_wins_and_preserves_owned_file tests/integration/test_upload_intake.py::test_receiving_cleanup_retires_before_unlink_and_accept_loses`；H0输出4 failed/AssertionError，H1同命令4 passed。 +- H1窄测`uv run --frozen pytest -q tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py`：35个测试点通过；另含A store/Resolver、URL API/history、runtime及maintenance的回归命令退出0。 +- 当前增量242行左右，仅原scope内7个文件；已先保存H0红验日志与受控两个文件顺序测试，接下来按显式路径小提交并push。 + +### 下一步唯一动作 -等待 Pi lead 对 draft PR #202 独立验收并托管CI/主审；执行器不标ready、不合并、不部署。 +提交并push H1增量；提交后针对R3 handoff与cleanup退休两个predicate做最小负向断言验证并恢复，再跑有界完整make test、核对clean/远端PR head。不得ready/merge。 diff --git a/goals/local-upload/M2-intake.md b/goals/local-upload/M2-intake.md index 91a17456..fc07ff2b 100644 --- a/goals/local-upload/M2-intake.md +++ b/goals/local-upload/M2-intake.md @@ -14,11 +14,12 @@ merged_pr: null - **预期产出**：Bearer 认证的能力查询、raw octet-stream 接收、有限 metadata header、真实额度准入、owner/key receipt 与诚实 HTTP 状态。 - **当前范围**：B 卡；不把 A 的 SQLite store 方法误报为已存在 HTTP 接收端。 -- **关键决策**：沿用 `docs/sessions/261008-local-upload/design.md` 的 wire 合同；新正式受理截止为 key 创建时间+24h（到截止时拒绝），未来容差5分钟；`receiving` 不等于 accepted/202，同意图只回原receipt、不消费新 body、不重入队，变化 metadata 409。 +- **关键决策**：沿用 `docs/sessions/261008-local-upload/design.md` 的 wire 合同；新正式受理截止为 key 创建时间+24h（到截止时拒绝），未来容差5分钟；`receiving` 不等于 accepted/202，同意图只回原receipt、不消费新 body、不重入队，变化 metadata 409。root/accepted与一次 `put_nowait` 位于同一个无await SQLite临界步骤，commit成功后才回202；SQL和queue并非共同事务，commit失败形成的stale项由dispatcher durable admission gate丢弃。 - **推进前必须拿到的证据**： - [x] 真实 FastAPI/ASGI 与临时 cache.db 上由 client 发 raw 文件字节及 headers；锁定实际落盘字节、服务端 SHA-256、Unicode metadata、root row 与 dispatcher 实际消费路径。producer fixture 保留在 `tests/integration/test_upload_intake.py`。 - [x] 缺/错 Bearer、默认关闭、额度未知、非 octet-stream、声明/实收超限、receiver/inflight/free-space 满载均在 body/file/ASR 边界前显式拒绝。 - - [x] 同 key 重放/变化 metadata、相同字节新 key、真实 ASGI 断开、队列满、SQLite commit fault、queue put fault、窗口跨期与丢失202回执重查均有 producer→consumer 测试；same-intent 不读 body/不重入队。 + - [x] 同 key 重放/变化 metadata、相同字节新 key、真实 ASGI 断开、队列满、SQLite commit fault（真实队列已发stale payload且dispatcher丢弃）、queue put fault、窗口跨期与丢失202回执重查均有 producer→consumer 测试；same-intent 不读 body/不重入队。 + - [x] receiving cleanup/accept 两个可控真实SQLite+文件顺序均锁定：accept先赢时源字节保留；cleanup先退休时accept失败且root不创建，之后才unlink已退休路径。 - [x] 未设置VTA开关的真实consumer测试在`env -i`裸shell与隔离的临时systemd user unit分别运行成功；未改真实systemd/service或生产环境。 - [ ] Hosted CI及实际生产unit环境传递仍由lead验收；本地隔离验证不等于生产部署/容量证明。 - **完成条件**：本地代码与测试已覆盖 202/失败边界；正式合并、CI 与真实部署环境变量/容量证明由 lead 后续托管，生产仍关闭。 diff --git a/src/video_transcript_api/api/routes/uploads.py b/src/video_transcript_api/api/routes/uploads.py index 5ab36593..b6a43a79 100644 --- a/src/video_transcript_api/api/routes/uploads.py +++ b/src/video_transcript_api/api/routes/uploads.py @@ -330,18 +330,17 @@ async def receive_upload( media_path=str(media_path), byte_size=observed_bytes, sha256=digest.hexdigest(), + enqueue=task_queue.put_nowait, ) except ValueError as exc: if "24-hour" in str(exc): failure_code = "acceptance_window_expired" raise _reject(410, failure_code, "上传幂等key已超过24小时受理截止时间") from exc raise - task_queue.put_nowait({"id": task_id, "url": metadata["source_url"] or "", "platform": "local_upload"}) transferred = True return JSONResponse(_receipt(cache_manager.get_local_upload_by_id(accepted["upload_id"])), status_code=202) except asyncio.QueueFull as exc: failure_code = "queue_full" - cache_manager.cancel_local_upload_acceptance(upload_id, task_id, failure_code) raise _reject(503, failure_code, "转录队列已满，上传未正式受理") from exc finally: if body_started and not transferred: diff --git a/src/video_transcript_api/cache/cache_manager.py b/src/video_transcript_api/cache/cache_manager.py index dfc7ec0b..c6fcc46b 100644 --- a/src/video_transcript_api/cache/cache_manager.py +++ b/src/video_transcript_api/cache/cache_manager.py @@ -6,7 +6,7 @@ import uuid import secrets from pathlib import Path -from typing import Optional, Dict, Any, List, Tuple, Union +from typing import Optional, Dict, Any, List, Tuple, Union, Callable from contextlib import contextmanager import threading import time @@ -753,23 +753,24 @@ def cleanup_expired_local_upload_receiving( resolved_path = media_path.resolve() if not resolved_path.is_relative_to(base_dir): raise ValueError("receiving upload media path escaped configured temp directory") - if media_path.exists(): - if not media_path.is_file(): - raise ValueError("receiving upload media mapping is not a file") - media_path.unlink() + if media_path.exists() and not media_path.is_file(): + raise ValueError("receiving upload media mapping is not a file") deleted = 0 for row in expired: + retired_path = None with self._get_cursor() as cursor: cursor.execute("BEGIN IMMEDIATE") cursor.execute( - "SELECT state, idempotency_key FROM local_uploads WHERE upload_id = ?", + """SELECT state, idempotency_key, media_path FROM local_uploads + WHERE upload_id = ?""", (row["upload_id"],), ) current = cursor.fetchone() if ( current is not None and current["state"] == "receiving" + and current["media_path"] == row["media_path"] and now >= self._upload_key_timestamp(current["idempotency_key"]) + self._UPLOAD_IDEMPOTENCY_WINDOW ): @@ -777,7 +778,12 @@ def cleanup_expired_local_upload_receiving( "DELETE FROM local_uploads WHERE upload_id = ? AND state = 'receiving'", (row["upload_id"],), ) - deleted += cursor.rowcount + row_deleted = cursor.rowcount + deleted += row_deleted + if row_deleted: + retired_path = current["media_path"] + if retired_path: + Path(retired_path).unlink(missing_ok=True) return deleted def accept_local_upload( @@ -794,9 +800,16 @@ def accept_local_upload( media_path: Optional[str] = None, byte_size: Optional[int] = None, sha256: Optional[str] = None, + enqueue: Optional[Callable[[Dict[str, Any]], None]] = None, now: Optional[datetime.datetime] = None, ) -> Dict[str, Any]: - """Atomically create the root and durable accepted admission for one upload.""" + """Create root/admission and optionally hand off once before SQL commit. + + The HTTP producer must supply its queue's synchronous ``put_nowait`` callback; + it executes inside this no-await SQL transaction. Existing store-level callers + may omit it when they are not producing an HTTP task. If commit fails after the + callback succeeds, the dispatcher rejects the stale item against durable state. + """ if not media_id: raise ValueError("media_id must be non-empty") now = now or datetime.datetime.now(datetime.timezone.utc) @@ -836,7 +849,14 @@ def accept_local_upload( if cursor.rowcount != 1: raise RuntimeError("local upload acceptance lost its receiving state") cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) - return dict(cursor.fetchone()) + record = dict(cursor.fetchone()) + if enqueue is not None: + enqueue({ + "id": task_id, + "url": source_url or "", + "platform": "local_upload", + }) + return record def get_admitted_local_upload_by_task( self, task_id: str @@ -865,29 +885,6 @@ def set_local_upload_error(self, upload_id: str, error_code: str) -> None: if cursor.rowcount != 1: raise ValueError("local upload does not exist") - def cancel_local_upload_acceptance( - self, upload_id: str, task_id: str, error_code: str - ) -> None: - """Undo the durable draft if the one-shot in-memory queue handoff fails.""" - with self._get_cursor() as cursor: - cursor.execute("BEGIN IMMEDIATE") - cursor.execute( - """UPDATE local_uploads - SET state = 'receiving', root_task_id = NULL, media_id = NULL, - view_token = NULL, expires_at = NULL, media_path = NULL, - byte_size = NULL, sha256 = NULL, error_code = ? - WHERE upload_id = ? AND root_task_id = ? AND state = 'accepted'""", - (error_code, upload_id, task_id), - ) - if cursor.rowcount != 1: - raise RuntimeError("cannot cancel missing local upload acceptance") - cursor.execute( - "DELETE FROM task_status WHERE task_id = ? AND status = 'queued'", - (task_id,), - ) - if cursor.rowcount != 1: - raise RuntimeError("cannot remove unqueued local upload root") - def get_local_upload_by_id(self, upload_id: str) -> Optional[Dict[str, Any]]: with self._get_cursor() as cursor: cursor.execute( diff --git a/tests/README.md b/tests/README.md index 23dde4b6..22fde657 100644 --- a/tests/README.md +++ b/tests/README.md @@ -52,7 +52,7 @@ loopback 与 Unix 域套接字放行，真实外网测试仍应放在 `tests/man uv run --frozen pytest -q tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py ``` -其中 `test_upload_intake.py` 由真实 Bearer `UserManager`、FastAPI/ASGI raw request、临时 cache.db 与真实 `process_task_queue` consumer 构成；断言落盘 bytes 与 SHA-256、Unicode wire metadata、同 key零body/零重复入队、distinct key独立任务、actual receiver/inflight/free-space/bytes限制、queue/SQLite失败、ASGI断开、丢回执查询原任务、owner stop write-once。ASR调用被替换为捕获真实 dispatcher 参数的本地消费者，生产 ASR/通知未调用；独立 `test_upload_dispatch.py` 证明 stale admission 不进PROCESSING/executor/ASR，并证明 local media未知/无音轨/超时长在ASR前拒绝且不走URL downloader。示例配置的四项 upload limits 为 null；这些测试不能被用作生产容量测量或部署授权。`tests/cache/test_cache_cleanup.py` 的缓存 fixture 仅写 `tmp_path`，并从缓存 producer 返回的实际文件路径删除，不触碰仓库 `test_cache_dir/`。 +其中 `test_upload_intake.py` 由真实 Bearer `UserManager`、FastAPI/ASGI raw request、临时 cache.db 与真实 `process_task_queue` consumer 构成；断言落盘 bytes 与 SHA-256、Unicode wire metadata、同 key零body/零重复入队、distinct key独立任务、actual receiver/inflight/free-space/bytes限制、queue/SQLite失败、ASGI断开、丢回执查询原任务、owner stop write-once。H1还断言独立SQLite连接在实际queue put时看见receiving/root-none；commit fault后真实入队的stale payload由dispatcher丢弃、不进PROCESSING/executor/ASR；cleanup/accept两顺序用真实SQLite与文件bytes/Event控制。ASR调用被替换为捕获真实 dispatcher 参数的本地消费者，生产 ASR/通知未调用；独立 `test_upload_dispatch.py` 证明 stale admission 不进PROCESSING/executor/ASR，并证明 local media未知/无音轨/超时长在ASR前拒绝且不走URL downloader。示例配置的四项 upload limits 为 null；这些测试不能被用作生产容量测量或部署授权。`tests/cache/test_cache_cleanup.py` 的缓存 fixture 仅写 `tmp_path`，并从缓存 producer 返回的实际文件路径删除，不触碰仓库 `test_cache_dir/`。 任务通知数量回归：`tests/unit/test_notification_budget.py` 从真实转录入口运行下载、平台字幕、YouTube API 字幕、部分/全部缓存命中与两种失败路径，断言成功路径携带完整总结和分享回执、失败不带回执；同文件还以 25 MiB 下载 fixture 锁定 generic 下载不发进度通知。`tests/unit/test_terminal_notification_outbox.py` 覆盖单行 outbox、缺失快照可见降级与标记 sent、两条成功消息共用快照及失败路径。`tests/unit/test_notification_e2e_delivery.py` 使用真实 `process_transcription → _TaskNotifier → NotificationRouter → WeComChannel/FeishuChannel → wecom-notifier FIFO → HTTP JSON`（只替换最终外部 POST），并覆盖即时与 outbox 补发入口；检查 SQLite 实际快照字节、双渠道最终 payload 中完整总结先于分享回执、精确 view URL 与网页首段。`tests/unit/test_completion_share.py` 锁定 Markdown→网页同源 HTML→首个 p 可见文本与缺总结状态。相关设计及 N1–N4 更新见 `docs/sessions/261008-completion-share/design.md`。 diff --git a/tests/integration/test_upload_intake.py b/tests/integration/test_upload_intake.py index ef74df7c..0ce42292 100644 --- a/tests/integration/test_upload_intake.py +++ b/tests/integration/test_upload_intake.py @@ -461,10 +461,79 @@ def test_queue_full_has_no_success_receipt_or_dispatch(upload_client, monkeypatc assert runtime._upload_reserved_bytes == {} -def test_sql_commit_failure_rolls_back_root_and_never_queues(upload_client, monkeypatch): +def test_http_queue_handoff_precedes_durable_acceptance_commit(upload_client, monkeypatch): + client, runtime, _body_reads, _worker_calls, worker_results = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + key = _key() + connection_path = runtime.config["storage"]["cache_dir"] + "/cache.db" + observed = [] + original_put = runtime.task_queue.put_nowait + + def observe_handoff(item): + connection = sqlite3.connect(connection_path, timeout=1) + connection.row_factory = sqlite3.Row + upload = connection.execute( + "SELECT state, root_task_id, view_token FROM local_uploads WHERE owner_user_id=? AND idempotency_key=?", + ("alice", key), + ).fetchone() + root = connection.execute( + "SELECT status FROM task_status WHERE task_id=?", (item["id"],) + ).fetchone() + connection.close() + observed.append((dict(upload) if upload else None, dict(root) if root else None, dict(item))) + original_put(item) + + monkeypatch.setattr(runtime.task_queue, "put_nowait", observe_handoff) + response = _post(client, key, PRODUCER_METADATA, RAW_PRODUCER_BYTES, "queue-order") + + assert response.status_code == 202 + assert observed == [( + {"state": "receiving", "root_task_id": None, "view_token": None}, + None, + { + "id": response.json()["task_id"], + "url": PRODUCER_METADATA["source_url"], + "platform": "local_upload", + }, + )] + assert runtime.cache_manager.get_local_upload_by_owner_key("alice", key)["state"] == "accepted" + assert worker_results.get(timeout=2)["task_id"] == response.json()["task_id"] + _wait_for_worker_release(runtime) + + +def test_sql_commit_failure_leaves_stale_queue_item_for_dispatcher_to_drop(upload_client, monkeypatch): client, runtime, body_reads, worker_calls, _ = upload_client monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") cache = runtime.cache_manager + queued_payloads = [] + queue_observations = [] + original_put = runtime.task_queue.put_nowait + database = runtime.config["storage"]["cache_dir"] + "/cache.db" + + def observe_real_queue_payload(item): + connection = sqlite3.connect(database, timeout=1) + connection.row_factory = sqlite3.Row + upload = connection.execute( + "SELECT state, root_task_id FROM local_uploads WHERE owner_user_id=? AND idempotency_key=?", + ("alice", key), + ).fetchone() + root = connection.execute( + "SELECT status FROM task_status WHERE task_id=?", (item["id"],) + ).fetchone() + connection.close() + queue_observations.append((dict(upload) if upload else None, dict(root) if root else None)) + queued_payloads.append(dict(item)) + original_put(item) + + monkeypatch.setattr(runtime.task_queue, "put_nowait", observe_real_queue_payload) + submitted = [] + original_submit = runtime.executor.submit + + def observe_submit(*args, **kwargs): + submitted.append((args, kwargs)) + return original_submit(*args, **kwargs) + + monkeypatch.setattr(runtime.executor, "submit", observe_submit) @contextmanager def fail_root_commit(): @@ -498,7 +567,14 @@ def __getattr__(self, name): response = _post(client, key, PRODUCER_METADATA, RAW_PRODUCER_BYTES, "commit-failure") assert response.status_code == 500 assert body_reads.get("commit-failure", 0) > 0 + assert len(queued_payloads) == 1, "producer must publish one item before the failed commit" + payload = queued_payloads[0] + assert payload["url"] == PRODUCER_METADATA["source_url"] + assert payload["platform"] == "local_upload" + assert queue_observations == [({"state": "receiving", "root_task_id": None}, None)] + client.portal.call(runtime.task_queue.join) assert runtime.task_queue.qsize() == 0 + assert submitted == [], "dispatcher must drop uncommitted queue payload before executor.submit" record = cache.get_local_upload_by_owner_key("alice", key) assert record["state"] == "receiving" assert record["root_task_id"] is None @@ -617,3 +693,109 @@ async def discard_response(message): assert body_reads.get("receipt-retry", 0) == 0 assert len(worker_calls) == 1 assert runtime.task_queue.qsize() == 0 + + +def _expired_receiving_fixture(tmp_path): + epoch_ms = int(datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000) + created = datetime.datetime.fromtimestamp(epoch_ms / 1000, datetime.timezone.utc) + key = f"{epoch_ms}-{uuid.uuid4()}" + manager = CacheManager(str(tmp_path / "cache")) + temp_dir = tmp_path / "temp" + source = temp_dir / "task_probe" / "upload-source.bin" + source.parent.mkdir(parents=True) + source.write_bytes(b"real accepted source bytes") + intent = manager.register_local_upload( + owner_user_id="alice", + idempotency_key=key, + retention="30d", + intent_metadata={"filename": "clip.mp4"}, + now=created + datetime.timedelta(seconds=1), + ) + manager.set_local_upload_receiving_path(intent["upload_id"], str(source)) + return manager, temp_dir, source, intent, key, created + + +def _accept_probe_intent(manager, intent, source, created): + return manager.accept_local_upload( + intent["upload_id"], + media_id="upload_probe_media", + task_id="task_probe_root", + filename="clip.mp4", + request_metadata={"filename": "clip.mp4"}, + media_path=str(source), + byte_size=26, + sha256="a" * 64, + now=created + datetime.timedelta(hours=24, seconds=-1), + ) + + +def test_receiving_cleanup_rechecks_after_accept_wins_and_preserves_owned_file( + tmp_path, monkeypatch +): + manager, temp_dir, source, intent, key, created = _expired_receiving_fixture(tmp_path) + candidate_selected = threading.Event() + allow_recheck = threading.Event() + parse_timestamp = CacheManager._upload_key_timestamp + + def pause_cleanup_after_candidate(candidate_key): + if candidate_key == key and threading.current_thread() is not threading.main_thread(): + candidate_selected.set() + if not allow_recheck.wait(timeout=5): + raise TimeoutError("test did not release cleanup candidate") + return parse_timestamp(candidate_key) + + monkeypatch.setattr(CacheManager, "_upload_key_timestamp", staticmethod(pause_cleanup_after_candidate)) + cleanup_now = created + datetime.timedelta(hours=24, seconds=2) + with ThreadPoolExecutor(max_workers=1) as executor: + cleanup = executor.submit( + manager.cleanup_expired_local_upload_receiving, str(temp_dir), now=cleanup_now + ) + assert candidate_selected.wait(timeout=5), "cleanup did not select expired receiving row" + accepted = _accept_probe_intent(manager, intent, source, created) + allow_recheck.set() + cleanup_count = cleanup.result(timeout=5) + + durable = manager.get_admitted_local_upload_by_task(accepted["root_task_id"]) + assert cleanup_count == 0 + assert durable is not None and durable["state"] == "accepted" + assert source.exists(), "an accepted media mapping must retain its owned source file" + assert source.read_bytes() == b"real accepted source bytes" + + +def test_receiving_cleanup_retires_before_unlink_and_accept_loses(tmp_path, monkeypatch): + manager, temp_dir, source, intent, key, created = _expired_receiving_fixture(tmp_path) + unlink_started = threading.Event() + allow_unlink = threading.Event() + original_unlink = Path.unlink + + def pause_retired_path_unlink(path, *args, **kwargs): + if path == source and threading.current_thread() is not threading.main_thread(): + unlink_started.set() + if not allow_unlink.wait(timeout=5): + raise TimeoutError("test did not release retired file unlink") + return original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", pause_retired_path_unlink) + cleanup_now = created + datetime.timedelta(hours=24, seconds=2) + with ThreadPoolExecutor(max_workers=1) as executor: + cleanup = executor.submit( + manager.cleanup_expired_local_upload_receiving, str(temp_dir), now=cleanup_now + ) + assert unlink_started.wait(timeout=5), "cleanup did not reach retired source unlink" + retired_receipt = manager.get_local_upload_by_id(intent["upload_id"]) + accepted = None + accept_error = None + try: + accepted = _accept_probe_intent(manager, intent, source, created) + except ValueError as exc: + accept_error = str(exc) + allow_unlink.set() + cleanup_count = cleanup.result(timeout=5) + + assert retired_receipt is None, "unlink is authorized only after SQLite retired the receiving row" + assert accepted is None and accept_error is not None + assert cleanup_count == 1 + assert not source.exists() + with manager._get_cursor() as cursor: + cursor.execute("SELECT COUNT(*) FROM task_status") + assert cursor.fetchone()[0] == 0 From edde0e3313899c16a0782cba50d5dc829d9236e3 Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 23:44:51 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E9=98=9F=E5=88=97=E4=BA=A4=E6=8E=A5=E4=B8=8E=E6=8E=A5=E6=94=B6?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E9=80=80=E4=BC=91=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-151543-50422d Task-Id: VideoTranscriptAPI-20261008-16 --- docs/sessions/261008-local-upload/progress/B-progress.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sessions/261008-local-upload/progress/B-progress.md b/docs/sessions/261008-local-upload/progress/B-progress.md index b9afab2f..9b6db83d 100644 --- a/docs/sessions/261008-local-upload/progress/B-progress.md +++ b/docs/sessions/261008-local-upload/progress/B-progress.md @@ -51,7 +51,7 @@ implementing；完成第一份独立可运行单元：bounded metadata decoder ### 当前阶段 -implementing；H0 `840facda8e0cb251a1197cf3647c5accd5c496ff` 后续审查指出两项原合同缺口。先在固定H0上固化四个真实AssertionError红测，之后完成最小顺序修复；H1窄测与A/URL回归已绿，本段准备作为H1小提交推送。 +implementing；固定H0上固化四个真实AssertionError红测，完成最小顺序修复。H1 narrow+A/URL回归已绿，本段准备作为H1小提交推送；全量make test未跑。 ### 本段结论（≤3句） @@ -63,7 +63,7 @@ implementing；H0 `840facda8e0cb251a1197cf3647c5accd5c496ff` 后续审查指出 - 新参数`enqueue`只服务HTTP队列跨SQLite原子发布边界；callback同步调用且无await，既有A store-level direct acceptance仍可不带queue callback。没有新增服务层/状态/锁/重试/fallback。 - 真实测试位于`tests/integration/test_upload_intake.py`：队列put hook用独立SQLite连接观察未提交receiving/root-none和实际payload；commit fault时实际queue收到payload，真实dispatcher drain后没有PROCESSING/executor submit/ASR；cleanup/accept两顺序均使用真实临时SQLite与文件字节，Pause仅位于测试fixture。 -- 红验命令：`uv run --frozen pytest -q tests/integration/test_upload_intake.py::test_http_queue_handoff_precedes_durable_acceptance_commit tests/integration/test_upload_intake.py::test_sql_commit_failure_leaves_stale_queue_item_for_dispatcher_to_drop tests/integration/test_upload_intake.py::test_receiving_cleanup_rechecks_after_accept_wins_and_preserves_owned_file tests/integration/test_upload_intake.py::test_receiving_cleanup_retires_before_unlink_and_accept_loses`；H0输出4 failed/AssertionError，H1同命令4 passed。 +- H0红验命令：`uv run --frozen pytest -q tests/integration/test_upload_intake.py::test_http_queue_handoff_precedes_durable_acceptance_commit tests/integration/test_upload_intake.py::test_sql_commit_failure_leaves_stale_queue_item_for_dispatcher_to_drop tests/integration/test_upload_intake.py::test_receiving_cleanup_rechecks_after_accept_wins_and_preserves_owned_file tests/integration/test_upload_intake.py::test_receiving_cleanup_retires_before_unlink_and_accept_loses`；H0四项均以AssertionError显示期望receiving/null或文件存在、实际accepted/root或文件已删；H1同四项4 passed。 - H1窄测`uv run --frozen pytest -q tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py`：35个测试点通过；另含A store/Resolver、URL API/history、runtime及maintenance的回归命令退出0。 - 当前增量242行左右，仅原scope内7个文件；已先保存H0红验日志与受控两个文件顺序测试，接下来按显式路径小提交并push。 From 37907b9872ed9f59721afb53760965d32c204227 Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 23:58:10 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E8=AE=B0=E5=BD=95B=E5=8D=A1H1=E9=A1=BA?= =?UTF-8?q?=E5=BA=8F=E5=9B=9E=E5=BD=92=E4=B8=8E=E5=85=A8=E9=87=8F=E9=AA=8C?= =?UTF-8?q?=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-151543-50422d Task-Id: VideoTranscriptAPI-20261008-16 --- docs/sessions/261008-local-upload/design.md | 1 + docs/sessions/261008-local-upload/progress/B-progress.md | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/sessions/261008-local-upload/design.md b/docs/sessions/261008-local-upload/design.md index 3c754d1e..60f3bc77 100644 --- a/docs/sessions/261008-local-upload/design.md +++ b/docs/sessions/261008-local-upload/design.md @@ -50,6 +50,7 @@ Resolver 每次新请求都查独立上传记录并核对开关、撤销和截 | 不变量 | 实现入口 | 锁定测试 | |---|---|---| | owner/key 唯一、窗口/时钟偏差、metadata conflict、旧 receipt 查询与过窗拒绝 | `cache_manager.py::register_local_upload` | `tests/unit/test_local_upload_policy.py`（临时真实 SQLite，含并发登记） | +| root/accepted+一次put_nowait提交顺序、commit失败stale item拒绝、过期receiving retirement与源文件所有权的两个真实顺序 | `cache_manager.py::accept_local_upload/cleanup_expired_local_upload_receiving`、`api/services/transcription.py::process_task_queue` | `tests/integration/test_upload_intake.py::test_http_queue_handoff_precedes_durable_acceptance_commit`、`test_sql_commit_failure_leaves_stale_queue_item_for_dispatcher_to_drop`、`test_receiving_cleanup_rechecks_after_accept_wins_and_preserves_owned_file`、`test_receiving_cleanup_retires_before_unlink_and_accept_loses` | | root `view_token=''`、独立 `upload_` token；blank 与上传 token 不走旧别名 | `cache_manager.py::accept_local_upload/get_task_by_view_token`、`audit.py::check_view_token_ownership` | `test_upload_task_uses_blank_legacy_column_and_separate_upload_token`、Resolver 与 ownership 用例 | | success/failed 首终态固定 30 天；never 不到期；撤销 write-once | `cache_manager.py::update_task_status/revoke_local_upload` | `test_root_terminal_sets_fixed_30_day_expiry_for_success_and_failure`、`test_never_expiry_and_revocation_are_write_once` | | Resolver 对 active 30d/never 读真实缓存；missing/off/revoked/expired 不读；`UPLOAD_DISABLED` 可 grep | `view_token_resolver.py::_local_upload_task_info` | `test_resolver_reads_active_upload_and_denies_disabled_revoked_or_expired`、`test_upload_progress_token_resolves_before_30_day_terminal_clock` | diff --git a/docs/sessions/261008-local-upload/progress/B-progress.md b/docs/sessions/261008-local-upload/progress/B-progress.md index 9b6db83d..1fbb7bbe 100644 --- a/docs/sessions/261008-local-upload/progress/B-progress.md +++ b/docs/sessions/261008-local-upload/progress/B-progress.md @@ -51,7 +51,7 @@ implementing；完成第一份独立可运行单元：bounded metadata decoder ### 当前阶段 -implementing；固定H0上固化四个真实AssertionError红测，完成最小顺序修复。H1 narrow+A/URL回归已绿，本段准备作为H1小提交推送；全量make test未跑。 +implementing；H1最小顺序修复已提交并push至原PR分支，随后完成窄测、A/URL回归及全量make test。PR保持draft，CI/独立审查由lead托管。 ### 本段结论（≤3句） @@ -65,8 +65,10 @@ implementing；固定H0上固化四个真实AssertionError红测，完成最小 - 真实测试位于`tests/integration/test_upload_intake.py`：队列put hook用独立SQLite连接观察未提交receiving/root-none和实际payload；commit fault时实际queue收到payload，真实dispatcher drain后没有PROCESSING/executor submit/ASR；cleanup/accept两顺序均使用真实临时SQLite与文件字节，Pause仅位于测试fixture。 - H0红验命令：`uv run --frozen pytest -q tests/integration/test_upload_intake.py::test_http_queue_handoff_precedes_durable_acceptance_commit tests/integration/test_upload_intake.py::test_sql_commit_failure_leaves_stale_queue_item_for_dispatcher_to_drop tests/integration/test_upload_intake.py::test_receiving_cleanup_rechecks_after_accept_wins_and_preserves_owned_file tests/integration/test_upload_intake.py::test_receiving_cleanup_retires_before_unlink_and_accept_loses`；H0四项均以AssertionError显示期望receiving/null或文件存在、实际accepted/root或文件已删；H1同四项4 passed。 - H1窄测`uv run --frozen pytest -q tests/unit/test_upload_routes.py tests/unit/test_upload_dispatch.py tests/integration/test_upload_intake.py`：35个测试点通过；另含A store/Resolver、URL API/history、runtime及maintenance的回归命令退出0。 -- 当前增量242行左右，仅原scope内7个文件；已先保存H0红验日志与受控两个文件顺序测试，接下来按显式路径小提交并push。 +- H1代码提交`5b1f3526fa958deffd704ba43cbc13be47db95df`已push；四个H0红测在H1已绿，H1末尾窄测+A/URL回归退出0。 +- 按红验安全要求，在代码已提交后分别临时把enqueue移到SQL commit之后、把unlink移到SQLite retirement之前；对应producer/consumer断言各自以AssertionError变红，恢复单个顺序后各自绿。变异输出记录在`/tmp/vta-upload-H1-r3-mutation.log`及`/tmp/vta-upload-H1-cleanup-mutation.log`。 +- H1 `make test` 与 `uv run --frozen pytest -q -rs tests` 均exit0；3个skip均因`tests/unit/test_capswriter_samples_total_contract.py`缺少真实preflight fixture。完整结果记录在`/tmp/vta-upload-H1-dlg-20261008-151543-50422d-*`。 ### 下一步唯一动作 -提交并push H1增量；提交后针对R3 handoff与cleanup退休两个predicate做最小负向断言验证并恢复，再跑有界完整make test、核对clean/远端PR head。不得ready/merge。 +push本进度证据并写终态报告；PR保持draft交lead做H0..H1增量review和CI，不ready/merge。 From 6a497263e67789b4ce8fef39d407e6e8f77faff3 Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Fri, 9 Oct 2026 01:01:19 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E5=A4=84=E7=90=86=E9=80=89=E9=A1=B9=E9=80=8F=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor Agent-Executor: cursor Agent-Model: gpt-5.6-luna-xhigh Agent-Effort: unknown Dispatch-Id: dlg-20261008-165700-787f04 Task-Id: VideoTranscriptAPI-20261009-02 --- .../progress/B-progress.md | 28 ++++++ .../cache/cache_manager.py | 1 + tests/integration/test_upload_intake.py | 86 +++++++++++++++++++ 3 files changed, 115 insertions(+) diff --git a/docs/sessions/261008-local-upload/progress/B-progress.md b/docs/sessions/261008-local-upload/progress/B-progress.md index 1fbb7bbe..297388e0 100644 --- a/docs/sessions/261008-local-upload/progress/B-progress.md +++ b/docs/sessions/261008-local-upload/progress/B-progress.md @@ -72,3 +72,31 @@ implementing；H1最小顺序修复已提交并push至原PR分支，随后完成 ### 下一步唯一动作 push本进度证据并写终态报告；PR保持draft交lead做H0..H1增量review和CI，不ready/merge。 + +## Milestone H2：公开处理选项透传 + +### 当前阶段 + +implementing；已完成真实 HTTP→持久 SQLite→runtime queue→dispatcher→捕获 consumer 的参数修正与窄测。 + +### 本段结论（≤3句） + +- producer 现在在同一 SQL 临界生成的 queue payload 中发布已写入 task/request metadata 的规范化 `processing_options`，consumer 收到的值不再回退为全开启。 +- 新增真实全关、混合、缺省三组边界测试，并断言 HTTP body/header、SQLite、queue payload 与 `process_transcription` 参数一致；旧 H1 交接和标准处理选项回归保持通过。 + +### 公开字段与实际消费映射（机械对照 decoder/normalize keys） + +| HTTP metadata | 持久记录 | queue payload | 实际 consumer 字段/用途 | +|---|---|---|---| +| `filename` | `local_uploads.filename`、`request_metadata` | 无 | `metadata_override.title` 缺省回退与展示 | +| `byte_size` | `local_uploads.byte_size` | 无 | 接收大小/哈希与临时预算验收 | +| `title/source_url/retention` | task title/url、upload source/retention、request metadata | `url`=source_url | consumer 的 `url`/title；retention 供回执分享期限 | +| `processing_options.{calibrate,summarize,infer_speaker_names,chapters,contradiction_scan}` | task `processing_options`、request metadata | `processing_options` | dispatcher normalize 后传 `process_transcription`，控制对应处理层 | + +### 复审接受不修 + +- P2 setup 故障已真实返回 HTTP 500 并释放资源；P3 SQLite commit 故障仍保留 `receive_incomplete` 诊断名；两项均非本卡阻断，不改接收接口或添加防御 catch。 + +### 下一步唯一动作 + +提交并推送 H2 小修；保持 draft，交 lead 做 H1→H2 增量复审与正式 CI，不 ready/merge。 diff --git a/src/video_transcript_api/cache/cache_manager.py b/src/video_transcript_api/cache/cache_manager.py index c6fcc46b..3e47ae71 100644 --- a/src/video_transcript_api/cache/cache_manager.py +++ b/src/video_transcript_api/cache/cache_manager.py @@ -855,6 +855,7 @@ def accept_local_upload( "id": task_id, "url": source_url or "", "platform": "local_upload", + "processing_options": json.loads(options_json), }) return record diff --git a/tests/integration/test_upload_intake.py b/tests/integration/test_upload_intake.py index 0ce42292..1a041dfd 100644 --- a/tests/integration/test_upload_intake.py +++ b/tests/integration/test_upload_intake.py @@ -25,6 +25,7 @@ unbind_runtime, validate_config, ) +from video_transcript_api.api.processing_options import normalize_processing_options from video_transcript_api.api.routes import uploads from video_transcript_api.api.services import transcription from video_transcript_api.cache.cache_manager import CacheManager @@ -130,6 +131,7 @@ def fake_process_transcription(*args, **kwargs): result = { "task_id": args[0], "url": args[1], "local_media_id": kwargs["local_media_id"], + "processing_options": kwargs["processing_options"], "bytes": Path(path).read_bytes(), "path": path, } with worker_lock: @@ -254,6 +256,9 @@ def test_real_http_stream_sqlite_dispatch_idempotency_and_owner_stop(upload_clie dispatched = worker_results.get(timeout=2) assert dispatched["task_id"] == receipt["task_id"] assert dispatched["local_media_id"] == stored["media_id"] + assert dispatched["processing_options"] == normalize_processing_options( + PRODUCER_METADATA["processing_options"] + ) assert dispatched["bytes"] == RAW_PRODUCER_BYTES assert "upload-source.bin" in dispatched["path"] _wait_for_worker_release(runtime) @@ -306,6 +311,81 @@ def test_real_http_stream_sqlite_dispatch_idempotency_and_owner_stop(upload_clie ).status_code == 404 +@pytest.mark.parametrize( + ("processing_options", "expected_options"), + [ + pytest.param( + { + "calibrate": False, + "summarize": False, + "infer_speaker_names": False, + "chapters": False, + }, + { + "calibrate": False, + "summarize": False, + "infer_speaker_names": False, + "chapters": False, + }, + id="all-disabled", + ), + pytest.param( + { + "calibrate": False, + "summarize": True, + "infer_speaker_names": True, + "chapters": False, + }, + { + "calibrate": False, + "summarize": True, + "infer_speaker_names": True, + "chapters": False, + }, + id="mixed", + ), + pytest.param({}, normalize_processing_options(None), id="default"), + ], +) +def test_processing_options_cross_real_upload_boundaries( + upload_client, monkeypatch, processing_options, expected_options +): + client, runtime, body_reads, _worker_calls, worker_results = upload_client + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + queue_payloads = [] + original_put = runtime.task_queue.put_nowait + + def capture_queue_payload(item): + queue_payloads.append(dict(item)) + original_put(item) + + monkeypatch.setattr(runtime.task_queue, "put_nowait", capture_queue_payload) + key = _key() + payload = {**PRODUCER_METADATA, "processing_options": processing_options} + + response = _post(client, key, payload, RAW_PRODUCER_BYTES, "options-boundary") + + assert response.status_code == 202 + assert body_reads["options-boundary"] > 0 + receipt = response.json() + stored = runtime.cache_manager.get_local_upload_by_owner_key("alice", key) + root = runtime.cache_manager.get_task_by_id(receipt["task_id"]) + dispatched = worker_results.get(timeout=2) + _wait_for_worker_release(runtime) + + assert json.loads(stored["request_metadata"])["processing_options"] == expected_options + assert root["processing_options"] == expected_options + assert queue_payloads == [ + { + "id": receipt["task_id"], + "url": payload["source_url"], + "platform": "local_upload", + "processing_options": expected_options, + } + ] + assert dispatched["processing_options"] == expected_options + + @pytest.mark.parametrize( ("field", "value"), [ @@ -494,6 +574,9 @@ def observe_handoff(item): "id": response.json()["task_id"], "url": PRODUCER_METADATA["source_url"], "platform": "local_upload", + "processing_options": normalize_processing_options( + PRODUCER_METADATA["processing_options"] + ), }, )] assert runtime.cache_manager.get_local_upload_by_owner_key("alice", key)["state"] == "accepted" @@ -571,6 +654,9 @@ def __getattr__(self, name): payload = queued_payloads[0] assert payload["url"] == PRODUCER_METADATA["source_url"] assert payload["platform"] == "local_upload" + assert payload["processing_options"] == normalize_processing_options( + PRODUCER_METADATA["processing_options"] + ) assert queue_observations == [({"state": "receiving", "root_task_id": None}, None)] client.portal.call(runtime.task_queue.join) assert runtime.task_queue.qsize() == 0