From 132daf968237fd46741e8d3c2e55f1b301656580 Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 18:31:08 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E5=BB=BA=E7=AB=8B=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=88=86=E4=BA=AB=E5=AE=89=E5=85=A8=E5=9F=BA?= =?UTF-8?q?=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-094534-0fd910 Task-Id: VideoTranscriptAPI-20261008-11 --- .../progress/A-progress.md | 22 + src/video_transcript_api/api/routes/audit.py | 6 + .../api/services/view_token_resolver.py | 48 +- .../cache/cache_manager.py | 322 ++++++++++- tests/README.md | 2 + tests/cache/test_cache_cleanup.py | 93 +--- tests/unit/test_local_upload_policy.py | 519 ++++++++++++++++++ 7 files changed, 935 insertions(+), 77 deletions(-) create mode 100644 docs/sessions/261008-local-upload/progress/A-progress.md create mode 100644 tests/unit/test_local_upload_policy.py diff --git a/docs/sessions/261008-local-upload/progress/A-progress.md b/docs/sessions/261008-local-upload/progress/A-progress.md new file mode 100644 index 00000000..49600d2a --- /dev/null +++ b/docs/sessions/261008-local-upload/progress/A-progress.md @@ -0,0 +1,22 @@ +# A 卡进度 + +## 当前阶段 + +implementing；安全基线代码与真实 SQLite/Resolver/cleanup 用例已落地，本地完整 gate 和窄测已通过，待红验、提交/推送与 PR 独立审查。HTTP接收、worker、网页、生产额度和恢复演练尚未实现，不能启用生产上传。 + +## 本段结论（≤3句） + +- A 使用现有 `CacheManager` 作为 store，新增 `local_uploads`；不增加 Service 转发层或账户/队列平台。 +- Resolver 通过独立 `upload_` token 读取，旧 NOT NULL token 列保持空字符串；禁用、撤销、到期拒读且不改变有效成果清理保护。 +- 最终窄测连续五次通过，指定 URL/history/cache/task/媒体锁清理回归与 `make test` 全量 gate 均通过；历史 `tests/cache/test_cache_cleanup.py` 会删除仓库 fixture，已改成 `tmp_path` 且消费 producer 返回的路径。 + +## 关键决策与否决方案 + +- 24 小时 key 窗口、5 分钟未来时钟偏差；重复 owner/key 通过 `BEGIN IMMEDIATE` 复用既有记录，metadata 指纹不同直接拒绝。 +- 30 天自 root 首个 success/failed 的 `completed_at` 固定起算；never 仅以显式选择无到期，撤销通过 SQLite trigger write-once。 +- 不将 `VTA_UPLOADS_ENABLED` 放进数据卷 JSON；生产额度、systemd恢复域未验证前保持关闭。 +- 上传 reprocess 暂不接线，拒绝走 task_status owner legacy fallback；原媒体仍走 TempFileManager 临时清理。 + +## 下一步唯一动作 + +提交已验证代码与本进度记录；提交后逐项移除 blank/source/cleanup predicate 做 AssertionError 红验，恢复实现后推送分支。 diff --git a/src/video_transcript_api/api/routes/audit.py b/src/video_transcript_api/api/routes/audit.py index a6470340..4b322cfa 100644 --- a/src/video_transcript_api/api/routes/audit.py +++ b/src/video_transcript_api/api/routes/audit.py @@ -657,6 +657,12 @@ def check_view_token_ownership( （或 legacy 审计行兜底通过），可以放行；否则 fail-closed 拒绝。 """ + # The upload baseline deliberately has no reprocessing implementation yet. + # Never reinterpret its independent public capability as a task_status + # view_token (including through the legacy audit-log ownership fallback). + if not view_token or view_token.startswith("upload_"): + return False + def _legacy_owns_task(candidate_task_id: str) -> bool: """纯 legacy 兜底：某个 task_id 在所有已知来源里都查不到 submitted_by（既不在 task_audit_snapshots，也不在 cache.db 的 diff --git a/src/video_transcript_api/api/services/view_token_resolver.py b/src/video_transcript_api/api/services/view_token_resolver.py index 3b856ff1..a767da1d 100644 --- a/src/video_transcript_api/api/services/view_token_resolver.py +++ b/src/video_transcript_api/api/services/view_token_resolver.py @@ -1,5 +1,6 @@ """View-token business logic coordinated through an existing CacheManager.""" +import os from typing import TYPE_CHECKING, Any, Dict, Optional from ...transcriber.segments import normalize_segments @@ -11,6 +12,7 @@ logger = setup_logger("view_token_resolver") +UPLOAD_DISABLED = "UPLOAD_DISABLED" class ViewTokenResolver: @@ -124,7 +126,7 @@ def _assemble_content_view( ) llm_config = self._cache_manager.get_task_llm_config(task_info["task_id"]) - if not llm_config: + if not llm_config and task_info.get("platform") != "local_upload": llm_config = self._get_llm_config_by_view_token(task_info["view_token"]) payload: Dict[str, Any] = { @@ -156,10 +158,42 @@ def _assemble_content_view( ) return payload + def _local_upload_task_info(self, view_token: str) -> Optional[Dict[str, Any]]: + """Resolve only an independently recorded upload capability, never a legacy blank token.""" + upload = self._cache_manager.get_local_upload_by_view_token(view_token) + if upload is None: + return None + if os.environ.get("VTA_UPLOADS_ENABLED", "").strip().lower() != "true": + logger.warning("{}: local upload content read denied while disabled", UPLOAD_DISABLED) + return None + if not self._cache_manager.local_upload_share_is_active(upload): + return None + return { + "task_id": upload["root_task_id"], + "view_token": view_token, + "url": upload.get("root_url") or "", + "download_url": upload.get("root_download_url"), + "platform": upload.get("root_platform"), + "media_id": upload.get("root_media_id"), + "use_speaker_recognition": bool(upload.get("root_speaker")), + "status": upload.get("root_status"), + "title": upload.get("root_title") or "本地上传", + "created_at": upload.get("root_created_at"), + "completed_at": upload.get("root_completed_at"), + "error_message": upload.get("root_error"), + "llm_config": upload.get("root_llm_config"), + } + def get_view_data_by_token(self, view_token: str) -> Optional[Dict[str, Any]]: """Return the view-page data associated with a view token.""" + if not view_token: + return None try: - task_info = self._cache_manager.get_task_by_view_token(view_token) + task_info = ( + self._local_upload_task_info(view_token) + if view_token.startswith("upload_") + else self._cache_manager.get_task_by_view_token(view_token) + ) if not task_info: return None @@ -228,8 +262,14 @@ def get_view_data_by_token(self, view_token: str) -> Optional[Dict[str, Any]]: def get_cache_by_view_token(self, view_token: str) -> Optional[Dict[str, Any]]: """Return complete cache data plus task information for a view token.""" + if not view_token: + return None try: - task_info = self._cache_manager.get_task_by_view_token(view_token) + task_info = ( + self._local_upload_task_info(view_token) + if view_token.startswith("upload_") + else self._cache_manager.get_task_by_view_token(view_token) + ) if not task_info: logger.warning(f"未找到 view_token 对应的任务: {view_token}") return None @@ -263,6 +303,8 @@ def _get_llm_config_by_view_token( self, view_token: str ) -> Optional[Dict[str, Any]]: """Find the newest non-empty LLM config for tasks sharing a token.""" + if not view_token or view_token.startswith("upload_"): + return None try: with self._cache_manager._get_cursor() as cursor: cursor.execute( diff --git a/src/video_transcript_api/cache/cache_manager.py b/src/video_transcript_api/cache/cache_manager.py index 168ab86a..6abbed7c 100644 --- a/src/video_transcript_api/cache/cache_manager.py +++ b/src/video_transcript_api/cache/cache_manager.py @@ -1,5 +1,6 @@ import os import json +import hashlib import sqlite3 import datetime import uuid @@ -341,6 +342,43 @@ def _init_database(self): ) ''') + cursor.execute(''' + CREATE TABLE IF NOT EXISTS local_uploads ( + upload_id TEXT PRIMARY KEY, + owner_user_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + idempotency_created_at TEXT NOT NULL, + metadata_fingerprint TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('receiving', 'accepted')), + retention TEXT NOT NULL CHECK(retention IN ('30d', 'never')), + root_task_id TEXT UNIQUE, + media_id TEXT UNIQUE, + view_token TEXT UNIQUE, + created_at TEXT NOT NULL, + expires_at TEXT, + revoked_at TEXT, + error_code TEXT, + UNIQUE(owner_user_id, idempotency_key), + FOREIGN KEY(root_task_id) REFERENCES task_status(task_id) + ) + ''') + cursor.execute( + 'CREATE INDEX IF NOT EXISTS idx_local_upload_view_token ' + 'ON local_uploads(view_token)' + ) + cursor.execute( + 'CREATE INDEX IF NOT EXISTS idx_local_upload_expiry ' + 'ON local_uploads(retention, expires_at, revoked_at)' + ) + cursor.execute(''' + CREATE TRIGGER IF NOT EXISTS local_upload_revocation_write_once + BEFORE UPDATE OF revoked_at ON local_uploads + WHEN OLD.revoked_at IS NOT NULL AND NEW.revoked_at IS NOT OLD.revoked_at + BEGIN + SELECT RAISE(ABORT, 'local upload revocation is write-once'); + END + ''') + # 创建索引以提高查询性能 cursor.execute('CREATE INDEX IF NOT EXISTS idx_platform_media_id ON video_cache(platform, media_id)') cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON video_cache(url)') @@ -558,6 +596,246 @@ def _rebuild_task_status_table(self, cursor): logger.info(f"数据库迁移完成，恢复了 {len(existing_data)} 条记录") + _UPLOAD_IDEMPOTENCY_WINDOW = datetime.timedelta(hours=24) + _UPLOAD_KEY_FUTURE_SKEW = datetime.timedelta(minutes=5) + + @staticmethod + def _upload_key_timestamp(idempotency_key: str) -> datetime.datetime: + """Parse the producer key `-` without accepting aliases.""" + if not isinstance(idempotency_key, str) or "-" not in idempotency_key: + raise ValueError("idempotency key must be -") + epoch_text, uuid_text = idempotency_key.split("-", 1) + if len(epoch_text) != 13 or not epoch_text.isascii() or not epoch_text.isdigit(): + raise ValueError("idempotency key must be -") + try: + parsed_uuid = uuid.UUID(uuid_text) + except ValueError as exc: + raise ValueError("idempotency key must be -") from exc + if str(parsed_uuid) != uuid_text.lower(): + raise ValueError("idempotency key must use canonical UUID text") + return datetime.datetime.fromtimestamp( + int(epoch_text) / 1000, tz=datetime.timezone.utc + ) + + @classmethod + def _validate_upload_key_window( + cls, idempotency_key: str, now: datetime.datetime + ) -> datetime.datetime: + created_at = cls._upload_key_timestamp(idempotency_key) + if now.tzinfo is None: + raise ValueError("now must be timezone-aware") + now = now.astimezone(datetime.timezone.utc) + if created_at > now + cls._UPLOAD_KEY_FUTURE_SKEW: + raise ValueError("idempotency key exceeds allowed clock skew") + if now - created_at > cls._UPLOAD_IDEMPOTENCY_WINDOW: + raise ValueError("idempotency key is outside the 24-hour acceptance window") + return created_at + + @staticmethod + def _utc_sql_timestamp(value: datetime.datetime) -> str: + if value.tzinfo is None: + raise ValueError("timestamp must be timezone-aware") + return value.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + def register_local_upload( + self, + *, + owner_user_id: str, + idempotency_key: str, + retention: str, + intent_metadata: Optional[Dict[str, Any]] = None, + now: Optional[datetime.datetime] = None, + ) -> Dict[str, Any]: + """Persist one owner's upload intent in cache.db; this does not receive media.""" + if not isinstance(owner_user_id, str) or not owner_user_id.strip(): + raise ValueError("owner_user_id must be non-empty") + if retention not in {"30d", "never"}: + raise ValueError("retention must be '30d' or 'never'") + canonical_metadata = json.dumps( + {"retention": retention, "metadata": intent_metadata or {}}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + metadata_fingerprint = hashlib.sha256(canonical_metadata.encode("utf-8")).hexdigest() + now = now or datetime.datetime.now(datetime.timezone.utc) + with self._get_cursor() as cursor: + cursor.execute("BEGIN IMMEDIATE") + cursor.execute( + "SELECT * FROM local_uploads WHERE owner_user_id = ? AND idempotency_key = ?", + (owner_user_id, idempotency_key), + ) + existing = cursor.fetchone() + if existing is not None: + record = dict(existing) + if record["metadata_fingerprint"] != metadata_fingerprint: + raise ValueError("idempotency key metadata conflicts with recorded intent") + return record + created_at = self._validate_upload_key_window(idempotency_key, now) + upload_id = f"upload_{uuid.uuid4().hex}" + cursor.execute( + """INSERT INTO local_uploads + (upload_id, owner_user_id, idempotency_key, idempotency_created_at, + metadata_fingerprint, state, retention, created_at) + VALUES (?, ?, ?, ?, ?, 'receiving', ?, ?)""", + ( + upload_id, + owner_user_id, + idempotency_key, + self._utc_sql_timestamp(created_at), + metadata_fingerprint, + retention, + self._utc_sql_timestamp(now), + ), + ) + cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) + return dict(cursor.fetchone()) + + def accept_local_upload( + self, + upload_id: str, + *, + media_id: str, + title: Optional[str] = None, + processing_options: Optional[Dict[str, Any]] = None, + task_id: Optional[str] = None, + now: Optional[datetime.datetime] = None, + ) -> Dict[str, Any]: + """Create a root task with a blank legacy token and persist its separate share token.""" + if not media_id: + raise ValueError("media_id must be non-empty") + now = now or datetime.datetime.now(datetime.timezone.utc) + task_id = task_id or self.generate_task_id() + view_token = f"upload_{secrets.token_urlsafe(32)}" + options_json = json.dumps(processing_options or {}, ensure_ascii=False, sort_keys=True) + with self._get_cursor() as cursor: + cursor.execute("BEGIN IMMEDIATE") + cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) + upload = cursor.fetchone() + if upload is None: + raise ValueError("local upload intent does not exist") + if upload["state"] == "accepted": + return dict(upload) + if upload["revoked_at"] is not None: + raise ValueError("revoked local upload cannot be accepted") + self._validate_upload_key_window(upload["idempotency_key"], now) + cursor.execute( + """INSERT INTO task_status + (task_id, view_token, url, platform, media_id, status, title, + processing_options, submitted_by) + VALUES (?, '', '', 'local_upload', ?, 'queued', ?, ?, ?)""", + (task_id, media_id, title, options_json, upload["owner_user_id"]), + ) + cursor.execute( + """UPDATE local_uploads + SET state = 'accepted', root_task_id = ?, media_id = ?, view_token = ? + WHERE upload_id = ? AND state = 'receiving' AND revoked_at IS NULL""", + (task_id, media_id, view_token, upload_id), + ) + if cursor.rowcount != 1: + raise RuntimeError("local upload acceptance lost its receiving state") + cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) + return dict(cursor.fetchone()) + + def get_local_upload_by_id(self, upload_id: str) -> Optional[Dict[str, Any]]: + with self._get_cursor() as cursor: + cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) + row = cursor.fetchone() + return dict(row) if row is not None else None + + def get_local_upload_by_owner_key( + self, owner_user_id: str, idempotency_key: str + ) -> Optional[Dict[str, Any]]: + with self._get_cursor() as cursor: + cursor.execute( + "SELECT * FROM local_uploads WHERE owner_user_id = ? AND idempotency_key = ?", + (owner_user_id, idempotency_key), + ) + row = cursor.fetchone() + return dict(row) if row is not None else None + + def get_local_upload_by_view_token( + self, view_token: str + ) -> Optional[Dict[str, Any]]: + if not view_token or not view_token.startswith("upload_"): + return None + with self._get_cursor() as cursor: + cursor.execute( + """SELECT u.*, t.status AS root_status, t.url AS root_url, + t.download_url AS root_download_url, t.platform AS root_platform, + t.media_id AS root_media_id, t.use_speaker_recognition AS root_speaker, + t.title AS root_title, t.created_at AS root_created_at, + t.completed_at AS root_completed_at, t.error_message AS root_error, + t.llm_config AS root_llm_config, t.calibration_status, + t.summary_status, t.chapters_status + FROM local_uploads u + JOIN task_status t ON t.task_id = u.root_task_id + WHERE u.view_token = ?""", + (view_token,), + ) + row = cursor.fetchone() + return dict(row) if row is not None else None + + @staticmethod + def local_upload_share_is_active( + upload: Dict[str, Any], now: Optional[datetime.datetime] = None + ) -> bool: + if upload.get("state") != "accepted" or upload.get("revoked_at") is not None: + return False + if upload.get("retention") == "never": + return upload.get("expires_at") is None + expires_at = upload.get("expires_at") + if not expires_at: + return upload.get("root_status") not in {"success", "failed", None} + now_text = CacheManager._utc_sql_timestamp( + now or datetime.datetime.now(datetime.timezone.utc) + ) + return expires_at > now_text + + def revoke_local_upload( + self, upload_id: str, *, now: Optional[datetime.datetime] = None + ) -> Dict[str, Any]: + revoked_at = self._utc_sql_timestamp( + now or datetime.datetime.now(datetime.timezone.utc) + ) + with self._get_cursor() as cursor: + cursor.execute( + "UPDATE local_uploads SET revoked_at = COALESCE(revoked_at, ?) WHERE upload_id = ?", + (revoked_at, upload_id), + ) + if cursor.rowcount == 0: + raise ValueError("local upload does not exist") + cursor.execute("SELECT * FROM local_uploads WHERE upload_id = ?", (upload_id,)) + return dict(cursor.fetchone()) + + @staticmethod + def _has_active_local_upload_for_task(cursor, task_id: str, now_text: str) -> bool: + cursor.execute( + """SELECT 1 FROM local_uploads + WHERE root_task_id = ? AND state = 'accepted' AND revoked_at IS NULL + AND ((retention = 'never' AND expires_at IS NULL) + OR (retention = '30d' AND expires_at > ?)) + LIMIT 1""", + (task_id, now_text), + ) + return cursor.fetchone() is not None + + @staticmethod + def _has_active_local_upload_for_media(cursor, platform: str, media_id: str, now_text: str) -> bool: + if platform != "local_upload": + return False + cursor.execute( + """SELECT 1 FROM local_uploads u + JOIN task_status t ON t.task_id = u.root_task_id + WHERE u.media_id = ? AND t.platform = 'local_upload' + AND u.state = 'accepted' AND u.revoked_at IS NULL + AND ((u.retention = 'never' AND u.expires_at IS NULL) + OR (u.retention = '30d' AND u.expires_at > ?)) + LIMIT 1""", + (media_id, now_text), + ) + return cursor.fetchone() is not None + def _get_file_path(self, platform: str, media_id: str, date: datetime.datetime = None) -> Path: """ 获取文件存储路径 @@ -1820,6 +2098,23 @@ def cleanup_old_cache(self, days: int = 30, now: Optional[datetime.datetime] = N ) continue + # Upload share qualification owns a longer lifetime than + # the generic cache age. This remains true when the + # feature flag is off; disabling access is not deletion. + with self._get_cursor() as cursor: + upload_share_is_active = self._has_active_local_upload_for_media( + cursor, + platform, + media_id, + self._utc_sql_timestamp(effective_now), + ) + if upload_share_is_active: + logger.info( + "Cache cleanup preserved an active local upload: {}", + media_id, + ) + continue + # 复核 2：重新查询该行 updated_at 是否仍早于 # cutoff，防止等锁期间被 save_cache() 并发刷新。 with self._get_cursor() as cursor: @@ -1987,15 +2282,19 @@ def cleanup_task_status( ).strftime('%Y-%m-%d %H:%M:%S') deleted_count = 0 + # Keyset pagination advances past protected rows too; otherwise a + # full batch of valid never-expiring uploads would be selected forever. + last_task_id = "" while True: with self._get_cursor() as cursor: cursor.execute(''' SELECT task_id FROM task_status WHERE status IN (?, ?) AND COALESCE(completed_at, created_at) < ? - ORDER BY COALESCE(completed_at, created_at) + AND task_id > ? + ORDER BY task_id LIMIT 500 - ''', (TaskStatus.SUCCESS, TaskStatus.FAILED, cutoff)) + ''', (TaskStatus.SUCCESS, TaskStatus.FAILED, cutoff, last_task_id)) task_ids = [row[0] for row in cursor.fetchall()] if not task_ids: break @@ -2031,6 +2330,11 @@ def cleanup_task_status( if row is None: connection.rollback() continue + if self._has_active_local_upload_for_task( + cursor, task_id, self._utc_sql_timestamp(effective_now) + ): + connection.rollback() + continue task = dict(row) for field in ("processing_options", "terminal_snapshot"): if task.get(field): @@ -2097,6 +2401,7 @@ def cleanup_task_status( raise finally: cursor.close() + last_task_id = task_ids[-1] if len(task_ids) < 500: break @@ -2462,6 +2767,17 @@ def update_task_status(self, task_id: str, status: str, platform: str = None, cursor.execute(query, params) updated = cursor.rowcount == 1 + if updated and status in ['success', 'failed']: + cursor.execute( + """UPDATE local_uploads + SET expires_at = ( + SELECT datetime(completed_at, '+30 days') + FROM task_status WHERE task_id = ? + ) + WHERE root_task_id = ? AND retention = '30d' + AND expires_at IS NULL""", + (task_id, task_id), + ) if not updated: logger.info( f"任务状态更新被终态黏性拦截(已是终态,跳过): {task_id} -> {status}" @@ -3091,6 +3407,8 @@ def get_task_by_view_token(self, view_token: str) -> Optional[Dict[str, Any]]: Returns: Dict: 任务信息 """ + if not view_token or view_token.startswith("upload_"): + return None try: with self._get_cursor() as cursor: # success 最高优先级；failed 垫底；其余状态居中按最新排序。 diff --git a/tests/README.md b/tests/README.md index 824b42d2..e65c40b0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -46,6 +46,8 @@ uv run pytest tests/deployment 默认门禁会阻断指向非 loopback 地址的 AF_INET/AF_INET6 出站连接；DNS 解析、 loopback 与 Unix 域套接字放行，真实外网测试仍应放在 `tests/manual/`。 +本地上传安全基线：`tests/unit/test_local_upload_policy.py` 用临时 SQLite、真实 `ViewTokenResolver` 与 cache/task cleanup 消费者验证 owner/key 幂等、blank legacy token 隔离、`upload_` token、固定终态期限、write-once 撤销、开关关闭后的拒读与有效成果保护；另启动真实 Python 子进程消费同一 SQLite，验证缺省关闭与显式环境开关。`tests/cache/test_cache_cleanup.py` 的缓存 fixture 仅写 `tmp_path`，并从缓存 producer 返回的实际文件路径删除，不触碰仓库 `test_cache_dir/`。 + 任务通知数量回归：`tests/unit/test_notification_budget.py` 从真实转录入口运行下载、平台字幕、YouTube API 字幕、部分/全部缓存命中与两种失败路径，断言成功路径携带完整总结和分享回执、失败不带回执；同文件还以 25 MiB 下载 fixture 锁定 generic 下载不发进度通知。`tests/unit/test_terminal_notification_outbox.py` 覆盖单行 outbox、缺失快照可见降级与标记 sent、两条成功消息共用快照及失败路径。`tests/unit/test_notification_e2e_delivery.py` 使用真实 `process_transcription → _TaskNotifier → NotificationRouter → WeComChannel/FeishuChannel → wecom-notifier FIFO → HTTP JSON`（只替换最终外部 POST），并覆盖即时与 outbox 补发入口；检查 SQLite 实际快照字节、双渠道最终 payload 中完整总结先于分享回执、精确 view URL 与网页首段。`tests/unit/test_completion_share.py` 锁定 Markdown→网页同源 HTML→首个 p 可见文本与缺总结状态。相关设计及 N1–N4 更新见 `docs/sessions/261008-completion-share/design.md`。 任务观测回归：`tests/unit/test_task_observability_report.py` 覆盖只读 CLI、旧 schema、 diff --git a/tests/cache/test_cache_cleanup.py b/tests/cache/test_cache_cleanup.py index 144b7f7e..04d354d8 100644 --- a/tests/cache/test_cache_cleanup.py +++ b/tests/cache/test_cache_cleanup.py @@ -2,28 +2,24 @@ 测试缓存清理逻辑 验证当文件不存在时，会自动清理数据库记录 """ -import os -import sys +from pathlib import Path import shutil -project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(project_root, "src")) +import pytest -from video_transcript_api.cache import CacheManager -from video_transcript_api.utils.logging import setup_logger +from src.video_transcript_api.cache.cache_manager import CacheManager -logger = setup_logger("test_cache_cleanup") +@pytest.fixture +def cache_manager(tmp_path): + manager = CacheManager(cache_dir=str(tmp_path / "cache")) + yield manager + manager.close() -def test_auto_cleanup(): + +def test_auto_cleanup(cache_manager): """测试自动清理无效记录""" - print("=== 测试缓存自动清理逻辑 ===\n") - - # 创建缓存管理器 - cache_manager = CacheManager("./test_cache_dir") - # 步骤1：创建正常的缓存 - print("1. 创建正常的缓存记录...") cache_result = cache_manager.save_cache( platform="youtube", url="https://www.youtube.com/watch?v=cleanup_test", @@ -36,55 +32,34 @@ def test_auto_cleanup(): description="测试描述" ) - if cache_result: - print(f" [OK] 缓存创建成功") - file_path = cache_result['transcript_file'] - print(f" 文件路径: {file_path}") + assert cache_result is not None # 验证缓存可以正常查询 cache_data = cache_manager.get_cache(platform="youtube", media_id="cleanup_test") - if cache_data: - print(" [OK] 缓存查询成功") + assert cache_data is not None # 获取初始统计 stats = cache_manager.get_cache_stats() initial_count = stats['total_records'] - print(f" 当前缓存记录数: {initial_count}\n") # 步骤2：删除转录文件 - print("2. 手动删除转录文件，模拟文件丢失...") - transcript_file = os.path.join( - cache_manager.cache_dir, - "youtube", "2025", "202508", "cleanup_test", - "transcript_capswriter.txt" - ) - if os.path.exists(transcript_file): - os.remove(transcript_file) - print(f" [OK] 已删除文件: {transcript_file}\n") + transcript_file = Path(cache_result["transcript_file"]) + transcript_file.unlink() # 步骤3：再次查询，应该返回 None 并删除记录 - print("3. 再次查询缓存（文件已丢失）...") cache_data = cache_manager.get_cache(platform="youtube", media_id="cleanup_test") - if cache_data is None: - print(" [OK] 查询返回 None（预期行为）") - else: - print(" [FAIL] 查询仍然返回了数据") + assert cache_data is None # 验证数据库记录已被删除 stats = cache_manager.get_cache_stats() final_count = stats['total_records'] - print(f" 清理后缓存记录数: {final_count}") - if final_count == initial_count - 1: - print(" [OK] 数据库记录已被自动删除\n") - else: - print(" [FAIL] 数据库记录未被删除\n") + assert final_count == initial_count - 1 # 步骤4：测试文件夹完全不存在的情况 - print("4. 测试文件夹完全不存在的情况...") # 先创建一个新缓存 - cache_manager.save_cache( + second_cache = cache_manager.save_cache( platform="bilibili", url="https://www.bilibili.com/video/BV1folder_test", media_id="BV1folder_test", @@ -96,36 +71,10 @@ def test_auto_cleanup(): description="" ) - # 删除整个文件夹 - folder_path = os.path.join( - cache_manager.cache_dir, - "bilibili", "2025", "202508", "BV1folder_test" - ) - if os.path.exists(folder_path): - shutil.rmtree(folder_path) - print(f" [OK] 已删除文件夹: {folder_path}") + # Delete the path returned by the actual cache producer. + assert second_cache is not None + shutil.rmtree(Path(second_cache["transcript_file"]).parent) # 查询应该返回 None cache_data = cache_manager.get_cache(platform="bilibili", media_id="BV1folder_test") - if cache_data is None: - print(" [OK] 文件夹不存在时，查询返回 None") - print(" [OK] 无效记录已被自动清理") - else: - print(" [FAIL] 查询仍然返回了数据") - - # 关闭数据库连接 - cache_manager.close() - - print("\n=== 测试完成 ===") - print("\n总结：") - print("1. 当缓存文件夹不存在时，会自动删除数据库记录") - print("2. 当转录文件不存在时，会自动删除数据库记录") - print("3. 这确保了数据库与文件系统的一致性") - - -if __name__ == "__main__": - test_auto_cleanup() - - # 清理测试目录 - if os.path.exists("./test_cache_dir"): - shutil.rmtree("./test_cache_dir") + assert cache_data is None diff --git a/tests/unit/test_local_upload_policy.py b/tests/unit/test_local_upload_policy.py new file mode 100644 index 00000000..867f23b1 --- /dev/null +++ b/tests/unit/test_local_upload_policy.py @@ -0,0 +1,519 @@ +"""Real SQLite contracts for the local-upload safety baseline.""" + +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +import os +from pathlib import Path +import sqlite3 +import subprocess +import sys +import threading +import uuid + +import pytest + +from src.video_transcript_api.api.routes.audit import check_view_token_ownership +from src.video_transcript_api.api.services import view_token_resolver as resolver_module +from src.video_transcript_api.api.services.view_token_resolver import ViewTokenResolver +from src.video_transcript_api.cache.cache_manager import CacheManager +from src.video_transcript_api.utils.tempfile_manager import TempFileManager + + +@pytest.fixture +def cm(tmp_path): + manager = CacheManager(cache_dir=str(tmp_path / "cache")) + yield manager + manager.close() + + +def _intent_key(now=None): + now = now or datetime.now(timezone.utc) + epoch_ms = int(now.timestamp() * 1000) + return f"{epoch_ms}-{uuid.uuid4()}" + + +def _accepted_upload(cm, *, retention="30d", owner="alice"): + upload = cm.register_local_upload( + owner_user_id=owner, + idempotency_key=_intent_key(), + retention=retention, + ) + return cm.accept_local_upload( + upload["upload_id"], + media_id=f"media-{uuid.uuid4().hex}", + title="本地上传", + ) + + +def _finish_with_transcript(cm, upload, status="success"): + media_id = upload["media_id"] + cm.save_cache( + platform="local_upload", + url="", + media_id=media_id, + use_speaker_recognition=False, + transcript_data="A real local transcript.", + transcript_type="capswriter", + title="本地上传", + author="", + description="", + ) + cm.update_task_status( + upload["root_task_id"], + status, + platform="local_upload", + media_id=media_id, + title="本地上传", + error_message="transcription failed" if status == "failed" else None, + ) + + +def _old_time(days=400): + return (datetime.now(timezone.utc) - timedelta(days=days)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + + +def test_real_sqlite_upload_identity_owner_key_and_intent_window(cm): + now = datetime(2026, 10, 8, 9, 0, tzinfo=timezone.utc) + key = _intent_key(now) + first = cm.register_local_upload( + owner_user_id="alice", + idempotency_key=key, + retention="30d", + intent_metadata={"filename": "clip.mp4"}, + now=now, + ) + replay = cm.register_local_upload( + owner_user_id="alice", + idempotency_key=key, + retention="30d", + intent_metadata={"filename": "clip.mp4"}, + now=now, + ) + another_owner = cm.register_local_upload( + owner_user_id="bob", idempotency_key=key, retention="30d", now=now + ) + + assert replay["upload_id"] == first["upload_id"] + assert another_owner["upload_id"] != first["upload_id"] + with pytest.raises(ValueError, match="metadata"): + cm.register_local_upload( + owner_user_id="alice", idempotency_key=key, retention="never", now=now + ) + with pytest.raises(ValueError, match="metadata"): + cm.register_local_upload( + owner_user_id="alice", + idempotency_key=key, + retention="30d", + intent_metadata={"filename": "different.mp4"}, + now=now, + ) + + too_old = _intent_key(now - timedelta(hours=24, milliseconds=1)) + too_far_future = _intent_key(now + timedelta(minutes=5, milliseconds=1)) + with pytest.raises(ValueError, match="24-hour"): + cm.register_local_upload( + owner_user_id="alice", idempotency_key=too_old, retention="30d", now=now + ) + with pytest.raises(ValueError, match="clock skew"): + cm.register_local_upload( + owner_user_id="alice", + idempotency_key=too_far_future, + retention="30d", + now=now, + ) + + # Existing receipts remain queryable after the acceptance window; once a + # retired row is removed, its timestamp cannot create a second upload. + old_receipt_key = _intent_key(now - timedelta(hours=23)) + old_receipt = cm.register_local_upload( + owner_user_id="alice", idempotency_key=old_receipt_key, retention="30d", now=now + ) + later = now + timedelta(hours=25) + assert cm.get_local_upload_by_owner_key("alice", old_receipt_key)["upload_id"] == old_receipt["upload_id"] + cm.register_local_upload( + owner_user_id="alice", idempotency_key=old_receipt_key, retention="30d", now=later + ) + with cm._get_cursor() as cursor: + cursor.execute("DELETE FROM local_uploads WHERE upload_id = ?", (old_receipt["upload_id"],)) + with pytest.raises(ValueError, match="24-hour"): + cm.register_local_upload( + owner_user_id="alice", idempotency_key=old_receipt_key, retention="30d", now=later + ) + + +def test_concurrent_same_owner_key_creates_one_sqlite_receipt(cm): + now = datetime.now(timezone.utc) + key = _intent_key(now) + start = threading.Barrier(2) + + def register(): + start.wait(timeout=5) + return cm.register_local_upload( + owner_user_id="alice", + idempotency_key=key, + retention="30d", + intent_metadata={"filename": "same.mp4"}, + now=now, + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(register) + second = pool.submit(register) + records = [first.result(timeout=10), second.result(timeout=10)] + + assert records[0]["upload_id"] == records[1]["upload_id"] + with cm._get_cursor() as cursor: + cursor.execute( + "SELECT COUNT(*) FROM local_uploads WHERE owner_user_id = ? AND idempotency_key = ?", + ("alice", key), + ) + assert cursor.fetchone()[0] == 1 + + +def test_upload_task_uses_blank_legacy_column_and_separate_upload_token(cm): + upload = _accepted_upload(cm) + + with cm._get_cursor() as cursor: + cursor.execute( + "SELECT view_token, platform, submitted_by FROM task_status WHERE task_id = ?", + (upload["root_task_id"],), + ) + task_row = dict(cursor.fetchone()) + + assert task_row == {"view_token": "", "platform": "local_upload", "submitted_by": "alice"} + assert upload["view_token"].startswith("upload_") + assert upload["view_token"] != task_row["view_token"] + replay = cm.accept_local_upload( + upload["upload_id"], media_id="ignored-replay-media", title="replay" + ) + assert replay["root_task_id"] == upload["root_task_id"] + assert replay["view_token"] == upload["view_token"] + assert cm.get_task_by_view_token("") is None + assert check_view_token_ownership( + "", upload["root_task_id"], "alice", cm, None + ) is False + assert cm.get_task_by_view_token(upload["view_token"]) is None + assert cm.get_local_upload_by_view_token(upload["view_token"])["upload_id"] == upload["upload_id"] + + # A legacy token collision cannot turn the upload capability into a task + # lookup fallback, even if a legacy row was inserted by old code. + with cm._get_cursor() as cursor: + cursor.execute( + """INSERT INTO task_status + (task_id, view_token, url, platform, media_id, status, llm_config) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + ( + "legacy-shadow", + upload["view_token"], + "https://legacy.example", + "youtube", + "legacy", + "success", + '{"legacy-secret": true}', + ), + ) + assert cm.get_task_by_view_token(upload["view_token"]) is None + assert ViewTokenResolver(cm)._get_llm_config_by_view_token(upload["view_token"]) is None + + +def test_root_terminal_sets_fixed_30_day_expiry_for_success_and_failure(cm): + for status in ("success", "failed"): + upload = _accepted_upload(cm, retention="30d") + _finish_with_transcript(cm, upload, status=status) + stored = cm.get_local_upload_by_id(upload["upload_id"]) + with cm._get_cursor() as cursor: + cursor.execute( + "SELECT datetime(completed_at, '+30 days') FROM task_status WHERE task_id = ?", + (upload["root_task_id"],), + ) + expected = cursor.fetchone()[0] + assert stored["expires_at"] == expected + + # Subsequent task writes, including late updates, cannot extend the + # root's original expiry because terminal task state is write-once. + cm.update_task_status(upload["root_task_id"], "failed", error_message="late") + assert cm.get_local_upload_by_id(upload["upload_id"])["expires_at"] == expected + + +def test_never_expiry_and_revocation_are_write_once(cm): + upload = _accepted_upload(cm, retention="never") + _finish_with_transcript(cm, upload) + assert cm.get_local_upload_by_id(upload["upload_id"])["expires_at"] is None + + first_time = datetime(2026, 10, 8, 9, 0, tzinfo=timezone.utc) + later_time = first_time + timedelta(days=3) + first = cm.revoke_local_upload(upload["upload_id"], now=first_time) + replay = cm.revoke_local_upload(upload["upload_id"], now=later_time) + assert first["revoked_at"] == "2026-10-08 09:00:00" + assert replay["revoked_at"] == first["revoked_at"] + with pytest.raises(sqlite3.IntegrityError, match="write-once"): + with cm._get_cursor() as cursor: + cursor.execute( + "UPDATE local_uploads SET revoked_at = NULL WHERE upload_id = ?", + (upload["upload_id"],), + ) + + +def test_resolver_uses_real_subprocess_environment_and_sqlite(tmp_path): + cache_root = tmp_path / "cache" + manager = CacheManager(cache_dir=str(cache_root)) + upload = _accepted_upload(manager, retention="never") + _finish_with_transcript(manager, upload) + manager.close() + + script = ( + "import sys; " + "from src.video_transcript_api.cache.cache_manager import CacheManager; " + "from src.video_transcript_api.api.services.view_token_resolver import ViewTokenResolver; " + "cm=CacheManager(cache_dir=sys.argv[1]); " + "data=ViewTokenResolver(cm).get_view_data_by_token(sys.argv[2]); " + "print('RESULT=' + str((data or {}).get('status', 'NONE'))); " + "print('BODY=' + str((data or {}).get('transcript', 'NONE'))); cm.close()" + ) + disabled_env = os.environ.copy() + disabled_env.pop("VTA_UPLOADS_ENABLED", None) + disabled = subprocess.run( + [sys.executable, "-c", script, str(cache_root), upload["view_token"]], + cwd=str(Path(__file__).resolve().parents[2]), + env=disabled_env, + capture_output=True, + text=True, + check=False, + ) + assert disabled.returncode == 0, disabled.stderr + assert "UPLOAD_DISABLED" in disabled.stdout + assert "RESULT=NONE" in disabled.stdout + assert "BODY=NONE" in disabled.stdout + + enabled_env = disabled_env | {"VTA_UPLOADS_ENABLED": "true"} + enabled = subprocess.run( + [sys.executable, "-c", script, str(cache_root), upload["view_token"]], + cwd=str(Path(__file__).resolve().parents[2]), + env=enabled_env, + capture_output=True, + text=True, + check=False, + ) + assert enabled.returncode == 0, enabled.stderr + assert "RESULT=success" in enabled.stdout + assert "BODY=A real local transcript." in enabled.stdout + + +def test_upload_progress_token_resolves_before_30_day_terminal_clock(cm, monkeypatch): + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + upload = _accepted_upload(cm, retention="30d") + + assert ViewTokenResolver(cm).get_view_data_by_token(upload["view_token"])["status"] == "processing" + assert cm.get_local_upload_by_id(upload["upload_id"])["expires_at"] is None + + +def test_resolver_reads_active_upload_and_denies_disabled_revoked_or_expired(cm, monkeypatch): + warnings = [] + monkeypatch.setattr(resolver_module.logger, "warning", lambda *args, **kwargs: warnings.append(args)) + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + short = _accepted_upload(cm, retention="30d") + long = _accepted_upload(cm, retention="never") + _finish_with_transcript(cm, short) + _finish_with_transcript(cm, long) + resolver = ViewTokenResolver(cm) + + assert resolver.get_view_data_by_token(short["view_token"])["transcript"] == "A real local transcript." + assert resolver.get_cache_by_view_token(long["view_token"])["transcript_data"] == "A real local transcript." + + monkeypatch.delenv("VTA_UPLOADS_ENABLED", raising=False) + assert resolver.get_view_data_by_token(short["view_token"]) is None + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "false") + assert resolver.get_view_data_by_token(short["view_token"]) is None + assert resolver.get_cache_by_view_token(long["view_token"]) is None + assert any(resolver_module.UPLOAD_DISABLED in str(args) for args in warnings) + assert check_view_token_ownership( + short["view_token"], short["root_task_id"], "alice", cm, None + ) is False + + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "true") + cm.revoke_local_upload(long["upload_id"]) + assert resolver.get_view_data_by_token(long["view_token"]) is None + + with cm._get_cursor() as cursor: + cursor.execute( + "UPDATE local_uploads SET expires_at = ? WHERE upload_id = ?", + (_old_time(31), short["upload_id"]), + ) + assert resolver.get_view_data_by_token(short["view_token"]) is None + expired_record = cm.get_local_upload_by_view_token(short["view_token"]) + exactly_expired = datetime.strptime( + expired_record["expires_at"], "%Y-%m-%d %H:%M:%S" + ).replace(tzinfo=timezone.utc) + assert cm.local_upload_share_is_active(expired_record, exactly_expired) is False + + +def test_cleanup_protects_active_upload_cache_even_when_feature_is_disabled(cm, monkeypatch): + monkeypatch.setenv("VTA_UPLOADS_ENABLED", "false") + for retention in ("30d", "never"): + upload = _accepted_upload(cm, retention=retention) + _finish_with_transcript(cm, upload) + with cm._get_cursor() as cursor: + cursor.execute( + "UPDATE video_cache SET updated_at = ? WHERE platform = 'local_upload' AND media_id = ?", + (_old_time(), upload["media_id"]), + ) + assert cm.cleanup_old_cache(days=1) == 0 + assert cm.get_cache("local_upload", upload["media_id"]) is not None + + +def test_cleanup_reclaims_revoked_or_expired_upload_cache(cm): + revoked = _accepted_upload(cm, retention="never") + _finish_with_transcript(cm, revoked) + cm.revoke_local_upload(revoked["upload_id"]) + + expired = _accepted_upload(cm, retention="30d") + _finish_with_transcript(cm, expired) + with cm._get_cursor() as cursor: + cursor.execute( + "UPDATE local_uploads SET expires_at = ? WHERE upload_id = ?", + (_old_time(31), expired["upload_id"]), + ) + cursor.execute( + "UPDATE video_cache SET updated_at = ? WHERE platform = 'local_upload'", + (_old_time(),), + ) + + assert cm.cleanup_old_cache(days=1) == 2 + assert cm.get_cache("local_upload", revoked["media_id"]) is None + assert cm.get_cache("local_upload", expired["media_id"]) is None + + +def test_task_cleanup_keeps_effective_share_root_for_audit_and_url_rows_still_expire(cm): + uploads = [ + _accepted_upload(cm, retention="never"), + _accepted_upload(cm, retention="30d"), + ] + for upload in uploads: + _finish_with_transcript(cm, upload) + url_task = cm.create_task("https://example.com/old", platform="youtube", media_id="old-url") + cm.update_task_status(url_task["task_id"], "success", platform="youtube", media_id="old-url") + with cm._get_cursor() as cursor: + for upload in uploads: + cursor.execute( + "UPDATE task_status SET completed_at = ?, created_at = ? WHERE task_id = ?", + (_old_time(), _old_time(), upload["root_task_id"]), + ) + cursor.execute( + "UPDATE task_status SET completed_at = ?, created_at = ? WHERE task_id = ?", + (_old_time(), _old_time(), url_task["task_id"]), + ) + archived = [] + cm.audit_logger = type( + "Audit", + (), + { + "archive_task_snapshot": lambda self, task: archived.append(task["task_id"]), + "expire_task_snapshot": lambda self, task_id: False, + "restore_live_task_snapshot": lambda self, task: None, + }, + )() + + assert cm.cleanup_task_status(retention_days=1) == 1 + assert all(cm.get_task_by_id(upload["root_task_id"]) is not None for upload in uploads) + assert cm.get_task_by_id(url_task["task_id"]) is None + assert all(upload["root_task_id"] not in archived for upload in uploads) + + +def test_revoke_and_task_cleanup_are_ordered_at_the_sqlite_guard(cm, monkeypatch): + upload = _accepted_upload(cm, retention="never") + _finish_with_transcript(cm, upload) + with cm._get_cursor() as cursor: + cursor.execute( + "UPDATE task_status SET completed_at = ?, created_at = ? WHERE task_id = ?", + (_old_time(), _old_time(), upload["root_task_id"]), + ) + cm.audit_logger = type( + "Audit", + (), + { + "archive_task_snapshot": lambda self, task: None, + "expire_task_snapshot": lambda self, task_id: False, + "restore_live_task_snapshot": lambda self, task: None, + }, + )() + + predicate_checked = threading.Event() + release_cleanup = threading.Event() + original_predicate = cm._has_active_local_upload_for_task + + def pause_after_active_check(cursor, task_id, now_text): + active = original_predicate(cursor, task_id, now_text) + predicate_checked.set() + assert release_cleanup.wait(timeout=10) + return active + + monkeypatch.setattr(cm, "_has_active_local_upload_for_task", pause_after_active_check) + with ThreadPoolExecutor(max_workers=2) as pool: + cleanup = pool.submit(cm.cleanup_task_status, 1) + assert predicate_checked.wait(timeout=10) + revoke_started = threading.Event() + + def revoke_after_cleanup_locked(): + revoke_started.set() + return cm.revoke_local_upload(upload["upload_id"]) + + revoke = pool.submit(revoke_after_cleanup_locked) + assert revoke_started.wait(timeout=10) + release_cleanup.set() + assert cleanup.result(timeout=10) == 0 + assert revoke.result(timeout=10)["revoked_at"] is not None + assert cm.get_task_by_id(upload["root_task_id"]) is not None + + # Opposite order: complete revocation before the cleaner takes its guarded + # snapshot. The now-inactive root follows the pre-existing retention rule. + second = _accepted_upload(cm, retention="never") + _finish_with_transcript(cm, second) + with cm._get_cursor() as cursor: + cursor.execute( + "UPDATE task_status SET completed_at = ?, created_at = ? WHERE task_id = ?", + (_old_time(), _old_time(), second["root_task_id"]), + ) + cm.revoke_local_upload(second["upload_id"]) + monkeypatch.setattr(cm, "_has_active_local_upload_for_task", original_predicate) + assert cm.cleanup_task_status(retention_days=1) == 2 + assert cm.get_task_by_id(upload["root_task_id"]) is None + assert cm.get_task_by_id(second["root_task_id"]) is None + + +def test_expired_or_revoked_upload_task_rows_follow_existing_cleanup(cm): + upload = _accepted_upload(cm, retention="never") + _finish_with_transcript(cm, upload) + cm.revoke_local_upload(upload["upload_id"]) + with cm._get_cursor() as cursor: + cursor.execute( + "UPDATE task_status SET completed_at = ?, created_at = ? WHERE task_id = ?", + (_old_time(), _old_time(), upload["root_task_id"]), + ) + cm.audit_logger = type( + "Audit", + (), + { + "archive_task_snapshot": lambda self, task: None, + "expire_task_snapshot": lambda self, task_id: False, + "restore_live_task_snapshot": lambda self, task: None, + }, + )() + + assert cm.cleanup_task_status(retention_days=1) == 1 + assert cm.get_task_by_id(upload["root_task_id"]) is None + + +def test_temp_source_media_remains_ephemeral_for_active_share(tmp_path, cm): + upload = _accepted_upload(cm, retention="never") + _finish_with_transcript(cm, upload) + manager = TempFileManager(str(tmp_path / "temp"), retention_hours=0) + task_dir = manager.create_task_dir("local-root") + source_media = task_dir / "source.mp4" + source_media.write_bytes(b"original media") + old = datetime.now(timezone.utc).timestamp() - 7200 + os.utime(source_media, (old, old)) + + assert manager.clean_up_old_files(hours=0) == 1 + assert not task_dir.exists() From cbad5ac62c5e3a1aeb7663a00e1d0385004d914c Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 18:34:14 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E8=AE=BE=E8=AE=A1=E5=90=88=E5=90=8C=E4=B8=8E?= =?UTF-8?q?=E5=85=AD=E9=98=B6=E6=AE=B5=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-094534-0fd910 Task-Id: VideoTranscriptAPI-20261008-11 --- docs/sessions/261008-local-upload/design.md | 65 +++++++++++++++++++ .../progress/A-progress.md | 6 +- goals/local-upload/M1-baseline.md | 23 +++++++ goals/local-upload/M2-intake.md | 23 +++++++ goals/local-upload/M3-worker.md | 23 +++++++ goals/local-upload/M4-web.md | 23 +++++++ goals/local-upload/M5-proof.md | 24 +++++++ goals/local-upload/M6-recovery.md | 24 +++++++ 8 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 docs/sessions/261008-local-upload/design.md create mode 100644 goals/local-upload/M1-baseline.md create mode 100644 goals/local-upload/M2-intake.md create mode 100644 goals/local-upload/M3-worker.md create mode 100644 goals/local-upload/M4-web.md create mode 100644 goals/local-upload/M5-proof.md create mode 100644 goals/local-upload/M6-recovery.md diff --git a/docs/sessions/261008-local-upload/design.md b/docs/sessions/261008-local-upload/design.md new file mode 100644 index 00000000..88462ac7 --- /dev/null +++ b/docs/sessions/261008-local-upload/design.md @@ -0,0 +1,65 @@ +# 本地媒体上传：A 卡安全基线 + +## 范围与批准边界 + +依据 2026-10-08 已批准的 v2（D16–D19、R1–R8）与最新授权“拆卡完整落地我们的设计方案”。本文件只持久化已批准的当前设计与跨卡 HTTP 合同；不是完整上传项目或生产启用许可。 + +A 卡交付：`cache.db` 内独立上传身份/期限/撤销记录；上传根任务与旧 `task_status.view_token` 分离；Resolver 只从上传记录解析 `upload_` capability；缓存/任务清理只在分享有效期间保护必要成果与根记录。`VTA_UPLOADS_ENABLED` 缺失或不是 `true` 时，上传分享读取拒绝，且写一条可 grep 的 `UPLOAD_DISABLED` 日志。开关关闭不改变清理保护。当前没有上传 HTTP 接收端、真实本地 ASR worker、上传 UI、生产额度或恢复证明。 + +主身份仍是现有 Bearer 用户。正文链接公开读取；提交归属由 owner 持有。每次正式上传独立成果，不跨上传去重；同一 owner 与同一幂等 key 是同一意图。上传开关不写入数据卷配置；没有完整恢复域证明，部署保持关闭。不得据代码测试宣称已授权生产部署。 + +## SQLite 与生命周期 + +单一存储仍是 `CacheManager`。新增 `local_uploads` 表含：`upload_id`、`owner_user_id`、幂等 key 和 key 创建时间、规范化请求 metadata 的 SHA-256 指纹、接收态、`retention`、唯一 root task / media / 独立 `upload_` token、创建时间、`expires_at`、write-once `revoked_at`、`error_code`。`UNIQUE(owner_user_id, idempotency_key)` 和 `BEGIN IMMEDIATE` 使并发重复登记只产生一个持久 receipt；同 key 元数据不同拒绝，不读新 body、不建第二个 root。 + +上传 root task 仍用现有 `task_status` 做进度和终态真相。由于旧列 `view_token TEXT NOT NULL`，上传根行写 `view_token=''`，`platform='local_upload'`；公开 token 只存 `local_uploads.view_token`。空 token 与 `upload_` token 都不准旧 `get_task_by_view_token` 或 audit 归属 legacy fallback 解析。写入撤销后不能复活；DB trigger 和 `COALESCE` 写入锁住 `revoked_at` 单调性。 + +接收 key 格式为 `-`。首次登记只接受当前时刻之前 24 小时内的 key，未来最多容许 5 分钟时钟偏差；超过 24 小时的已有 owner/key 仍可查原记录，记录退休/清除后同 key 不得重新受理。此窗口是幂等记录生命周期，不是分享期限。当前 A 不做记录 GC；后续维护只能在确认查询/历史消费者及唯一记录窗口后回收。 + +`30d` 从 root 首次 `success` 或 `failed` 终态的 SQLite `completed_at` 固定加 30 天；未到终态时 `expires_at=NULL` 不等于长期，尚未终态的 root 只呈现处理中。`never` 才以 `expires_at=NULL` 表示不自动到期。子任务、重复终态写和重启不续期。撤销时间只写一次；关闭分享不取消已接受处理，也不专属擦除正文。 + +Resolver 每次新请求都查独立上传记录并核对开关、撤销和截止边界（`expires_at <= now` 已失效）；它不经旧 token fallback。有效 `never` 与有效 `30d` 均从既有 CacheManager 读真实缓存。失效、撤销、禁用均不返回新正文；失效内容随后按原全局清理规则处理。上传 reprocess 业务尚未实现，`upload_` token 不能走既有 task owner fallback，当前拒绝重处理。 + +## 清理与数据边界 + +- `cleanup_old_cache` 在现有媒体锁路径下检查有效上传 root 与 media 关系；有效短期和长期正文都保护，跟 `VTA_UPLOADS_ENABLED` 无关。撤销/到期后，缓存按已有 `video_cache.updated_at` 全局年龄规则回收。URL 缓存仍走原规则。 +- `cleanup_task_status` 在 `BEGIN IMMEDIATE` 内再次检查 root 的有效分享资格；有效 root 留存，audit snapshot 因现有 `task_exists` 依赖仍可解析。inactive 上传和 URL task 按原 task retention 及快照过期次序清理。Keyset 扫描越过被保护记录，避免有效长期记录阻塞后续批次。 +- 审计 ownership 不接受 `upload_` token 的 legacy fallback；有效分享所需根 task/snapshot 留存由现有 audit 消费者关系维持。历史展示仍由后续 B/D 卡实现。 +- `data/temp` 中原媒体、转码副本和任务暂存文件仍是临时输入，不因长期正文分享而保护；现有 `TempFileManager` 保持其清理行为。 + +## 跨卡 HTTP 合同（B/C/D/E 消费） + +### 请求与幂等 + +- 原生 raw `File`/`Blob` body；`Content-Type: application/octet-stream`；现有 `Authorization: Bearer ...`。 +- `Idempotency-Key: -`；服务器核验 24 小时受理窗口与最多 5 分钟未来时钟偏差。窗口内同 owner/key 唯一记录不能 GC；已有 key 的 receipt 仍可查；过窗且唯一记录已退休时拒绝，不能重新受理。 +- `X-Upload-Metadata` 为 base64url 编码的 UTF-8 compact JSON，必须有界；字段 `filename`、`byte_size`、`title`（可空）、`source_url`（可空）、`retention`（`30d` 或 `never`）、`processing_options`（沿现有处理选项）。不得含本地路径、`user_id` 或 public token。 +- 首次提交和接受动作必须由真实服务端边界验证 metadata；重复相同意图不读新 body、不重入队，只返回原 receipt；同 key metadata 变化返回 409。未知回执只查询，不自动换 key 或重发。 + +### 响应 + +- `GET /api/uploads/capabilities`（Bearer）：`{enabled:boolean, default_retention:'30d', retention_options:['30d','never'], limits:{max_file_mib,max_media_hours,receive_concurrency,upload_temp_budget_mib}}`。四项额度必须正且有限、来自经过真实消费环境验证的配置；当前生产值未知/未测，缺一项即 `enabled=false`。不得用示例值声称已测。 +- `POST /api/uploads`：只有数据库正式接受后才返回 202；同意图返回已有 receipt，变化 metadata 返回 409。FastAPI 错误必须明确 status/detail，不得假 202。 +- `GET /api/uploads/by-idempotency-key/{key}`（Bearer/owner）：已存在 receipt 或 404；不能借查询创建任务。 +- receipt：`upload_id`、`state`（接收阶段，不复制 `task_status`）、`task_id`（未正式受理时 null）、`view_token`（正式受理后独立 `upload_` token，否则 null）、`retention`、`expires_at`（终态前与 `never` 为 null）、`share_active`、`error_code`（无错误为 null）。处理进度继续由既有 `GET /api/task/{task_id}` 承载。 +- `DELETE /api/uploads/{upload_id}/share`（Bearer/owner）：`{upload_id, share_active:false, revoked_at}`。重复关闭返回原截止，不得复活。关闭/到期后所有新 public read 和新 reprocess 拒绝；已经授权的传输和已接受处理可继续。 +- 本人历史增加 `source=upload` 过滤，先过滤再分页；upload item 可含 `upload_id/retention/expires_at/share_active/revoked_at/title/source_url`。bodygate 未实现前不得提供旁路摘要。 + +## 不变量与真实验收入口 + +| 不变量 | 实现入口 | 锁定测试 | +|---|---|---| +| owner/key 唯一、窗口/时钟偏差、metadata conflict、旧 receipt 查询与过窗拒绝 | `cache_manager.py::register_local_upload` | `tests/unit/test_local_upload_policy.py`（临时真实 SQLite，含并发登记） | +| root `view_token=''`、独立 `upload_` token；blank 与上传 token 不走旧别名 | `cache_manager.py::accept_local_upload/get_task_by_view_token`、`audit.py::check_view_token_ownership` | `test_upload_task_uses_blank_legacy_column_and_separate_upload_token`、Resolver 与 ownership 用例 | +| success/failed 首终态固定 30 天；never 不到期；撤销 write-once | `cache_manager.py::update_task_status/revoke_local_upload` | `test_root_terminal_sets_fixed_30_day_expiry_for_success_and_failure`、`test_never_expiry_and_revocation_are_write_once` | +| Resolver 对 active 30d/never 读真实缓存；missing/off/revoked/expired 不读；`UPLOAD_DISABLED` 可 grep | `view_token_resolver.py::_local_upload_task_info` | `test_resolver_reads_active_upload_and_denies_disabled_revoked_or_expired`、`test_upload_progress_token_resolves_before_30_day_terminal_clock` | +| 有效产物不被 cache/task/audit 清理删除；开关 off 不影响保护；关闭后恢复原清理；媒体仍临时 | `cleanup_old_cache/cleanup_task_status`；现有 `task_exists` 与 `TempFileManager` | local policy cleanup/controlled ordering tests、`tests/cache/test_task_status_cleanup.py`、`tests/cache/test_cache_cleanup.py` | +| URL 读、去重、history 与 task 行为不改变 | 原有 URL routes/CacheManager | `tests/unit/test_view_token_resolver.py`、`test_api_routes.py`、`test_history_routes.py` 及 cache cleanup 套件 | + +当前真实开发验证入口：`uv sync --frozen`；重复五次 `uv run --frozen pytest -q tests/unit/test_local_upload_policy.py tests/unit/test_view_token_resolver.py tests/cache/test_task_status_cleanup.py`；完整 gate `make test`。对环境变量依赖的测试必须在实际消费 Resolver 的环境运行。部署/systemd 恢复及容量验证仍是上线前置条件；本卡不触及生产 unit、数据卷或服务。 + +## 非目标与依赖 + +A 不做 HTTP 接收/流式限额、队列原子交接、ASR/LLM 接线、上传完整 reprocess、网页、部署/备份恢复实测、生产容量验证、source URL fetch 或原媒体长期留存。B 消费该 store 与 Resolver；C 消费 root/终态与媒体清理合同；D 消费本节 wire/history 合同；E 证明真实 producer 到 HTTP/ASR/正文消费者；F 证明独立恢复域与单向关闭。 + +不增加账户平台、`LocalUploadService` 转发层、outbox、broker、重试/fallback、外部撤销系统或新配置源。没有 `config.py` 加载器；`VTA_UPLOADS_ENABLED` 是部署环境变量，故不写入 `config/config.example.jsonc` 或数据卷配置。没有正有限额度和恢复证明时，B 必须报告关闭，而不是启用默认占位值。 diff --git a/docs/sessions/261008-local-upload/progress/A-progress.md b/docs/sessions/261008-local-upload/progress/A-progress.md index 49600d2a..dc504604 100644 --- a/docs/sessions/261008-local-upload/progress/A-progress.md +++ b/docs/sessions/261008-local-upload/progress/A-progress.md @@ -2,13 +2,13 @@ ## 当前阶段 -implementing；安全基线代码与真实 SQLite/Resolver/cleanup 用例已落地，本地完整 gate 和窄测已通过，待红验、提交/推送与 PR 独立审查。HTTP接收、worker、网页、生产额度和恢复演练尚未实现，不能启用生产上传。 +implementing；store/Resolver/cleanup 代码已提交为 `132daf96`，窄测五连绿、指定回归与 `make test` 均通过；blank/source/cleanup 三项变异均由 AssertionError 命中且恢复后绿。待设计/六里程碑文件提交、push 与 PR 独立审查。HTTP接收、worker、网页、生产额度和恢复演练尚未实现，不能启用生产上传。 ## 本段结论（≤3句） - A 使用现有 `CacheManager` 作为 store，新增 `local_uploads`；不增加 Service 转发层或账户/队列平台。 - Resolver 通过独立 `upload_` token 读取，旧 NOT NULL token 列保持空字符串；禁用、撤销、到期拒读且不改变有效成果清理保护。 -- 最终窄测连续五次通过，指定 URL/history/cache/task/媒体锁清理回归与 `make test` 全量 gate 均通过；历史 `tests/cache/test_cache_cleanup.py` 会删除仓库 fixture，已改成 `tmp_path` 且消费 producer 返回的路径。 +- 最终窄测连续五次通过，指定 URL/history/cache/task/媒体锁清理回归与 `make test` 全量 gate 均通过；blank lookup、上传 source fallback、active-cache cleanup predicate 的移除各自触发普通 AssertionError，恢复后对应测试各绿。历史 `tests/cache/test_cache_cleanup.py` 会删除仓库 fixture，已改成 `tmp_path` 且消费 producer 返回的路径。 ## 关键决策与否决方案 @@ -19,4 +19,4 @@ implementing；安全基线代码与真实 SQLite/Resolver/cleanup 用例已落 ## 下一步唯一动作 -提交已验证代码与本进度记录；提交后逐项移除 blank/source/cleanup predicate 做 AssertionError 红验，恢复实现后推送分支。 +提交设计合同、六个里程碑和更新后的本进度记录，再推送两笔提交并创建 draft PR。 diff --git a/goals/local-upload/M1-baseline.md b/goals/local-upload/M1-baseline.md new file mode 100644 index 00000000..ffa5279c --- /dev/null +++ b/goals/local-upload/M1-baseline.md @@ -0,0 +1,23 @@ +--- +lane: local-upload +id: M1 +slug: baseline +status: 实现中 +owner: delegate implementer dlg-20261008-094534-0fd910 +order: 1 +priority: 高 +depends_on: [] +merged_pr: null +--- + +# 里程碑进度：local-upload/M1：上传身份、分享资格与清理基线 + +- **预期产出**：cache.db 持有独立上传身份/期限/撤销记录；旧 NOT NULL `view_token` 不再承载公开上传 token；实际 Resolver 与 cache/task/audit/temp 维护消费者按资格保护。 +- **当前范围**：A 卡 guard-only 基线。不包含 HTTP 接收、真实 ASR worker、网页、生产额度和恢复证明；未合并、未部署、未启用。 +- **关键决策**：用现有 `CacheManager` 做 store；`view_token=''` 只作旧列占位，公开 `upload_` token 只从新表解析；`VTA_UPLOADS_ENABLED` 缺省/非 true 即关闭读取，不影响清理保护。 +- **推进前必须拿到的证据**： + - [x] 卡树 `uv sync --frozen` 后，窄测连续五次：`uv run --frozen pytest -q tests/unit/test_local_upload_policy.py tests/unit/test_view_token_resolver.py tests/cache/test_task_status_cleanup.py`，五轮 exit=0；环境：本地隔离 worktree 与临时 SQLite，日志 `/tmp/vta-upload-A-narrow-final-{1..5}.log`。 + - [x] `make test` 完整 gate exit=0；URL Resolver、API、history、cache/task cleanup 与媒体锁竞态整文件回归共 196 tests，exit=0；环境：当前 card worktree。全量日志 `/tmp/vta-upload-A-make-test-final.log`。 + - [ ] Resolver 开关测试同时经过当前进程和真实子进程，使用同一份临时 SQLite 验证变量缺省与显式 true；CI 和裸 shell 需运行。当前仓无 upload systemd consumer；不得把本地结果声称为 systemd/生产验证。 + - [ ] 独立审查与 PR run/job 实际 conclusion；draft 的 SKIPPED 不算 primary 通过。 +- **完成条件**：上述证据全部来自真实 Resolver 与维护入口，代码位置/测试锁定成对；PR merge 之前由 lead 验收。生产启用仍依赖 M2–M6 的部署、容量及恢复门。 diff --git a/goals/local-upload/M2-intake.md b/goals/local-upload/M2-intake.md new file mode 100644 index 00000000..b4958da4 --- /dev/null +++ b/goals/local-upload/M2-intake.md @@ -0,0 +1,23 @@ +--- +lane: local-upload +id: M2 +slug: intake +status: 未开始 +owner: delegate implementer dlg-20261008-094534-0fd910 +order: 2 +priority: 高 +depends_on: [local-upload/M1] +merged_pr: null +--- + +# 里程碑进度：local-upload/M2：认证接收与幂等回执 + +- **预期产出**：Bearer 认证的能力查询、raw octet-stream 接收、有限 metadata header、真实额度准入、owner/key receipt 与诚实 HTTP 状态。 +- **当前范围**：B 卡；不把 A 的 SQLite store 方法误报为已存在 HTTP 接收端。 +- **关键决策**：沿用 `docs/sessions/261008-local-upload/design.md` 的 wire 合同；key 窗口24小时、未来容差5分钟；同意图不消费新 body、不重入队，变化 metadata 409。 +- **推进前必须拿到的证据**： + - [ ] 真实 FastAPI/ASGI 与临时 cache.db 上由 client 发 raw 文件字节及所有 headers；断言 handler 实收字节、metadata 解码、commit receipt 与真实 task/queue producer 参数。跨进程 payload 保留 producer fixture。 + - [ ] 无开关、关闭开关、任一额度缺失/无效时实际返回 disabled 并拒绝接收；四项额度全为正有限数才测 enabled。生产测量前不填真实容量结论。 + - [ ] 同 key 重放时 consumer 证明没有读取新 body、没有第二次 queue 入队；查询只读回原 receipt；丢回执不自动生成新 key/重传。 + - [ ] 在 CI、无会话身份裸 shell 和获准隔离 systemd unit 分别验证真实消费环境变量与接收行为。systemd/生产改动另需部署授权。 +- **完成条件**：202 只在持久正式接受后返回；未知/失败不假成功；queue/SQLite 交接窗口由真实入口测试锁死，不加 retry/fallback/outbox。 diff --git a/goals/local-upload/M3-worker.md b/goals/local-upload/M3-worker.md new file mode 100644 index 00000000..1fbdf2d9 --- /dev/null +++ b/goals/local-upload/M3-worker.md @@ -0,0 +1,23 @@ +--- +lane: local-upload +id: M3 +slug: worker +status: 未开始 +owner: delegate implementer dlg-20261008-094534-0fd910 +order: 3 +priority: 高 +depends_on: [local-upload/M1, local-upload/M2] +merged_pr: null +--- + +# 里程碑进度：local-upload/M3：本地输入进入既有转录 worker + +- **预期产出**：本地 owned media 按唯一 root task 接入既有 `process_transcription`、ASR、LLM、通知、终态与临时文件生命周期。 +- **当前范围**：C 卡；不另建 ASR/LLM/通知状态机，不实现第二套重试/预算，不把公开 upload token 写入旧 `task_status.view_token`。 +- **关键决策**：复用现有处理链；root首个 success/failed 起算固定期限，子任务/重启不续期；原媒体及中间文件仍按既有 finally 清理。 +- **推进前必须拿到的证据**： + - [ ] 真实 `process_transcription` 消费临时文件与 SQLite root；只替换 ASR/LLM/通知外部边缘。保留 producer fixture 并断言实际 ASR 参数/argv、文件字节、root id、终态与通知 payload。 + - [ ] success/failure/invalid result 均有诚实终态；stop-share 不取消已接受处理、不专属删正文；重启/迟到写不能改变 root TTL 或复活撤销。 + - [ ] queue、数据库、ASR/LLM 失败通过具名状态/日志可见；不吞错、不假成功、不自动重试或切换引擎。 + - [ ] 在 CI、裸 shell、获准 systemd 隔离 unit 测真实环境变量和 worker 启动消费；不碰生产配置/数据。 +- **完成条件**：真实处理者消费的是接收器写出的媒体与选项，阶段与 URL 路径保持一致；正文资格和任务进度分离。 diff --git a/goals/local-upload/M4-web.md b/goals/local-upload/M4-web.md new file mode 100644 index 00000000..f8b5c159 --- /dev/null +++ b/goals/local-upload/M4-web.md @@ -0,0 +1,23 @@ +--- +lane: local-upload +id: M4 +slug: web +status: 未开始 +owner: delegate implementer dlg-20261008-094534-0fd910 +order: 4 +priority: 中 +depends_on: [local-upload/M1, local-upload/M2, local-upload/M3] +merged_pr: null +--- + +# 里程碑进度：local-upload/M4：上传、回执与本人历史网页 + +- **预期产出**：原生单文件 picker/drop、metadata headers、期限选择、真实 receipt/处理中/结果状态与本人 history source filter。 +- **当前范围**：D 卡；不得从 mock-only fixture 发明 wire 形状，不改旧 URL 任务页面规则。 +- **关键决策**：沿用设计文件中的 HTTP 字段；默认30天、长期显式选择 never；进度继续走既有 task API；source URL 仅展示，不 fetch。 +- **推进前必须拿到的证据**： + - [ ] 浏览器/harness 调用真实 API，断言 producer 发出的 raw Blob 字节、headers 和完整服务端 receipt；保留实际请求 fixture。 + - [ ] picker/drop 同路；100% 传输不等于正式接受；覆盖 401、网络未知、duplicate receipt、处理中、失败、关闭和到期状态。 + - [ ] 真实 history query 先按 source 过滤再分页；断言实际 ID 集、total、页边界与 owner 隔离；未实现 bodygate 时无摘要旁路。 + - [ ] filename/source 输入经真实模板安全渲染；恶意路径、Unicode、引号、无效 URL、键盘和小屏测试。CI browser/API 环境运行；systemd验证仅随另行授权部署。 +- **完成条件**：浏览器状态只反映服务端 receipt/进度事实；失效 token 仍由服务端拒绝，文案不许承诺回收通知副本。 diff --git a/goals/local-upload/M5-proof.md b/goals/local-upload/M5-proof.md new file mode 100644 index 00000000..2bf51f84 --- /dev/null +++ b/goals/local-upload/M5-proof.md @@ -0,0 +1,24 @@ +--- +lane: local-upload +id: M5 +slug: proof +status: 未开始 +owner: delegate implementer dlg-20261008-094534-0fd910 +order: 5 +priority: 高 +depends_on: [local-upload/M1, local-upload/M2, local-upload/M3, local-upload/M4] +merged_pr: null +--- + +# 里程碑进度：local-upload/M5：全链路安全与兼容性证明 + +- **预期产出**：从真实 HTTP producer 到 owned bytes、task/queue、既有处理链、Resolver、公正文、本人 history 与 cleanup 的端到端证据。 +- **当前范围**：E 卡；整理方案、helper 单测或邻侧 response 不算实际消费证明。 +- **关键决策**：每项不变式必须指向真实生产代码与能被 mutation 打红的测试；跨文件/进程边界保存 producer 实际 payload。 +- **推进前必须拿到的证据**： + - [ ] 真 FastAPI、临时 SQLite/文件系统、真实 dispatcher、Resolver 与现有 consumer；断言 HTTP/body、subprocess argv/env、写入文件等实际 producer 输出。 + - [ ] 同意图恰好一个受理，两次明确上传恰好两个独立成果；短期、never、expiry、revoke、disabled 全有 active positive control。 + - [ ] controlled pause 证明撤销/迟到发布与清理/发布两顺序；每种均由 fresh consumer 检查关闭不复活和有效成果不丢。 + - [ ] URL dedup/read/history/notification 与全局 cleanup 整文件回归；`make test` 通过。CI只看 run/job `conclusion`，skip 不算通过。 + - [ ] CI、裸 shell、实际 systemd 消费环境分别跑环境相关测试；systemd 环境必须是本地 consumer 实际运行的 unit。 +- **完成条件**：任何“已覆盖”都有执行证据；矛盾/未知机制重新开受影响设计，不以 fixture 命名掩盖差异。 diff --git a/goals/local-upload/M6-recovery.md b/goals/local-upload/M6-recovery.md new file mode 100644 index 00000000..c0f5083e --- /dev/null +++ b/goals/local-upload/M6-recovery.md @@ -0,0 +1,24 @@ +--- +lane: local-upload +id: M6 +slug: recovery +status: 未开始 +owner: delegate implementer dlg-20261008-094534-0fd910 +order: 6 +priority: 高 +depends_on: [local-upload/M1, local-upload/M2, local-upload/M3, local-upload/M4, local-upload/M5] +merged_pr: null +--- + +# 里程碑进度：local-upload/M6：恢复域、单向撤销与容量恢复 + +- **预期产出**：证明独立恢复开关、cache.db/正文/audit/share记录完整恢复与容量阈值；未证明前上传读写关闭。 +- **当前范围**：F 卡，生产前置证明；A 不做恢复演练、容量实测或部署授权。 +- **关键决策**：`VTA_UPLOADS_ENABLED` 在数据卷外；可信撤销状态/完整恢复域不能证明时保持关闭；长期成果不能通过删除腾容量。 +- **推进前必须拿到的证据**： + - [ ] 用与目标服务一致但隔离的真实 systemd unit、身份、mount、配置与数据格式演练完整备份/恢复；启动新服务前检查开关关闭。仅获授权后操作。 + - [ ] fresh Resolver 证明有效30d/never正文和必要解析记录保留、已撤销/关闭状态不复活；缺一个恢复域时拒绝打开；回滚不能以旧卷重开分享。 + - [ ] 在实际磁盘配额、ASR并发和URL负载下测 max_file_mib、max_media_hours、receive_concurrency、upload_temp_budget_mib；保留真实文件/调用证据，拒绝用示例数字冒充实测。 + - [ ] 容量不足拒绝新受理且可见；中断/孤儿清理有界，原媒体仍按既有处理阶段清理；不删除有效成果释放空间。 + - [ ] 单列CI、裸 shell、获准 systemd unit 环境结果；文档、配置值或模拟介质不能替代恢复演练。 +- **完成条件**：证明完整、通过独立审查并经用户/部署授权后才能评估启用；任何关键证明缺失都维持默认关闭。 From 3a25dccc2cb92e22b4031715c37a584c041e582d Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Thu, 8 Oct 2026 18:43:04 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E8=AE=B0=E5=BD=95=20A=20=E5=8D=A1=E9=AA=8C?= =?UTF-8?q?=E6=94=B6=E4=B8=8E=20draft=20PR=20=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Executor: codex-pi Agent-Model: gpt-6-luna Agent-Effort: xhigh Dispatch-Id: dlg-20261008-094534-0fd910 Task-Id: VideoTranscriptAPI-20261008-11 --- docs/sessions/261008-local-upload/progress/A-progress.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/sessions/261008-local-upload/progress/A-progress.md b/docs/sessions/261008-local-upload/progress/A-progress.md index dc504604..cefd8ab3 100644 --- a/docs/sessions/261008-local-upload/progress/A-progress.md +++ b/docs/sessions/261008-local-upload/progress/A-progress.md @@ -17,6 +17,12 @@ implementing；store/Resolver/cleanup 代码已提交为 `132daf96`，窄测五 - 不将 `VTA_UPLOADS_ENABLED` 放进数据卷 JSON；生产额度、systemd恢复域未验证前保持关闭。 - 上传 reprocess 暂不接线，拒绝走 task_status owner legacy fallback；原媒体仍走 TempFileManager 临时清理。 +## 本段结论：推送与 draft PR + +- 代码提交 `132daf96`、设计/里程碑提交 `cbad5ac6` 均已 push；PR #201 为 OPEN draft，当前 tip `cbad5ac62c5e3a1aeb7663a00e1d0385004d914c`。 +- CI run `37764505752` 的 run conclusion 为 SUCCESS，quality/draft-gate/ledger job 为 SUCCESS；primary 为 SKIPPED（draft），不能等同独立主审或 ready。 +- 实现与本地验证完成，仍待 Pi lead 独立审查；未 ready/merge、未碰生产。 + ## 下一步唯一动作 -提交设计合同、六个里程碑和更新后的本进度记录，再推送两笔提交并创建 draft PR。 +由 Pi lead 独立审查 PR #201 并决定后续 gate 节点；执行器不把 PR 标 ready、不 merge、不启用生产。