diff --git a/backend/src/api/patchnotes_routes.py b/backend/src/api/patchnotes_routes.py index cb13967e..de7ae32a 100644 --- a/backend/src/api/patchnotes_routes.py +++ b/backend/src/api/patchnotes_routes.py @@ -24,19 +24,29 @@ PatchnotesConfigError, PatchnotesDBError, approve_bullet, + add_folder_rule, current_week, deny_bullet, + ensure_folder, + get_folder, insert_bullet, insert_sourced_bullet, list_approved, + list_folders, list_pending, list_published_notes, list_published_weeks, + list_sync_tasks, load_preview, migrate, + observe_folder, parse_week, + queue_sync_task, + remove_added_folder, replace_preview, + request_added_folder, ) +from src.patchnotes.folders import catalog_entries, clean_folder_name, safe_rule from src.patchnotes.safety import hidden_knowledge_warning from src.patchnotes.summarize import signature_ok, summarize_push from src.skins.auth import HEADER_STAFF_KEY, require_staff_key @@ -304,10 +314,14 @@ def staff_create_bullet(body: CreateBulletBody): try: migrate() if body.source_key: + source_key = body.source_key + # One server-edit note per plugin per week. The watcher omits the week. + if source_key.startswith("watch:") and source_key.count(":") == 1: + source_key = f"{source_key}:{current_week()}" row = insert_sourced_bullet( section=body.section, body=body.body, - source_key=body.source_key, + source_key=source_key, ) if row is None: return {"duplicate": True, "source_key": body.source_key} @@ -392,6 +406,211 @@ def staff_read_preview(): return _preview_payload(row) +def _folder_payload(row: dict[str, Any], syncing: set[str]) -> dict[str, Any]: + rules = row.get("rules") if isinstance(row.get("rules"), list) else [] + unclassified = row.get("unclassified") if isinstance(row.get("unclassified"), list) else [] + return { + "name": row["name"], + "origin": row["origin"], + "repo": row.get("repo"), + "status": row["status"], + "dangerous": bool(row.get("dangerous")), + "reject_reason": row.get("reject_reason"), + "rules": rules, + "unclassified": unclassified, + "sync_task": row["name"] in syncing, + } + + +class FolderNameBody(BaseModel): + name: str + + @field_validator("name") + @classmethod + def _check_name(cls, value: str) -> str: + return clean_folder_name(value) + + +class FolderCatalogBody(BaseModel): + present: list[str] = Field(default_factory=list) + + @field_validator("present") + @classmethod + def _check_present(cls, value: list[str]) -> list[str]: + return [clean_folder_name(item) for item in value] + + +class FolderObservationBody(BaseModel): + status: Literal["pending", "active", "rejected"] + rules: list[str] | None = None + unclassified: list[str] = Field(default_factory=list) + reject_reason: str | None = None + dangerous: bool | None = None + + @field_validator("rules") + @classmethod + def _check_rules(cls, value: list[str] | None) -> list[str] | None: + if value is None: + return None + return [safe_rule(item) for item in value] + + @field_validator("reject_reason") + @classmethod + def _check_reason(cls, value: str | None) -> str | None: + if value is None: + return None + text = value.strip() + return text[:300] or None + + +class FolderTrackBody(BaseModel): + glob: str + + @field_validator("glob") + @classmethod + def _check_glob(cls, value: str) -> str: + return safe_rule(value) + + +class SyncTaskBody(BaseModel): + folder: str + + @field_validator("folder") + @classmethod + def _check_folder(cls, value: str) -> str: + return clean_folder_name(value) + + +@patchnotes_router.get("/staff/folders", dependencies=[Depends(_staff_guard)]) +def staff_list_folders(): + """Repo plugins, content packs, and folders staff added.""" + try: + migrate() + rows = list_folders() + syncing = set(list_sync_tasks(current_week())) + except (PatchnotesDBError, PatchnotesConfigError) as e: + logger.exception("staff_list_folders failed") + raise HTTPException(status_code=502, detail=_client_detail(e)) from e + return {"folders": [_folder_payload(row, syncing) for row in rows]} + + +@patchnotes_router.post("/staff/folders", dependencies=[Depends(_staff_guard)]) +def staff_add_folder(body: FolderNameBody): + """Ask the host watcher to verify a plugin folder on TFMCMain.""" + try: + migrate() + row = request_added_folder(body.name) + except (PatchnotesDBError, PatchnotesConfigError) as e: + logger.exception("staff_add_folder failed") + raise HTTPException(status_code=502, detail=_client_detail(e)) from e + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) from e + return _folder_payload(row, set()) + + +@patchnotes_router.delete("/staff/folders/{name}", dependencies=[Depends(_staff_guard)]) +def staff_remove_folder(name: str): + try: + folder_name = clean_folder_name(name) + migrate() + remove_added_folder(folder_name) + except BulletNotFound as e: + raise HTTPException(status_code=404, detail="Folder not found") from e + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) from e + except (PatchnotesDBError, PatchnotesConfigError) as e: + logger.exception("staff_remove_folder failed") + raise HTTPException(status_code=502, detail=_client_detail(e)) from e + return {"removed": folder_name} + + +@patchnotes_router.post("/staff/folders/catalog", dependencies=[Depends(_staff_guard)]) +def staff_ensure_catalog(body: FolderCatalogBody): + """Keep repo plugins and content packs that exist on TFMCMain.""" + present = set(body.present) + try: + migrate() + kept = [] + for entry in catalog_entries(): + if entry["name"] not in present: + continue + row = ensure_folder( + name=entry["name"], + origin=entry["origin"], + repo=entry["repo"], + dangerous=entry["dangerous"], + rules=entry["rules"], + ) + kept.append(row["name"]) + except (PatchnotesDBError, PatchnotesConfigError) as e: + logger.exception("staff_ensure_catalog failed") + raise HTTPException(status_code=502, detail=_client_detail(e)) from e + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) from e + return {"ensured": kept} + + +@patchnotes_router.post("/staff/folders/{name}/observation", dependencies=[Depends(_staff_guard)]) +def staff_observe_folder(name: str, body: FolderObservationBody): + try: + folder_name = clean_folder_name(name) + migrate() + row = observe_folder( + name=folder_name, + status=body.status, + rules=body.rules, + unclassified=body.unclassified, + reject_reason=body.reject_reason, + dangerous=body.dangerous, + ) + except BulletNotFound as e: + raise HTTPException(status_code=404, detail="Folder not found") from e + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) from e + except (PatchnotesDBError, PatchnotesConfigError) as e: + logger.exception("staff_observe_folder failed") + raise HTTPException(status_code=502, detail=_client_detail(e)) from e + return _folder_payload(row, set()) + + +@patchnotes_router.post("/staff/folders/{name}/track", dependencies=[Depends(_staff_guard)]) +def staff_track_rule(name: str, body: FolderTrackBody): + """Extend a folder's rules so a new kind of file can be watched.""" + try: + folder_name = clean_folder_name(name) + migrate() + row = add_folder_rule(folder_name, body.glob) + except BulletNotFound as e: + raise HTTPException(status_code=404, detail="Folder not found") from e + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) from e + except (PatchnotesDBError, PatchnotesConfigError) as e: + logger.exception("staff_track_rule failed") + raise HTTPException(status_code=502, detail=_client_detail(e)) from e + return _folder_payload(row, set()) + + +@patchnotes_router.post("/staff/sync-tasks", dependencies=[Depends(_staff_guard)]) +def staff_queue_sync_task(body: SyncTaskBody): + """Queue one Friday task to update GitHub from the main-server files.""" + try: + migrate() + row = get_folder(body.folder) + if row is None or row["origin"] != "repo" or not row.get("repo"): + raise HTTPException(status_code=404, detail="Folder is not a GitHub plugin") + created = queue_sync_task( + folder_name=body.folder, + repo=str(row["repo"]), + week=current_week(), + ) + except HTTPException: + raise + except (PatchnotesDBError, PatchnotesConfigError) as e: + logger.exception("staff_queue_sync_task failed") + raise HTTPException(status_code=502, detail=_client_detail(e)) from e + return {"folder": body.folder, "created": created} + + @patchnotes_router.get("/staff/queue", dependencies=[Depends(_staff_guard)]) def staff_queue(week: str | None = None): """Pending bullets, oldest first.""" diff --git a/backend/src/api/test_patchnotes_routes.py b/backend/src/api/test_patchnotes_routes.py index b9db85be..69634880 100644 --- a/backend/src/api/test_patchnotes_routes.py +++ b/backend/src/api/test_patchnotes_routes.py @@ -62,6 +62,11 @@ def test_staff_routes_require_auth(self) -> None: self.assertEqual(self.client.get("/patchnotes/staff/queue").status_code, 401) self.assertEqual(self.client.get("/patchnotes/staff/preview").status_code, 401) self.assertEqual(self.client.post("/patchnotes/staff/preview", json={}).status_code, 401) + self.assertEqual(self.client.get("/patchnotes/staff/folders").status_code, 401) + self.assertEqual( + self.client.post("/patchnotes/staff/folders", json={"name": "Essentials"}).status_code, + 401, + ) self.assertEqual( self.client.post(f"/patchnotes/staff/bullets/{_BULLET_ID}/approve").status_code, 401 ) @@ -427,3 +432,63 @@ def test_public_notes_do_not_read_the_preview(self, _notes) -> None: res = self.client.get("/patchnotes") self.assertEqual(res.status_code, 200) self.assertEqual(res.json()["weeks"], []) + + +class FolderRouteTest(unittest.TestCase): + def setUp(self) -> None: + self.client = TestClient(app) + patcher = mock.patch("src.api.patchnotes_routes.migrate") + self.addCleanup(patcher.stop) + patcher.start() + + def tearDown(self) -> None: + self.client.close() + + def test_a_path_is_not_a_folder_name(self) -> None: + res = self.client.post( + "/patchnotes/staff/folders", + headers=_HEADERS, + json={"name": "../TFMCDev01"}, + ) + self.assertEqual(res.status_code, 422) + + @mock.patch("src.api.patchnotes_routes.request_added_folder") + def test_add_folder_is_pending_until_the_watcher_checks_it(self, mock_add) -> None: + mock_add.return_value = { + "name": "Essentials", + "origin": "added", + "repo": None, + "status": "pending", + "dangerous": False, + "reject_reason": None, + "rules": [], + "unclassified": [], + } + res = self.client.post( + "/patchnotes/staff/folders", + headers=_HEADERS, + json={"name": "Essentials"}, + ) + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json()["status"], "pending") + self.assertEqual(res.json()["origin"], "added") + + @mock.patch("src.api.patchnotes_routes.current_week", return_value="2026-W39") + @mock.patch("src.api.patchnotes_routes.insert_sourced_bullet", return_value=_row()) + def test_a_watch_note_is_once_per_plugin_per_week(self, mock_insert, _week) -> None: + res = self.client.post( + "/patchnotes/staff/bullets", + headers=_HEADERS, + json={ + "section": "adjusted", + "body": "A dungeon was adjusted on the main server.", + "source_key": "watch:MythicDungeons", + }, + ) + self.assertEqual(res.status_code, 200) + self.assertNotIn("MythicDungeons", res.json()["body"]) + mock_insert.assert_called_once() + self.assertEqual( + mock_insert.call_args.kwargs["source_key"], + "watch:MythicDungeons:2026-W39", + ) diff --git a/backend/src/patchnotes/db.py b/backend/src/patchnotes/db.py index 6bf5391d..b24fd881 100644 --- a/backend/src/patchnotes/db.py +++ b/backend/src/patchnotes/db.py @@ -251,6 +251,37 @@ def migrate() -> None: ) """ ) + cur.execute( + """ + CREATE TABLE IF NOT EXISTS patchnote_folders ( + name TEXT PRIMARY KEY, + origin TEXT NOT NULL, + repo TEXT, + status TEXT NOT NULL, + dangerous BOOLEAN NOT NULL DEFAULT FALSE, + reject_reason TEXT, + rules JSONB NOT NULL DEFAULT '[]'::jsonb, + unclassified JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + verified_at TIMESTAMPTZ, + CONSTRAINT patchnote_folders_origin_chk + CHECK (origin IN ('repo', 'content', 'added')), + CONSTRAINT patchnote_folders_status_chk + CHECK (status IN ('pending', 'active', 'rejected')) + ) + """ + ) + cur.execute( + """ + CREATE TABLE IF NOT EXISTS patchnote_sync_tasks ( + folder_name TEXT NOT NULL, + week TEXT NOT NULL, + repo TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (folder_name, week) + ) + """ + ) finally: conn.close() _MIGRATED = True @@ -477,3 +508,197 @@ def load_preview() -> dict[str, Any] | None: return dict(row) if row else None finally: conn.close() + + +_FOLDER_COLUMNS = ( + "name, origin, repo, status, dangerous, reject_reason, rules, unclassified, created_at, verified_at" +) + + +def list_folders() -> list[dict[str, Any]]: + conn = _connect() + try: + with conn, conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + f""" + SELECT {_FOLDER_COLUMNS} + FROM patchnote_folders + ORDER BY origin, name + """ + ) + return [dict(row) for row in cur.fetchall()] + finally: + conn.close() + + +def get_folder(name: str) -> dict[str, Any] | None: + conn = _connect() + try: + with conn, conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + f"SELECT {_FOLDER_COLUMNS} FROM patchnote_folders WHERE name = %s", + (name,), + ) + row = cur.fetchone() + return dict(row) if row else None + finally: + conn.close() + + +def ensure_folder( + *, + name: str, + origin: str, + repo: str | None, + dangerous: bool, + rules: list[str] | None = None, +) -> dict[str, Any]: + """Insert a watched folder if it is missing. An existing row is left as staff set it.""" + if origin not in {"repo", "content", "added"}: + raise ValueError("origin is invalid") + conn = _connect() + try: + with conn, conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + f""" + INSERT INTO patchnote_folders + (name, origin, repo, status, dangerous, rules) + VALUES (%s, %s, %s, 'pending', %s, %s) + ON CONFLICT (name) DO NOTHING + RETURNING {_FOLDER_COLUMNS} + """, + (name, origin, repo, dangerous, psycopg2.extras.Json(rules or [])), + ) + row = cur.fetchone() + if row is not None: + return dict(row) + cur.execute( + f"SELECT {_FOLDER_COLUMNS} FROM patchnote_folders WHERE name = %s", + (name,), + ) + return dict(cur.fetchone()) + finally: + conn.close() + + +def request_added_folder(name: str) -> dict[str, Any]: + """Staff asked to watch a plugin folder. The host watcher accepts or rejects it.""" + existing = get_folder(name) + if existing is not None: + raise ValueError("That folder is already on the list") + return ensure_folder(name=name, origin="added", repo=None, dangerous=False, rules=[]) + + +def remove_added_folder(name: str) -> None: + row = get_folder(name) + if row is None: + raise BulletNotFound(name) + if row["origin"] != "added": + raise ValueError("GitHub plugins and content packs stay on the list") + conn = _connect() + try: + with conn, conn.cursor() as cur: + cur.execute("DELETE FROM patchnote_folders WHERE name = %s AND origin = 'added'", (name,)) + finally: + conn.close() + + +def observe_folder( + *, + name: str, + status: str, + rules: list[str] | None, + unclassified: list[str], + reject_reason: str | None, + dangerous: bool | None = None, +) -> dict[str, Any]: + """Record what the host watcher found. Rules already stored are kept when omitted.""" + if status not in {"pending", "active", "rejected"}: + raise ValueError("status is invalid") + conn = _connect() + try: + with conn, conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + f""" + UPDATE patchnote_folders + SET status = %s, + reject_reason = %s, + unclassified = %s, + rules = COALESCE(%s, rules), + dangerous = COALESCE(%s, dangerous), + verified_at = now() + WHERE name = %s + RETURNING {_FOLDER_COLUMNS} + """, + ( + status, + reject_reason, + psycopg2.extras.Json(unclassified), + psycopg2.extras.Json(rules) if rules is not None else None, + dangerous, + name, + ), + ) + row = cur.fetchone() + if row is None: + raise BulletNotFound(name) + return dict(row) + finally: + conn.close() + + +def add_folder_rule(name: str, pattern: str) -> dict[str, Any]: + row = get_folder(name) + if row is None: + raise BulletNotFound(name) + if row["status"] != "active": + raise ValueError("That folder is not being watched yet") + rules = list(row["rules"] or []) + if pattern not in rules: + rules.append(pattern) + return observe_folder( + name=name, + status="active", + rules=rules, + unclassified=list(row["unclassified"] or []), + reject_reason=None, + ) + + +def queue_sync_task(*, folder_name: str, repo: str, week: str) -> bool: + """One GitHub update task per repo plugin per week. True when this call created it.""" + week_key = parse_week(week) + conn = _connect() + try: + with conn, conn.cursor() as cur: + cur.execute( + """ + INSERT INTO patchnote_sync_tasks (folder_name, week, repo) + VALUES (%s, %s, %s) + ON CONFLICT (folder_name, week) DO NOTHING + RETURNING folder_name + """, + (folder_name, week_key, repo), + ) + return cur.fetchone() is not None + finally: + conn.close() + + +def list_sync_tasks(week: str) -> list[str]: + week_key = parse_week(week) + conn = _connect() + try: + with conn, conn.cursor() as cur: + cur.execute( + """ + SELECT folder_name + FROM patchnote_sync_tasks + WHERE week = %s + ORDER BY folder_name + """, + (week_key,), + ) + return [row[0] for row in cur.fetchall()] + finally: + conn.close() diff --git a/backend/src/patchnotes/folders.py b/backend/src/patchnotes/folders.py new file mode 100644 index 00000000..3bb77bb2 --- /dev/null +++ b/backend/src/patchnotes/folders.py @@ -0,0 +1,395 @@ +"""Which plugin files on TFMCMain are worth a patch note. + +Rules name kinds of files (`vehicles/*.yml`), not the files that existed when +watching started. A later file that matches is tracked. Player saves, databases, +logs, and lore files never match, even if a rule would otherwise include them. + +MythicDungeons is dangerous: a change can be noted, but the note must not say +which dungeon changed or what it does. +""" + +from __future__ import annotations + +import fnmatch +import hashlib +import re +from dataclasses import dataclass, field +from pathlib import Path + +from .summarize import player_text + +_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,60}$") +_UUID = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + re.IGNORECASE, +) +_LORE_PART = re.compile(r"lore", re.IGNORECASE) +_SECRET_NAME = re.compile(r"(password|secret|token|apikey|api_key|keystore)", re.IGNORECASE) + +# Directory names that are runtime data at any depth. +HARD_DIRS = frozenset( + { + "userdata", + "playerdata", + "player-data", + "globalplayerdata", + "players", + "logs", + "log", + "cache", + "backups", + "backup", + "libs", + "plugindata", + "region", + "data", + ".codex-backups", + "tmp", + "temp", + } +) +HARD_SUFFIXES = ( + ".db", + ".db-wal", + ".db-shm", + ".jar", + ".log", + ".mca", + ".dat", + ".zip", + ".lock", +) +SKIP_FILE_NAMES = frozenset({"permissions.yml", "plugin.yml", "session.lock"}) + +# Plugin folder on TFMCMain -> GitHub repo name. Aliases cover folders whose +# names are not the repository name. +REPO_FOLDERS: dict[str, str] = { + "AACommandsFiller": "AACommandsFiller", + "AdvancedCrafting": "AdvancedCrafting", + "Archaeo": "Archaeo", + "ArmourShop": "ArmourShop", + "BarterShops": "BarterShops", + "BirdMessenger": "BirdMessenger", + "Cooking": "Cooking", + "CoreProtect": "CoreProtect", + "DenarEconomy": "DenarEconomy", + "Dowsing": "Dowsing", + "DrinkBuilder": "DrinkBuilder", + "Games": "Games", + "Gathering": "Gathering", + "GemInfusion": "GemInfusion", + "GunsAndGadgets": "GunsAndGadgets", + "Infestations": "Infestations", + "InteractibleFurniture": "InteractibleFurniture", + "Magic": "Magic", + "MarketBlock": "MarketBlock", + "MusicalInstruments": "MusicalInstruments", + "RPCharacters": "RPCharacters", + "Recycler": "Recycler", + "Research": "Research", + "SimpleFactions": "SimpleFactions", + "TFMCCore": "TFMCCore", + "TFMCWeb": "TFMCWeb", + "TLibs": "TLibs", + "Thievery": "Thievery", + "TrialRooms": "TrialRooms", + "VFBuilders": "VFBuilders", + "VehicleFramework": "VehicleFramework", + "Woodworking": "Woodworking", + "WorldBorder": "WorldBorder", + "activity": "ActivityTF", + "geiger_counter": "GeigerCounters", + "surgery": "Surgery", +} + +# Content packs are not GitHub repos. Rules are kinds of files, so a new +# items.yml or vehicle file is included. Dungeons stay opaque. +CONTENT_FOLDERS: dict[str, dict] = { + "MMOItems": { + "dangerous": False, + "rules": ["*.yml", "item/*.yml", "crafting-stations/*.yml", "modifiers/**/*.yml"], + }, + "MMOCore": { + "dangerous": False, + "rules": [ + "*.yml", + "attributes/**/*.yml", + "classes/**/*.yml", + "drop-tables/**/*.yml", + "exp-curves/**/*.yml", + "exp-tables/**/*.yml", + "gui/**/*.yml", + "professions/**/*.yml", + "quests/**/*.yml", + "skill-trees/**/*.yml", + "waypoints/**/*.yml", + "loot-chests/**/*.yml", + ], + }, + "MythicMobs": { + "dangerous": False, + "rules": [ + "*.yml", + "config/**/*.yml", + "items/**/*.yml", + "droptables/**/*.yml", + "mobs/**/*.yml", + "skills/**/*.yml", + "spawners/**/*.yml", + "randomspawns/**/*.yml", + "menus/**/*.yml", + "dialogs/**/*.yml", + "cutscenes/**/*.yml", + "packs/**/*.yml", + ], + }, + "MythicDungeons": { + "dangerous": True, + "rules": ["*.yml", "maps/*/config.yml", "maps/*/functions.yml"], + }, +} + + +@dataclass(frozen=True) +class FolderChange: + added: tuple[str, ...] = () + edited: tuple[str, ...] = () + removed: tuple[str, ...] = () + + @property + def any(self) -> bool: + return bool(self.added or self.edited or self.removed) + + +@dataclass +class Classified: + rules: list[str] = field(default_factory=list) + unclassified: list[str] = field(default_factory=list) + + +def clean_folder_name(name: str) -> str: + """A plugin directory name. Slashes and parent traversal are rejected.""" + text = name.strip() + if not _NAME.fullmatch(text) or text.startswith("."): + raise ValueError("Folder name is not a plugin directory") + return text + + +def assert_main_plugins_dir(path: Path) -> Path: + """The plugins directory on TFMCMain. The dev server is refused.""" + resolved = path.resolve() + parts = resolved.parts + if any(part.lower().startswith("tfmcdev") for part in parts): + raise ValueError("The dev server is not watched") + if "TFMCMain01" not in parts or resolved.name != "plugins": + raise ValueError("Only TFMCMain plugin folders are watched") + if not resolved.is_dir(): + raise ValueError("Plugins directory is not available") + return resolved + + +def plugin_folder(plugins: Path, name: str) -> Path | None: + """The plugin directory, or None when that folder is not on TFMCMain.""" + folder_name = clean_folder_name(name) + root = assert_main_plugins_dir(plugins) + folder = (root / folder_name).resolve() + if folder.parent != root or not folder.is_dir(): + return None + return folder + + +def rule_matches(pattern: str, relative: str) -> bool: + """True when a relative path matches a rule. `*` does not cross `/`.""" + return _match_parts(pattern.split("/"), relative.split("/")) + + +def _match_parts(pattern: list[str], parts: list[str]) -> bool: + if not pattern: + return not parts + head, rest = pattern[0], pattern[1:] + if head == "**": + if not rest: + return True + return any(_match_parts(rest, parts[index:]) for index in range(len(parts) + 1)) + if not parts: + return False + if head != "*" and not fnmatch.fnmatchcase(parts[0], head): + return False + return _match_parts(rest, parts[1:]) + + +def skip_directory(name: str) -> bool: + """Runtime directories the walker does not open.""" + return name in HARD_DIRS or name.startswith(".") or bool(_LORE_PART.search(name)) + + +def excluded_relative(relative: str) -> bool: + """Player data, databases, logs, lore, and secrets are never tracked.""" + path = Path(relative) + if path.is_absolute() or ".." in path.parts: + return True + if any(part in HARD_DIRS or part.startswith(".") for part in path.parts): + return True + if any(_LORE_PART.search(part) for part in path.parts): + return True + name = path.name + if name in SKIP_FILE_NAMES or name.lower().endswith(HARD_SUFFIXES): + return True + if _UUID.fullmatch(path.stem) or _SECRET_NAME.search(name): + return True + return False + + +def safe_rule(pattern: str) -> str: + """A tracking rule staff can add. Hard-excluded pieces are rejected.""" + text = pattern.strip().replace("\\", "/").lstrip("/") + if not text or text.startswith("/") or ".." in text.split("/"): + raise ValueError("That rule is not inside the plugin folder") + if not text.endswith((".yml", ".yaml")): + raise ValueError("Only yaml files can be tracked") + sample = text.replace("**", "sample").replace("*", "sample") + if excluded_relative(sample): + raise ValueError("That rule would include data that is not tracked") + return text + + +def covered(relative: str, rules: list[str]) -> bool: + if excluded_relative(relative): + return False + return any(rule_matches(rule, relative) for rule in rules) + + +def classify_tree(files: dict[str, bytes]) -> Classified: + """Rules for yaml worth tracking, plus top-level dirs that were left out.""" + rules: list[str] = [] + root_yaml = False + tops: set[str] = set() + seen_tops: set[str] = set() + unclassified: set[str] = set() + for relative in sorted(files): + path = Path(relative) + if not path.parts or path.parts[0].startswith("."): + continue + top = path.parts[0] if len(path.parts) > 1 else "" + if top: + seen_tops.add(top) + if excluded_relative(relative): + continue + if path.suffix.lower() not in {".yml", ".yaml"}: + if top and top not in HARD_DIRS: + unclassified.add(top) + continue + if len(path.parts) == 1: + root_yaml = True + else: + tops.add(path.parts[0]) + if root_yaml: + rules.append("*.yml") + for top in sorted(tops): + rules.append(f"{top}/**/*.yml") + left = sorted(name for name in unclassified if name not in tops and name not in HARD_DIRS) + return Classified(rules=rules, unclassified=left[:20]) + + +def tracked_files(files: dict[str, bytes], rules: list[str]) -> dict[str, str]: + """Hash of each file a rule includes. Excluded paths are omitted.""" + found: dict[str, str] = {} + for relative, payload in files.items(): + if covered(relative, rules): + found[relative] = hashlib.sha256(payload).hexdigest()[:16] + return found + + +def look_truncated(previous: dict[str, str], current: dict[str, str]) -> bool: + """A partial write should not look like every file was deleted.""" + before = len(previous) + return before >= 5 and len(current) * 2 < before + + +def diff_tracked( + previous: dict[str, str] | None, + current: dict[str, str], + *, + rules_changed: bool, +) -> tuple[dict[str, str], FolderChange | None, bool]: + """Return the snapshot to store, a note-worthy change, and whether this baselines. + + The first sight of a folder stores hashes and drafts nothing. A new rule + adopts files that are already there without calling them new. + """ + if previous is None: + return current, None, True + if look_truncated(previous, current): + return previous, None, False + added = tuple(sorted(path for path in current if path not in previous)) + removed = tuple(sorted(path for path in previous if path not in current)) + edited = tuple( + sorted(path for path in current if path in previous and current[path] != previous[path]) + ) + if rules_changed: + # Files already on disk are adopted by the new rule. They are not news. + added = () + removed = () + change = FolderChange(added=added, edited=edited, removed=removed) + if not change.any: + return current, None, False + return current, change, False + + +def note_text(folder: str, change: FolderChange, *, dangerous: bool) -> tuple[str, str] | None: + """A pending line, or None when the line would reveal something it should not.""" + if dangerous: + body = "A dungeon was adjusted on the main server." + section = "adjusted" + elif change.added and not change.edited and not change.removed: + body = f"New files were added under {folder} on the main server." + section = "new" + elif change.removed and not change.added and not change.edited: + body = f"Files were removed under {folder} on the main server." + section = "adjusted" + else: + body = f"{folder} was edited on the main server." + section = "adjusted" + if player_text(body) is None: + return None + for secret in (*change.added, *change.edited, *change.removed): + if secret and secret in body: + return None + return section, body + + +def catalog_entries() -> list[dict]: + """Repo plugins and the four content packs the watcher always keeps.""" + entries = [ + { + "name": name, + "origin": "repo", + "repo": repo, + "dangerous": False, + "rules": [], + } + for name, repo in sorted(REPO_FOLDERS.items()) + ] + for name, spec in CONTENT_FOLDERS.items(): + entries.append( + { + "name": name, + "origin": "content", + "repo": None, + "dangerous": bool(spec["dangerous"]), + "rules": list(spec["rules"]), + } + ) + return entries + + +def preset_rules(name: str) -> list[str] | None: + spec = CONTENT_FOLDERS.get(name) + if spec is None: + return None + return list(spec["rules"]) + + +def is_dangerous(name: str) -> bool: + spec = CONTENT_FOLDERS.get(name) + return bool(spec and spec["dangerous"]) diff --git a/backend/src/patchnotes/test_folders.py b/backend/src/patchnotes/test_folders.py new file mode 100644 index 00000000..235c8b84 --- /dev/null +++ b/backend/src/patchnotes/test_folders.py @@ -0,0 +1,142 @@ +"""Rules for TFMCMain plugin folders.""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +_BACKEND_SRC = Path(__file__).resolve().parents[1] +if str(_BACKEND_SRC) not in sys.path: + sys.path.insert(0, str(_BACKEND_SRC)) + +from patchnotes.folders import ( # noqa: E402 + classify_tree, + diff_tracked, + note_text, + plugin_folder, + rule_matches, + safe_rule, + tracked_files, +) + + +class RuleMatchTest(unittest.TestCase): + def test_a_new_file_in_a_tracked_directory_matches(self) -> None: + self.assertTrue(rule_matches("vehicles/*.yml", "vehicles/airship.yml")) + self.assertTrue(rule_matches("item/*.yml", "item/sword.yml")) + self.assertTrue(rule_matches("maps/*/functions.yml", "maps/newdungeon/functions.yml")) + self.assertFalse(rule_matches("maps/*/functions.yml", "maps/newdungeon/players/save.yml")) + self.assertFalse(rule_matches("maps/*/config.yml", "maps/newdungeon/paper-world.yml")) + self.assertFalse(rule_matches("*.yml", "item/sword.yml")) + + def test_userdata_and_lore_are_never_tracked(self) -> None: + rules = ["**/*.yml", "userdata/**/*.yml", "*.yml"] + files = { + "config.yml": b"spawn: 1", + "userdata/player.yml": b"home: 1", + "item/sword.yml": b"name: sword", + "language/lore-formats/item.yml": b"lore", + "maps/dungeon1/players/11111111-1111-1111-1111-111111111111.yml": b"x", + "data/vehicles.db": b"db", + } + tracked = tracked_files(files, rules) + self.assertEqual(set(tracked), {"config.yml", "item/sword.yml"}) + + def test_safe_rule_rejects_player_data(self) -> None: + with self.assertRaises(ValueError): + safe_rule("userdata/**/*.yml") + with self.assertRaises(ValueError): + safe_rule("../config.yml") + self.assertEqual(safe_rule("vehicles/*.yml"), "vehicles/*.yml") + + +class ClassifyTest(unittest.TestCase): + def test_yaml_directories_are_rules_and_json_dumps_are_unclassified(self) -> None: + classified = classify_tree( + { + "config.yml": b"a", + "vehicles/sloop.yml": b"b", + "templates/notes.json": b"{}", + "userdata/player.yml": b"c", + "data/vehicles.db": b"d", + } + ) + self.assertIn("*.yml", classified.rules) + self.assertIn("vehicles/**/*.yml", classified.rules) + self.assertNotIn("userdata/**/*.yml", classified.rules) + self.assertIn("templates", classified.unclassified) + self.assertNotIn("userdata", classified.unclassified) + self.assertNotIn("data", classified.unclassified) + + +class DiffTest(unittest.TestCase): + def test_first_scan_baselines(self) -> None: + current = {"vehicles/sloop.yml": "aaa"} + snapshot, change, baselined = diff_tracked(None, current, rules_changed=False) + self.assertTrue(baselined) + self.assertIsNone(change) + self.assertEqual(snapshot, current) + + def test_a_new_matching_file_is_a_change(self) -> None: + previous = {"vehicles/sloop.yml": "aaa"} + current = {"vehicles/sloop.yml": "aaa", "vehicles/airship.yml": "bbb"} + _snapshot, change, baselined = diff_tracked(previous, current, rules_changed=False) + self.assertFalse(baselined) + self.assertIsNotNone(change) + assert change is not None + self.assertEqual(change.added, ("vehicles/airship.yml",)) + + def test_a_partial_directory_is_not_a_wipe(self) -> None: + previous = {f"item/{index}.yml": "a" for index in range(6)} + current = {"item/0.yml": "a"} + snapshot, change, _baselined = diff_tracked(previous, current, rules_changed=False) + self.assertIsNone(change) + self.assertEqual(snapshot, previous) + + def test_dungeon_notes_do_not_name_the_dungeon(self) -> None: + from patchnotes.folders import FolderChange + + change = FolderChange( + edited=("maps/secretmaze/functions.yml", "maps/secretmaze/config.yml"), + ) + drafted = note_text("MythicDungeons", change, dangerous=True) + self.assertIsNotNone(drafted) + assert drafted is not None + _section, body = drafted + self.assertEqual(body, "A dungeon was adjusted on the main server.") + self.assertNotIn("secretmaze", body) + self.assertNotIn("functions", body) + self.assertNotIn("MythicDungeons", body) + + def test_an_ordinary_edit_names_the_plugin_only(self) -> None: + from patchnotes.folders import FolderChange + + drafted = note_text( + "MMOItems", + FolderChange(edited=("item/sword.yml",)), + dangerous=False, + ) + assert drafted is not None + self.assertEqual(drafted[1], "MMOItems was edited on the main server.") + self.assertNotIn("sword", drafted[1]) + + +class PathTest(unittest.TestCase): + def test_only_a_tfmcmain_plugin_folder_is_accepted(self) -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "TFMCMain01" / "Minecraft" / "plugins" + (root / "Cooking").mkdir(parents=True) + found = plugin_folder(root, "Cooking") + self.assertIsNotNone(found) + self.assertIsNone(plugin_folder(root, "Missing")) + dev = Path(tmp) / "TFMCDev01" / "Minecraft" / "plugins" + (dev / "Cooking").mkdir(parents=True) + with self.assertRaises(ValueError): + plugin_folder(dev, "Cooking") + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/src/patchnotes/watch_plugins.py b/backend/src/patchnotes/watch_plugins.py new file mode 100644 index 00000000..3753aff6 --- /dev/null +++ b/backend/src/patchnotes/watch_plugins.py @@ -0,0 +1,252 @@ +"""Watch TFMCMain plugin folders and queue pending patch notes. + +Repo plugins and the content packs are kept automatically. Staff can add +another folder. The first scan of a folder only stores hashes. A later edit +becomes one pending line. MythicDungeons notes do not name the dungeon. + + python -m patchnotes.watch_plugins --once \\ + --plugins-dir /path/TFMCMain01/Minecraft/plugins \\ + --state /var/lib/tfmc/plugin-patchnotes.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import time +import urllib.error +import urllib.request +from pathlib import Path + +from .folders import ( + FolderChange, + assert_main_plugins_dir, + classify_tree, + diff_tracked, + excluded_relative, + is_dangerous, + note_text, + plugin_folder, + preset_rules, + skip_directory, + tracked_files, +) + +_MAX_READ = 2_000_000 + + +def read_plugin_files(folder: Path) -> dict[str, bytes]: + """File bytes under one plugin. Excluded and huge files are not read.""" + files: dict[str, bytes] = {} + root = folder.resolve() + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [name for name in dirnames if not skip_directory(name)] + for filename in filenames: + path = Path(dirpath) / filename + relative = path.relative_to(root).as_posix() + if excluded_relative(relative): + files[relative] = b"" + continue + try: + if path.stat().st_size > _MAX_READ: + files[relative] = b"" + continue + files[relative] = path.read_bytes() + except OSError: + continue + return files + + +def load_state(path: Path) -> dict: + if not path.is_file(): + return {"folders": {}} + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("State file must be a JSON object") + folders = data.get("folders") + if not isinstance(folders, dict): + folders = {} + return {"folders": folders} + + +def save_state(path: Path, state: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(state, sort_keys=True), encoding="utf-8") + temporary.replace(path) + + +def _request(api_base: str, staff_key: str, method: str, path: str, payload: dict | None) -> dict: + data = None if payload is None else json.dumps(payload).encode() + headers = {"User-Agent": "TFMC-PatchNotes/1.0", "X-Staff-Key": staff_key} + if data is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + f"{api_base.rstrip('/')}{path}", + data=data, + headers=headers, + method=method, + ) + try: + with urllib.request.urlopen(request, timeout=20) as response: + raw = response.read().decode() or "{}" + except urllib.error.HTTPError as exc: + detail = exc.read(300).decode(errors="replace") + raise RuntimeError(f"Patch note API returned HTTP {exc.code}: {detail}") from exc + body = json.loads(raw) + if not isinstance(body, dict): + raise RuntimeError("Patch note API returned an unexpected body") + return body + + +def _present_names(plugins: Path) -> list[str]: + return sorted(path.name for path in plugins.iterdir() if path.is_dir() and not path.name.startswith(".")) + + +def _verify(plugins: Path, folder: dict) -> dict: + name = str(folder["name"]) + found = plugin_folder(plugins, name) + if found is None: + return { + "status": "rejected", + "rules": None, + "unclassified": [], + "reject_reason": "No folder with that name on TFMCMain", + "dangerous": True if is_dangerous(name) else None, + } + files = read_plugin_files(found) + preset = preset_rules(name) + if preset is not None: + rules = preset + unclassified = classify_tree(files).unclassified + elif folder.get("rules"): + rules = None + unclassified = classify_tree(files).unclassified + else: + classified = classify_tree(files) + rules = classified.rules + unclassified = classified.unclassified + if rules is not None and not rules: + return { + "status": "rejected", + "rules": [], + "unclassified": unclassified, + "reject_reason": "Nothing in that folder is safe to track", + "dangerous": True if is_dangerous(name) else None, + } + return { + "status": "active", + "rules": rules, + "unclassified": unclassified, + "reject_reason": None, + "dangerous": True if is_dangerous(name) else None, + } + + +def _scan_active(plugins: Path, folder: dict, previous: dict | None) -> tuple[dict, FolderChange | None, bool]: + name = str(folder["name"]) + found = plugin_folder(plugins, name) + rules = list(folder.get("rules") or []) + if found is None or not rules: + stored = previous or {"files": {}, "rules": rules} + return stored, None, False + current = tracked_files(read_plugin_files(found), rules) + prev_files = None if previous is None else previous.get("files") + if prev_files is not None and not isinstance(prev_files, dict): + prev_files = None + rules_changed = previous is not None and list(previous.get("rules") or []) != rules + snapshot, change, baselined = diff_tracked( + prev_files if isinstance(prev_files, dict) else None, + current, + rules_changed=rules_changed, + ) + return {"files": snapshot, "rules": rules}, change, baselined + + +def scan(plugins: Path, state_path: Path, api_base: str, staff_key: str) -> dict: + root = assert_main_plugins_dir(plugins) + _request(api_base, staff_key, "POST", "/patchnotes/staff/folders/catalog", {"present": _present_names(root)}) + listed = _request(api_base, staff_key, "GET", "/patchnotes/staff/folders", None) + folders = listed.get("folders") if isinstance(listed.get("folders"), list) else [] + state = load_state(state_path) + stored = state["folders"] + baselined = 0 + noted = 0 + for folder in folders: + if not isinstance(folder, dict) or not folder.get("name"): + continue + name = str(folder["name"]) + if folder.get("status") == "rejected": + continue + if folder.get("status") != "active" or not folder.get("rules"): + observation = _verify(root, folder) + updated = _request( + api_base, + staff_key, + "POST", + f"/patchnotes/staff/folders/{name}/observation", + observation, + ) + folder = updated + if folder.get("status") != "active": + continue + previous = stored.get(name) if isinstance(stored.get(name), dict) else None + snapshot, change, did_baseline = _scan_active(root, folder, previous) + stored[name] = snapshot + if did_baseline: + baselined += 1 + continue + if change is None: + continue + drafted = note_text(name, change, dangerous=bool(folder.get("dangerous"))) + if drafted is None: + continue + section, body = drafted + week_key = f"watch:{name}" + _request( + api_base, + staff_key, + "POST", + "/patchnotes/staff/bullets", + {"section": section, "body": body, "source_key": week_key[:200]}, + ) + noted += 1 + if folder.get("origin") == "repo" and folder.get("repo"): + _request( + api_base, + staff_key, + "POST", + "/patchnotes/staff/sync-tasks", + {"folder": name}, + ) + save_state(state_path, state) + return {"baselined": baselined, "noted": noted} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Queue TFMCMain plugin edits as patch notes") + parser.add_argument("--plugins-dir", default=os.environ.get("PATCHNOTES_PLUGINS_DIR", "")) + parser.add_argument("--state", default=os.environ.get("PATCHNOTES_PLUGINS_STATE", "")) + parser.add_argument("--api-base", default=os.environ.get("API_BASE_URL", "http://127.0.0.1:8000")) + parser.add_argument("--staff-key", default=os.environ.get("STAFF_KEY", "")) + parser.add_argument("--once", action="store_true") + parser.add_argument("--interval", type=int, default=0) + args = parser.parse_args() + if not args.plugins_dir or not args.state or not args.staff_key: + parser.error("--plugins-dir, --state, and --staff-key are required") + plugins = Path(args.plugins_dir) + state_path = Path(args.state) + + def run() -> None: + result = scan(plugins, state_path, args.api_base, args.staff_key) + print(json.dumps(result), flush=True) + + run() + while args.interval > 0 and not args.once: + time.sleep(args.interval) + run() + + +if __name__ == "__main__": + main() diff --git a/scripts/patchnotes-watch.sh b/scripts/patchnotes-watch.sh new file mode 100755 index 00000000..eb1d322c --- /dev/null +++ b/scripts/patchnotes-watch.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Scan TFMCMain plugin folders once. The staff key stays in the API container. +set -euo pipefail +cd /home/tfmc/ProvinceSystem/backend +export PYTHONPATH=src +export API_BASE_URL=http://127.0.0.1:8000 +export PATCHNOTES_PLUGINS_DIR=/home/amp/.ampdata/instances/TFMCMain01/Minecraft/plugins +export PATCHNOTES_PLUGINS_STATE=/var/lib/tfmc/plugin-patchnotes.json +export STAFF_KEY +STAFF_KEY="$(docker exec provincesystem-backend-1 printenv STAFF_KEY)" +exec python3 -m patchnotes.watch_plugins --once