diff --git a/README.md b/README.md index 0e9c07a..37a96da 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ PYTHONPATH=scripts python3 -m czdev --help | `czdev logout` | Remove the stored GitHub credentials. | | `czdev bump [--deb PATH]` | Print the next patch version for a package (reads the version from the `.deb`). Defaults to `./build/*.deb`. | | `czdev publish [--deb PATH]` | Validate the `.deb` and open a publish PR against the `packages` repo. Defaults to `./build/*.deb`. | +| `czdev publish --meta-only` | Update only the store listing (the `store` section of `app-builder.json`, screenshots, icon) of an already-published package, without a new `.deb` or version. Merges automatically once ownership and the metadata policy pass. | | `czdev unpublish NAME --version V [--arch arm64]` | Open a PR that removes a published package version. | ### Ownership model diff --git a/scripts/czdev/__main__.py b/scripts/czdev/__main__.py index 7216ce6..f5db14f 100644 --- a/scripts/czdev/__main__.py +++ b/scripts/czdev/__main__.py @@ -39,6 +39,8 @@ def main(): # publish pub_parser = subparsers.add_parser("publish", help="Publish a .deb package to the CardputerZero app store.") pub_parser.add_argument("--deb", default=None, help="Path to .deb file. If omitted, searches ./build/*.deb") + pub_parser.add_argument("--meta-only", action="store_true", + help="Only update the store listing (app-builder.json store section, screenshots, icon); no new .deb.") # unpublish unpub_parser = subparsers.add_parser("unpublish", help="Create a PR to remove a published package.") @@ -66,8 +68,12 @@ def main(): from .bump import run run(deb=args.deb) elif args.command == "publish": - from .publish import run - run(deb=args.deb) + if args.meta_only: + from .publish import run_meta_only + run_meta_only() + else: + from .publish import run + run(deb=args.deb) elif args.command == "unpublish": from .unpublish import run run(package=args.package, version=args.version, arch=args.arch) diff --git a/scripts/czdev/bump.py b/scripts/czdev/bump.py index ad13ff4..0ded1f8 100644 --- a/scripts/czdev/bump.py +++ b/scripts/czdev/bump.py @@ -1,12 +1,13 @@ """Show next version (patch bump) for a package — mirrors the Rust bump module.""" -import re import subprocess import sys import urllib.request from pathlib import Path from typing import Optional +from .versions import compare_versions + PACKAGES_INDEX_URL = "https://cardputerzero.github.io/packages/dists/stable/main/binary-arm64/Packages" @@ -101,14 +102,3 @@ def bump_patch(version: str) -> str: except ValueError: patch = 0 return f"{parts[0]}.{parts[1]}.{patch + 1}" - - -def compare_versions(a: str, b: str) -> int: - def parse(v): - return [int(x) for x in re.split(r'[.\-~]', v) if x.isdigit()] - pa, pb = parse(a), parse(b) - if pa > pb: - return 1 - elif pa < pb: - return -1 - return 0 diff --git a/scripts/czdev/publish.py b/scripts/czdev/publish.py index 3fd388d..29db5a8 100644 --- a/scripts/czdev/publish.py +++ b/scripts/czdev/publish.py @@ -3,12 +3,12 @@ import hashlib import json import os -import re import shutil import subprocess import sys import tempfile import time +import urllib.error import urllib.request from datetime import datetime, timezone from pathlib import Path @@ -16,6 +16,7 @@ from . import auth from .github_client import GitHubClient, Permission +from .versions import asset_safe, compare_versions, ref_safe TARGET_OWNER = "CardputerZero" TARGET_REPO = "packages" @@ -95,35 +96,19 @@ def run(deb: Optional[str] = None): precheck_store_meta(meta_out, app_dir, store_meta, meta["package"], deb_maintainer=meta.get("maintainer", "")) - # Determine target: direct push or fork - perm = gh.check_permission(TARGET_OWNER, TARGET_REPO) - if perm >= Permission.WRITE: - push_owner = TARGET_OWNER - push_repo = TARGET_REPO - pr_head = None - else: - print(f"You don't have write access to {TARGET_OWNER}/{TARGET_REPO}.") - print(" → Forking to your account... ", end="", flush=True) - fork_name = gh.fork_repo(TARGET_OWNER, TARGET_REPO) - print(f"done ({fork_name})") - parts = fork_name.split("/") - push_owner = parts[0] - push_repo = parts[1] - branch = branch_name(meta) - pr_head = f"{user.login}:{branch}" - - # Integrity + canonical asset name. GitHub sanitizes '~' to '.' in release - # asset names, so the on-release/manifest filename may differ from the - # Debian version string (which keeps '~'). + push_owner, push_repo, pr_head_owner = push_target(gh, user) + + # Integrity + canonical asset name. GitHub sanitizes '~' and ':' to '.' in + # release asset names, so the on-release/manifest filename may differ from + # the Debian version string (which keeps them). file_bytes = Path(deb_path).read_bytes() sha256_hash = hashlib.sha256(file_bytes).hexdigest() file_size = len(file_bytes) deb_name = f"{meta['package']}_{meta['version']}_{meta['architecture']}.deb" - asset_name = deb_name.replace("~", ".") + asset_name = asset_safe(deb_name) manifest_name = f"{asset_name}.release.json" manifest_path_in_repo = f"pool/main/{meta['package']}/{manifest_name}" branch = branch_name(meta) - remote_url = f"git@github.com:{push_owner}/{push_repo}.git" # 1) Upload the .deb to a buffer Release on the push target. For third-party # contributors this is their own fork — it uses THEIR free Release storage @@ -151,27 +136,123 @@ def run(deb: Optional[str] = None): # 2) Commit only metadata (meta.json, screenshots, icon, manifest) to a PR # branch — no .deb, no LFS. + push_pool_branch(gh, push_owner, push_repo, branch, meta["package"], app_dir, store_meta, meta_out, + extra={manifest_name: json.dumps(manifest, indent=2) + "\n"}, + message=f"publish: {meta['package']} {meta['version']} ({meta['architecture']})") + + pr_body = ( + f"## Package: `{meta['package']}`\n\n" + f"| Field | Value |\n" + f"|-------|-------|\n" + f"| Version | {meta['version']} |\n" + f"| Architecture | {meta['architecture']} |\n" + f"| Maintainer | {meta['maintainer']} |\n" + f"| Size | {size_mb:.1f} MB |\n" + f"| SHA-256 | `{sha256_hash}` |\n" + f"| Manifest | `{manifest_path_in_repo}` |\n" + f"| Binary | [{asset_name}]({download_url}) |\n\n" + f"Submitted via `czdev publish`. The `.deb` is hosted on the contributor's " + f"buffer release; CI verifies the sha256 and, on merge, promotes it into the " + f"official `apt-pool` release." + ) + open_pr(gh, pr_head_owner, branch, f"publish: {meta['package']} {meta['version']}", pr_body) + + +def run_meta_only(): + """Update the store listing (meta.json, screenshots, icon) without a new + .deb. The version and binary stay as published; CI checks ownership and the + metadata policy exactly as for a release, and merges it automatically.""" + check_git_installed() + + raw, app_dir = load_app_builder(Path.cwd()) + pkg = raw.get("package_name", "") + if not is_valid_package_name(pkg): + print(f"ERROR: app-builder.json has no valid package_name (got '{pkg}')", file=sys.stderr) + sys.exit(1) + store_meta = store_meta_from(raw) + print(f"Package: {pkg} (store listing only, no new .deb)") + print() + + if not fetch_url(f"{RAW_BASE}/pool/main/{pkg}/meta.json"): + print(f"ERROR: '{pkg}' is not published yet. Publish a .deb first with `czdev publish`.", file=sys.stderr) + sys.exit(1) + + gh = GitHubClient(auth.load_token()) + user = gh.get_user() + + print("Preflight checks:") + meta_out = build_merged_meta(store_meta, {"package": pkg, "maintainer": ""}, user) + precheck_store_meta(meta_out, app_dir, store_meta, pkg) + + push_owner, push_repo, pr_head_owner = push_target(gh, user) + branch = f"metadata/{pkg}-{int(time.time())}" + push_pool_branch(gh, push_owner, push_repo, branch, pkg, app_dir, store_meta, meta_out, + extra={}, message=f"metadata: {pkg} store listing") + pr_body = ( + f"## Store listing update: `{pkg}`\n\n" + f"Only `meta.json`, screenshots and the icon change; the published version and " + f"binary stay as they are.\n\n" + f"Submitted via `czdev publish --meta-only`." + ) + open_pr(gh, pr_head_owner, branch, f"metadata: {pkg} (store listing)", pr_body) + + +def push_target(gh: GitHubClient, user) -> tuple: + """Where to push the PR branch: the official repo for maintainers, the + contributor's fork otherwise. Returns (owner, repo, PR head owner or None).""" + perm = gh.check_permission(TARGET_OWNER, TARGET_REPO) + if perm >= Permission.WRITE: + return TARGET_OWNER, TARGET_REPO, None + print(f"You don't have write access to {TARGET_OWNER}/{TARGET_REPO}.") + print(" → Forking to your account... ", end="", flush=True) + fork_name = gh.fork_repo(TARGET_OWNER, TARGET_REPO) + print(f"done ({fork_name})") + owner, repo = fork_name.split("/") + return owner, repo, user.login + + +def push_pool_branch(gh: GitHubClient, push_owner: str, push_repo: str, branch: str, pkg: str, + app_dir: str, store_meta: dict, meta_out: dict, extra: dict, message: str): + """Commit pool/main// (meta.json, screenshots, icon and `extra` files) + on a new branch of the push target, based on the official repo's main. + + The branch is created through the API at upstream main's commit rather + than from the fork's own main: a fork that has not been synced in a while + would otherwise base the PR on old history and conflict. A fork shares its + object store with upstream, so that commit is already there. + """ print(f"Uploading metadata to {TARGET_OWNER}/{TARGET_REPO}...") + upstream_sha = gh.get_ref_sha(TARGET_OWNER, TARGET_REPO, "heads/main") + print(f" → Creating branch {branch} at upstream main ({upstream_sha[:7]})... ", end="", flush=True) + # A fork that was created a moment ago may not accept refs yet. + for attempt in range(6): + try: + gh.create_ref(push_owner, push_repo, branch, upstream_sha) + break + except urllib.error.HTTPError: + if attempt == 5: + raise + time.sleep(5) + print("done") tmp_dir = Path(tempfile.mkdtemp(prefix="czdev-publish-")) try: run_cmd_in(tmp_dir, ["git", "init"]) - run_cmd_in(tmp_dir, ["git", "remote", "add", "origin", remote_url]) + run_cmd_in(tmp_dir, ["git", "remote", "add", "origin", f"git@github.com:{push_owner}/{push_repo}.git"]) print(" → git fetch (minimal)... ", end="", flush=True) - run_cmd_in(tmp_dir, ["git", "fetch", "--depth=1", "--filter=blob:none", "origin", "main"]) - print("done") - - print(f" → Creating branch {branch}... ", end="", flush=True) - run_cmd_in(tmp_dir, ["git", "checkout", "-b", branch, "origin/main"]) + run_cmd_in(tmp_dir, ["git", "fetch", "--depth=1", "--filter=blob:none", "origin", branch]) + run_cmd_in(tmp_dir, ["git", "checkout", "-b", branch, "FETCH_HEAD"]) print("done") - dest_dir = tmp_dir / "pool" / "main" / meta["package"] + dest_dir = tmp_dir / "pool" / "main" / pkg dest_dir.mkdir(parents=True, exist_ok=True) # Copy screenshots if store_meta.get("screenshots"): screenshots_dest = dest_dir / "screenshots" + if screenshots_dest.is_dir(): + shutil.rmtree(screenshots_dest) screenshots_dest.mkdir(parents=True, exist_ok=True) for shot in store_meta["screenshots"]: src = Path(app_dir) / shot @@ -186,54 +267,35 @@ def run(deb: Optional[str] = None): if icon_src.exists(): shutil.copy2(icon_src, dest_dir / icon_src.name) - # meta.json + release manifest (the binary stays in the buffer release). - # meta_out was built and pre-checked before the upload. + # meta.json was built and pre-checked before anything was uploaded. (dest_dir / "meta.json").write_text(json.dumps(meta_out, indent=2, ensure_ascii=False) + "\n") - (dest_dir / manifest_name).write_text(json.dumps(manifest, indent=2) + "\n") + for name, text in extra.items(): + (dest_dir / name).write_text(text) print(" → Creating commit... ", end="", flush=True) - run_cmd_in(tmp_dir, ["git", "add", f"pool/main/{meta['package']}"]) - run_cmd_in(tmp_dir, ["git", "commit", "-m", - f"publish: {meta['package']} {meta['version']} ({meta['architecture']})"]) + run_cmd_in(tmp_dir, ["git", "add", "-A", f"pool/main/{pkg}"]) + run_cmd_in(tmp_dir, ["git", "commit", "-m", message]) print("done") print(" → Pushing branch... ", end="", flush=True) run_cmd_in(tmp_dir, ["git", "push", "origin", branch]) print("done") - finally: shutil.rmtree(tmp_dir, ignore_errors=True) - # Create PR via API - head = pr_head if pr_head else branch - pr_body = ( - f"## Package: `{meta['package']}`\n\n" - f"| Field | Value |\n" - f"|-------|-------|\n" - f"| Version | {meta['version']} |\n" - f"| Architecture | {meta['architecture']} |\n" - f"| Maintainer | {meta['maintainer']} |\n" - f"| Size | {size_mb:.1f} MB |\n" - f"| SHA-256 | `{sha256_hash}` |\n" - f"| Manifest | `{manifest_path_in_repo}` |\n" - f"| Binary | [{asset_name}]({download_url}) |\n\n" - f"Submitted via `czdev publish`. The `.deb` is hosted on the contributor's " - f"buffer release; CI verifies the sha256 and, on merge, promotes it into the " - f"official `apt-pool` release." - ) + +def open_pr(gh: GitHubClient, pr_head_owner: Optional[str], branch: str, title: str, body: str): + head = f"{pr_head_owner}:{branch}" if pr_head_owner else branch print(" → Creating pull request... ", end="", flush=True) - pr = gh.create_pull_request( - TARGET_OWNER, TARGET_REPO, - f"publish: {meta['package']} {meta['version']}", - pr_body, head, "main", - ) + pr = gh.create_pull_request(TARGET_OWNER, TARGET_REPO, title, body, head, "main") print("done") print() print("✓ Pull request created:") print(f" {pr.html_url}") print() - print(" The PR will be validated by CI. A maintainer will review and merge it.") + print(" The PR will be validated by CI. Updates to your own packages merge automatically;") + print(" a first release waits for a maintainer.") def resolve_deb(deb: Optional[str]) -> str: @@ -260,42 +322,50 @@ def load_store_meta(deb_path: str) -> tuple: Path(deb_path).parent, ] + raw, app_dir = load_app_builder(*search_dirs) + return store_meta_from(raw), app_dir + + +def load_app_builder(*search_dirs: Path) -> tuple: + """The first app-builder.json found, parsed, with the directory it is in.""" for d in search_dirs: manifest_path = d / "app-builder.json" if manifest_path.exists(): - raw = json.loads(manifest_path.read_text()) - - if "store" not in raw: - print("app-builder.json found but missing \"store\" section.", file=sys.stderr) - print(' Add a "store" field with title, summary, categories, screenshots, etc.', file=sys.stderr) - sys.exit(1) - - store = raw["store"] - meta = { - "title": raw.get("app_name", ""), - "summary": store.get("summary", ""), - "categories": store.get("categories", []), - "screenshots": store.get("screenshots", []), - } - for key in ("description", "locales", "license", "source_repo", - "icon", "permissions", "share_code"): - if store.get(key): - meta[key] = store[key] - if isinstance(store.get("author"), dict): - meta["author"] = store["author"] - - if not meta["screenshots"]: - print("No screenshots defined in app-builder.json store.screenshots.", file=sys.stderr) - print(" Add at least one 320x170 screenshot to publish.", file=sys.stderr) - sys.exit(1) - - return meta, str(d) + return json.loads(manifest_path.read_text()), str(d) print("app-builder.json not found in current directory.", file=sys.stderr) print(" Run `czdev publish` from your app's project directory.", file=sys.stderr) sys.exit(1) +def store_meta_from(raw: dict) -> dict: + """The store listing fields of an app-builder.json.""" + if "store" not in raw: + print("app-builder.json found but missing \"store\" section.", file=sys.stderr) + print(' Add a "store" field with title, summary, categories, screenshots, etc.', file=sys.stderr) + sys.exit(1) + + store = raw["store"] + meta = { + "title": raw.get("app_name", ""), + "summary": store.get("summary", ""), + "categories": store.get("categories", []), + "screenshots": store.get("screenshots", []), + } + for key in ("description", "locales", "license", "source_repo", + "icon", "permissions", "share_code"): + if store.get(key): + meta[key] = store[key] + if isinstance(store.get("author"), dict): + meta["author"] = store["author"] + + if not meta["screenshots"]: + print("No screenshots defined in app-builder.json store.screenshots.", file=sys.stderr) + print(" Add at least one 320x170 screenshot to publish.", file=sys.stderr) + sys.exit(1) + return meta + + RAW_BASE = f"https://raw.githubusercontent.com/{TARGET_OWNER}/{TARGET_REPO}/main" @@ -520,7 +590,7 @@ def is_valid_package_name(name: str) -> bool: def branch_name(meta: dict) -> str: ts = int(time.time()) - return f"publish/{meta['package']}-{meta['version']}-{ts}" + return f"publish/{meta['package']}-{ref_safe(meta['version'])}-{ts}" def check_version_newer(meta: dict): @@ -559,16 +629,6 @@ def check_version_newer(meta: dict): print(" Later version updates merge automatically.") -def compare_versions(a: str, b: str) -> int: - def parse(v): - return [int(x) for x in re.split(r'[.\-~]', v) if x.isdigit()] - - pa, pb = parse(a), parse(b) - if pa > pb: - return 1 - elif pa < pb: - return -1 - return 0 def check_git_installed(): diff --git a/scripts/czdev/unpublish.py b/scripts/czdev/unpublish.py index 44c01b7..3a7906e 100644 --- a/scripts/czdev/unpublish.py +++ b/scripts/czdev/unpublish.py @@ -11,6 +11,7 @@ from . import auth from .github_client import GitHubClient, Permission +from .versions import asset_safe, ref_safe TARGET_OWNER = "CardputerZero" TARGET_REPO = "packages" @@ -22,8 +23,8 @@ def run(package: str, version: str, arch: str = "arm64"): user = gh.get_user() # Packages are referenced by a manifest in git; the .deb itself lives in a - # Release. GitHub sanitizes '~' to '.' in release asset names. - asset_name = f"{package}_{version}_{arch}.deb".replace("~", ".") + # Release. GitHub sanitizes '~' and ':' to '.' in release asset names. + asset_name = asset_safe(f"{package}_{version}_{arch}.deb") file_path = f"pool/main/{package}/{asset_name}.release.json" print(f"Checking ownership of {package} {version}...") @@ -81,9 +82,11 @@ def run(package: str, version: str, arch: str = "arm64"): print("Creating removal PR...") - # Get base - base_sha = gh.get_ref_sha(push_owner, push_repo, "heads/main") - _, base_tree_sha = gh.get_commit(push_owner, push_repo, base_sha) + # Base on the official repo's main, not the fork's: an unsynced fork may not + # even contain the manifest yet. A fork shares upstream's objects, so the + # tree and commit below can be built on it directly. + base_sha = gh.get_ref_sha(TARGET_OWNER, TARGET_REPO, "heads/main") + _, base_tree_sha = gh.get_commit(TARGET_OWNER, TARGET_REPO, base_sha) # Create tree with file removed (sha: None deletes the entry) tree_sha = gh.create_tree(push_owner, push_repo, base_tree_sha, file_path, None) @@ -118,7 +121,7 @@ def run(package: str, version: str, arch: str = "arm64"): def branch_name(package: str, version: str) -> str: ts = int(time.time()) - return f"unpublish/{package}-{version}-{ts}" + return f"unpublish/{package}-{ref_safe(version)}-{ts}" def extract_email(maintainer: str) -> str: diff --git a/scripts/czdev/versions.py b/scripts/czdev/versions.py new file mode 100644 index 0000000..e92f482 --- /dev/null +++ b/scripts/czdev/versions.py @@ -0,0 +1,77 @@ +"""Debian version strings: comparing them, and the names derived from them. + +A Debian version may contain '~' and ':', which a git ref cannot and a GitHub +release asset name silently turns into '.'. Both channels of the store (the +portal and `czdev publish`) derive names the same way, so they stay here. +""" + +import re + + +def ref_safe(version: str) -> str: + """The version as it may appear in a branch name.""" + return re.sub(r"[~:]", "_", version) + + +def asset_safe(name: str) -> str: + """The file name GitHub will store a release asset under.""" + return re.sub(r"[~:]", ".", name) + + +def _order(c: str) -> int: + if c == "~": + return -1 + if c.isdigit(): + return 0 + if c.isalpha(): + return ord(c) + return ord(c) + 256 + + +def _verrevcmp(a: str, b: str) -> int: + ia = ib = 0 + while ia < len(a) or ib < len(b): + first_diff = 0 + while (ia < len(a) and not a[ia].isdigit()) or (ib < len(b) and not b[ib].isdigit()): + ac = _order(a[ia]) if ia < len(a) else 0 + bc = _order(b[ib]) if ib < len(b) else 0 + if ac != bc: + return ac - bc + ia += 1 + ib += 1 + while ia < len(a) and a[ia] == "0": + ia += 1 + while ib < len(b) and b[ib] == "0": + ib += 1 + while ia < len(a) and a[ia].isdigit() and ib < len(b) and b[ib].isdigit(): + if not first_diff: + first_diff = ord(a[ia]) - ord(b[ib]) + ia += 1 + ib += 1 + if ia < len(a) and a[ia].isdigit(): + return 1 + if ib < len(b) and b[ib].isdigit(): + return -1 + if first_diff: + return first_diff + return 0 + + +def _split(v: str): + v = v.strip() + epoch = 0 + head, sep, rest = v.partition(":") + if sep and head.isdigit(): + epoch, v = int(head), rest + upstream, sep, revision = v.rpartition("-") + return (epoch, upstream, revision) if sep else (epoch, v, "") + + +def compare_versions(a: str, b: str) -> int: + """`dpkg --compare-versions` semantics: -1, 0 or 1. 1.0~rc1 sorts before 1.0.""" + ea, ua, ra = _split(a) + eb, ub, rb = _split(b) + if ea != eb: + return 1 if ea > eb else -1 + rc = _verrevcmp(ua, ub) or _verrevcmp(ra, rb) + return (rc > 0) - (rc < 0)