diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 08b8064..e545b1f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -9,7 +9,9 @@ original product vision see [docs/DESIGN.md](docs/DESIGN.md). ## Design principles 1. **CLI-first.** The primary interface is the `vp` command, scriptable in - notebooks, training servers, and CI. No web app in the core. + notebooks, training servers, and CI. No web app in the core. For Python + callers the same workflow is exposed as a facade (`visionpack.sdk`) with the + CLI's locking and result shapes. 2. **The manifest is the source of truth.** `visionpack.yaml` declares the dataset (classes, sources, splits, validation policy, pack profiles). Behaviour is driven by the manifest + the internal index, never by "a folder with the right @@ -32,8 +34,10 @@ visionpack/ cli/ main.py # argparse wiring; registers every subcommand commands/ # one module per command (init, import, sync, validate, - # fsck, stats, split, snapshot, diff, export, pack, - # annotate, eval, autolabel, queue) + # audit, fsck, stats, split, snapshot, diff, export, + # pack, annotate, eval, autolabel, queue) + sdk/ # VisionPackClient: the Python facade over the whole + # workflow (same locking + result shapes as the CLI) core/ project.py # Project: manifest + index + object store handle manifest.py # Manifest dataclass + pydantic schema (sources, classes…) @@ -63,6 +67,8 @@ visionpack/ eval.py # vp eval: AP/mAP, accuracy, confusion matrix vs a split set autolabel.py # vp autolabel: persist confident predictions as annotations curation.py # vp queue: active-learning ranking + label-quality audit + audit.py # vp audit: label-health findings (duplicate/tiny/edge boxes, + # aspect outliers, class imbalance) ``` --- @@ -259,17 +265,21 @@ The roadmap is sequenced so each phase unblocks the next. - [x] semantic segmentation (per-class mask PNGs via `vp export --format masks`) - [x] YOLO-seg import-export (polygon label lines; `--seg` / segmentation-task default) - [ ] YOLO-pose import-export; dedicated keypoint importer -- [ ] `--format auto` task/format detection on import (predictions already auto-detect) +- [x] `--format auto` format detection on import (now the default; structural + detection of YOLO/COCO/ImageFolder, explicit `--format` when ambiguous) ### Phase B — Differentiators - [x] near-duplicate & cross-split leakage detection (perceptual-hash tier) - [ ] optional embedding tier (CLIP/DINOv2) for semantic near-duplicates -- [ ] label-health audit (`vp audit`): duplicate/degenerate/edge-pinned boxes, - aspect-ratio outliers, class imbalance +- [x] label-health audit (`vp audit`): duplicate/degenerate/edge-pinned boxes, + aspect-ratio outliers, class imbalance (advisory by default; + `--fail-on-findings` for CI) - [x] model-in-the-loop quality (`vp queue --include-labeled`: confident detections with no matching label, and labels the model never finds) -- [ ] distribution-drift diff between snapshots (per-class deltas / KL) -- [ ] dataset → model lineage (`vp snapshot tag v4 trained:`) +- [x] distribution-drift diff between snapshots (`vp diff --drift`: per-class + share deltas, smoothed KL + Jensen–Shannon divergence) +- [x] dataset → model lineage (`vp snapshot tag v4 trained:`, free-form + tags; `snapshots_by_tag` lookup in the SDK) ### Benchmarking - [x] `vp eval` — score predictions against a split set (per-class AP@50, diff --git a/CHANGELOG.md b/CHANGELOG.md index 76db67f..200c788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,40 @@ All notable changes to VisionPack are tracked here. ## [Unreleased] +- Distribution drift between snapshots (`vp diff v1 v2 --drift`, SDK + `ds.drift("v1", "v2")`): per-class object counts and distribution-share + deltas (biggest movers first) plus smoothed KL and Jensen–Shannon divergence, + computed from the stats frozen in each snapshot — reproducible forever. +- Dataset → model lineage (`vp snapshot tag v4 trained:run-812`): free-form + tags on snapshots (add/`--remove`, shown in `snapshot list`/`show`); SDK + `tag_snapshot`/`untag_snapshot`/`snapshots_by_tag` (a bare `key:` prefix + matches any value). +- `vp import --format auto` (now the default): the import format is detected + from the dataset's structure — instances-style JSON means COCO (a directory + with the JSON next to the images works too), `.txt` labels or + classes/data.yaml furniture means YOLO, folder-per-class means ImageFolder; + ambiguous layouts ask for an explicit `--format` instead of guessing. The + SDK's `import_dir` defaults to `auto` as well. +- Python SDK (`visionpack.sdk`): the whole dataset lifecycle behind one class, + `VisionPackClient` — init/open, import, sync, validate, audit, stats, splits, + snapshots (including read-only `checkout(version)` views), export, and the + model loop (evaluate/autolabel/annotation queue). Mutating methods take the + same project lock the CLI takes, and summaries come back as the same + JSON-friendly shapes the `--json` envelopes carry. +- `vp audit`: label-health audit (roadmap Phase B) — duplicate same-class boxes, + degenerate (tiny) boxes, edge-pinned and whole-image boxes, aspect-ratio + outliers, rare classes, and class imbalance. Findings are advisory by default + (`--fail-on-findings` gates CI); thresholds configurable via flags or + `validation.audit` in `visionpack.yaml`; `--json` supported. +- Security: class names arriving from imported data (folder names, COCO + categories, `classes.txt`) are sanitized before being used as export path + components, so a hostile name like `../../x` can no longer write outside the + export directory. +- Robustness: a decompression-bomb image (header claiming absurd dimensions) + now records a per-file ingest failure instead of aborting the whole + import/sync batch. +- Performance: sync/import now read only asset *ids* when checking which assets + already exist (`SELECT id`), instead of materializing every asset record. - Model-in-the-loop foundation: a shared predictions loader (`visionpack/predictions.py`) reads model output in three formats — vp-native JSON, COCO results/instances JSON, and YOLO txt directories (what Ultralytics diff --git a/README.md b/README.md index 29ac90f..7311203 100644 --- a/README.md +++ b/README.md @@ -83,11 +83,12 @@ uv run vp --help # 1. create a project (the manifest is visionpack.yaml) vp init --name factory-defects --task detection -# 2. bring in a YOLO dataset -vp import ./raw --format yolo +# 2. bring in a dataset (YOLO/COCO/ImageFolder auto-detected) +vp import ./raw # 3. check it for real problems -vp validate +vp validate # invalid labels, duplicates, leakage +vp audit # suspicious labels: double boxes, tiny boxes, imbalance # 4. a deterministic, reproducible split vp split create --train 0.8 --val 0.1 --test 0.1 --strategy stratified @@ -103,6 +104,20 @@ vp stats --by split vp export --format yolo --split ``` +Prefer Python? The same workflow is one class away +([Python SDK docs](https://caiowing.github.io/VisionPack/sdk/)): + +```python +from visionpack.sdk import VisionPackClient + +ds = VisionPackClient.open(".") +ds.validate() +ds.create_split(train=0.8, val=0.1, test=0.1, strategy="stratified") +ds.snapshot("baseline") +ds.export("./exports/yolo", format="yolo", split="default") +metrics = ds.evaluate("runs/predict/labels", format="yolo") +``` + --- ## Works across the common CV tasks @@ -182,9 +197,20 @@ See the [Cloud Sync guide](https://caiowing.github.io/VisionPack/cloud-sync/). single-pass verified relay across providers (S3→GCS, local→S3, …); one fast-list instead of per-object lookups, retries with backoff on every remote call, tunable concurrency (`--jobs`); streaming export. -- **Content-addressed snapshots & diff** — reproducible versions; compare any two. +- **Content-addressed snapshots & diff** — reproducible versions; compare any + two, including class-distribution drift (`vp diff v1 v2 --drift`: per-class + share deltas, KL/JS divergence). +- **Dataset → model lineage** — `vp snapshot tag v4 trained:` links a + training run to the exact dataset version it consumed. - **Strong validation** — unreadable images, missing/orphan labels, unknown classes, invalid/out-of-bounds boxes, exact + near duplicates, split leakage. +- **Label-health audit (`vp audit`)** — the labels that are valid but usually + wrong: the same object boxed twice, tiny/degenerate boxes, boxes pinned to + image borders or covering the whole image, extreme aspect ratios, rare + classes and class imbalance. Advisory by default, `--fail-on-findings` for CI. +- **Python SDK (`visionpack.sdk`)** — the whole workflow programmatically: + `VisionPackClient` wraps import/sync/validate/audit/split/snapshot/export and + the model loop with the same locking and result shapes as the CLI. - **Comparable metrics** — per-split stats so class balance stays auditable as data grows. - **Benchmarking (`vp eval`)** — score model predictions (vp/COCO JSON or YOLO diff --git a/docs/json-output.md b/docs/json-output.md index 9f0d190..b283ab1 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -53,10 +53,12 @@ Rules a consumer can rely on: | `vp sync --dry-run --json` | `plans[]` (per source: `images_found`, `labels_found`, `matched`, `class_names[]`) | | `vp import ... --json` | `assets`, `annotations`, `objects`, `classes_added`, `recorded_source`, `failures[]` | | `vp validate --json` | `ok`, `errors`, `warnings`, `issues[]` (severity, code, message, asset_id, path) | +| `vp audit --json` | `ok`, `findings`, `by_code` (code → count), `class_counts`, `items[]` (code, message, asset_id, path, class_id) | | `vp stats --json` | `stats` (counts, `class_distribution`, `resolutions`), `splits` (per-split breakdowns) | | `vp split create/lock/list/show --json` | `id`, `strategy`, `locked`, `sets` (name → count); `show` adds `asset_ids` | -| `vp snapshot create/list/show --json` | snapshot records (`version`, `message`, `created_at`, `stats`) | -| `vp diff v1 v2 --json` | `assets_added/removed`, `annotations_added/removed/modified`, `classes_added/removed`, `splits_changed` | +| `vp snapshot create/list/show --json` | snapshot records (`version`, `message`, `created_at`, `stats`, `tags[]`) | +| `vp snapshot tag ... --json` | `version`, `tag`, `removed`, `tags[]` (the snapshot's tags after the change) | +| `vp diff v1 v2 --json` | `assets_added/removed`, `annotations_added/removed/modified`, `classes_added/removed`, `splits_changed`; with `--drift`: `drift` (`classes[]` share deltas, `kl_divergence`, `js_divergence`) | | `vp export --json` | `format`, `output`, per-format counts (`images`, `objects`, `sets`, `streamed`) | | `vp pack --json` | `profile`, `format`, shard/archive counts and paths | | `vp fsck --json` | `ok`, `mode`, `checked_assets`, `checked_objects`, `issues[]` | diff --git a/docs/sdk.md b/docs/sdk.md new file mode 100644 index 0000000..bd0c4ac --- /dev/null +++ b/docs/sdk.md @@ -0,0 +1,90 @@ +--- +title: Python SDK +nav_order: 7 +--- + +# Python SDK + +Everything the CLI does is available from Python through the SDK — the same +project on disk, the same locking, the same results the `--json` envelopes +carry, without subprocess plumbing. Use it from notebooks, training scripts, +labeling services, or CI jobs: + +```python +from visionpack.sdk import VisionPackClient + +ds = VisionPackClient.init("./factory-defects", task="detection") # or .open(".") +ds.import_dir("./raw", format="yolo") + +report = ds.validate() # ValidationReport (errors/warnings/ok) +audit = ds.audit() # AuditReport (label-health findings) +print(ds.stats()["class_distribution"]) + +ds.create_split(train=0.8, val=0.1, test=0.1, strategy="stratified") +ds.lock_split() +ds.snapshot("baseline") +ds.export("./exports/yolo", format="yolo", split="default") +``` + +Close the model-in-the-loop cycle with the same handle: + +```python +metrics = ds.evaluate("runs/predict/labels", format="yolo") # mAP on the test set +ds.autolabel("preds.json", min_confidence=0.6) # confident preds -> labels +for item in ds.annotation_queue("preds.json")[:20]: # what to label next + print(item["score"], item["path"]) +``` + +## The client + +| Area | Methods | +|------|---------| +| Lifecycle | `VisionPackClient.init(root, name=..., task=...)`, `VisionPackClient.open(root)` (also `sdk.init` / `sdk.open`) | +| Ingest | `import_dir(source, format="yolo"\|"coco"\|"imagefolder", images=..., copy_mode=...)`, `sync(source=..., jobs=...)`, `plan_sync()` | +| Quality | `validate(strict=...)`, `audit(**thresholds)`, `stats()`, `split_stats()` | +| Splits | `create_split(...)`, `lock_split()`, `split()` | +| Versions | `snapshot(message)`, `snapshots()`, `get_snapshot(version)`, `checkout(version)`, `diff(v1, v2)`, `drift(v1, v2)`, `tag_snapshot(v, tag)`, `untag_snapshot(v, tag)`, `snapshots_by_tag(tag)` | +| Outputs | `export(output, format="yolo"\|"coco"\|"imagefolder"\|"masks", split=..., seg=...)` | +| Model loop | `load_predictions(...)`, `evaluate(...)`, `autolabel(...)`, `annotation_queue(...)` | +| Data access | `assets()`, `annotations()`, `samples()` (streaming iterator), `len(ds)`, `for asset, ann in ds:` | + +## Guarantees + +- **Safe next to the CLI.** Every mutating method takes the same project lock + `vp` takes, so an SDK caller and a CLI process can never corrupt each + other's writes — the second writer fails fast with a clear error. +- **Stable, JSON-friendly returns.** Summaries come back as plain dicts that + mirror the [`--json` contract]({% link json-output.md %}), so a service can + switch between shelling out to `vp` and importing the SDK without + re-parsing anything. +- **Read-only snapshot views.** `ds.checkout("v2")` returns a client pinned to + that snapshot: exports, stats, and evaluation reflect the frozen state, and + mutating methods raise instead of silently writing into live history. + +## Versions, drift, and lineage + +```python +ds.snapshot("after week-30 batch") # v5 +print(ds.drift("v4", "v5")["js_divergence"]) # did the class mix shift? + +# after training, link the run to the exact dataset version it consumed: +ds.tag_snapshot("v5", "trained:run-812") +ds.snapshots_by_tag("trained:") # every version a model trained on +``` + +## Streaming a dataset + +`samples()` iterates straight off the index without materializing it, so a +training-adjacent script can walk 100k+ assets in bounded memory: + +```python +for asset, annotation in ds.samples(): + if annotation is None: + continue + for obj in annotation.objects: + box = obj.bbox # enclosing BBox for any geometry + ... +``` + +The lower-level `Project` object stays reachable as `ds.project` for anything +the facade doesn't cover yet. diff --git a/docs/usage.md b/docs/usage.md index 3e81f25..b2f0acd 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -88,9 +88,16 @@ raw/ Import the dataset: ```bash -vp import ./raw --format yolo +vp import ./raw # format auto-detected from the layout +vp import ./raw --format yolo # or say it explicitly ``` +`--format` defaults to `auto`: a `.json` annotations file (or a directory with +an instances-style JSON at its root) is COCO, `.txt` labels or +`classes.txt`/`data.yaml` mean YOLO, and images living only under +folder-per-class subdirectories mean ImageFolder. When the layout is ambiguous +the command asks you to pass `--format` instead of guessing. + By default, VisionPack ingests images into `.vp/objects/sha256` and indexes them by content hash. You can choose another copy mode: ```bash @@ -166,6 +173,40 @@ vp validate --report reports/validation.json The current validator checks image readability, missing annotations, orphan labels, unknown classes, invalid boxes, boxes outside image bounds, duplicate exact assets, and split leakage. +## Audit Label Health (vp audit) + +`vp validate` catches labels that are *invalid*; `vp audit` finds labels that +are valid but *suspicious* — the ones that usually turn out to be annotation +mistakes: + +```bash +vp audit +``` + +It reports duplicate boxes (the same object labeled twice), degenerate (tiny) +boxes, boxes pinned to two or more image borders, whole-image boxes, extreme +aspect-ratio outliers, rare classes, and dataset-level class imbalance. + +Findings are advisory (exit code 0). Gate CI on them explicitly: + +```bash +vp audit --fail-on-findings +``` + +Thresholds can be tuned per run (`--min-box-px`, `--duplicate-iou`, +`--max-aspect-ratio`, `--imbalance-ratio`, `--min-class-count`) or persisted in +`visionpack.yaml`: + +```yaml +validation: + audit: + min_box_px: 12 + duplicate_iou: 0.85 +``` + +Like every pipeline command, `vp audit --json` prints a machine-readable +envelope with per-code counts and the full findings list. + ## Show Statistics Print a summary: @@ -208,6 +249,19 @@ vp snapshot show v1 Snapshots store hashes for the manifest, assets, annotations, splits, and summary stats. They are written to `.vp/snapshots/`. +### Lineage tags (which dataset trained this model?) + +After a training run, stamp the snapshot it consumed: + +```bash +vp snapshot tag v4 trained:run-812 +``` + +Tags are free-form (the `key:value` convention is just a convention), show up +in `vp snapshot list`/`show`, and can be removed with `--remove`. From the SDK, +`ds.snapshots_by_tag("trained:")` lists every version any run trained on — so +"which dataset produced this model?" is a lookup, not archaeology. + ## Diff Snapshots Compare two snapshots: @@ -224,6 +278,20 @@ vp diff v1 v2 --json The diff reports added and removed assets, added/removed/modified annotations, class changes, split changes, and before/after stats. +### Distribution drift + +`--drift` adds a class-distribution comparison: per-class object counts and +distribution-share deltas (biggest movers first), plus KL and Jensen–Shannon +divergence as single drift scores a CI job can threshold: + +```bash +vp diff v1 v2 --drift +vp diff v1 v2 --drift --json # adds a "drift" object to the diff payload +``` + +Because it derives from the stats frozen inside each snapshot, the drift +between two versions is reproducible forever. + ## Export YOLO Export the indexed dataset back to YOLO format: @@ -323,25 +391,27 @@ The archive includes: ## Python API -The public API is intentionally small while the project is early: +The supported programmatic surface is the SDK — the whole CLI workflow behind +one Python class, with the same locking and result shapes (see the +[Python SDK]({% link sdk.md %}) page): ```python -from visionpack import Dataset +from visionpack.sdk import VisionPackClient -ds = Dataset.open(".") -print(ds.manifest.name) -print(len(ds.index.assets())) +ds = VisionPackClient.open(".") +print(ds.name, len(ds)) +ds.validate() +ds.export("./exports/yolo", format="yolo", split="default") ``` -More stable SDK methods will be added as the internal workflows settle. +The lower-level `visionpack.Dataset` / `Project` handle stays available as +`ds.project` for anything the facade doesn't cover yet. ## Current Limitations - segmentation metrics in `vp eval` use each polygon's enclosing box (mask IoU is planned); YOLO-pose import/export and a dedicated keypoint importer are not implemented yet -- `--format auto` detection on import is not implemented; pass `--format` - explicitly (predictions for `vp eval`/`autolabel`/`queue` *are* auto-detected) - `vp annotate` is scaffolded but not implemented yet - cloud sync (S3/GCS/Azure) is **same-provider** in v1 — cross-cloud transfer (S3↔GCS) and remote COCO/ImageFolder sync are planned; `pack` is local-only diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..2f3325c --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import io +import json +import tempfile +import unittest +from pathlib import Path + +from PIL import Image + +from visionpack.audit import AuditThresholds, audit_project +from visionpack.core.errors import FormatError +from visionpack.core.project import Project +from visionpack.formats.base import safe_path_component +from visionpack.formats.classification import ImageFolderImporter, export_imagefolder +from visionpack.formats.yolo import YoloImporter +from visionpack.media import image_info_from_bytes + + +def _png(path: Path, seed: int, size: tuple[int, int] = (40, 40)) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", size, (seed * 7 % 256, seed * 13 % 256, seed * 29 % 256)).save(path, format="PNG") + + +def _seed_detection(root: Path, labels: dict[str, str]) -> Project: + data = root / "raw" + for index, (name, text) in enumerate(labels.items(), start=1): + _png(data / f"{name}.png", index) + (data / f"{name}.txt").write_text(text, encoding="utf-8") + (data / "classes.txt").write_text("alpha\nbeta\n", encoding="utf-8") + project = Project.init(root, name="audit", task="detection") + YoloImporter(project, data).run() + return Project.open(root) + + +class AuditBoxChecksTest(unittest.TestCase): + def _codes(self, project: Project, thresholds: AuditThresholds | None = None) -> dict[str, int]: + return audit_project(project, thresholds or AuditThresholds()).counts_by_code() + + def test_duplicate_boxes_same_class_are_flagged(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + project = _seed_detection(Path(tmp), {"a": "0 0.5 0.5 0.5 0.5\n0 0.5 0.5 0.5 0.5\n"}) + codes = self._codes(project) + self.assertEqual(codes.get("box.duplicate"), 1) + + def test_overlapping_boxes_of_different_classes_are_not_duplicates(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + project = _seed_detection(Path(tmp), {"a": "0 0.5 0.5 0.5 0.5\n1 0.5 0.5 0.5 0.5\n"}) + codes = self._codes(project) + self.assertNotIn("box.duplicate", codes) + + def test_tiny_box_is_degenerate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + # 4x4 px on a 40x40 image, below the 8 px default. + project = _seed_detection(Path(tmp), {"a": "0 0.5 0.5 0.1 0.1\n"}) + codes = self._codes(project) + self.assertEqual(codes.get("box.degenerate"), 1) + + def test_aspect_ratio_outlier(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + # 36x9 px: ratio 4:1 — an outlier once the threshold is lowered. + project = _seed_detection(Path(tmp), {"a": "0 0.5 0.5 0.9 0.225\n"}) + self.assertNotIn("box.aspect_outlier", self._codes(project)) + codes = self._codes(project, AuditThresholds(max_aspect_ratio=3.0)) + self.assertEqual(codes.get("box.aspect_outlier"), 1) + + def test_edge_pinned_box(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + # Full-width band at the top: pinned to three borders, covers 25%. + project = _seed_detection(Path(tmp), {"a": "0 0.5 0.125 1.0 0.25\n"}) + codes = self._codes(project) + self.assertEqual(codes.get("box.edge_pinned"), 1) + self.assertNotIn("box.covers_image", codes) + + def test_full_image_box_reports_coverage_not_edges(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + project = _seed_detection(Path(tmp), {"a": "0 0.5 0.5 1.0 1.0\n"}) + codes = self._codes(project) + self.assertEqual(codes.get("box.covers_image"), 1) + self.assertNotIn("box.edge_pinned", codes) + + def test_box_touching_one_border_is_fine(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + # 20x20 box flush with the left edge only. + project = _seed_detection(Path(tmp), {"a": "0 0.25 0.5 0.5 0.5\n"}) + codes = self._codes(project) + self.assertNotIn("box.edge_pinned", codes) + + +class AuditClassBalanceTest(unittest.TestCase): + def test_rare_class_and_imbalance(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + labels = {f"img{i}": "0 0.5 0.5 0.5 0.5\n" for i in range(12)} + labels["odd"] = "1 0.5 0.5 0.5 0.5\n" + project = _seed_detection(Path(tmp), labels) + report = audit_project(project, AuditThresholds(min_class_count=5, imbalance_ratio=10.0)) + codes = report.counts_by_code() + self.assertEqual(codes.get("class.rare"), 1) # beta has 1 < 5 + self.assertEqual(codes.get("class.imbalance"), 1) # 12:1 > 10:1 + rare = next(f for f in report.findings if f.code == "class.rare") + self.assertEqual(rare.class_id, "beta") + + def test_balanced_dataset_is_clean(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + labels = {f"a{i}": "0 0.5 0.5 0.5 0.5\n" for i in range(3)} + labels.update({f"b{i}": "1 0.5 0.5 0.5 0.5\n" for i in range(3)}) + project = _seed_detection(Path(tmp), labels) + report = audit_project(project, AuditThresholds(min_class_count=2, imbalance_ratio=10.0)) + self.assertTrue(report.ok, [f.message for f in report.findings]) + self.assertEqual(report.images_audited, 6) + self.assertEqual(report.objects_audited, 6) + + +class AuditThresholdsConfigTest(unittest.TestCase): + def test_manifest_and_overrides_layering(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + project = _seed_detection(Path(tmp), {"a": "0 0.5 0.5 0.5 0.5\n"}) + project.manifest.validation["audit"] = {"min_box_px": 12, "unknown_key": True} + thresholds = AuditThresholds.from_project(project, duplicate_iou=0.5, max_aspect_ratio=None) + self.assertEqual(thresholds.min_box_px, 12) # from manifest + self.assertEqual(thresholds.duplicate_iou, 0.5) # explicit override + self.assertEqual(thresholds.max_aspect_ratio, 20.0) # None override ignored + + +class AuditCliTest(unittest.TestCase): + def test_cli_json_envelope_and_exit_codes(self) -> None: + import contextlib + import io as _io + import os + + from visionpack.cli.main import main + + with tempfile.TemporaryDirectory() as tmp: + project = _seed_detection(Path(tmp), {"a": "0 0.5 0.5 0.1 0.1\n"}) + self.assertIsNotNone(project) + cwd = os.getcwd() + os.chdir(tmp) + try: + buffer = _io.StringIO() + with contextlib.redirect_stdout(buffer): + self.assertEqual(main(["audit", "--json"]), 0) # advisory by default + envelope = json.loads(buffer.getvalue()) + self.assertEqual(envelope["command"], "audit") + self.assertGreaterEqual(envelope["data"]["by_code"]["box.degenerate"], 1) + + buffer = _io.StringIO() + with contextlib.redirect_stdout(buffer): + self.assertEqual(main(["audit", "--json", "--fail-on-findings"]), 1) + finally: + os.chdir(cwd) + + +class SafePathComponentTest(unittest.TestCase): + def test_traversal_and_separators_are_neutralized(self) -> None: + self.assertEqual(safe_path_component(".."), "unnamed") + self.assertEqual(safe_path_component("."), "unnamed") + self.assertEqual(safe_path_component(""), "unnamed") + self.assertEqual(safe_path_component("../../etc"), ".._.._etc") # separators replaced, no traversal + self.assertNotIn("/", safe_path_component("a/b")) + self.assertNotIn("\\", safe_path_component("a\\b")) + self.assertEqual(safe_path_component("dog"), "dog") + + def test_export_with_hostile_class_name_stays_inside_output(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + data = root / "data" + for i in range(2): + _png(data / "cat" / f"c{i}.png", i) + project = Project.init(root, name="cls", task="classification") + ImageFolderImporter(project, data).run() + project = Project.open(root) + # Simulate a hostile class name arriving from imported metadata. + project.manifest.classes[0].name = "../../escape" + output = root / "exports" / "out" + export_imagefolder(project, output) + written = [p for p in output.rglob("*") if p.is_file()] + self.assertTrue(written) + for path in written: + self.assertTrue(path.resolve().is_relative_to(output.resolve()), path) + self.assertFalse((root / "escape").exists()) + + +class DecompressionBombTest(unittest.TestCase): + def test_bomb_is_reported_as_format_error(self) -> None: + buffer = io.BytesIO() + Image.new("RGB", (40, 40)).save(buffer, format="PNG") + original = Image.MAX_IMAGE_PIXELS + Image.MAX_IMAGE_PIXELS = 10 # force the bomb check to trip on a tiny image + try: + with self.assertRaises(FormatError): + image_info_from_bytes(buffer.getvalue(), Path("bomb.png")) + finally: + Image.MAX_IMAGE_PIXELS = original + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_drift_lineage.py b/tests/test_drift_lineage.py new file mode 100644 index 0000000..d299797 --- /dev/null +++ b/tests/test_drift_lineage.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import contextlib +import io +import json +import os +import tempfile +import unittest +from pathlib import Path + +from PIL import Image + +from visionpack.core.errors import VisionPackError +from visionpack.drift import drift_from_stats +from visionpack.formats.detect import coco_json_in, detect_import_format +from visionpack.sdk import VisionPackClient + + +def _png(path: Path, seed: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (40, 40), (seed * 7 % 256, seed * 13 % 256, seed * 29 % 256)).save(path, format="PNG") + + +def _seed_yolo(root: Path, spec: dict[str, str]) -> Path: + data = root / "raw" + for index, (name, label) in enumerate(spec.items(), start=1): + _png(data / f"{name}.png", index) + (data / f"{name}.txt").write_text(label, encoding="utf-8") + (data / "classes.txt").write_text("alpha\nbeta\n", encoding="utf-8") + return data + + +class DriftMathTest(unittest.TestCase): + def test_identical_distributions_have_near_zero_divergence(self) -> None: + stats = {"class_distribution": {"a": 10, "b": 10}, "assets": 20} + drift = drift_from_stats(stats, stats) + self.assertAlmostEqual(drift["kl_divergence"], 0.0, places=6) + self.assertAlmostEqual(drift["js_divergence"], 0.0, places=6) + self.assertTrue(all(item["delta"] == 0 for item in drift["classes"])) + + def test_shift_reports_deltas_and_positive_divergence(self) -> None: + before = {"class_distribution": {"a": 10, "b": 10}, "assets": 20} + after = {"class_distribution": {"a": 30, "b": 5}, "assets": 35} + drift = drift_from_stats(before, after) + self.assertGreater(drift["kl_divergence"], 0.0) + self.assertGreater(drift["js_divergence"], 0.0) + by_class = {item["class_id"]: item for item in drift["classes"]} + self.assertEqual(by_class["a"]["delta"], 20) + self.assertEqual(by_class["b"]["delta"], -5) + self.assertGreater(by_class["a"]["share_delta"], 0) + self.assertLess(by_class["b"]["share_delta"], 0) + # sorted by |share_delta| descending + self.assertEqual(drift["classes"][0]["class_id"], "a") + + def test_new_class_stays_finite(self) -> None: + before = {"class_distribution": {"a": 10}, "assets": 10} + after = {"class_distribution": {"a": 10, "b": 10}, "assets": 20} + drift = drift_from_stats(before, after) + self.assertIsNotNone(drift["kl_divergence"]) + self.assertLess(drift["kl_divergence"], 100) # smoothing keeps it finite + + def test_empty_side_yields_none_divergence(self) -> None: + drift = drift_from_stats({"class_distribution": {}}, {"class_distribution": {"a": 5}}) + self.assertIsNone(drift["kl_divergence"]) + self.assertIsNone(drift["js_divergence"]) + + +class DriftEndToEndTest(unittest.TestCase): + def test_sdk_drift_and_cli_diff_drift(self) -> None: + from visionpack.cli.main import main + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + data = _seed_yolo(root, {"a": "0 0.5 0.5 0.5 0.5\n", "b": "0 0.5 0.5 0.5 0.5\n"}) + ds = VisionPackClient.init(root, name="drift", task="detection") + ds.import_dir(data, format="yolo") + ds.snapshot("v1: two alpha") + + for i in range(3): + _png(data / f"new{i}.png", 50 + i) + (data / f"new{i}.txt").write_text("1 0.5 0.5 0.5 0.5\n", encoding="utf-8") + ds.import_dir(data, format="yolo") + ds.snapshot("v2: beta arrives") + + drift = ds.drift("v1", "v2") + self.assertEqual(drift["from"], "v1") + by_class = {item["class_id"]: item for item in drift["classes"]} + self.assertEqual(by_class["beta"]["before"], 0) + self.assertEqual(by_class["beta"]["after"], 3) + self.assertGreater(drift["js_divergence"], 0.0) + + cwd = os.getcwd() + os.chdir(tmp) + try: + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + self.assertEqual(main(["diff", "v1", "v2", "--drift", "--json"]), 0) + envelope = json.loads(buffer.getvalue()) + self.assertIn("drift", envelope["data"]) + self.assertEqual(envelope["data"]["drift"]["to"], "v2") + finally: + os.chdir(cwd) + + +class SnapshotLineageTest(unittest.TestCase): + def _dataset(self, root: Path) -> VisionPackClient: + data = _seed_yolo(root, {"a": "0 0.5 0.5 0.5 0.5\n"}) + ds = VisionPackClient.init(root, name="lineage", task="detection") + ds.import_dir(data, format="yolo") + ds.snapshot("baseline") + return ds + + def test_tag_untag_and_find(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + ds = self._dataset(Path(tmp)) + snap = ds.tag_snapshot("v1", "trained:run-812") + self.assertEqual(snap["tags"], ["trained:run-812"]) + # idempotent + snap = ds.tag_snapshot("v1", "trained:run-812") + self.assertEqual(snap["tags"], ["trained:run-812"]) + ds.tag_snapshot("v1", "release:2026-07") + + self.assertEqual(len(ds.snapshots_by_tag("trained:run-812")), 1) + self.assertEqual(len(ds.snapshots_by_tag("trained:")), 1) # bare key: prefix + self.assertEqual(ds.snapshots_by_tag("trained:other"), []) + + snap = ds.untag_snapshot("v1", "trained:run-812") + self.assertEqual(snap["tags"], ["release:2026-07"]) + # removing a missing tag is a no-op + snap = ds.untag_snapshot("v1", "nope") + self.assertEqual(snap["tags"], ["release:2026-07"]) + + def test_empty_tag_is_rejected_and_readonly_view_cannot_tag(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + ds = self._dataset(Path(tmp)) + with self.assertRaises(VisionPackError): + ds.tag_snapshot("v1", " ") + view = ds.checkout("v1") + with self.assertRaises(VisionPackError): + view.tag_snapshot("v1", "trained:x") + + def test_cli_tag_and_list(self) -> None: + from visionpack.cli.main import main + + with tempfile.TemporaryDirectory() as tmp: + self._dataset(Path(tmp)) + cwd = os.getcwd() + os.chdir(tmp) + try: + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + self.assertEqual(main(["snapshot", "tag", "v1", "trained:run-9", "--json"]), 0) + envelope = json.loads(buffer.getvalue()) + self.assertEqual(envelope["data"]["tags"], ["trained:run-9"]) + + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + self.assertEqual(main(["snapshot", "list"]), 0) + self.assertIn("[trained:run-9]", buffer.getvalue()) + + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + self.assertEqual(main(["snapshot", "tag", "v1", "trained:run-9", "--remove", "--json"]), 0) + envelope = json.loads(buffer.getvalue()) + self.assertEqual(envelope["data"]["tags"], []) + finally: + os.chdir(cwd) + + +class FormatDetectionTest(unittest.TestCase): + def test_yolo_by_labels_and_furniture(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _png(root / "img.png", 1) + (root / "img.txt").write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8") + self.assertEqual(detect_import_format(root), "yolo") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _png(root / "images" / "img.png", 1) + (root / "classes.txt").write_text("a\n", encoding="utf-8") + self.assertEqual(detect_import_format(root), "yolo") + + def test_imagefolder_by_class_dirs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _png(root / "cat" / "c0.png", 1) + _png(root / "dog" / "d0.png", 2) + self.assertEqual(detect_import_format(root), "imagefolder") + + def test_coco_json_file_and_directory(self) -> None: + document = {"images": [], "annotations": [], "categories": []} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + annotations = root / "instances.json" + annotations.write_text(json.dumps(document), encoding="utf-8") + self.assertEqual(detect_import_format(annotations), "coco") + _png(root / "img.png", 1) + self.assertEqual(detect_import_format(root), "coco") + self.assertEqual(coco_json_in(root), annotations) + + def test_plain_images_fall_back_to_yolo(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _png(root / "img.png", 1) + self.assertEqual(detect_import_format(root), "yolo") + + def test_undetectable_raises(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(VisionPackError): + detect_import_format(Path(tmp)) # empty dir + with tempfile.TemporaryDirectory() as tmp: + stray = Path(tmp) / "notes.md" + stray.write_text("hi", encoding="utf-8") + with self.assertRaises(VisionPackError): + detect_import_format(stray) + + def test_sdk_import_auto_detects_coco_directory(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + dataset = root / "dataset" + _png(dataset / "img0.png", 3) + document = { + "images": [{"id": 1, "file_name": "img0.png", "width": 40, "height": 40}], + "annotations": [{"id": 1, "image_id": 1, "category_id": 1, "bbox": [5, 5, 10, 10], "area": 100, "iscrowd": 0}], + "categories": [{"id": 1, "name": "thing"}], + } + (dataset / "instances.json").write_text(json.dumps(document), encoding="utf-8") + ds = VisionPackClient.init(root, name="auto", task="detection") + summary = ds.import_dir(dataset) # format defaults to auto + self.assertEqual(summary["format"], "coco") + self.assertEqual(summary["assets"], 1) + self.assertEqual([c.name for c in ds.classes], ["thing"]) + + def test_cli_import_auto_detects_yolo(self) -> None: + from visionpack.cli.main import main + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + data = _seed_yolo(root, {"a": "0 0.5 0.5 0.5 0.5\n"}) + VisionPackClient.init(root, name="cli-auto", task="detection") + cwd = os.getcwd() + os.chdir(tmp) + try: + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + self.assertEqual(main(["import", str(data), "--json"]), 0) + envelope = json.loads(buffer.getvalue()) + self.assertEqual(envelope["data"]["format"], "yolo") + self.assertEqual(envelope["data"]["assets"], 1) + finally: + os.chdir(cwd) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sdk.py b/tests/test_sdk.py new file mode 100644 index 0000000..547c702 --- /dev/null +++ b/tests/test_sdk.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from PIL import Image + +from visionpack import sdk +from visionpack.core.errors import VisionPackError +from visionpack.sdk import VisionPackClient + + +def _png(path: Path, seed: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (40, 40), (seed * 7 % 256, seed * 13 % 256, seed * 29 % 256)).save(path, format="PNG") + + +def _seed_yolo(root: Path, count: int = 6) -> Path: + data = root / "raw" + for i in range(count): + _png(data / f"img{i}.png", i + 1) + (data / f"img{i}.txt").write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8") + (data / "classes.txt").write_text("widget\n", encoding="utf-8") + return data + + +class SdkLifecycleTest(unittest.TestCase): + def test_full_dataset_lifecycle(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + data = _seed_yolo(root) + + ds = VisionPackClient.init(root, name="lifecycle", task="detection") + self.assertEqual(ds.name, "lifecycle") + self.assertEqual(ds.task, "detection") + + summary = ds.import_dir(data, format="yolo") + self.assertEqual(summary["assets"], 6) + self.assertEqual(summary["failures"], []) + self.assertEqual(len(ds), 6) + self.assertEqual([c.name for c in ds.classes], ["widget"]) + + report = ds.validate() + self.assertTrue(report.ok) + audit = ds.audit(min_class_count=1) + self.assertTrue(audit.ok, [f.message for f in audit.findings]) + self.assertEqual(ds.stats()["assets"], 6) + + split = ds.create_split(train=0.5, val=0.25, test=0.25, strategy="random") + self.assertEqual(sum(len(ids) for ids in split.sets.values()), 6) + ds.lock_split() + self.assertTrue(ds.split().locked) + + snap = ds.snapshot("baseline") + self.assertEqual(snap["version"], "v1") + self.assertEqual(len(ds.snapshots()), 1) + + out = root / "exports" / "yolo" + result = ds.export(out, format="yolo", split="default") + self.assertEqual(result["images"], 6) + self.assertTrue((out / "data.yaml").exists()) + + # streaming access + pairs = list(ds.samples()) + self.assertEqual(len(pairs), 6) + self.assertTrue(all(ann is not None for _, ann in pairs)) + + def test_module_level_init_and_open(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sdk.init(root, name="mod", task="classification") + ds = sdk.open(root) + self.assertEqual(ds.name, "mod") + self.assertEqual(ds.task, "classification") + + def test_import_coco_requires_images(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + ds = VisionPackClient.init(Path(tmp), name="x") + with self.assertRaises(VisionPackError): + ds.import_dir("annotations.json", format="coco") + + def test_unknown_formats_raise(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + ds = VisionPackClient.init(Path(tmp), name="x") + with self.assertRaises(VisionPackError): + ds.import_dir("nowhere", format="voc") + with self.assertRaises(VisionPackError): + ds.export(Path(tmp) / "out", format="tfrecord") + + +class SdkModelLoopTest(unittest.TestCase): + def _dataset(self, root: Path) -> VisionPackClient: + data = _seed_yolo(root, count=4) + ds = VisionPackClient.init(root, name="loop", task="detection") + ds.import_dir(data, format="yolo") + return ds + + def _predictions_file(self, ds: VisionPackClient, root: Path) -> Path: + items = [ + { + "image": asset.id, + "objects": [{"class": "widget", "confidence": 0.9, "bbox": [10, 10, 20, 20]}], + } + for asset in ds.assets() + ] + path = root / "preds.json" + path.write_text(json.dumps({"predictions": items}), encoding="utf-8") + return path + + def test_evaluate_autolabel_and_queue(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + ds = self._dataset(root) + ds.create_split(train=0.5, val=0.25, test=0.25, strategy="random") + preds = self._predictions_file(ds, root) + + metrics = ds.evaluate(preds, split="default", set_name="test") + self.assertEqual(metrics["task"], "detection") + self.assertIn("map50", json.dumps(metrics).lower()) + + queue = ds.annotation_queue(preds, include_labeled=True) + self.assertIsInstance(queue, list) + + result = ds.autolabel(preds, min_confidence=0.5) + self.assertEqual(result["skipped_existing"], 4) # all already labeled + + def test_predictions_accepts_loaded_set(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + ds = self._dataset(root) + loaded = ds.load_predictions(self._predictions_file(ds, root)) + self.assertIs(ds.load_predictions(loaded), loaded) + + +class SdkSnapshotViewTest(unittest.TestCase): + def test_checkout_is_readonly_and_pinned(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + data = _seed_yolo(root, count=3) + ds = VisionPackClient.init(root, name="ver", task="detection") + ds.import_dir(data, format="yolo") + ds.snapshot("three images") + + # grow the live dataset past the snapshot + _png(data / "extra.png", 99) + (data / "extra.txt").write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8") + ds.import_dir(data, format="yolo") + self.assertEqual(len(ds), 4) + + view = ds.checkout("v1") + self.assertTrue(view.readonly) + self.assertEqual(len(view), 3) + with self.assertRaises(VisionPackError): + view.snapshot("nope") + with self.assertRaises(VisionPackError): + view.create_split() + + out = root / "exports" / "v1" + result = view.export(out, format="yolo") + self.assertEqual(result["images"], 3) + + +class SdkLockTest(unittest.TestCase): + def test_mutations_take_the_project_lock(self) -> None: + from visionpack.core.lock import project_lock + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + data = _seed_yolo(root, count=2) + ds = VisionPackClient.init(root, name="locked") + with project_lock(ds.root): + with self.assertRaises(VisionPackError): + ds.import_dir(data, format="yolo") + + +if __name__ == "__main__": + unittest.main() diff --git a/visionpack/__init__.py b/visionpack/__init__.py index b097eae..1e9b37f 100644 --- a/visionpack/__init__.py +++ b/visionpack/__init__.py @@ -1,4 +1,11 @@ -"""VisionPack public Python API.""" +"""VisionPack public Python API. + +``Project``/``Dataset`` are the low-level handles; the supported programmatic +surface is the SDK (:mod:`visionpack.sdk`), which adds locking, stable return +shapes, and snapshot views: + + from visionpack.sdk import VisionPackClient +""" from visionpack.core.project import Dataset, Project diff --git a/visionpack/audit.py b/visionpack/audit.py new file mode 100644 index 0000000..8493809 --- /dev/null +++ b/visionpack/audit.py @@ -0,0 +1,259 @@ +"""Label-health audit (``vp audit``): find labels that are *suspicious*, not invalid. + +``vp validate`` catches labels that are wrong by construction (zero area, out of +bounds, unknown class). This audit targets the quieter failure mode: labels that +pass validation but usually mean an annotation mistake or a dataset-level risk — + +- **duplicate boxes**: two same-class objects on one image with near-total + overlap (double-click, double-import, or a stale autolabel pass); +- **degenerate boxes**: valid but tiny — usually annotation noise the model can + never learn from; +- **edge-pinned boxes**: boxes pinned to two or more image borders, or covering + nearly the whole image — typical of clipping bugs and auto-label artifacts; +- **aspect-ratio outliers**: extreme sliver boxes (a 40:1 "person") that are + almost always drawing errors; +- **class imbalance**: a dataset-level warning when the most frequent class + dwarfs the rarest, plus per-class warnings for classes too rare to learn. + +Every finding is advisory (a warning, never an error): the audit is a review +queue, not a gate — though ``vp audit --fail-on-findings`` lets CI treat it as +one. Thresholds come from ``validation.audit`` in ``visionpack.yaml`` and can be +overridden per run. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from typing import Any + +from visionpack.core.models import Annotation, Asset, BBox +from visionpack.core.project import Project +from visionpack.eval import bbox_iou + +# Defaults, overridable via manifest `validation.audit` and the CLI flags. +DEFAULT_MIN_BOX_PX = 8.0 # boxes thinner/shorter than this are degenerate +DEFAULT_DUPLICATE_IOU = 0.9 # same-class overlap at/above this is a duplicate +DEFAULT_MAX_ASPECT_RATIO = 20.0 # long-side / short-side beyond this is an outlier +DEFAULT_EDGE_TOLERANCE_PX = 1.0 # how close to a border counts as touching it +DEFAULT_COVERS_IMAGE_RATIO = 0.98 # box area / image area at/above this covers it +DEFAULT_IMBALANCE_RATIO = 20.0 # most-frequent / least-frequent class warning +DEFAULT_MIN_CLASS_COUNT = 10 # classes with fewer instances are flagged rare + + +@dataclass(slots=True) +class AuditThresholds: + """Tunable limits for every audit check. See the module docstring.""" + + min_box_px: float = DEFAULT_MIN_BOX_PX + duplicate_iou: float = DEFAULT_DUPLICATE_IOU + max_aspect_ratio: float = DEFAULT_MAX_ASPECT_RATIO + edge_tolerance_px: float = DEFAULT_EDGE_TOLERANCE_PX + covers_image_ratio: float = DEFAULT_COVERS_IMAGE_RATIO + imbalance_ratio: float = DEFAULT_IMBALANCE_RATIO + min_class_count: int = DEFAULT_MIN_CLASS_COUNT + + @classmethod + def from_project(cls, project: Project, **overrides: Any) -> AuditThresholds: + """Manifest ``validation.audit`` values, overlaid by explicit overrides. + + Precedence (lowest to highest): built-in defaults, ``visionpack.yaml``, + keyword overrides (the CLI flags). ``None`` overrides are ignored so + callers can pass optional flags straight through. + """ + config = dict(project.manifest.validation.get("audit", {})) + config.update({key: value for key, value in overrides.items() if value is not None}) + known = {f: config[f] for f in cls.__dataclass_fields__ if f in config} # ignore unknown keys + return cls(**known) + + +@dataclass(slots=True) +class AuditFinding: + code: str + message: str + asset_id: str | None = None + path: str | None = None + class_id: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"code": self.code, "message": self.message, "asset_id": self.asset_id, "path": self.path, "class_id": self.class_id} + + +@dataclass(slots=True) +class AuditReport: + findings: list[AuditFinding] + images_audited: int = 0 + objects_audited: int = 0 + class_counts: dict[str, int] = field(default_factory=dict) + + @property + def ok(self) -> bool: + return not self.findings + + def counts_by_code(self) -> dict[str, int]: + counts = Counter(finding.code for finding in self.findings) + return dict(sorted(counts.items())) + + def to_dict(self) -> dict[str, Any]: + return { + "ok": self.ok, + "findings": len(self.findings), + "by_code": self.counts_by_code(), + "images_audited": self.images_audited, + "objects_audited": self.objects_audited, + "class_counts": dict(sorted(self.class_counts.items())), + "items": [finding.to_dict() for finding in self.findings], + } + + +def audit_project(project: Project, thresholds: AuditThresholds | None = None) -> AuditReport: + """Run every label-health check over the whole dataset in one streamed pass.""" + limits = thresholds or AuditThresholds.from_project(project) + findings: list[AuditFinding] = [] + class_counts: Counter[str] = Counter() + images = 0 + objects = 0 + + for asset, annotation in project.index.iter_assets_with_annotations(): + images += 1 + if annotation is None or not annotation.objects: + continue + objects += len(annotation.objects) + class_counts.update(obj.class_id for obj in annotation.objects) + findings.extend(_audit_image(asset, annotation, limits)) + + findings.extend(_audit_class_balance(class_counts, limits)) + return AuditReport(findings=findings, images_audited=images, objects_audited=objects, class_counts=dict(class_counts)) + + +# --- per-image checks --------------------------------------------------------- + + +def _audit_image(asset: Asset, annotation: Annotation, limits: AuditThresholds) -> list[AuditFinding]: + findings: list[AuditFinding] = [] + boxed = [(obj, obj.bbox) for obj in annotation.objects if obj.bbox is not None] + + for obj, box in boxed: + findings.extend(_audit_box(asset, obj.class_id, box, limits)) + + # Duplicate detection is O(n²) per image over same-class pairs — n is the + # object count of one image, so this stays trivial even on dense scenes. + for i in range(len(boxed)): + for j in range(i + 1, len(boxed)): + obj_a, box_a = boxed[i] + obj_b, box_b = boxed[j] + if obj_a.class_id != obj_b.class_id: + continue + iou = bbox_iou(box_a, box_b) + if iou >= limits.duplicate_iou: + findings.append( + AuditFinding( + "box.duplicate", + f"Two {obj_a.class_id!r} boxes overlap at IoU {iou:.2f} in {asset.original_path} — " + "likely the same object labeled twice", + asset.id, + asset.original_path, + obj_a.class_id, + ) + ) + return findings + + +def _audit_box(asset: Asset, class_id: str, box: BBox, limits: AuditThresholds) -> list[AuditFinding]: + findings: list[AuditFinding] = [] + if box.width <= 0 or box.height <= 0: + return findings # invalid, vp validate's territory + + if box.width < limits.min_box_px or box.height < limits.min_box_px: + findings.append( + AuditFinding( + "box.degenerate", + f"Tiny {class_id!r} box ({box.width:.0f}x{box.height:.0f} px, threshold {limits.min_box_px:.0f}) in {asset.original_path}", + asset.id, + asset.original_path, + class_id, + ) + ) + + ratio = max(box.width / box.height, box.height / box.width) + if ratio > limits.max_aspect_ratio: + findings.append( + AuditFinding( + "box.aspect_outlier", + f"Extreme aspect ratio {ratio:.1f}:1 for {class_id!r} box in {asset.original_path}", + asset.id, + asset.original_path, + class_id, + ) + ) + + if asset.width > 0 and asset.height > 0: + coverage = (box.width * box.height) / (asset.width * asset.height) + if coverage >= limits.covers_image_ratio: + findings.append( + AuditFinding( + "box.covers_image", + f"{class_id!r} box covers {coverage:.0%} of {asset.original_path} — whole-image boxes are usually labeling artifacts", + asset.id, + asset.original_path, + class_id, + ) + ) + else: + tol = limits.edge_tolerance_px + edges = sum( + ( + box.x <= tol, + box.y <= tol, + box.x + box.width >= asset.width - tol, + box.y + box.height >= asset.height - tol, + ) + ) + # One touched border is normal (an object leaving the frame); two or + # more usually means clipping bugs or coordinate-space mistakes. + if edges >= 2: + findings.append( + AuditFinding( + "box.edge_pinned", + f"{class_id!r} box is pinned to {edges} image borders in {asset.original_path}", + asset.id, + asset.original_path, + class_id, + ) + ) + return findings + + +# --- dataset-level checks ----------------------------------------------------- + + +def _audit_class_balance(class_counts: Counter[str], limits: AuditThresholds) -> list[AuditFinding]: + findings: list[AuditFinding] = [] + if not class_counts: + return findings + + for class_id, count in sorted(class_counts.items()): + if count < limits.min_class_count: + findings.append( + AuditFinding( + "class.rare", + f"Class {class_id!r} has only {count} labeled object(s) (threshold {limits.min_class_count}) — " + "metrics on it will be noise", + class_id=class_id, + ) + ) + + if len(class_counts) >= 2: + most = class_counts.most_common() + top_class, top_count = most[0] + bottom_class, bottom_count = most[-1] + if bottom_count > 0 and top_count / bottom_count > limits.imbalance_ratio: + findings.append( + AuditFinding( + "class.imbalance", + f"Class imbalance {top_count / bottom_count:.1f}:1 — {top_class!r} has {top_count} objects, " + f"{bottom_class!r} has {bottom_count} (threshold {limits.imbalance_ratio:.0f}:1)", + class_id=bottom_class, + ) + ) + return findings diff --git a/visionpack/cli/commands/audit.py b/visionpack/cli/commands/audit.py new file mode 100644 index 0000000..5319884 --- /dev/null +++ b/visionpack/cli/commands/audit.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import argparse + +from visionpack.audit import AuditThresholds, audit_project +from visionpack.cli.output import emit_json +from visionpack.core.project import Project + + +def register(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None: + parser = subparsers.add_parser("audit", help="Label-health audit: suspicious (but valid) labels and class-balance risks") + parser.add_argument("--min-box-px", type=float, default=None, help="Boxes thinner/shorter than this are degenerate (default: 8)") + parser.add_argument("--duplicate-iou", type=float, default=None, help="Same-class IoU at/above this is a duplicate (default: 0.9)") + parser.add_argument( + "--max-aspect-ratio", type=float, default=None, help="Long/short side ratio beyond this is an outlier (default: 20)" + ) + parser.add_argument("--imbalance-ratio", type=float, default=None, help="Most/least frequent class ratio that warns (default: 20)") + parser.add_argument("--min-class-count", type=int, default=None, help="Classes with fewer objects are flagged rare (default: 10)") + parser.add_argument("--limit", type=int, default=50, help="Max findings to print in human output (default: 50)") + parser.add_argument( + "--fail-on-findings", + action="store_true", + help="Exit non-zero when the audit finds anything (findings are advisory by default)", + ) + parser.add_argument("--json", action="store_true", help="Print a machine-readable JSON result") + parser.set_defaults(func=run) + + +def run(args: argparse.Namespace) -> int: + project = Project.open(".") + thresholds = AuditThresholds.from_project( + project, + min_box_px=args.min_box_px, + duplicate_iou=args.duplicate_iou, + max_aspect_ratio=args.max_aspect_ratio, + imbalance_ratio=args.imbalance_ratio, + min_class_count=args.min_class_count, + ) + report = audit_project(project, thresholds) + exit_code = 1 if (args.fail_on_findings and not report.ok) else 0 + + if args.json: + emit_json("audit", report.to_dict()) + return exit_code + + print(f"Audit: {len(report.findings)} finding(s) over {report.images_audited} image(s), {report.objects_audited} object(s)") + for code, count in report.counts_by_code().items(): + print(f" {code}: {count}") + for finding in report.findings[: args.limit]: + print(f"[{finding.code}] {finding.message}") + if len(report.findings) > args.limit: + print(f"... {len(report.findings) - args.limit} more findings (raise --limit or use --json)") + if report.ok: + print("No label-health findings.") + return exit_code diff --git a/visionpack/cli/commands/diff.py b/visionpack/cli/commands/diff.py index 056449e..9c5f09b 100644 --- a/visionpack/cli/commands/diff.py +++ b/visionpack/cli/commands/diff.py @@ -5,12 +5,18 @@ from visionpack.cli.output import emit_json from visionpack.core.project import Project from visionpack.diff import diff_snapshots +from visionpack.drift import drift_from_stats def register(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None: parser = subparsers.add_parser("diff", help="Diff two snapshots") parser.add_argument("left") parser.add_argument("right") + parser.add_argument( + "--drift", + action="store_true", + help="Also report class-distribution drift (per-class share deltas, KL/JS divergence)", + ) parser.add_argument("--json", action="store_true", help="Print a machine-readable JSON result") parser.set_defaults(func=run) @@ -18,8 +24,15 @@ def register(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> def run(args: argparse.Namespace) -> int: project = Project.open(".") result = diff_snapshots(project, args.left, args.right) + drift = None + if args.drift: + # Reuse the stats the diff already loaded; no second snapshot read. + drift = drift_from_stats(result["stats_before"], result["stats_after"], left=args.left, right=args.right) if args.json: - emit_json("diff", {"left": args.left, "right": args.right, **result}) + payload = {"left": args.left, "right": args.right, **result} + if drift is not None: + payload["drift"] = drift + emit_json("diff", payload) return 0 print(f"Diff {args.left} -> {args.right}") for key in ( @@ -33,4 +46,28 @@ def run(args: argparse.Namespace) -> int: ): print(f"{key}: {len(result[key])}") print(f"splits_changed: {result['splits_changed']}") + if drift is not None: + _print_drift(drift) return 0 + + +def _print_drift(drift: dict) -> None: + print( + f"\nDrift {drift['from']} -> {drift['to']}: " + f"{drift['images_before']} -> {drift['images_after']} images, " + f"{drift['objects_before']} -> {drift['objects_after']} objects" + ) + if drift["kl_divergence"] is not None: + print(f"KL divergence (after || before): {drift['kl_divergence']}") + print(f"JS divergence (symmetric, max ln2 = 0.693): {drift['js_divergence']}") + if not drift["classes"]: + print("No labeled objects in either snapshot; nothing to compare.") + return + print(f"{'class':<24} {'before':>8} {'after':>8} {'delta':>7} {'share delta':>12}") + for item in drift["classes"][:20]: + print( + f"{item['class_id']:<24} {item['before']:>8} {item['after']:>8} " + f"{item['delta']:>+7} {item['share_delta']:>+11.2%}" + ) + if len(drift["classes"]) > 20: + print(f"... {len(drift['classes']) - 20} more classes (use --json for all)") diff --git a/visionpack/cli/commands/import_.py b/visionpack/cli/commands/import_.py index 384e00d..f9c1671 100644 --- a/visionpack/cli/commands/import_.py +++ b/visionpack/cli/commands/import_.py @@ -18,7 +18,12 @@ def register(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None: parser = subparsers.add_parser("import", help="Import a dataset") parser.add_argument("source", help="Input dataset path (YOLO/ImageFolder root, or COCO annotation JSON)") - parser.add_argument("--format", required=True, choices=["yolo", "coco", "imagefolder"], help="Input format") + parser.add_argument( + "--format", + default="auto", + choices=["auto", "yolo", "coco", "imagefolder"], + help="Input format (default: auto — detected from the dataset's structure)", + ) parser.add_argument("--images", help="Image directory (required for --format coco)") parser.add_argument( "--task", @@ -49,6 +54,8 @@ def _run_locked(project: Project, args: argparse.Namespace) -> int: project.manifest.task = args.task project.save_manifest() + _resolve_format(args) + if args.format == "coco": if not args.images: raise VisionPackError("--images is required when importing COCO (the directory holding the image files)") @@ -102,6 +109,30 @@ def _run_locked(project: Project, args: argparse.Namespace) -> int: return 0 +def _resolve_format(args: argparse.Namespace) -> None: + """Turn ``--format auto`` into a concrete format (mutating ``args`` so the + recorded source and the JSON output carry what was actually imported). + + When COCO is detected from a *directory*, the annotations JSON found inside + becomes the source and the directory doubles as ``--images`` unless one was + given — so ``vp import ./dataset`` works on the common "instances.json next + to the images" layout. + """ + if args.format != "auto": + return + from visionpack.formats.detect import coco_json_in, detect_import_format + + source = Path(args.source) + detected = detect_import_format(source) + if detected == "coco" and source.is_dir(): + annotations = coco_json_in(source) + assert annotations is not None # detection said coco, so the JSON is there + args.source = str(annotations) + if not args.images: + args.images = str(source) + args.format = detected + + def _report_failures(failures: list) -> None: print(f"Skipped {len(failures)} unreadable/corrupt image(s):") for failure in failures[:20]: diff --git a/visionpack/cli/commands/snapshot.py b/visionpack/cli/commands/snapshot.py index 543076d..797197c 100644 --- a/visionpack/cli/commands/snapshot.py +++ b/visionpack/cli/commands/snapshot.py @@ -6,7 +6,7 @@ from visionpack.cli.output import emit_json from visionpack.core.lock import project_lock from visionpack.core.project import Project -from visionpack.snapshot import create_snapshot, list_snapshots, load_snapshot +from visionpack.snapshot import create_snapshot, list_snapshots, load_snapshot, tag_snapshot, untag_snapshot def register(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None: @@ -27,6 +27,13 @@ def register(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> show.add_argument("--json", action="store_true", help="Print the machine-readable JSON envelope") show.set_defaults(func=run_show) + tag = nested.add_parser("tag", help="Tag a snapshot for lineage (e.g. trained:run-812)") + tag.add_argument("version", help="Snapshot version, e.g. v4") + tag.add_argument("tag", help="Free-form tag; convention is key:value (trained:)") + tag.add_argument("--remove", action="store_true", help="Remove the tag instead of adding it") + tag.add_argument("--json", action="store_true", help="Print a machine-readable JSON result") + tag.set_defaults(func=run_tag) + def run_create(args: argparse.Namespace) -> int: project = Project.open(".") @@ -48,12 +55,28 @@ def run_list(args: argparse.Namespace) -> int: for item in snapshots: stats = item.get("stats", {}) counts = f"{stats.get('assets', '?')} imgs, {stats.get('objects', '?')} objs" - print(f"{item['version']:<5} {item['created_at']} {counts:<20} {item['message']}") + tags = f" [{', '.join(item['tags'])}]" if item.get("tags") else "" + print(f"{item['version']:<5} {item['created_at']} {counts:<20} {item['message']}{tags}") if not snapshots: print("No snapshots. Create one with: vp snapshot create -m ") return 0 +def run_tag(args: argparse.Namespace) -> int: + project = Project.open(".") + with project_lock(project.root): + if args.remove: + snapshot = untag_snapshot(project, args.version, args.tag) + else: + snapshot = tag_snapshot(project, args.version, args.tag) + if args.json: + emit_json("snapshot.tag", {"version": args.version, "tag": args.tag, "removed": args.remove, "tags": snapshot.get("tags", [])}) + return 0 + action = "Removed tag" if args.remove else "Tagged" + print(f"{action} {args.version} {args.tag!r}; tags now: {', '.join(snapshot.get('tags', [])) or '(none)'}") + return 0 + + def run_show(args: argparse.Namespace) -> int: project = Project.open(".") snapshot = load_snapshot(project, args.version) diff --git a/visionpack/cli/main.py b/visionpack/cli/main.py index f3dca97..2f3cf3e 100644 --- a/visionpack/cli/main.py +++ b/visionpack/cli/main.py @@ -7,6 +7,7 @@ from visionpack.cli.commands import ( annotate, + audit, autolabel, diff, export, @@ -45,6 +46,7 @@ def build_parser() -> argparse.ArgumentParser: import_.register(subparsers) sync.register(subparsers) validate.register(subparsers) + audit.register(subparsers) fsck.register(subparsers) stats.register(subparsers) split.register(subparsers) diff --git a/visionpack/drift.py b/visionpack/drift.py new file mode 100644 index 0000000..67473a8 --- /dev/null +++ b/visionpack/drift.py @@ -0,0 +1,112 @@ +"""Distribution drift between two snapshots (``vp diff --drift``). + +A dataset that grows is supposed to change; what needs watching is *how* its +class distribution changes. A new batch of images that doubles one class and +starves another silently shifts what the next model optimizes for — metrics +move and nobody knows whether the model or the data changed. + +This module compares the class distributions captured in two snapshots' +``stats`` blocks (no inventory rehydration needed — snapshot stats are computed +at create time) and reports: + +- **per-class deltas**: object counts and distribution share before/after, so + the classes driving the shift are named, not just scored; +- **divergence scores**: Kullback–Leibler (after vs. before, additively + smoothed so new/vanished classes stay finite) and Jensen–Shannon divergence + (symmetric, bounded to [0, ln 2]) as single drift numbers a CI job can + threshold. + +Everything derives from data already frozen in the snapshots, so drift between +``v1`` and ``v2`` is reproducible forever. +""" + +from __future__ import annotations + +import math +from typing import Any + +from visionpack.core.project import Project +from visionpack.snapshot import load_snapshot + +# Additive smoothing applied to both distributions before KL: keeps a class +# that appears (or disappears) between snapshots from producing infinity while +# barely perturbing well-populated classes. +_SMOOTHING = 0.5 + + +def drift_between(project: Project, left: str, right: str) -> dict[str, Any]: + """Class-distribution drift from snapshot ``left`` to snapshot ``right``.""" + old = load_snapshot(project, left) + new = load_snapshot(project, right) + return drift_from_stats(old.get("stats", {}), new.get("stats", {}), left=left, right=right) + + +def drift_from_stats( + old_stats: dict[str, Any], new_stats: dict[str, Any], *, left: str = "before", right: str = "after" +) -> dict[str, Any]: + old_counts = {str(k): int(v) for k, v in old_stats.get("class_distribution", {}).items()} + new_counts = {str(k): int(v) for k, v in new_stats.get("class_distribution", {}).items()} + classes = sorted(set(old_counts) | set(new_counts)) + old_total = sum(old_counts.values()) + new_total = sum(new_counts.values()) + + per_class: list[dict[str, Any]] = [] + for class_id in classes: + before = old_counts.get(class_id, 0) + after = new_counts.get(class_id, 0) + share_before = before / old_total if old_total else 0.0 + share_after = after / new_total if new_total else 0.0 + per_class.append( + { + "class_id": class_id, + "before": before, + "after": after, + "delta": after - before, + "share_before": round(share_before, 6), + "share_after": round(share_after, 6), + "share_delta": round(share_after - share_before, 6), + } + ) + # Biggest distribution movers first, so the head of the list is the story. + per_class.sort(key=lambda item: (-abs(item["share_delta"]), item["class_id"])) + + return { + "from": left, + "to": right, + "classes": per_class, + "objects_before": old_total, + "objects_after": new_total, + "images_before": int(old_stats.get("assets", 0)), + "images_after": int(new_stats.get("assets", 0)), + "kl_divergence": _kl(old_counts, new_counts, classes), + "js_divergence": _js(old_counts, new_counts, classes), + } + + +def _distribution(counts: dict[str, int], classes: list[str]) -> list[float]: + smoothed = [counts.get(class_id, 0) + _SMOOTHING for class_id in classes] + total = sum(smoothed) + return [value / total for value in smoothed] + + +def _kl(old_counts: dict[str, int], new_counts: dict[str, int], classes: list[str]) -> float | None: + """KL(after || before), smoothed. ``None`` when either side has no labels.""" + if not classes or not sum(old_counts.values()) or not sum(new_counts.values()): + return None + p = _distribution(new_counts, classes) + q = _distribution(old_counts, classes) + return round(sum(pi * math.log(pi / qi) for pi, qi in zip(p, q, strict=True)), 6) + + +def _js(old_counts: dict[str, int], new_counts: dict[str, int], classes: list[str]) -> float | None: + """Jensen–Shannon divergence: symmetric, bounded to [0, ln 2].""" + if not classes or not sum(old_counts.values()) or not sum(new_counts.values()): + return None + p = _distribution(new_counts, classes) + q = _distribution(old_counts, classes) + m = [(pi + qi) / 2 for pi, qi in zip(p, q, strict=True)] + + def kl(a: list[float], b: list[float]) -> float: + return sum(ai * math.log(ai / bi) for ai, bi in zip(a, b, strict=True)) + + return round((kl(p, m) + kl(q, m)) / 2, 6) diff --git a/visionpack/formats/base.py b/visionpack/formats/base.py index cc99182..8e9eb84 100644 --- a/visionpack/formats/base.py +++ b/visionpack/formats/base.py @@ -2,6 +2,25 @@ from dataclasses import dataclass, field +# Characters that would let a name escape (or misbehave in) an export tree when +# used as a single path component: separators, traversal, and Windows-reserved. +_UNSAFE_COMPONENT_CHARS = str.maketrans({ch: "_" for ch in '/\\:*?"<>|\0'}) + + +def safe_path_component(name: str, fallback: str = "unnamed") -> str: + """Make ``name`` safe to use as one directory/file name inside an export. + + Class names come from *imported data* (folder names, COCO ``categories``, + ``classes.txt``), so a name like ``../../x`` or ``a/b`` must never be able + to place files outside — or in unexpected subtrees of — the export + directory. Separators and reserved characters are replaced, traversal names + collapse to the fallback, and the result is never empty. + """ + cleaned = name.translate(_UNSAFE_COMPONENT_CHARS).strip() + if cleaned.strip(".") == "": # "", ".", "..", "..." — traversal or hidden-empty + return fallback + return cleaned + @dataclass(slots=True) class IngestFailure: diff --git a/visionpack/formats/classification.py b/visionpack/formats/classification.py index fe68e67..d0c844a 100644 --- a/visionpack/formats/classification.py +++ b/visionpack/formats/classification.py @@ -8,7 +8,7 @@ from visionpack.core.errors import FormatError, VisionPackError from visionpack.core.models import Annotation, Asset, ObjectAnnotation, utc_now from visionpack.core.project import Project -from visionpack.formats.base import ImportSummary, IngestFailure +from visionpack.formats.base import ImportSummary, IngestFailure, safe_path_component from visionpack.media import image_info_from_bytes, is_image_path from visionpack.perceptual import dhash_bytes from visionpack.progress import ProgressCallback @@ -161,7 +161,7 @@ def export_imagefolder( if label_obj is None: skipped += 1 continue - class_name = id_to_name.get(label_obj.class_id, label_obj.class_id) + class_name = safe_path_component(id_to_name.get(label_obj.class_id, label_obj.class_id)) parts = [output, set_name, class_name] if split_id else [output, class_name] target_dir = Path(*[str(part) for part in parts]) target_dir.mkdir(parents=True, exist_ok=True) diff --git a/visionpack/formats/detect.py b/visionpack/formats/detect.py new file mode 100644 index 0000000..8d73979 --- /dev/null +++ b/visionpack/formats/detect.py @@ -0,0 +1,91 @@ +"""Import-format detection for ``vp import --format auto``. + +Given the path a user points ``vp import`` at, decide whether it is a COCO +annotations file, a YOLO dataset root, or an ImageFolder root. Detection is by +*structure*, in order of decreasing certainty: + +1. a ``.json`` file (or a directory whose root holds an instances-style JSON) + is **COCO**; +2. any ``.txt`` label files, or YOLO furniture (``classes.txt``, ``obj.names``, + ``data.yaml``), means **YOLO**; +3. otherwise, images living only under first-level subdirectories (the + folder-per-class convention) means **ImageFolder**; +4. images sitting directly in the root (no labels at all) fall back to + **YOLO**, which imports unlabeled images fine. + +Anything that fits none of these raises a :class:`VisionPackError` telling the +user to pass ``--format`` explicitly — a wrong silent guess would import the +dataset with the wrong task, which costs far more than one flag. +""" + +from __future__ import annotations + +from pathlib import Path + +from visionpack.core.errors import VisionPackError +from visionpack.media import is_image_path + +_YOLO_FURNITURE = ("classes.txt", "obj.names", "data.yaml") + + +def detect_import_format(source: Path) -> str: + """The import format (``yolo`` | ``coco`` | ``imagefolder``) of ``source``.""" + source = source.resolve() + if not source.exists(): + raise VisionPackError(f"Import source does not exist: {source}") + + if source.is_file(): + if source.suffix.lower() == ".json": + return "coco" + raise VisionPackError( + f"Cannot auto-detect the format of a single file: {source}. " + "Pass --format explicitly (a COCO annotations file must end in .json)." + ) + + # COCO: an instances-style JSON at the root. + if coco_json_in(source) is not None: + return "coco" + + # YOLO: label files or the classes/data furniture anywhere relevant. + if any((source / name).exists() for name in _YOLO_FURNITURE): + return "yolo" + has_txt = any(path.suffix.lower() == ".txt" for path in source.rglob("*.txt")) + if has_txt: + return "yolo" + + top_level_images = any(path.is_file() and is_image_path(path) for path in source.iterdir()) + class_dirs = [path for path in source.iterdir() if path.is_dir()] + nested_images = any( + any(child.is_file() and is_image_path(child) for child in class_dir.rglob("*")) for class_dir in class_dirs + ) + if not top_level_images and nested_images: + return "imagefolder" + if top_level_images: + return "yolo" # plain images, no labels: YOLO import handles unlabeled sets + + raise VisionPackError( + f"Cannot auto-detect the dataset format under {source} (no labels, no images found). " + "Pass --format yolo|coco|imagefolder explicitly." + ) + + +def coco_json_in(source: Path) -> Path | None: + """The first instances-style JSON directly under ``source``, if any.""" + for path in sorted(source.glob("*.json")): + if _looks_like_coco(path): + return path + return None + + +def _looks_like_coco(path: Path) -> bool: + """Cheap structural sniff: a JSON object mentioning images+annotations. + + Reads only the head of the file, so a multi-hundred-MB instances JSON + doesn't get fully parsed just to answer "is this COCO?". + """ + try: + with path.open("r", encoding="utf-8", errors="replace") as handle: + head = handle.read(4096) + except OSError: + return False + return head.lstrip().startswith("{") and any(key in head for key in ('"images"', '"annotations"', '"categories"')) diff --git a/visionpack/index/json_index.py b/visionpack/index/json_index.py index 8e716c3..328d1a3 100644 --- a/visionpack/index/json_index.py +++ b/visionpack/index/json_index.py @@ -79,6 +79,9 @@ def annotations(self) -> list[Annotation]: def splits(self) -> list[Split]: return [Split.from_dict(item) for item in self._data.get("splits", {}).values()] + def asset_ids(self) -> set[str]: + return set(self._data.get("assets", {})) + def annotation_for_asset(self, asset_id: str) -> Annotation | None: if self._annotation_by_asset is None: self._annotation_by_asset = {item.asset_id: item for item in self.annotations()} diff --git a/visionpack/index/sqlite_index.py b/visionpack/index/sqlite_index.py index c2c2719..6a00844 100644 --- a/visionpack/index/sqlite_index.py +++ b/visionpack/index/sqlite_index.py @@ -265,6 +265,21 @@ def splits(self) -> list[Split]: self._splits[split_id] = split return list(self._splits.values()) + def asset_ids(self) -> set[str]: + """The ids of every indexed asset, without materializing the records. + + Sync/import only need "is this asset already known?", so a bare id + query keeps that check cheap on large stores — no per-row JSON parse, + no ``Asset`` construction. Unsaved writes are overlaid; an + already-materialized cache is reused instead of re-querying. + """ + if self._assets is not None: + return set(self._assets) + with closing(self._connect()) as conn: + ids = {row[0] for row in conn.execute("SELECT id FROM assets")} + ids.update(self._dirty_assets) + return ids + def annotation_for_asset(self, asset_id: str) -> Annotation | None: if self._annotation_by_asset is None: self._annotation_by_asset = {item.asset_id: item for item in self.annotations()} diff --git a/visionpack/media.py b/visionpack/media.py index 02a57a6..4f1426e 100644 --- a/visionpack/media.py +++ b/visionpack/media.py @@ -27,7 +27,7 @@ def image_info(path: Path) -> tuple[int, int, int | None, str]: try: with Image.open(path) as img: return _probe(img, path) - except (UnidentifiedImageError, OSError, ValueError) as exc: + except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as exc: raise FormatError(f"File is not a readable image: {path} ({exc})") from exc @@ -36,11 +36,15 @@ def image_info_from_bytes(data: bytes, source: Path) -> tuple[int, int, int | No Used by import, where the bytes have just been read to compute the content hash, so dimensions and hash come from a single read of the file. + + ``DecompressionBombError`` (a header claiming absurd dimensions) subclasses + neither ``OSError`` nor ``ValueError``, so it is caught explicitly — one + hostile file must become a per-file ingest failure, not abort the batch. """ try: with Image.open(BytesIO(data)) as img: return _probe(img, source) - except (UnidentifiedImageError, OSError, ValueError) as exc: + except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as exc: raise FormatError(f"File is not a readable image: {source} ({exc})") from exc diff --git a/visionpack/sdk/__init__.py b/visionpack/sdk/__init__.py new file mode 100644 index 0000000..02f174f --- /dev/null +++ b/visionpack/sdk/__init__.py @@ -0,0 +1,411 @@ +"""VisionPack SDK: drive the framework from Python instead of the shell. + +The CLI's ``--json`` envelopes are the contract for driving VisionPack from +*other processes*; this SDK is the same contract for Python code — notebooks, +training scripts, labeling services, CI jobs — with none of the subprocess +plumbing. One class, :class:`VisionPackClient`, wraps the whole dataset +lifecycle behind stable, typed methods: + + from visionpack.sdk import VisionPackClient + + ds = VisionPackClient.init("./factory-defects", task="detection") + ds.import_dir("./raw", format="yolo") + report = ds.validate() + ds.create_split(train=0.8, val=0.1, test=0.1, strategy="stratified") + ds.lock_split() + ds.snapshot("baseline") + ds.export("./exports/yolo", format="yolo", split="default") + + # ...train, predict, then close the loop: + metrics = ds.evaluate("runs/predict/labels", format="yolo") + ds.autolabel("preds.json", min_confidence=0.6) + for item in ds.annotation_queue("preds.json")[:20]: + print(item["path"], item["score"]) + +Guarantees the SDK adds on top of the internal modules: + +- **Concurrency safety**: every mutating method takes the same project lock the + CLI takes, so an SDK caller and a ``vp`` process can never corrupt each + other's writes. +- **Stable returns**: methods return plain dicts/dataclasses that mirror the + ``--json`` CLI output, so a service can switch between shelling out and + importing the SDK without re-parsing. +- **Snapshot views**: :meth:`VisionPackClient.checkout` returns a read-only + client pinned to a snapshot, for exporting or evaluating historical versions. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from visionpack.audit import AuditReport, AuditThresholds, audit_project +from visionpack.autolabel import apply_predictions +from visionpack.core.errors import VisionPackError +from visionpack.core.lock import project_lock +from visionpack.core.models import Annotation, Asset, ClassDef, Split +from visionpack.core.project import Project +from visionpack.curation import rank_for_annotation +from visionpack.diff import diff_snapshots +from visionpack.drift import drift_between +from visionpack.eval import evaluate as _evaluate +from visionpack.predictions import PredictionSet, load_predictions +from visionpack.snapshot import ( + create_snapshot, + find_snapshots_by_tag, + list_snapshots, + load_snapshot, + open_snapshot, + tag_snapshot, + untag_snapshot, +) +from visionpack.split import create_split as _create_split +from visionpack.split import get_split, lock_split +from visionpack.stats import collect_stats, split_breakdown +from visionpack.validation import ValidationReport, validate_project + +__all__ = ["VisionPackClient", "init", "open"] + +EXPORT_FORMATS = ("yolo", "coco", "imagefolder", "masks") +IMPORT_FORMATS = ("yolo", "coco", "imagefolder") + + +class VisionPackClient: + """A Python handle on one VisionPack dataset (one ``visionpack.yaml``). + + Construct through :meth:`init` (create) or :meth:`open` (existing). The + client is a thin, stateless facade: every call reads/writes the same + on-disk project the CLI uses, so the two can be mixed freely. + """ + + def __init__(self, project: Project, *, _readonly: bool = False, _snapshot: str | None = None) -> None: + self._project = project + self._readonly = _readonly + self._snapshot_version = _snapshot + + # -- lifecycle ------------------------------------------------------------- + + @classmethod + def init(cls, root: str | Path = ".", *, name: str | None = None, task: str = "detection") -> VisionPackClient: + """Create a project at ``root`` (idempotent) and return a client on it.""" + return cls(Project.init(root, name=name, task=task)) + + @classmethod + def open(cls, root: str | Path = ".") -> VisionPackClient: + """Open the project at (or above) ``root``.""" + return cls(Project.open(root)) + + @property + def project(self) -> Project: + """The underlying :class:`Project`, for advanced/internal use.""" + return self._project + + @property + def root(self) -> Path: + return self._project.root + + @property + def name(self) -> str: + return self._project.manifest.name + + @property + def task(self) -> str: + return self._project.manifest.task + + @property + def classes(self) -> list[ClassDef]: + return list(self._project.manifest.classes) + + @property + def readonly(self) -> bool: + """True for snapshot views returned by :meth:`checkout`.""" + return self._readonly + + def __repr__(self) -> str: + pinned = f", snapshot={self._snapshot_version!r}" if self._snapshot_version else "" + return f"VisionPackClient({str(self.root)!r}, name={self.name!r}, task={self.task!r}{pinned})" + + # -- data access ----------------------------------------------------------- + + def assets(self) -> list[Asset]: + return self._project.index.assets() + + def annotations(self) -> list[Annotation]: + return self._project.index.annotations() + + def samples(self) -> Iterator[tuple[Asset, Annotation | None]]: + """Stream ``(asset, annotation)`` pairs without materializing the index.""" + yield from self._project.index.iter_assets_with_annotations() + + def __len__(self) -> int: + return self._project.index.count_assets() + + def __iter__(self) -> Iterator[tuple[Asset, Annotation | None]]: + return self.samples() + + # -- ingest ---------------------------------------------------------------- + + def import_dir( + self, + source: str | Path, + *, + format: str = "auto", + images: str | Path | None = None, + copy_mode: str = "ingest", + ) -> dict[str, Any]: + """Import a dataset from disk (mirrors ``vp import``). + + ``source`` is the YOLO/ImageFolder root — or, for ``format="coco"``, + the annotations JSON, with ``images`` pointing at the image directory. + The default ``format="auto"`` detects the layout from the dataset's + structure. Returns the import summary as a dict (including per-file + ``failures``). + """ + if format == "auto": + from visionpack.formats.detect import coco_json_in, detect_import_format + + source_path = Path(source) + format = detect_import_format(source_path) + if format == "coco" and source_path.is_dir(): + # "instances.json next to the images" layout: the JSON becomes + # the source and the directory doubles as the images root. + annotations = coco_json_in(source_path) + assert annotations is not None # detection said coco, so the JSON is there + source = annotations + images = images or source_path + if format not in IMPORT_FORMATS: + raise VisionPackError(f"Unknown import format {format!r}. Use one of: auto, {', '.join(IMPORT_FORMATS)}.") + from visionpack.formats.classification import ImageFolderImporter + from visionpack.formats.coco import CocoImporter + from visionpack.formats.yolo import YoloImporter + + with self._write_lock(): + if format == "coco": + if images is None: + raise VisionPackError("import_dir(format='coco') needs images=.") + importer = CocoImporter(self._project, Path(source), Path(images), copy_mode=copy_mode) + elif format == "imagefolder": + importer = ImageFolderImporter(self._project, Path(source), copy_mode=copy_mode) + else: + importer = YoloImporter(self._project, Path(source), copy_mode=copy_mode) + summary = importer.run() + return { + "format": format, + "assets": summary.assets, + "annotations": summary.annotations, + "objects": summary.objects, + "classes_added": summary.classes_added, + "orphan_labels": summary.orphan_labels, + "failures": [asdict(failure) for failure in summary.failures], + } + + def sync(self, *, source: str | None = None, jobs: int | None = None) -> list[dict[str, Any]]: + """Pull every source declared in ``visionpack.yaml`` (mirrors ``vp sync``).""" + from visionpack.sources.importer import sync_sources + + with self._write_lock(): + summaries = sync_sources(self._project, source_name=source, max_workers=jobs) + return [asdict(summary) for summary in summaries] + + def plan_sync(self, *, source: str | None = None) -> list[dict[str, Any]]: + """What :meth:`sync` would do, without writing (``vp sync --dry-run``).""" + from visionpack.sources.importer import plan_sources + + return [asdict(plan) for plan in plan_sources(self._project, source_name=source)] + + # -- quality --------------------------------------------------------------- + + def validate(self, *, strict: bool = False) -> ValidationReport: + """Correctness checks: corrupt images, bad boxes, duplicates, leakage.""" + return validate_project(self._project, strict=strict) + + def audit(self, **thresholds: Any) -> AuditReport: + """Label-health audit (mirrors ``vp audit``). + + Keyword thresholds override ``validation.audit`` from the manifest: + ``min_box_px``, ``duplicate_iou``, ``max_aspect_ratio``, + ``edge_tolerance_px``, ``covers_image_ratio``, ``imbalance_ratio``, + ``min_class_count``. + """ + return audit_project(self._project, AuditThresholds.from_project(self._project, **thresholds)) + + def stats(self) -> dict[str, Any]: + return collect_stats(self._project) + + def split_stats(self, split_id: str = "default") -> dict[str, Any] | None: + return split_breakdown(self._project, split_id) + + # -- splits ---------------------------------------------------------------- + + def create_split( + self, + *, + train: float = 0.8, + val: float = 0.1, + test: float = 0.1, + strategy: str = "stratified", + seed: int = 0, + split_id: str = "default", + force: bool = False, + ) -> Split: + with self._write_lock(): + return _create_split( + self._project, train=train, val=val, test=test, strategy=strategy, seed=seed, split_id=split_id, force=force + ) + + def lock_split(self, split_id: str = "default") -> Split: + with self._write_lock(): + return lock_split(self._project, split_id) + + def split(self, split_id: str = "default") -> Split | None: + return get_split(self._project, split_id) + + # -- versions ---------------------------------------------------------------- + + def snapshot(self, message: str) -> dict[str, Any]: + """Freeze the current dataset state as a new version (``vp snapshot create``).""" + with self._write_lock(): + return create_snapshot(self._project, message) + + def snapshots(self) -> list[dict[str, Any]]: + return list_snapshots(self._project) + + def get_snapshot(self, version: str) -> dict[str, Any]: + return load_snapshot(self._project, version) + + def tag_snapshot(self, version: str, tag: str) -> dict[str, Any]: + """Attach a lineage tag (convention ``key:value``, e.g. ``trained:run-812``).""" + with self._write_lock(): + return tag_snapshot(self._project, version, tag) + + def untag_snapshot(self, version: str, tag: str) -> dict[str, Any]: + with self._write_lock(): + return untag_snapshot(self._project, version, tag) + + def snapshots_by_tag(self, tag: str) -> list[dict[str, Any]]: + """Snapshots carrying ``tag`` (a bare ``key:`` prefix matches any value).""" + return find_snapshots_by_tag(self._project, tag) + + def diff(self, left: str, right: str) -> dict[str, Any]: + """Structural diff between two snapshots (mirrors ``vp diff``).""" + return diff_snapshots(self._project, left, right) + + def drift(self, left: str, right: str) -> dict[str, Any]: + """Class-distribution drift between two snapshots (``vp diff --drift``): + per-class share deltas plus KL/JS divergence.""" + return drift_between(self._project, left, right) + + def checkout(self, version: str) -> VisionPackClient: + """A read-only client pinned to snapshot ``version``. + + Reads (``assets``, ``stats``, ``export``, ``evaluate``, ...) reflect + that exact state; mutating methods raise. + """ + return VisionPackClient(open_snapshot(self._project, version), _readonly=True, _snapshot=version) + + # -- outputs --------------------------------------------------------------- + + def export( + self, + output: str | Path, + *, + format: str = "yolo", + split: str | None = None, + seg: bool | None = None, + ) -> dict[str, Any]: + """Write a ready-to-train layout (mirrors ``vp export``).""" + from visionpack.formats.classification import export_imagefolder + from visionpack.formats.coco import export_coco + from visionpack.formats.masks import export_masks + from visionpack.formats.yolo import export_yolo + + output_path = Path(output) + if format == "coco": + return export_coco(self._project, output_path, split_id=split) + if format == "imagefolder": + return export_imagefolder(self._project, output_path, split_id=split) + if format == "masks": + return export_masks(self._project, output_path, split_id=split) + if format == "yolo": + return export_yolo(self._project, output_path, split_id=split, seg=seg) + raise VisionPackError(f"Unknown export format {format!r}. Use one of: {', '.join(EXPORT_FORMATS)}.") + + # -- model in the loop ------------------------------------------------------- + + def load_predictions(self, predictions: str | Path | PredictionSet, *, format: str = "auto") -> PredictionSet: + """Load model output (vp JSON / COCO JSON / YOLO txt dir) into a + :class:`PredictionSet` resolved against this dataset's assets.""" + if isinstance(predictions, PredictionSet): + return predictions + return load_predictions(self._project, Path(predictions), fmt=format) + + def evaluate( + self, + predictions: str | Path | PredictionSet, + *, + format: str = "auto", + split: str | None = "default", + set_name: str = "test", + conf_threshold: float = 0.25, + ) -> dict[str, Any]: + """Score predictions against a split's labels (mirrors ``vp eval``).""" + loaded = self.load_predictions(predictions, format=format) + return _evaluate(self._project, loaded, split_id=split, set_name=set_name, conf_threshold=conf_threshold) + + def autolabel( + self, + predictions: str | Path | PredictionSet, + *, + format: str = "auto", + min_confidence: float = 0.5, + replace: bool = False, + ) -> dict[str, Any]: + """Persist confident predictions as annotations (mirrors ``vp autolabel``).""" + loaded = self.load_predictions(predictions, format=format) + with self._write_lock(): + return apply_predictions(self._project, loaded, min_confidence=min_confidence, replace=replace) + + def annotation_queue( + self, + predictions: str | Path | PredictionSet | None = None, + *, + format: str = "auto", + include_labeled: bool = False, + confident: float = 0.5, + iou_threshold: float = 0.5, + ) -> list[dict[str, Any]]: + """Rank images by annotation value (mirrors ``vp queue``).""" + loaded = self.load_predictions(predictions, format=format) if predictions is not None else None + return rank_for_annotation( + self._project, loaded, include_labeled=include_labeled, confident=confident, iou_threshold=iou_threshold + ) + + # -- internal ---------------------------------------------------------------- + + @contextmanager + def _write_lock(self) -> Iterator[None]: + """The project lock every mutating SDK call runs under. + + Snapshot views are frozen history: mutating them would silently write + into the *live* index (they share the root), so it's refused outright. + """ + if self._readonly: + raise VisionPackError( + f"This client is a read-only view of snapshot {self._snapshot_version!r}; " + "open the live dataset with VisionPackClient.open() to modify it." + ) + with project_lock(self._project.root): + yield + + +def init(root: str | Path = ".", *, name: str | None = None, task: str = "detection") -> VisionPackClient: + """Module-level alias: ``visionpack.sdk.init(...)``.""" + return VisionPackClient.init(root, name=name, task=task) + + +def open(root: str | Path = ".") -> VisionPackClient: # noqa: A001 - deliberate, mirrors Project.open + """Module-level alias: ``visionpack.sdk.open(...)``.""" + return VisionPackClient.open(root) diff --git a/visionpack/snapshot.py b/visionpack/snapshot.py index bb1e040..14f8002 100644 --- a/visionpack/snapshot.py +++ b/visionpack/snapshot.py @@ -45,6 +45,9 @@ def create_snapshot(project: Project, message: str) -> dict[str, Any]: # Stored directly (small) so opening a snapshot doesn't need to load the # whole inventory blob just to recover the class list. "classes": [item.to_dict() for item in project.manifest.classes], + # Free-form lineage tags added later via `vp snapshot tag` (e.g. + # trained:run-812); created empty so the shape is stable. + "tags": [], } (snapshot_dir / f"{version}.json").write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") return load_snapshot(project, version) @@ -68,6 +71,52 @@ def _read_snapshot_file(project: Project, version: str) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) +def tag_snapshot(project: Project, version: str, tag: str) -> dict[str, Any]: + """Attach a free-form tag to a snapshot (idempotent), returning its record. + + This is the dataset -> model lineage link: after training, stamp the exact + dataset version the run consumed (``vp snapshot tag v4 trained:run-812``), + and "which dataset trained this model?" becomes a lookup instead of a + guess. Tags are free-form strings; ``key:value`` is the convention. + """ + cleaned = tag.strip() + if not cleaned: + raise VisionPackError("Tag must be a non-empty string (convention: key:value, e.g. trained:run-812).") + payload = _read_snapshot_file(project, version) + tags = list(payload.get("tags", [])) + if cleaned not in tags: + tags.append(cleaned) + payload["tags"] = tags + _write_snapshot_file(project, version, payload) + return load_snapshot(project, version) + + +def untag_snapshot(project: Project, version: str, tag: str) -> dict[str, Any]: + """Remove a tag from a snapshot (no error if it wasn't there).""" + payload = _read_snapshot_file(project, version) + tags = [item for item in payload.get("tags", []) if item != tag.strip()] + if tags != payload.get("tags", []): + payload["tags"] = tags + _write_snapshot_file(project, version, payload) + return load_snapshot(project, version) + + +def find_snapshots_by_tag(project: Project, tag: str) -> list[dict[str, Any]]: + """Snapshots carrying ``tag`` exactly, or any ``key:*`` tag when ``tag`` is + a bare ``key:`` prefix (so ``trained:`` lists every trained version).""" + matches = [] + for snapshot in list_snapshots(project): + tags = snapshot.get("tags", []) + if tag in tags or (tag.endswith(":") and any(item.startswith(tag) for item in tags)): + matches.append(snapshot) + return matches + + +def _write_snapshot_file(project: Project, version: str, payload: dict[str, Any]) -> None: + path = project.root / ".vp" / "snapshots" / f"{version}.json" + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + def load_snapshot(project: Project, version: str) -> dict[str, Any]: payload = _read_snapshot_file(project, version) # Rehydrate the inventory from its blob unless an older snapshot still diff --git a/visionpack/sources/importer.py b/visionpack/sources/importer.py index 1a008c8..d8d7b55 100644 --- a/visionpack/sources/importer.py +++ b/visionpack/sources/importer.py @@ -218,7 +218,7 @@ def _drain_pool(self, process, items, summary: SourceSyncSummary, progress: Prog stays on this thread; ``pool.map`` preserves input order, so the result is deterministic regardless of scheduling. """ - existing_ids = {asset.id for asset in self.project.index.assets()} + existing_ids = self.project.index.asset_ids() total = len(items) with ThreadPoolExecutor(max_workers=self._pool_size()) as pool: for done, outcome in enumerate(pool.map(process, items), 1): @@ -460,7 +460,7 @@ def _run_coco(self, progress: ProgressCallback | None = None) -> SourceSyncSumma labels_path = label_res.local_path(labels_loc.resolved_uri()) if images_path is None or labels_path is None: return self._run_coco_remote(images_loc, labels_loc, image_res, label_res, progress) - before = {asset.id for asset in self.project.index.assets()} + before = self.project.index.asset_ids() result = CocoImporter(self.project, labels_path, images_path, copy_mode=self.source.copy).run(progress) added = self._tag_provenance(before) return SourceSyncSummary( @@ -600,7 +600,7 @@ def _run_imagefolder(self, progress: ProgressCallback | None = None) -> SourceSy root_path = resolver.local_path(root.resolved_uri()) if root_path is None: return self._run_imagefolder_remote(root, resolver, progress) - before = {asset.id for asset in self.project.index.assets()} + before = self.project.index.asset_ids() result = ImageFolderImporter(self.project, root_path, copy_mode=self.source.copy).run(progress) added = self._tag_provenance(before) return SourceSyncSummary(