diff --git a/.claude/skills/mblt-vision/SKILL.md b/.claude/skills/mblt-vision/SKILL.md index d326291..1b9b9c5 100644 --- a/.claude/skills/mblt-vision/SKILL.md +++ b/.claude/skills/mblt-vision/SKILL.md @@ -49,9 +49,11 @@ description: >- argmax. - Preserve decoded-output layout provenance through NMS. For ambiguous tensors without provenance, prioritize channels-first raw-output normalization. -- Keep NMS candidate sorts on Ultralytics' unstable `argsort(descending=True)` (a stable sort - shifts MXQ mAP through tied scores; see AGENTS.md), and suppress only IoU above the threshold - so NaN from zero-area boxes keeps the box, as torchvision does. Rank ImageNet top-1/top-5 from +- Rank NMS and end-to-end candidates with `common.descending_order` (stable), which is + Ultralytics' CUDA validation order on every device; never a plain `argsort` or `torch.topk` + (see AGENTS.md). `dual_topk`'s second stage returns up to `max_det` (anchor, class) pairs. + Suppress only IoU above the threshold so NaN from zero-area boxes keeps the box, as + torchvision does. Rank ImageNet top-1/top-5 from one stable sort. Decode-true pose MXQ visibility is sigmoided; copy NMS rows before any in-place rescale for evaluation. - Normalize dense depth and semantic outputs before inverse letterboxing. Validate baked semantic @@ -100,6 +102,9 @@ description: >- together. - Rank NYU Depth evaluation by delta1. Expose abs_rel then RMSE (m) as secondary metrics, with median-aligned metrics averaged per image. +- Rank DOTAv1 evaluation by rotated mAP50, with mAP50-95 secondary (Ultralytics' published + metric). Load difficult objects as ordinary targets, as Ultralytics' validation does. +- Run YOLOv7 models at `iou_thres: 0.65`, WongKinYiu/yolov7's own `test.py` default. ## Runtime and Packaging diff --git a/AGENTS.md b/AGENTS.md index f22e4c7..5fd0aa6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,13 +75,22 @@ The current ownership boundary is deliberate: suffix conflicts with an explicitly selected framework. - Preserve anchorless decoded-output layout provenance through NMS. When a tensor is ambiguous and provenance is unavailable, normalize it as raw channels-first before candidates-first. -- Keep NMS candidate ordering on Ultralytics' unstable `argsort(descending=True)`, not - mblt-model-ops' `kind="stable"` sorts. Quantized MXQ scores tie often, and the call Ultralytics - makes reproduces its tie order; switching to a stable sort measured YOLOv8m -0.00047, - YOLOv8m-pose -0.0038 and YOLOv8m-seg +0.00029 mAP50-95 on 500 COCO val images (aries-rb), - entirely from ties. `non_max_suppression` suppresses only an IoU *above* `iou_thres`, as - `torchvision.ops.nms` does: two zero-area boxes give a NaN IoU, which must keep the - candidate, not drop it (no change on those three models' real outputs). +- Rank every NMS and end-to-end candidate with `common.descending_order`, a stable descending + sort, never a plain `argsort(descending=True)` or `torch.topk`. That is Ultralytics' validation + order: `non_max_suppression` passes up to 30000 candidates straight to `torchvision.ops.nms`, + which sorts stably on CPU and CUDA, and the `argsort` above that cap and an end-to-end head's + `torch.topk` (at `k = 300`) are stable on CUDA, where it validates. On CPU those calls order + tied scores differently, and quantized MXQ scores tie often, so the unstable sort this rule + used to require made mblt-vision's CPU results diverge from Ultralytics; the earlier + measurement (YOLOv8m -0.00047, YOLOv8m-pose -0.0038, YOLOv8m-seg +0.00029 mAP50-95 for the + stable sort) compared two tie orders, not either against Ultralytics. + `tests/test_ultralytics_selection_order.py` pins the order. `non_max_suppression` suppresses + only an IoU *above* `iou_thres`, as `torchvision.ops.nms` does: two zero-area boxes give a NaN + IoU, which must keep the candidate, not drop it. +- `dual_topk` reproduces `Detect.get_topk_index`: it may rank only the anchors that clear the + confidence threshold, but its second stage still returns up to `max_det` (anchor, class) pairs. + Capping that stage at the surviving anchor count dropped every further class of those anchors + whenever fewer than `max_det` of them cleared the threshold. - Rank ImageNet predictions once with a stable sort and take top-1 as the head of top-5, so tied scores favour the higher class index as in mblt-model-ops' evaluator, and top-1 is always inside top-5. @@ -154,6 +163,15 @@ The current ownership boundary is deliberate: - For NYU Depth evaluation, rank results by delta1 and retain abs_rel then RMSE (m) as secondary metrics. Median-align each image and average every metric per image, following Ultralytics' depth-validation convention. +- For DOTAv1 evaluation, rank results by rotated mAP50 and retain mAP50-95 as the secondary + metric, because rotated mAP50 is what Ultralytics publishes for its OBB models. Load difficult + objects (flag `1` or `2`) as ordinary targets, as Ultralytics' validation does: its + `convert_dota_to_yolo_obb` drops the flag. `evaluate_dota_predictions` still honours ignore + regions a caller supplies explicitly (the DOTA devkit protocol), but the loader produces none. +- Run YOLOv7 models (`YOLOv7`, `-x`, `-w6`, `-e6`, `-d6`, `-e6e`) at `iou_thres: 0.65`, the + value WongKinYiu/yolov7's `test.py --iou-thres` defaults to and its README reproduces every + published COCO number with. The `iou_thres=0.6` in that script's `test()` signature is always + overridden by the command line. - Reuse `wrapper.download_hub_artifact` (extracted from `MBLT_Engine._download_hub_artifact`) for any future model needing more than one Hub artifact, rather than duplicating Hub-resolution logic. diff --git a/benchmark/benchmark_vision_models.py b/benchmark/benchmark_vision_models.py index bb8c977..a5c086d 100644 --- a/benchmark/benchmark_vision_models.py +++ b/benchmark/benchmark_vision_models.py @@ -317,10 +317,10 @@ def _evaluate( ) return ( float(result.primary_score), - "map50_95", + "map50", { - "map50_95": float(result.map5095), "map50": float(result.map50), + "map50_95": float(result.map5095), }, ) if args.task == "face_detection": diff --git a/mblt_vision/README.md b/mblt_vision/README.md index c006702..30bbddc 100644 --- a/mblt_vision/README.md +++ b/mblt_vision/README.md @@ -375,6 +375,9 @@ validation split, following Ultralytics' depth-validation convention. follows its December 2022 checkpoints, the only ones still downloadable: it decodes with PIL (`Reader.style: pil`) and restores each axis by its own ratio (`LetterBox.per_axis_ratio: true`). +- YOLOv7 models run NMS at `iou_thres` 0.65, the value WongKinYiu/yolov7's `test.py` defaults + to and its README reproduces with. Their scores above were measured at 0.6 and are pending + re-measurement. @@ -439,15 +442,16 @@ validation split, following Ultralytics' depth-validation convention. ### OBB -| Model | Input Size
(H,W,C) | $\underset{\texttt{50-95}}{\texttt{mAP}_{\texttt{val}}^{\texttt{obb}}}$
(NPU) | $\underset{\texttt{50-95}}{\texttt{mAP}_{\texttt{val}}^{\texttt{obb}}}$
(GPU) | FLOPs (B) | params (M) | Source | Note | +| Model | Input Size
(H,W,C) | $\underset{\texttt{50}}{\texttt{mAP}_{\texttt{val}}^{\texttt{obb}}}$
(NPU) | $\underset{\texttt{50}}{\texttt{mAP}_{\texttt{val}}^{\texttt{obb}}}$
(GPU) | FLOPs (B) | params (M) | Source | Note | | --- | --- | --- | --- | --- | --- | --- | --- |
OBB (DOTA v1.0) -- $\underset{\texttt{50-95}}{\texttt{mAP}_{\texttt{val}}^{\texttt{obb}}}$ is the primary metric +- $\underset{\texttt{50}}{\texttt{mAP}_{\texttt{val}}^{\texttt{obb}}}$ is the primary metric for single-model single-scale validation on the - [DOTA v1.0](https://docs.ultralytics.com/datasets/obb/dota-v2#) dataset. Validation also reports - rotated mAP50 as the secondary metric. + [DOTA v1.0](https://docs.ultralytics.com/datasets/obb/dota-v2#) dataset, the metric Ultralytics + publishes for its OBB models. Validation also reports rotated mAP50-95 as the secondary metric. +- Difficult objects count as ordinary targets, as in Ultralytics' validation.
diff --git a/mblt_vision/__init__.py b/mblt_vision/__init__.py index e56c12a..924e798 100644 --- a/mblt_vision/__init__.py +++ b/mblt_vision/__init__.py @@ -19,7 +19,7 @@ from ._api import list_tasks as list_tasks from .wrapper import MBLT_Engine as MBLT_Engine -__version__ = "0.0.6" +__version__ = "0.0.7" _TASK_MODULES = ( face_detection, diff --git a/mblt_vision/cli/val.py b/mblt_vision/cli/val.py index 5939aae..e4a9611 100644 --- a/mblt_vision/cli/val.py +++ b/mblt_vision/cli/val.py @@ -433,8 +433,8 @@ def _run_validation(args: argparse.Namespace) -> float: ) print( "Validation score " - f"(rotated mAP50-95): {dota_result.map5095:.5f}, " - f"(rotated mAP50): {dota_result.map50:.5f}" + f"(rotated mAP50): {dota_result.map50:.5f}, " + f"(rotated mAP50-95): {dota_result.map5095:.5f}" ) return dota_result.primary_score diff --git a/mblt_vision/models/YOLOv7.yaml b/mblt_vision/models/YOLOv7.yaml index 65f88c7..42d18a3 100644 --- a/mblt_vision/models/YOLOv7.yaml +++ b/mblt_vision/models/YOLOv7.yaml @@ -37,7 +37,7 @@ DEFAULT: - 459 - 401 conf_thres: 0.001 - iou_thres: 0.6 + iou_thres: 0.65 TURBO: update: DEFAULT file_cfg: diff --git a/mblt_vision/models/YOLOv7d6.yaml b/mblt_vision/models/YOLOv7d6.yaml index 7c777d7..1607536 100644 --- a/mblt_vision/models/YOLOv7d6.yaml +++ b/mblt_vision/models/YOLOv7d6.yaml @@ -43,7 +43,7 @@ DEFAULT: - 925 - 792 conf_thres: 0.001 - iou_thres: 0.6 + iou_thres: 0.65 TURBO: update: DEFAULT file_cfg: diff --git a/mblt_vision/models/YOLOv7e6.yaml b/mblt_vision/models/YOLOv7e6.yaml index 9c2dd47..db66aca 100644 --- a/mblt_vision/models/YOLOv7e6.yaml +++ b/mblt_vision/models/YOLOv7e6.yaml @@ -43,7 +43,7 @@ DEFAULT: - 925 - 792 conf_thres: 0.001 - iou_thres: 0.6 + iou_thres: 0.65 TURBO: update: DEFAULT file_cfg: diff --git a/mblt_vision/models/YOLOv7e6e.yaml b/mblt_vision/models/YOLOv7e6e.yaml index cc13bcb..801fd92 100644 --- a/mblt_vision/models/YOLOv7e6e.yaml +++ b/mblt_vision/models/YOLOv7e6e.yaml @@ -43,7 +43,7 @@ DEFAULT: - 925 - 792 conf_thres: 0.001 - iou_thres: 0.6 + iou_thres: 0.65 TURBO: update: DEFAULT file_cfg: diff --git a/mblt_vision/models/YOLOv7w6.yaml b/mblt_vision/models/YOLOv7w6.yaml index 776dc21..eb9b154 100644 --- a/mblt_vision/models/YOLOv7w6.yaml +++ b/mblt_vision/models/YOLOv7w6.yaml @@ -43,7 +43,7 @@ DEFAULT: - 925 - 792 conf_thres: 0.001 - iou_thres: 0.6 + iou_thres: 0.65 TURBO: update: DEFAULT file_cfg: diff --git a/mblt_vision/models/YOLOv7x.yaml b/mblt_vision/models/YOLOv7x.yaml index ae4e3b2..476c359 100644 --- a/mblt_vision/models/YOLOv7x.yaml +++ b/mblt_vision/models/YOLOv7x.yaml @@ -37,7 +37,7 @@ DEFAULT: - 459 - 401 conf_thres: 0.001 - iou_thres: 0.6 + iou_thres: 0.65 TURBO: update: DEFAULT file_cfg: diff --git a/mblt_vision/utils/evaluation/eval_dota.py b/mblt_vision/utils/evaluation/eval_dota.py index b229061..3741dfd 100644 --- a/mblt_vision/utils/evaluation/eval_dota.py +++ b/mblt_vision/utils/evaluation/eval_dota.py @@ -56,13 +56,13 @@ class DOTAResult(NamedTuple): @property def primary_score(self) -> float: - """Return the primary DOTAv1 validation metric.""" - return self.map5095 + """Return rotated mAP50, the metric Ultralytics publishes for its DOTAv1 models.""" + return self.map50 @property def secondary_score(self) -> float: - """Return the secondary DOTAv1 validation metric.""" - return self.map50 + """Return rotated mAP50-95.""" + return self.map5095 def _label_to_index(label: str) -> int: @@ -134,9 +134,15 @@ def _load_ground_truths( data_path: DOTAv1 root directory. dataset: Dataset containing image IDs and image paths. + Difficult objects (flag ``1`` or ``2``) load as ordinary targets, as Ultralytics, + whose checkpoints these are, validates them: its ``convert_dota_to_yolo_obb`` keeps + the coordinates and class and drops the flag. The DOTA devkit behind the test server + ignores them instead; callers who want that protocol can still pass ignore regions to + ``evaluate_dota_predictions``. The flag is validated either way. + Returns: - Mapping from image ID to tensors for positive and ignored classes, polygons, - and ``xywhr`` boxes. + Mapping from image ID to tensors for target classes, polygons, and ``xywhr`` + boxes, with empty ignored-region tensors. """ label_dir = Path(data_path) / "labels" / "val" original_label_dir = Path(data_path) / "labels" / "val_original" @@ -213,12 +219,8 @@ def _load_ground_truths( f"{label_path}:{line_number}." ) seen_targets.add(target_key) - if len(parts) >= 10 and parts[9] in {"1", "2"}: - ignore_classes.append(cls) - ignore_polygons.append(coords) - else: - classes.append(cls) - polygons.append(coords) + classes.append(cls) + polygons.append(coords) elif original_label_path.is_file(): for line_number, line in enumerate( original_label_path.read_text(encoding="utf-8").splitlines(), start=1 @@ -267,12 +269,8 @@ def _load_ground_truths( f"{original_label_path}:{line_number}." ) seen_targets.add(target_key) - if parts[9] in {"1", "2"}: - ignore_classes.append(cls) - ignore_polygons.append(coords) - else: - classes.append(cls) - polygons.append(coords) + classes.append(cls) + polygons.append(coords) else: raise FileNotFoundError( @@ -674,8 +672,8 @@ def evaluate_dota_predictions( Returns: Rotated mAP at IoU ``0.50`` followed by mAP averaged across ``0.50:0.95``. - The ``primary_score`` and ``secondary_score`` properties expose mAP50-95 - and mAP50, respectively. + The ``primary_score`` and ``secondary_score`` properties expose mAP50 + and mAP50-95, respectively. """ iouv = torch.linspace(0.5, 0.95, 10) stats = _empty_stats() diff --git a/mblt_vision/utils/postprocess/common.py b/mblt_vision/utils/postprocess/common.py index 7daedc9..e8dc2da 100644 --- a/mblt_vision/utils/postprocess/common.py +++ b/mblt_vision/utils/postprocess/common.py @@ -417,6 +417,20 @@ def batch_probiou( ROTATED_NMS_BLOCK = 512 +def descending_order(values: torch.Tensor) -> torch.Tensor: + """Return the indices that sort ``values`` high to low, ties in ascending index order. + + This is Ultralytics' validation order, which every candidate ranking here must + reproduce. ``non_max_suppression`` hands its candidates to ``torchvision.ops.nms``, + whose sort is stable on CPU and CUDA, and the ``argsort(descending=True)`` it runs + above ``max_nms`` candidates, like the ``torch.topk`` of an end-to-end head, is stable + on CUDA, its normal validation device. Those unstable calls order ties differently on + CPU, and quantized MXQ scores tie often, so a plain ``argsort`` made the result depend + on the device. A stable sort gives Ultralytics' CUDA order everywhere. + """ + return torch.sort(values, descending=True, stable=True).indices + + def rotated_nms( boxes: torch.Tensor, scores: torch.Tensor, @@ -436,7 +450,7 @@ def rotated_nms( """ if boxes.numel() == 0: return torch.empty((0,), dtype=torch.int64, device=boxes.device) - sorted_idx = torch.argsort(scores, descending=True) + sorted_idx = descending_order(scores) sorted_boxes = boxes[sorted_idx] count = int(sorted_boxes.shape[0]) suppressed = torch.zeros(count, dtype=torch.bool, device=sorted_boxes.device) @@ -568,15 +582,20 @@ def dual_topk( return torch.zeros( (0, 6 + n_extra), dtype=torch.float32, device=pre_topk.device ) - max_det = min(pre_topk.shape[0], max_det) - - row_index = torch.topk( - pre_topk[:, score_start:score_end].amax(dim=-1), max_det, dim=0 - ).indices + # Ultralytics' Detect.get_topk_index ranks every anchor and keeps the best + # ``max_det`` (anchor, class) pairs; the validator then drops the rows at or + # below the threshold. Ranking only the anchors that clear it gives the same + # rows provided the second stage may still return ``max_det`` pairs: capping it + # at the anchor count dropped every further class of those anchors whenever + # fewer than ``max_det`` of them cleared the threshold (100 anchors with three + # classes each kept 100 rows where Ultralytics keeps 300). + row_index = descending_order(pre_topk[:, score_start:score_end].amax(dim=-1))[ + :max_det + ] selected = pre_topk[row_index] - top_scores, flat_index = torch.topk( - selected[:, score_start:score_end].reshape(-1), max_det - ) + flat_scores = selected[:, score_start:score_end].reshape(-1) + flat_index = descending_order(flat_scores)[:max_det] + top_scores = flat_scores[flat_index] keep = top_scores > threshold if not torch.any(keep): return torch.zeros( diff --git a/mblt_vision/utils/postprocess/yolo_anchor_post.py b/mblt_vision/utils/postprocess/yolo_anchor_post.py index 1894378..3f6c873 100644 --- a/mblt_vision/utils/postprocess/yolo_anchor_post.py +++ b/mblt_vision/utils/postprocess/yolo_anchor_post.py @@ -9,7 +9,12 @@ import torch from .base import YOLODetectionPostBase -from .common import YOLOFaceDetectionMixin, YOLOSegPostMixin, non_max_suppression +from .common import ( + YOLOFaceDetectionMixin, + YOLOSegPostMixin, + descending_order, + non_max_suppression, +) class YOLOAnchorDetectionPost(YOLODetectionPostBase): @@ -252,7 +257,7 @@ def _nms_single( out[:, 5] = j.to(rows.dtype) if self.n_extra > 0: out[:, 6:] = rows[:, mi:] - out = out[out[:, 4].argsort(descending=True)[:max_nms]] + out = out[descending_order(out[:, 4])[:max_nms]] c = out[:, 5:6] * max_wh boxes, score = out[:, :4] + c, out[:, 4] i_idx = non_max_suppression(boxes, score, self.iou_thres, max_det) diff --git a/mblt_vision/utils/postprocess/yolo_anchorless_post.py b/mblt_vision/utils/postprocess/yolo_anchorless_post.py index 1c1c213..b6b5b98 100644 --- a/mblt_vision/utils/postprocess/yolo_anchorless_post.py +++ b/mblt_vision/utils/postprocess/yolo_anchorless_post.py @@ -18,6 +18,7 @@ YOLOSegPostMixin, concat_converted_obb_outputs, decode_split_converted_obb_outputs, + descending_order, dist2bbox, dist2rbox, non_max_suppression, @@ -363,7 +364,7 @@ def _nms_single_legacy_rows( xi_out[:, 5] = cls_idx.to(xi.dtype) if self.n_extra > 0: xi_out[:, 6:] = extra - xi_out = xi_out[torch.argsort(xi_out[:, 4], descending=True)[:max_nms]] + xi_out = xi_out[descending_order(xi_out[:, 4])[:max_nms]] c = xi_out[:, 5:6] * max_wh boxes, scores = xi_out[:, :4] + c, xi_out[:, 4] i_idx = non_max_suppression(boxes, scores, self.iou_thres, max_det) @@ -617,7 +618,7 @@ def extract_final_outputs( retained_batches = self._final_detection_batches(detections) selected_batches = [] for batch in retained_batches: - order = torch.argsort(batch[:, 4], descending=True) + order = descending_order(batch[:, 4]) ordered = batch[order] keep = non_max_suppression( ordered[:, :4], ordered[:, 4], self.iou_thres, max_output=300 @@ -998,7 +999,7 @@ def _nms_single_legacy_rows( xi_out = yolo_multilabel_candidates(xi, self.nc, self.n_extra, self.conf_thres) if xi_out.numel() == 0: return torch.zeros((0, 7), dtype=torch.float32, device=self.device) - xi_out = xi_out[torch.argsort(xi_out[:, 4], descending=True)[:max_nms]] + xi_out = xi_out[descending_order(xi_out[:, 4])[:max_nms]] c = xi_out[:, 5:6] * max_wh boxes = torch.cat([xi_out[:, :2] + c, xi_out[:, 2:4], xi_out[:, 6:7]], dim=-1) keep = rotated_nms(boxes, xi_out[:, 4], self.iou_thres)[:max_det] diff --git a/mblt_vision/utils/postprocess/yolo_dflfree_post.py b/mblt_vision/utils/postprocess/yolo_dflfree_post.py index 9c3c45b..235e12e 100644 --- a/mblt_vision/utils/postprocess/yolo_dflfree_post.py +++ b/mblt_vision/utils/postprocess/yolo_dflfree_post.py @@ -13,6 +13,7 @@ YOLOSegPostMixin, concat_converted_obb_outputs, decode_split_converted_obb_outputs, + descending_order, dist2bbox, dist2rbox, dual_topk, @@ -589,7 +590,7 @@ def extract_final_outputs( retained_batches = self._final_detection_batches(detections) selected_batches = [] for batch in retained_batches: - order = torch.argsort(batch[:, 4], descending=True) + order = descending_order(batch[:, 4]) ordered = batch[order] keep = non_max_suppression( ordered[:, :4], ordered[:, 4], self.iou_thres, max_output=300 @@ -1016,7 +1017,7 @@ def nms( if xi.numel() == 0: output.append(xi.new_zeros((0, 7))) continue - xi = xi[torch.argsort(xi[:, 4], descending=True)[:max_nms]] + xi = xi[descending_order(xi[:, 4])[:max_nms]] c = xi[:, 5:6] * max_wh boxes = torch.cat([xi[:, :2] + c, xi[:, 2:4], xi[:, 6:7]], dim=-1) keep = rotated_nms(boxes, xi[:, 4], self.iou_thres)[:max_det] diff --git a/mblt_vision/utils/postprocess/yolo_nmsfree_post.py b/mblt_vision/utils/postprocess/yolo_nmsfree_post.py index 2a835c3..ef00bf8 100644 --- a/mblt_vision/utils/postprocess/yolo_nmsfree_post.py +++ b/mblt_vision/utils/postprocess/yolo_nmsfree_post.py @@ -10,7 +10,7 @@ import numpy as np import torch -from .common import YOLOFaceDetectionMixin, dist2bbox, dual_topk +from .common import YOLOFaceDetectionMixin, descending_order, dist2bbox, dual_topk from .yolo_anchorless_post import YOLOAnchorlessDetectionPost, _AnchorlessNMSInput @@ -162,7 +162,7 @@ def _decoded_output_batches( labels = torch.zeros_like(confidence) detections = torch.cat((boxes, confidence, labels), dim=-1) return [ - batch[torch.argsort(batch[:, 4], descending=True)[: self.max_det]] + batch[descending_order(batch[:, 4])[: self.max_det]] for batch in self._final_detection_batches(detections) ] diff --git a/tests/test_benchmark_vision.py b/tests/test_benchmark_vision.py index 7dec064..6db73c3 100644 --- a/tests/test_benchmark_vision.py +++ b/tests/test_benchmark_vision.py @@ -157,8 +157,9 @@ class FakeModel: FakeModel(), args, tmp_path ) - assert (score, score_name) == (0.5, "map50_95") - assert metrics == {"map50_95": 0.5, "map50": 0.7} + # Ultralytics publishes its DOTAv1 models by rotated mAP50. + assert (score, score_name) == (0.7, "map50") + assert metrics == {"map50": 0.7, "map50_95": 0.5} def test_benchmark_ranks_widerface_by_hard_ap( diff --git a/tests/test_eval_dota.py b/tests/test_eval_dota.py index 4393903..37b26a4 100644 --- a/tests/test_eval_dota.py +++ b/tests/test_eval_dota.py @@ -106,8 +106,12 @@ def test_dota_ground_truth_requires_annotation_for_every_image(tmp_path) -> None _load_ground_truths(str(tmp_path), dataset) -def test_normalized_difficult_flag_loads_as_an_ignored_region(tmp_path) -> None: - """Read organizer-produced difficult metadata before selecting evaluation targets.""" +def test_normalized_difficult_flag_loads_as_an_ordinary_target(tmp_path) -> None: + """Count difficult objects as targets, as Ultralytics' validation does. + + Its ``convert_dota_to_yolo_obb`` drops the difficulty flag, so a difficult object is + found by a true positive and missed as a false negative, never ignored. + """ label_dir = tmp_path / "labels" / "val" label_dir.mkdir(parents=True) @@ -126,8 +130,8 @@ def test_normalized_difficult_flag_loads_as_an_ignored_region(tmp_path) -> None: ground_truth = _load_ground_truths(str(tmp_path), dataset)["image"] - assert ground_truth["cls"].tolist() == [0] - assert ground_truth["ignore_cls"].tolist() == [0] + assert ground_truth["cls"].tolist() == [0, 0] + assert ground_truth["ignore_cls"].tolist() == [] def test_normalized_truncated_annotation_raises_with_file_and_line(tmp_path) -> None: @@ -215,6 +219,31 @@ def test_dota_annotations_reject_equivalent_reordered_targets( _load_ground_truths(str(tmp_path), dataset) +def test_original_difficult_label_loads_as_an_ordinary_target(tmp_path) -> None: + """Official ``val_original`` labels count difficult objects as targets too.""" + + original_label_dir = tmp_path / "labels" / "val_original" + original_label_dir.mkdir(parents=True) + (original_label_dir / "image.txt").write_text( + "imagesource:GoogleEarth\ngsd:0.1\n" + "0 0 20 0 20 20 0 20 plane 0\n40 40 60 40 60 60 40 60 plane 1\n", + encoding="utf-8", + ) + dataset = cast( + CustomDOTAv1, + SimpleNamespace( + ids=["image"], + image_paths=["unused"], + _load_image=lambda _: np.zeros((100, 100, 3), dtype=np.uint8), + ), + ) + + ground_truth = _load_ground_truths(str(tmp_path), dataset)["image"] + + assert ground_truth["cls"].tolist() == [0, 0] + assert ground_truth["ignore_cls"].tolist() == [] + + def test_original_truncated_annotation_raises_with_file_and_line(tmp_path) -> None: """Require a difficulty flag when loading original DOTAv1 labels directly.""" @@ -445,8 +474,8 @@ def test_dota_annotations_reject_unknown_difficulty_flags( _load_ground_truths(str(tmp_path), dataset) -def test_difficult_regions_do_not_count_as_positive_or_false_positive() -> None: - """Ignore a detection on a difficult region while retaining positive matching.""" +def test_supplied_ignore_regions_do_not_count_as_positive_or_false_positive() -> None: + """Ignore a detection on a caller-supplied ignore region, the DOTA devkit protocol.""" ground_truths = { "image": { diff --git a/tests/test_ultralytics_selection_order.py b/tests/test_ultralytics_selection_order.py new file mode 100644 index 0000000..dedff0e --- /dev/null +++ b/tests/test_ultralytics_selection_order.py @@ -0,0 +1,106 @@ +"""Candidate ranking reproduces Ultralytics' validation order on every device. + +Ultralytics validates on CUDA, where ``torchvision.ops.nms``, the ``argsort`` it runs above +``max_nms`` candidates and an end-to-end head's ``torch.topk`` all order tied scores by +ascending index. Quantized MXQ scores tie often, so these tests pin that order on CPU too, +and the end-to-end selection's row count, against a direct reading of +``Detect.get_topk_index``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import torch +import yaml + +import mblt_vision +from mblt_vision.utils.postprocess import build_postprocess +from mblt_vision.utils.postprocess.common import descending_order, dual_topk + +DEVICES = ["cpu", *(["cuda"] if torch.cuda.is_available() else [])] + + +def _model_cfg(name: str) -> tuple[dict, dict]: + path = Path(mblt_vision.__file__).parent / "models" / f"{name}.yaml" + config = yaml.safe_load(path.read_text())["DEFAULT"] + return config["pre_cfg"], config["post_cfg"] + + +@pytest.mark.parametrize("device", DEVICES) +def test_descending_order_keeps_tied_scores_in_index_order(device: str) -> None: + # Large enough that an unstable CPU sort reorders the ties. + values = (torch.arange(5000) % 7 == 0).float().to(device) + + order = descending_order(values).cpu() + + expected = torch.cat( + (torch.arange(0, 5000, 7), torch.tensor([i for i in range(5000) if i % 7])) + ) + torch.testing.assert_close(order, expected) + + +def _ultralytics_end2end(rows: torch.Tensor, nc: int, conf: float) -> torch.Tensor: + """``Detect.get_topk_index`` plus the validator's threshold, with CUDA's tie order.""" + scores = rows[:, 4 : 4 + nc] + k = min(300, rows.shape[0]) + anchors = descending_order(scores.amax(dim=-1))[:k] + flat = scores[anchors].reshape(-1) + top = descending_order(flat)[:k] + keep = flat[top] > conf + top = top[keep] + return torch.stack( + (flat[top], (top % nc).to(rows.dtype), anchors[top // nc].to(rows.dtype)), dim=1 + ) + + +@pytest.mark.parametrize("device", DEVICES) +def test_dual_topk_keeps_every_class_of_the_anchors_that_clear_the_threshold( + device: str, +) -> None: + """100 anchors with three classes each keep 300 rows, as Ultralytics does, not 100.""" + generator = torch.Generator().manual_seed(0) + rows = torch.zeros((8400, 4 + 80)) + rows[:, :4] = torch.rand((8400, 4), generator=generator) * 640 + hot = torch.randperm(8400, generator=generator)[:100] + for anchor in hot: + classes = torch.randperm(80, generator=generator)[:3] + rows[anchor, 4 + classes] = torch.rand(3, generator=generator) * 0.9 + 0.01 + + selected = dual_topk(rows.to(device), 80, 0, max_det=300, conf_thres=0.001).cpu() + + expected = _ultralytics_end2end(rows, 80, 0.001) + assert len(selected) == 300 + torch.testing.assert_close(selected[:, 4], expected[:, 0]) + torch.testing.assert_close(selected[:, 5], expected[:, 1]) + torch.testing.assert_close(selected[:, :4], rows[expected[:, 2].long(), :4]) + + +@pytest.mark.parametrize("device", DEVICES) +def test_dual_topk_orders_tied_scores_by_index(device: str) -> None: + rows = torch.zeros((8400, 4 + 80)) + rows[:, 4:] = 0.5 + + selected = dual_topk(rows.to(device), 80, 0, max_det=300, conf_thres=0.001).cpu() + + # Every (anchor, class) ties, so both rankings keep index order: anchor 0's + # 80 classes first, then anchor 1's. + expected = torch.arange(300) % 80 + torch.testing.assert_close(selected[:, 5], expected.float()) + + +@pytest.mark.parametrize("device", DEVICES) +def test_nms_keeps_the_lowest_index_of_tied_overlapping_candidates(device: str) -> None: + """Greedy NMS keeps whichever tied candidate it ranks first; Ultralytics ranks index 0.""" + post = build_postprocess(*_model_cfg("YOLOv8n")) + rows = torch.zeros((8400, 84)) + shift = torch.arange(2000, dtype=torch.float32) * 1e-3 + rows[:2000, 0], rows[:2000, 2] = 100.0 + shift, 200.0 + shift + rows[:2000, 1], rows[:2000, 3] = 100.0, 200.0 + rows[:2000, 4] = 0.5 + + (detections,) = post.nms_multilabel([rows.to(device)]) + + assert detections.shape[0] == 1 + assert detections[0, 0].item() == pytest.approx(100.0)