From 12d1fdf9a52e5c9ad7e479bd13b36257b93e99ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o?= Date: Wed, 4 Nov 2026 13:39:22 -0500 Subject: [PATCH] Add Workflow Eval Dataset search to the SDK --- docs/core/workspace.md | 26 +++++++++++++++++++++ roboflow/core/workflow_evals.py | 28 ++++++++++++++++++++++ tests/test_workflow_evals.py | 41 ++++++++++++++++++++++++++++++--- 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/docs/core/workspace.md b/docs/core/workspace.md index 62706ac5..a70d9c45 100644 --- a/docs/core/workspace.md +++ b/docs/core/workspace.md @@ -1 +1,27 @@ :::roboflow.core.workspace + +## Search Workflow Eval Dataset Cases + +```python +evals = workspace.workflow_evals() +query = {"groundTruth": {"output": "boxes", "className": "car", "count": {"min": 1}}} +cursor = None +while True: + page = evals.search_dataset("DATASET_ID", query=query, limit=24, cursor=cursor) + for case in page["items"]: + print(case["id"], case["name"]) + cursor = page.get("nextCursor") + if not cursor: + break +``` + +`query` also accepts `semantic`, Source `filename`, `tags`, `width`/`height` ranges, +and ground-truth `state`, `slices`, `sliceMatch` and exact `scalar: {output, value}`. +Source and semantic filters require `image:read` in addition to `workflow-evals:read` +and only match Cases backed by platform Sources. Ground-truth filters also cover +artifact-only Cases. Class/count filters require an output name. + +Pages include Dataset revisions, coverage counts and a possibly unknown (`None`) +`total`. Follow `nextCursor` even when `items` is empty, keeping the same query and +limit (1–48, default 24). On HTTP 409, restart without a cursor. Preview URLs expire. +This search does not execute a Workflow or create embeddings. diff --git a/roboflow/core/workflow_evals.py b/roboflow/core/workflow_evals.py index 00fc842b..a993f8ca 100644 --- a/roboflow/core/workflow_evals.py +++ b/roboflow/core/workflow_evals.py @@ -287,6 +287,34 @@ def create_dataset(self, body: Dict[str, Any], *, idempotency_key: Optional[str] def get_dataset(self, dataset_id: str) -> Dict[str, Any]: return self._call("GET", f"/eval-datasets/{_seg(dataset_id)}") + def search_dataset( + self, + dataset_id: str, + *, + query: Optional[Dict[str, Any]] = None, + limit: Optional[int] = None, + cursor: Optional[str] = None, + ) -> Dict[str, Any]: + """Search Cases using semantic, Source and saved ground-truth filters. + + ``query`` accepts ``semantic``, ``source`` (filename, tags, width/height + ranges), and ``groundTruth`` (state, slices, sliceMatch, output, + className, count range, scalar: {output, value}). Class/count filters + require an output. Source/semantic filters additionally need image:read + and match only Source-backed Cases; GT filters also cover artifact-only Cases. + + Returns items, dataset revisions, coverage, total (possibly None), and + optional nextCursor. Keep query and limit unchanged when continuing, + even after an empty page; stop only when nextCursor is absent. Page size + defaults to 24 (maximum 48). Restart without a cursor on HTTP 409. + Preview URLs expire. This read-only POST needs no idempotency key. + """ + return self._call( + "POST", + f"/eval-datasets/{_seg(dataset_id)}/search", + body=_body(query=query, limit=limit, cursor=cursor), + ) + def update_dataset(self, dataset_id: str, body: Dict[str, Any], *, revision: Any) -> Dict[str, Any]: """Update Dataset metadata (``name``, ``description``).""" return self._call("PATCH", f"/eval-datasets/{_seg(dataset_id)}", body=body, revision=revision) diff --git a/tests/test_workflow_evals.py b/tests/test_workflow_evals.py index 4aca881b..0d14d428 100644 --- a/tests/test_workflow_evals.py +++ b/tests/test_workflow_evals.py @@ -20,8 +20,8 @@ SUBJECT = {"kind": "savedWorkflow", "workflowId": "wf", "subjectKey": "wf"} # (client method, args, kwargs, HTTP method, path relative to BASE, sends Idempotency-Key, If-Match value) -# One row per public JSON route in the Workflow Evals OpenAPI registry. The 71st route, -# POST /specs/ai-draft, streams SSE and is covered by the ai_draft tests below. +# Public JSON routes in the Workflow Evals OpenAPI registry. +# POST /specs/ai-draft streams SSE and is covered by the ai_draft tests below. ROUTES = [ ("capabilities", (), {}, "GET", "/capabilities", False, None), ("openapi", (), {}, "GET", "/openapi.json", False, None), @@ -50,6 +50,7 @@ ("list_datasets", (), {}, "GET", "/eval-datasets", False, None), ("create_dataset", ({"name": "d"},), {}, "POST", "/eval-datasets", True, None), ("get_dataset", (D,), {}, "GET", f"/eval-datasets/{D}", False, None), + ("search_dataset", (D,), {}, "POST", f"/eval-datasets/{D}/search", False, None), ("update_dataset", (D, {"name": "d"}), {"revision": 1}, "PATCH", f"/eval-datasets/{D}", False, "1"), ( "mutate_dataset_contract", @@ -164,7 +165,7 @@ def setUp(self): self.client = WorkflowEvals(KEY, WS) def test_covers_every_public_json_route(self): - self.assertEqual(len({(row[3], row[4]) for row in ROUTES}), 70) + self.assertEqual(len({(row[3], row[4]) for row in ROUTES}), 71) @responses.activate def test_each_method_calls_its_route(self): @@ -210,6 +211,40 @@ def test_case_results_filters_repeat_slices_and_encode_booleans(self): self.assertIn("failedCheck=true", query) self.assertIn("state=failed", query) + @responses.activate + def test_search_dataset_preserves_filters_and_empty_page_continuation(self): + url = f"{BASE}/eval-datasets/{D}/search" + query = { + "semantic": "cars on a street", + "source": {"filename": "street", "tags": ["daytime"], "width": {"min": 640}}, + "groundTruth": { + "state": "complete", + "slices": ["daytime"], + "sliceMatch": "all", + "output": "boxes", + "className": "car", + "count": {"min": 0, "max": 3}, + "scalar": {"output": "occluded", "value": False}, + }, + } + page = { + "items": [], + "nextCursor": "continue", + "dataset": {"id": D, "generation": 1, "contentRevision": 2}, + "coverage": {"totalCases": 50, "sourceBackedCases": 40}, + "total": None, + } + responses.add(responses.POST, url, json=page, status=200) + responses.add(responses.POST, url, json={**page, "nextCursor": None}, status=200) + self.assertEqual(self.client.search_dataset(D, query=query, limit=12), page) + self.client.search_dataset(D, query=query, limit=12, cursor=page["nextCursor"]) + self.assertEqual(json.loads(responses.calls[0].request.body), {"query": query, "limit": 12}) + self.assertEqual( + json.loads(responses.calls[1].request.body), + {"query": query, "limit": 12, "cursor": "continue"}, + ) + self.assertNotIn("Idempotency-Key", responses.calls[1].request.headers) + @responses.activate def test_validate_spec_wraps_bare_spec(self): responses.add(responses.POST, f"{BASE}/specs/validate", json={"valid": True}, status=200)