feat(web): BlockedError — block_kind, vendor and the gating CAPTCHA, 0.61.0 (SCR-306)

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.61.0 — 2026-10-14
+
+- **`BlockedError` — why a web scrape was blocked.** A `web.scrape` whose every attempt hit an anti-bot wall (`422`, `error: "blocking_page_detected"`) now raises `BlockedError`, a subclass of `ValidationError`, so existing `except ValidationError` handlers still catch it. It exposes `block_kind` (`"antibot"`, `"captcha"`, `"rate_limit"`, `"ip_ban"`, `"geo_block"`, `"auth_wall"`, `"empty"` or `"other"`), `vendor` (the protection that served the wall, e.g. `"cloudflare"`, `"datadome"`, `"akamai"`, `"aws_waf"`), `captcha` (the CAPTCHA gating the page when `block_kind == "captcha"`: `system`, `version`, `gate`, `solvable`, `params` such as `sitekey`) and `data` (the scrape result returned with the block). Blocked requests are not charged.
+- `captcha_systems` entries on web results may now carry `version`, `gate`, `solvable` and `params`, and `blocking_details` may carry `vendor`, `kind` and `signals` (server-side; already passed through as dicts).
+
 ## 0.60.1 — 2026-10-13
 
 - **Gumtree South Africa ids are refs.** gumtree.co.za re-platformed on 2026-10-12 and the API now serves `market="za"` from the new site: category, location and listing ids are refs such as `"p6dnh"` (Cars & Bakkies), `"dkp8y"` (Gauteng) and `"ls26bvmcpnsqmz7l9jz"` (a listing), e.g. `search(market="za", category="p6dnh", location="dkp8y", filters="make:toyota")`. 0.60.0 typed those ids as `int`, so za responses failed validation: `TreeNode.id`, `TreeResponse.id`, `CategoryRef.id`, `CategoryFacet.id`, `LocationSuggestion.id`, `category_id` and `category_ids` are now `int | str` (uk/au still return ints), and `list_categories` / `list_locations` take `parent_id: int | str`. za filter keys are the site's own (`make`, `model`, `fuelType`, `year`, `mileageKm` …), and za search pages through the first 375 matches.

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapebadger"
-version = "0.60.1"
+version = "0.61.0"
 description = "Official Python SDK for ScrapeBadger - Async web scraping APIs for Twitter and more"
 readme = "README.md"
 license = { text = "MIT" }

src/scrapebadger/__init__.py

@@ -32,6 +32,7 @@ async def main():
 from scrapebadger._internal.config import ClientConfig
 from scrapebadger._internal.exceptions import (
     AuthenticationError,
+    BlockedError,
     InsufficientCreditsError,
     IPNotAllowedError,
     NotFoundError,
@@ -1335,7 +1336,7 @@ async def main():
     ZestimateHistoryPoint as ZillowZestimateHistoryPoint,
 )
 
-__version__ = "0.60.1"
+__version__ = "0.61.0"
 
 __all__ = [
     # TikTok core models
@@ -1394,6 +1395,7 @@ async def main():
     "BingSearchResponse",
     "BingVideoResult",
     "BingVideosResponse",
+    "BlockedError",
     # Vinted response envelopes
     "BrandsResponse",
     "CategoriesResponse",

src/scrapebadger/_internal/__init__.py

@@ -4,6 +4,7 @@
 from scrapebadger._internal.config import ClientConfig
 from scrapebadger._internal.exceptions import (
     AuthenticationError,
+    BlockedError,
     InsufficientCreditsError,
     IPNotAllowedError,
     NotFoundError,
@@ -18,6 +19,7 @@
 __all__ = [
     "AuthenticationError",
     "BaseClient",
+    "BlockedError",
     "ClientConfig",
     "IPNotAllowedError",
     "InsufficientCreditsError",

src/scrapebadger/_internal/client.py

@@ -14,6 +14,7 @@
 
 from scrapebadger._internal.exceptions import (
     AuthenticationError,
+    BlockedError,
     InsufficientCreditsError,
     IPNotAllowedError,
     NotFoundError,
@@ -46,7 +47,7 @@
 T = TypeVar("T")
 
 # User agent for SDK requests
-SDK_VERSION = "0.60.1"
+SDK_VERSION = "0.61.0"
 USER_AGENT = f"scrapebadger-python/{SDK_VERSION}"
 
 
@@ -156,7 +157,8 @@ def _handle_error_response(
             PermissionDeniedError: For 403 responses from a key restricted
                 away from the API it called.
             NotFoundError: For 404 responses.
-            ValidationError: For 422 responses.
+            BlockedError: For a 422 blocking_page_detected (web scraping).
+            ValidationError: For other 422 responses.
             RateLimitError: For 429 responses.
             ServerError: For 5xx responses.
             ScrapeBadgerError: For other error responses.
@@ -186,6 +188,9 @@ def _handle_error_response(
         if status_code == 404:
             raise NotFoundError(message, status_code, data)
 
+        if status_code == 422 and data.get("error") == "blocking_page_detected":
+            raise BlockedError(message, status_code, data)
+
         if status_code == 422:
             raise ValidationError(message, status_code, data)
 

src/scrapebadger/_internal/exceptions.py

@@ -244,6 +244,53 @@ def __init__(
         super().__init__(message, status_code, response_data)
 
 
+class BlockedError(ValidationError):
+    """Raised when the Web Scraping API reports the target blocked the request.
+
+    A ``422`` with ``error == "blocking_page_detected"``: every retry hit an
+    anti-bot wall. It subclasses :class:`ValidationError`, so existing
+    ``except ValidationError`` handlers keep catching it. Blocked requests are
+    not charged.
+
+    Attributes:
+        block_kind: The wall's class: ``"antibot"``, ``"captcha"``,
+            ``"rate_limit"``, ``"ip_ban"``, ``"geo_block"``, ``"auth_wall"``,
+            ``"empty"`` or ``"other"`` (``None`` on older responses).
+        vendor: The protection vendor that served it, e.g. ``"cloudflare"``,
+            ``"datadome"``, ``"akamai"``, ``"aws_waf"`` (``None`` when unknown).
+        captcha: The CAPTCHA gating the page when ``block_kind == "captcha"``:
+            ``system``, ``version``, ``gate``, ``solvable`` and ``params``
+            (e.g. ``sitekey``), else ``None``.
+        data: The scrape result the API returned with the block.
+
+    Example:
+        ```python
+        try:
+            page = await client.web.scrape("https://example.com")
+        except BlockedError as e:
+            print(e.block_kind, e.vendor)
+            if e.captcha:
+                print(e.captcha["system"], e.captcha.get("params"))
+        ```
+    """
+
+    @property
+    def block_kind(self) -> str | None:
+        return self.response_data.get("block_kind")
+
+    @property
+    def vendor(self) -> str | None:
+        return self.response_data.get("vendor")
+
+    @property
+    def captcha(self) -> dict[str, Any] | None:
+        return self.response_data.get("captcha")
+
+    @property
+    def data(self) -> dict[str, Any] | None:
+        return self.response_data.get("data")
+
+
 class ServerError(ScrapeBadgerError):
     """Raised when the server encounters an error.
 

tests/test_client.py

@@ -189,3 +189,61 @@ async def test_ip_not_allowed_raises_ip_error(self, api_key: str) -> None:
             await client.get("/v1/google/search")
         assert exc.value.client_ip == "203.0.113.9"
         assert isinstance(exc.value, PermissionDeniedError)
+
+
+class TestBlocked:
+    async def _raise(self, api_key: str, body: dict) -> Exception:
+        import httpx
+
+        from scrapebadger._internal.client import BaseClient
+        from scrapebadger._internal.config import ClientConfig
+
+        client = BaseClient(ClientConfig(api_key=api_key))
+        client._client = httpx.AsyncClient(
+            transport=httpx.MockTransport(lambda r: httpx.Response(422, json=body)),
+            base_url="https://api.test",
+        )
+        with pytest.raises(Exception) as exc:
+            await client.post("/v1/web/scrape", json={"url": "https://example.com"})
+        return exc.value
+
+    async def test_blocking_422_raises_blocked_error(self, api_key: str) -> None:
+        from scrapebadger import BlockedError, ValidationError
+
+        captcha = {
+            "system": "recaptcha_v2",
+            "version": "v2_checkbox",
+            "gate": True,
+            "params": {"sitekey": "6Lc"},
+        }
+        err = await self._raise(
+            api_key,
+            {
+                "error": "blocking_page_detected",
+                "detail": "blocked",
+                "block_kind": "captcha",
+                "vendor": "google",
+                "captcha": captcha,
+                "data": {"status_code": 403},
+            },
+        )
+        assert isinstance(err, BlockedError)
+        assert isinstance(err, ValidationError)
+        assert err.block_kind == "captcha"
+        assert err.vendor == "google"
+        assert err.captcha == captcha
+        assert err.data == {"status_code": 403}
+
+    async def test_older_blocking_body_has_none_fields(self, api_key: str) -> None:
+        from scrapebadger import BlockedError
+
+        err = await self._raise(api_key, {"error": "blocking_page_detected", "detail": "blocked"})
+        assert isinstance(err, BlockedError)
+        assert err.block_kind is None and err.vendor is None and err.captcha is None
+
+    async def test_other_422_stays_validation_error(self, api_key: str) -> None:
+        from scrapebadger import BlockedError, ValidationError
+
+        err = await self._raise(api_key, {"detail": [{"loc": ["body", "url"], "msg": "required"}]})
+        assert isinstance(err, ValidationError)
+        assert not isinstance(err, BlockedError)