diff --git a/CHANGELOG.md b/CHANGELOG.md index b1f4591..847dada 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.61.0 — 2026-10-14 + +- **`BlockedError` — why a web scrape was blocked.** A `web.scrape` whose every attempt hit an anti-bot wall (`422`, `error: "blocking_page_detected"`) now raises `BlockedError`, a subclass of `ValidationError`, so existing `except ValidationError` handlers still catch it. It exposes `block_kind` (`"antibot"`, `"captcha"`, `"rate_limit"`, `"ip_ban"`, `"geo_block"`, `"auth_wall"`, `"empty"` or `"other"`), `vendor` (the protection that served the wall, e.g. `"cloudflare"`, `"datadome"`, `"akamai"`, `"aws_waf"`), `captcha` (the CAPTCHA gating the page when `block_kind == "captcha"`: `system`, `version`, `gate`, `solvable`, `params` such as `sitekey`) and `data` (the scrape result returned with the block). Blocked requests are not charged. +- `captcha_systems` entries on web results may now carry `version`, `gate`, `solvable` and `params`, and `blocking_details` may carry `vendor`, `kind` and `signals` (server-side; already passed through as dicts). + ## 0.60.1 — 2026-10-13 - **Gumtree South Africa ids are refs.** gumtree.co.za re-platformed on 2026-10-12 and the API now serves `market="za"` from the new site: category, location and listing ids are refs such as `"p6dnh"` (Cars & Bakkies), `"dkp8y"` (Gauteng) and `"ls26bvmcpnsqmz7l9jz"` (a listing), e.g. `search(market="za", category="p6dnh", location="dkp8y", filters="make:toyota")`. 0.60.0 typed those ids as `int`, so za responses failed validation: `TreeNode.id`, `TreeResponse.id`, `CategoryRef.id`, `CategoryFacet.id`, `LocationSuggestion.id`, `category_id` and `category_ids` are now `int | str` (uk/au still return ints), and `list_categories` / `list_locations` take `parent_id: int | str`. za filter keys are the site's own (`make`, `model`, `fuelType`, `year`, `mileageKm` …), and za search pages through the first 375 matches. diff --git a/pyproject.toml b/pyproject.toml index a8b4677..490568d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "scrapebadger" -version = "0.60.1" +version = "0.61.0" description = "Official Python SDK for ScrapeBadger - Async web scraping APIs for Twitter and more" readme = "README.md" license = { text = "MIT" } diff --git a/src/scrapebadger/__init__.py b/src/scrapebadger/__init__.py index cafe7a4..c17f90e 100644 --- a/src/scrapebadger/__init__.py +++ b/src/scrapebadger/__init__.py @@ -32,6 +32,7 @@ async def main(): from scrapebadger._internal.config import ClientConfig from scrapebadger._internal.exceptions import ( AuthenticationError, + BlockedError, InsufficientCreditsError, IPNotAllowedError, NotFoundError, @@ -1335,7 +1336,7 @@ async def main(): ZestimateHistoryPoint as ZillowZestimateHistoryPoint, ) -__version__ = "0.60.1" +__version__ = "0.61.0" __all__ = [ # TikTok core models @@ -1394,6 +1395,7 @@ async def main(): "BingSearchResponse", "BingVideoResult", "BingVideosResponse", + "BlockedError", # Vinted response envelopes "BrandsResponse", "CategoriesResponse", diff --git a/src/scrapebadger/_internal/__init__.py b/src/scrapebadger/_internal/__init__.py index 62c569e..3c68366 100644 --- a/src/scrapebadger/_internal/__init__.py +++ b/src/scrapebadger/_internal/__init__.py @@ -4,6 +4,7 @@ from scrapebadger._internal.config import ClientConfig from scrapebadger._internal.exceptions import ( AuthenticationError, + BlockedError, InsufficientCreditsError, IPNotAllowedError, NotFoundError, @@ -18,6 +19,7 @@ __all__ = [ "AuthenticationError", "BaseClient", + "BlockedError", "ClientConfig", "IPNotAllowedError", "InsufficientCreditsError", diff --git a/src/scrapebadger/_internal/client.py b/src/scrapebadger/_internal/client.py index 91d0dc1..5381e1b 100644 --- a/src/scrapebadger/_internal/client.py +++ b/src/scrapebadger/_internal/client.py @@ -14,6 +14,7 @@ from scrapebadger._internal.exceptions import ( AuthenticationError, + BlockedError, InsufficientCreditsError, IPNotAllowedError, NotFoundError, @@ -46,7 +47,7 @@ T = TypeVar("T") # User agent for SDK requests -SDK_VERSION = "0.60.1" +SDK_VERSION = "0.61.0" USER_AGENT = f"scrapebadger-python/{SDK_VERSION}" @@ -156,7 +157,8 @@ def _handle_error_response( PermissionDeniedError: For 403 responses from a key restricted away from the API it called. NotFoundError: For 404 responses. - ValidationError: For 422 responses. + BlockedError: For a 422 blocking_page_detected (web scraping). + ValidationError: For other 422 responses. RateLimitError: For 429 responses. ServerError: For 5xx responses. ScrapeBadgerError: For other error responses. @@ -186,6 +188,9 @@ def _handle_error_response( if status_code == 404: raise NotFoundError(message, status_code, data) + if status_code == 422 and data.get("error") == "blocking_page_detected": + raise BlockedError(message, status_code, data) + if status_code == 422: raise ValidationError(message, status_code, data) diff --git a/src/scrapebadger/_internal/exceptions.py b/src/scrapebadger/_internal/exceptions.py index 0254e60..57ddfef 100644 --- a/src/scrapebadger/_internal/exceptions.py +++ b/src/scrapebadger/_internal/exceptions.py @@ -244,6 +244,53 @@ def __init__( super().__init__(message, status_code, response_data) +class BlockedError(ValidationError): + """Raised when the Web Scraping API reports the target blocked the request. + + A ``422`` with ``error == "blocking_page_detected"``: every retry hit an + anti-bot wall. It subclasses :class:`ValidationError`, so existing + ``except ValidationError`` handlers keep catching it. Blocked requests are + not charged. + + Attributes: + block_kind: The wall's class: ``"antibot"``, ``"captcha"``, + ``"rate_limit"``, ``"ip_ban"``, ``"geo_block"``, ``"auth_wall"``, + ``"empty"`` or ``"other"`` (``None`` on older responses). + vendor: The protection vendor that served it, e.g. ``"cloudflare"``, + ``"datadome"``, ``"akamai"``, ``"aws_waf"`` (``None`` when unknown). + captcha: The CAPTCHA gating the page when ``block_kind == "captcha"``: + ``system``, ``version``, ``gate``, ``solvable`` and ``params`` + (e.g. ``sitekey``), else ``None``. + data: The scrape result the API returned with the block. + + Example: + ```python + try: + page = await client.web.scrape("https://example.com") + except BlockedError as e: + print(e.block_kind, e.vendor) + if e.captcha: + print(e.captcha["system"], e.captcha.get("params")) + ``` + """ + + @property + def block_kind(self) -> str | None: + return self.response_data.get("block_kind") + + @property + def vendor(self) -> str | None: + return self.response_data.get("vendor") + + @property + def captcha(self) -> dict[str, Any] | None: + return self.response_data.get("captcha") + + @property + def data(self) -> dict[str, Any] | None: + return self.response_data.get("data") + + class ServerError(ScrapeBadgerError): """Raised when the server encounters an error. diff --git a/tests/test_client.py b/tests/test_client.py index c00eeaa..b273cfd 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -189,3 +189,61 @@ async def test_ip_not_allowed_raises_ip_error(self, api_key: str) -> None: await client.get("/v1/google/search") assert exc.value.client_ip == "203.0.113.9" assert isinstance(exc.value, PermissionDeniedError) + + +class TestBlocked: + async def _raise(self, api_key: str, body: dict) -> Exception: + import httpx + + from scrapebadger._internal.client import BaseClient + from scrapebadger._internal.config import ClientConfig + + client = BaseClient(ClientConfig(api_key=api_key)) + client._client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda r: httpx.Response(422, json=body)), + base_url="https://api.test", + ) + with pytest.raises(Exception) as exc: + await client.post("/v1/web/scrape", json={"url": "https://example.com"}) + return exc.value + + async def test_blocking_422_raises_blocked_error(self, api_key: str) -> None: + from scrapebadger import BlockedError, ValidationError + + captcha = { + "system": "recaptcha_v2", + "version": "v2_checkbox", + "gate": True, + "params": {"sitekey": "6Lc"}, + } + err = await self._raise( + api_key, + { + "error": "blocking_page_detected", + "detail": "blocked", + "block_kind": "captcha", + "vendor": "google", + "captcha": captcha, + "data": {"status_code": 403}, + }, + ) + assert isinstance(err, BlockedError) + assert isinstance(err, ValidationError) + assert err.block_kind == "captcha" + assert err.vendor == "google" + assert err.captcha == captcha + assert err.data == {"status_code": 403} + + async def test_older_blocking_body_has_none_fields(self, api_key: str) -> None: + from scrapebadger import BlockedError + + err = await self._raise(api_key, {"error": "blocking_page_detected", "detail": "blocked"}) + assert isinstance(err, BlockedError) + assert err.block_kind is None and err.vendor is None and err.captcha is None + + async def test_other_422_stays_validation_error(self, api_key: str) -> None: + from scrapebadger import BlockedError, ValidationError + + err = await self._raise(api_key, {"detail": [{"loc": ["body", "url"], "msg": "required"}]}) + assert isinstance(err, ValidationError) + assert not isinstance(err, BlockedError) diff --git a/uv.lock b/uv.lock index 62695e4..85d2cde 100644 --- a/uv.lock +++ b/uv.lock @@ -930,7 +930,7 @@ wheels = [ [[package]] name = "scrapebadger" -version = "0.60.1" +version = "0.61.0" source = { editable = "." } dependencies = [ { name = "httpx" },