Add strict mypy checking and type hints

.github/workflows/main.yml

@@ -42,6 +42,9 @@ jobs:
           - python-version: "3.14"
             env:
               TOXENV: "docs"
+          - python-version: "3.14"
+            env:
+              TOXENV: "mypy"
 
           # https://github.com/kevin1024/vcrpy/issues/885
           # - python-version: "pypy3.11-v7.3.20"

MANIFEST.in

@@ -2,3 +2,4 @@ include README.rst
 include LICENSE
 include MANIFEST.in
 include scrapinghub/VERSION
+include scrapinghub/py.typed

pyproject.toml

@@ -2,6 +2,13 @@
 # Keep in sync with .github/workflows/main.yml
 python-version = "3.14"
 
+[tool.mypy]
+strict = true
+
+[[tool.mypy.overrides]]
+module = ["msgpack", "retrying", "vcr"]
+ignore_missing_imports = true
+
 [dependency-groups]
 docs = [
     "sphinx-reredirects",

scrapinghub/__init__.py

@@ -3,10 +3,8 @@
            "DuplicateJobError", "BadRequest", "NotFound",
            "Unauthorized", "ValueTooLarge", "ServerError"]
 
-import pkgutil
-__version__ = pkgutil.get_data(__package__, 'VERSION')
-__version__ = str(__version__.decode('ascii').strip())
-del pkgutil
+from importlib.metadata import version as _version
+__version__ = _version(__name__)
 
 
 from .legacy import *

scrapinghub/client/__init__.py

@@ -1,8 +1,16 @@
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from requests import Response
+
 from scrapinghub import Connection as _Connection
 from scrapinghub import HubstorageClient as _HubstorageClient
 
 from .exceptions import _wrap_http_errors
-from .projects import Projects
+from .jobs import Job
+from .projects import Project, Projects
 from .utils import parse_auth
 from .utils import parse_project_id, parse_job_key
 
@@ -15,14 +23,14 @@
 class Connection(_Connection):
 
     @_wrap_http_errors
-    def _request(self, *args, **kwargs):
+    def _request(self, *args: Any, **kwargs: Any) -> Any:
         return super(Connection, self)._request(*args, **kwargs)
 
 
 class HubstorageClient(_HubstorageClient):
 
     @_wrap_http_errors
-    def request(self, *args, **kwargs):
+    def request(self, *args: Any, **kwargs: Any) -> Response:
         return super(HubstorageClient, self).request(*args, **kwargs)
 
 
@@ -62,9 +70,11 @@ class ScrapinghubClient(object):
         <scrapinghub.client.ScrapinghubClient at 0x1047af2e8>
     """
 
-    def __init__(self, auth=None, dash_endpoint=None,
-                 connection_timeout=DEFAULT_CONNECTION_TIMEOUT,
-                 dotenv_path=None, **kwargs):
+    def __init__(self, auth: str | tuple[str, str] | None = None,
+                 dash_endpoint: str | None = None,
+                 connection_timeout: float = DEFAULT_CONNECTION_TIMEOUT,
+                 dotenv_path: str | os.PathLike[str] | None = None,
+                 **kwargs: Any) -> None:
         self.projects = Projects(self)
         login, password = parse_auth(auth, dotenv_path=dotenv_path)
         timeout = connection_timeout or DEFAULT_CONNECTION_TIMEOUT
@@ -75,7 +85,7 @@ def __init__(self, auth=None, dash_endpoint=None,
         self._hsclient = HubstorageClient(auth=(login, password),
                                           connection_timeout=timeout, **kwargs)
 
-    def get_project(self, project_id):
+    def get_project(self, project_id: int | str) -> Project:
         """Get :class:`scrapinghub.client.projects.Project` instance with
         a given project id.
 
@@ -93,7 +103,7 @@ def get_project(self, project_id):
         """
         return self.projects.get(parse_project_id(project_id))
 
-    def get_job(self, job_key):
+    def get_job(self, job_key: str) -> Job:
         """Get :class:`~scrapinghub.client.jobs.Job` with a given job key.
 
         :param job_key: job key string in format ``project_id/spider_id/job_id``,
@@ -110,7 +120,7 @@ def get_job(self, job_key):
         project_id = parse_job_key(job_key).project_id
         return self.projects.get(project_id).jobs.get(job_key)
 
-    def close(self, timeout=None):
+    def close(self, timeout: float | None = None) -> None:
         """Close client instance.
 
         :param timeout: (optional) float timeout secs to stop gracefully.

scrapinghub/client/activity.py

@@ -1,10 +1,14 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, annotations
 
+from collections.abc import Iterable, Iterator
+from typing import Any
+
+from ..hubstorage.activity import Activity as _Activity
 from .proxy import _Proxy
 from .utils import parse_job_key, update_kwargs
 
 
-class Activity(_Proxy):
+class Activity(_Proxy[_Activity]):
     """Representation of collection of job activity events.
 
     Not a public constructor: use :class:`~scrapinghub.client.projects.Project`
@@ -44,7 +48,8 @@ class Activity(_Proxy):
         >>> project.activity.add(events)
 
     """
-    def iter(self, count=None, **params):
+    def iter(self, count: int | None = None,
+             **params: Any) -> Iterator[Any]:
         """Iterate over activity events.
 
         :param count: limit amount of elements.
@@ -55,7 +60,7 @@ def iter(self, count=None, **params):
         params = self._modify_iter_params(params)
         return self._origin.list(**params)
 
-    def add(self, values, **kwargs):
+    def add(self, values: Iterable[dict[str, Any]], **kwargs: Any) -> None:
         """Add new event to the project activity.
 
         :param values: a single event or a list of events, where event is

scrapinghub/client/collections.py

@@ -1,15 +1,24 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, annotations
 
+from collections.abc import Callable, Iterable, Iterator
+from typing import TYPE_CHECKING, Any
+
+import requests
 from six import string_types
 from six.moves import collections_abc
 
 from ..hubstorage.collectionsrt import Collection as _Collection
+from ..hubstorage.collectionsrt import Collections as _Collections
 
 from .proxy import _Proxy
 from .utils import update_kwargs
 
+if TYPE_CHECKING:
+    from ..hubstorage.batchuploader import _BatchWriter
+    from . import ScrapinghubClient
+
 
-class Collections(_Proxy):
+class Collections(_Proxy[_Collections]):
     """Access to project collections.
 
     Not a public constructor: use :class:`~scrapinghub.client.projects.Project`
@@ -24,7 +33,7 @@ class Collections(_Proxy):
         >>> foo_store = collections.get_store('foo_store')
     """
 
-    def get(self, type_, name):
+    def get(self, type_: str, name: str) -> Collection:
         """Base method to get a collection with a given type and name.
 
         :param `type_`: a collection type string.
@@ -35,7 +44,7 @@ def get(self, type_, name):
         self._origin._validate_collection(type_, name)
         return Collection(self._client, self, type_, name)
 
-    def get_store(self, name):
+    def get_store(self, name: str) -> Collection:
         """Method to get a store collection by name.
 
         :param name: a collection name string.
@@ -44,7 +53,7 @@ def get_store(self, name):
         """
         return self.get('s', name)
 
-    def get_cached_store(self, name):
+    def get_cached_store(self, name: str) -> Collection:
         """Method to get a cashed-store collection by name.
 
         The collection type means that items expire after a month.
@@ -55,7 +64,7 @@ def get_cached_store(self, name):
         """
         return self.get('cs', name)
 
-    def get_versioned_store(self, name):
+    def get_versioned_store(self, name: str) -> Collection:
         """Method to get a versioned-store collection by name.
 
         The collection type retains up to 3 copies of each item.
@@ -66,7 +75,7 @@ def get_versioned_store(self, name):
         """
         return self.get('vs', name)
 
-    def get_versioned_cached_store(self, name):
+    def get_versioned_cached_store(self, name: str) -> Collection:
         """Method to get a versioned-cached-store collection by name.
 
         Multiple copies are retained, and each one expires after a month.
@@ -77,7 +86,7 @@ def get_versioned_cached_store(self, name):
         """
         return self.get('vcs', name)
 
-    def iter(self):
+    def iter(self) -> Iterator[Any]:
         """Iterate through collections of a project.
 
         :return: an iterator over collections list where each collection is
@@ -86,7 +95,7 @@ def iter(self):
         """
         return self._origin.apiget('list')
 
-    def list(self):
+    def list(self) -> list[Any]:
         """List collections of a project.
 
         :return: a list of collections where each collection is
@@ -151,12 +160,13 @@ class Collection(object):
         >>> foo_store.truncate()
     """
 
-    def __init__(self, client, collections, type_, name):
+    def __init__(self, client: ScrapinghubClient, collections: Collections,
+                 type_: str, name: str) -> None:
         self._client = client
         self._collections = collections
         self._origin = _Collection(type_, name, collections._origin)
 
-    def get(self, key, **params):
+    def get(self, key: str, **params: Any) -> Any:
         """Get item from collection by key.
 
         :param key: string item key.
@@ -168,7 +178,7 @@ def get(self, key, **params):
             raise ValueError("key cannot be None")
         return self._origin.get(key, **params)
 
-    def set(self, value):
+    def set(self, value: Any) -> None:
         """Set item to collection by key.
 
         :param value: a dict representing a collection item.
@@ -177,7 +187,7 @@ def set(self, value):
         """
         self._origin.set(value)
 
-    def delete(self, keys):
+    def delete(self, keys: str | Iterable[str]) -> None:
         """Delete item(s) from collection by key(s).
 
         :param keys: a single key or a list of keys.
@@ -190,14 +200,14 @@ def delete(self, keys):
                              "object providing string keys")
         self._origin.delete(keys)
 
-    def truncate(self):
+    def truncate(self) -> None:
         """Remove the entire collection with a single API call.
 
         The method returns ``None`` (original method returns an empty generator).
         """
         self._origin.truncate()
 
-    def count(self, *args, **kwargs):
+    def count(self, *args: Any, **kwargs: Any) -> int:
         """Count collection items with a given filters.
 
         :return: amount of elements in collection.
@@ -207,8 +217,12 @@ def count(self, *args, **kwargs):
         return self._origin._collections.count(
             self._origin.coltype, self._origin.colname, *args, **kwargs)
 
-    def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
-             endts=None, requests_params=None, **params):
+    def iter(self, key: str | Iterable[str] | None = None,
+             prefix: str | Iterable[str] | None = None,
+             prefixcount: int | None = None, startts: int | None = None,
+             endts: int | None = None,
+             requests_params: dict[str, Any] | None = None,
+             **params: Any) -> Iterator[Any]:
         """A method to iterate through collection items.
 
         :param key: a string key or a list of keys to filter with.
@@ -228,8 +242,12 @@ def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
         return self._origin._collections.iter_values(
             self._origin.coltype, self._origin.colname, **params)
 
-    def list(self, key=None, prefix=None, prefixcount=None, startts=None,
-             endts=None, requests_params=None, **params):
+    def list(self, key: str | Iterable[str] | None = None,
+             prefix: str | Iterable[str] | None = None,
+             prefixcount: int | None = None, startts: int | None = None,
+             endts: int | None = None,
+             requests_params: dict[str, Any] | None = None,
+             **params: Any) -> list[Any]:
         """Convenient shortcut to list iter results.
 
         Please note that :meth:`list` method can use a lot of memory and for a
@@ -251,9 +269,12 @@ def list(self, key=None, prefix=None, prefixcount=None, startts=None,
                       startts=startts, endts=endts)
         return list(self.iter(requests_params=requests_params, **params))
 
-    def create_writer(self, start=0, auth=None, size=1000, interval=15,
-                      qsize=None, content_encoding='identity',
-                      maxitemsize=1024 ** 2, callback=None):
+    def create_writer(
+        self, start: int = 0, auth: str | tuple[str, str] | None = None,
+        size: int = 1000, interval: float = 15, qsize: int | None = None,
+        content_encoding: str = 'identity', maxitemsize: int = 1024 ** 2,
+        callback: Callable[[requests.Response | None], object] | None = None,
+    ) -> _BatchWriter:
         """Create a new writer for a collection.
 
         :param start: (optional) initial offset for writer thread.
@@ -270,7 +291,7 @@ def create_writer(self, start=0, auth=None, size=1000, interval=15,
         If provided - calllback shouldn't try to inject more items in the queue,
         otherwise it can lead to deadlocks.
         """
-        kwargs = {}
+        kwargs: dict[str, Any] = {}
         update_kwargs(kwargs, start=start, auth=auth, size=size, interval=interval,
                       qsize=qsize, content_encoding=content_encoding,
                       maxitemsize=maxitemsize, callback=callback)