From 5ea813cf663866c69a9cf5e91b39a0a7479ff69e Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 28 Sep 2026 13:28:47 +0200 Subject: [PATCH] Accept datetime objects for startts and endts --- docs/use/overview.rst | 4 +++- scrapinghub/client/collections.py | 12 ++++++++---- scrapinghub/client/jobs.py | 16 ++++++++-------- scrapinghub/hubstorage/client.py | 8 +++++++- tests/hubstorage/test_client.py | 18 ++++++++++++++++++ 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/docs/use/overview.rst b/docs/use/overview.rst index 528ad213..af930857 100644 --- a/docs/use/overview.rst +++ b/docs/use/overview.rst @@ -239,7 +239,9 @@ in batches, using the ``start`` parameter:: >>> jobs_summary = spider.jobs.iter(start=1000) There are several filters like ``spider``, ``state``, ``has_tag``, -``lacks_tag``, ``startts`` and ``endts`` (check `list endpoint`_ for more details). +``lacks_tag``, ``startts`` and ``endts`` (check `list endpoint`_ for more +details). ``startts`` and ``endts`` also accept :class:`~datetime.datetime` +objects, and interpret naive ones as UTC. To get jobs filtered by tags:: diff --git a/scrapinghub/client/collections.py b/scrapinghub/client/collections.py index 9cdc061b..0b7e9ede 100644 --- a/scrapinghub/client/collections.py +++ b/scrapinghub/client/collections.py @@ -214,8 +214,10 @@ def iter(self, key=None, prefix=None, prefixcount=None, startts=None, :param key: a string key or a list of keys to filter with. :param prefix: a string prefix to filter items. :param prefixcount: maximum number of values to return per prefix. - :param startts: UNIX timestamp at which to begin results. - :param endts: UNIX timestamp at which to end results. + :param startts: UNIX timestamp in milliseconds, or + :class:`~datetime.datetime`, at which to begin results. + :param endts: UNIX timestamp in milliseconds, or + :class:`~datetime.datetime`, at which to end results. :param requests_params: (optional) a dict with optional requests params. :param params: (optional) additional query params for the request. :return: an iterator over items list. @@ -240,8 +242,10 @@ def list(self, key=None, prefix=None, prefixcount=None, startts=None, :param key: a string key or a list of keys to filter with. :param prefix: a string prefix to filter items. :param prefixcount: maximum number of values to return per prefix. - :param startts: UNIX timestamp at which to begin results. - :param endts: UNIX timestamp at which to end results. + :param startts: UNIX timestamp in milliseconds, or + :class:`~datetime.datetime`, at which to begin results. + :param endts: UNIX timestamp in milliseconds, or + :class:`~datetime.datetime`, at which to end results. :param requests_params: (optional) a dict with optional requests params. :param params: (optional) additional query params for the request. :return: a list of items where each item is represented with a dict. diff --git a/scrapinghub/client/jobs.py b/scrapinghub/client/jobs.py index 75d0acbc..f0a988a6 100644 --- a/scrapinghub/client/jobs.py +++ b/scrapinghub/client/jobs.py @@ -53,10 +53,10 @@ def count(self, spider=None, state=None, has_tag=None, lacks_tag=None, or a list of strings. :param lacks_tag: (optional) filter results by missing tag(s), a string or a list of strings. - :param startts: (optional) UNIX timestamp at which to begin results, - in milliseconds. - :param endts: (optional) UNIX timestamp at which to end results, - in milliseconds. + :param startts: (optional) UNIX timestamp in milliseconds, or + :class:`~datetime.datetime`, at which to begin results. + :param endts: (optional) UNIX timestamp in milliseconds, or + :class:`~datetime.datetime`, at which to end results. :param params: (optional) other filter params. :return: jobs count. @@ -221,10 +221,10 @@ def list(self, count=None, start=None, spider=None, state=None, or a list of strings. :param lacks_tag: (optional) filter results by missing tag(s), a string or a list of strings. - :param startts: (optional) UNIX timestamp at which to begin results, - in milliseconds. - :param endts: (optional) UNIX timestamp at which to end results, - in milliseconds. + :param startts: (optional) UNIX timestamp in milliseconds, or + :class:`~datetime.datetime`, at which to begin results. + :param endts: (optional) UNIX timestamp in milliseconds, or + :class:`~datetime.datetime`, at which to end results. :param meta: (optional) request for additional fields, a single field name or a list of field names to return. :param params: (optional) other filter params. diff --git a/scrapinghub/hubstorage/client.py b/scrapinghub/hubstorage/client.py index 72d2a556..5116860a 100644 --- a/scrapinghub/hubstorage/client.py +++ b/scrapinghub/hubstorage/client.py @@ -3,6 +3,7 @@ """ import logging import os +from datetime import datetime from requests import session, HTTPError, ConnectionError, Timeout from retrying import Retrying @@ -12,7 +13,7 @@ from .jobq import JobQ from .batchuploader import BatchUploader from .resourcetype import ResourceType -from .serialization import MSGPACK_AVAILABLE +from .serialization import MSGPACK_AVAILABLE, jsondefault __all__ = ["HubstorageClient"] @@ -120,6 +121,11 @@ def request(self, is_idempotent=False, **kwargs): Use the retry policy configured in the client when is_idempotent is True """ kwargs.setdefault('timeout', self.connection_timeout) + if kwargs.get('params'): + kwargs['params'] = { + k: int(jsondefault(v)) if isinstance(v, datetime) else v + for k, v in kwargs['params'].items() + } def invoke_request(): r = self.session.request(**kwargs) diff --git a/tests/hubstorage/test_client.py b/tests/hubstorage/test_client.py index d6533c31..57e94575 100644 --- a/tests/hubstorage/test_client.py +++ b/tests/hubstorage/test_client.py @@ -1,6 +1,9 @@ """ Test Client """ +from datetime import datetime, timedelta, timezone +from unittest import mock + from scrapinghub import HubstorageClient from scrapinghub.hubstorage.utils import apipoll @@ -57,3 +60,18 @@ def test_timestamp(hsclient): ts2 = hsclient.server_timestamp() assert ts1 > 0 assert ts1 <= ts2 + + +def test_datetime_params(): + client = HubstorageClient(auth=TEST_AUTH, endpoint=TEST_ENDPOINT) + with mock.patch.object(client.session, 'request') as request: + client.request(url='x', params={ + 'startts': datetime(2017, 5, 4, 3, 2, 1, 123456), + 'endts': datetime(2017, 5, 4, 5, 2, 1, tzinfo=timezone(timedelta(hours=2))), + 'count': 1, + }) + assert request.call_args.kwargs['params'] == { + 'startts': 1493866921123, + 'endts': 1493866921000, + 'count': 1, + }