Accept datetime objects for startts and endts

docs/use/overview.rst

@@ -239,7 +239,9 @@ in batches, using the ``start`` parameter::
     >>> jobs_summary = spider.jobs.iter(start=1000)
 
 There are several filters like ``spider``, ``state``, ``has_tag``,
-``lacks_tag``, ``startts`` and ``endts`` (check `list endpoint`_ for more details).
+``lacks_tag``, ``startts`` and ``endts`` (check `list endpoint`_ for more
+details). ``startts`` and ``endts`` also accept :class:`~datetime.datetime`
+objects, and interpret naive ones as UTC.
 
 To get jobs filtered by tags::
 

scrapinghub/client/collections.py

@@ -214,8 +214,10 @@ def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
         :param key: a string key or a list of keys to filter with.
         :param prefix: a string prefix to filter items.
         :param prefixcount: maximum number of values to return per prefix.
-        :param startts: UNIX timestamp at which to begin results.
-        :param endts: UNIX timestamp at which to end results.
+        :param startts: UNIX timestamp in milliseconds, or
+            :class:`~datetime.datetime`, at which to begin results.
+        :param endts: UNIX timestamp in milliseconds, or
+            :class:`~datetime.datetime`, at which to end results.
         :param requests_params: (optional) a dict with optional requests params.
         :param params: (optional) additional query params for the request.
         :return: an iterator over items list.
@@ -240,8 +242,10 @@ def list(self, key=None, prefix=None, prefixcount=None, startts=None,
         :param key: a string key or a list of keys to filter with.
         :param prefix: a string prefix to filter items.
         :param prefixcount: maximum number of values to return per prefix.
-        :param startts: UNIX timestamp at which to begin results.
-        :param endts: UNIX timestamp at which to end results.
+        :param startts: UNIX timestamp in milliseconds, or
+            :class:`~datetime.datetime`, at which to begin results.
+        :param endts: UNIX timestamp in milliseconds, or
+            :class:`~datetime.datetime`, at which to end results.
         :param requests_params: (optional) a dict with optional requests params.
         :param params: (optional) additional query params for the request.
         :return: a list of items where each item is represented with a dict.

scrapinghub/client/jobs.py

@@ -53,10 +53,10 @@ def count(self, spider=None, state=None, has_tag=None, lacks_tag=None,
             or a list of strings.
         :param lacks_tag: (optional) filter results by missing tag(s), a string
             or a list of strings.
-        :param startts: (optional) UNIX timestamp at which to begin results,
-            in milliseconds.
-        :param endts: (optional) UNIX timestamp at which to end results,
-            in milliseconds.
+        :param startts: (optional) UNIX timestamp in milliseconds, or
+            :class:`~datetime.datetime`, at which to begin results.
+        :param endts: (optional) UNIX timestamp in milliseconds, or
+            :class:`~datetime.datetime`, at which to end results.
         :param params: (optional) other filter params.
 
         :return: jobs count.
@@ -221,10 +221,10 @@ def list(self, count=None, start=None, spider=None, state=None,
             or a list of strings.
         :param lacks_tag: (optional) filter results by missing tag(s), a string
             or a list of strings.
-        :param startts: (optional) UNIX timestamp at which to begin results,
-            in milliseconds.
-        :param endts: (optional) UNIX timestamp at which to end results,
-            in milliseconds.
+        :param startts: (optional) UNIX timestamp in milliseconds, or
+            :class:`~datetime.datetime`, at which to begin results.
+        :param endts: (optional) UNIX timestamp in milliseconds, or
+            :class:`~datetime.datetime`, at which to end results.
         :param meta: (optional) request for additional fields, a single
             field name or a list of field names to return.
         :param params: (optional) other filter params.

scrapinghub/hubstorage/client.py

@@ -3,6 +3,7 @@
 """
 import logging
 import os
+from datetime import datetime
 
 from requests import session, HTTPError, ConnectionError, Timeout
 from retrying import Retrying
@@ -12,7 +13,7 @@
 from .jobq import JobQ
 from .batchuploader import BatchUploader
 from .resourcetype import ResourceType
-from .serialization import MSGPACK_AVAILABLE
+from .serialization import MSGPACK_AVAILABLE, jsondefault
 
 
 __all__ = ["HubstorageClient"]
@@ -120,6 +121,11 @@ def request(self, is_idempotent=False, **kwargs):
         Use the retry policy configured in the client when is_idempotent is True
         """
         kwargs.setdefault('timeout', self.connection_timeout)
+        if kwargs.get('params'):
+            kwargs['params'] = {
+                k: int(jsondefault(v)) if isinstance(v, datetime) else v
+                for k, v in kwargs['params'].items()
+            }
 
         def invoke_request():
             r = self.session.request(**kwargs)

tests/hubstorage/test_client.py

@@ -1,6 +1,9 @@
 """
 Test Client
 """
+from datetime import datetime, timedelta, timezone
+from unittest import mock
+
 from scrapinghub import HubstorageClient
 from scrapinghub.hubstorage.utils import apipoll
 
@@ -57,3 +60,18 @@ def test_timestamp(hsclient):
     ts2 = hsclient.server_timestamp()
     assert ts1 > 0
     assert ts1 <= ts2
+
+
+def test_datetime_params():
+    client = HubstorageClient(auth=TEST_AUTH, endpoint=TEST_ENDPOINT)
+    with mock.patch.object(client.session, 'request') as request:
+        client.request(url='x', params={
+            'startts': datetime(2017, 5, 4, 3, 2, 1, 123456),
+            'endts': datetime(2017, 5, 4, 5, 2, 1, tzinfo=timezone(timedelta(hours=2))),
+            'count': 1,
+        })
+    assert request.call_args.kwargs['params'] == {
+        'startts': 1493866921123,
+        'endts': 1493866921000,
+        'count': 1,
+    }