From b3779f44cb1b11314a0c62519d14bcaf9ae0863a Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 28 Sep 2026 09:40:16 +0200 Subject: [PATCH 1/2] Fix list_iter repeats with filter, document iter params --- .pre-commit-config.yaml | 2 +- .readthedocs.yml | 4 +--- docs/conf.py | 5 +++++ docs/requirements.in | 2 +- docs/requirements.txt | 27 ++++++++++++++++++--------- scrapinghub/client/items.py | 31 ++++++++++++++++++++++++------- scrapinghub/client/proxy.py | 11 +++++++++-- tests/client/test_items.py | 22 ++++++++++++++++++++++ tox.ini | 2 +- 9 files changed, 82 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 53802d64..90def964 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.7.1 + rev: 0.12.0 hooks: - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index a2773dcf..fa9aa073 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,4 @@ build: python: "3.14" commands: - pip install tox - - tox -e docs - - mkdir -p $READTHEDOCS_OUTPUT/html - - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ + - tox -e docs -- $READTHEDOCS_OUTPUT/html diff --git a/docs/conf.py b/docs/conf.py index d3889c1e..97e241cc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,6 +41,11 @@ # ones. extensions = ['sphinx_scrapy', 'sphinx_reredirects'] +intersphinx_mapping = { + 'requests': ('https://requests.readthedocs.io/en/latest/', None), +} +scrapy_intersphinx_enable = ['zyte'] + redirects = { "quickstart": "intro/install.html", "client/overview": "../use/overview.html", diff --git a/docs/requirements.in b/docs/requirements.in index 536c0aff..942aac5e 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -1,3 +1,3 @@ sphinx-reredirects sphinx-rtd-theme -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.1 +sphinx-scrapy==0.12.0 diff --git a/docs/requirements.txt b/docs/requirements.txt index 394f6bfe..83190216 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -11,8 +11,9 @@ charset-normalizer==3.4.6 docutils==0.22.4 # via # sphinx - # sphinx-markdown-builder + # sphinx-llm-friendly # sphinx-rtd-theme + # sphinx-scrapy idna==3.11 # via requests imagesize==2.0.0 @@ -27,8 +28,12 @@ packaging==26.0 # sphinx-scrapy pygments==2.19.2 # via sphinx +regex==2026.9.10 + # via tiktoken requests==2.32.5 - # via sphinx + # via + # sphinx + # tiktoken roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 @@ -36,24 +41,26 @@ snowballstemmer==3.0.1 sphinx==9.1.0 # via # sphinx-copybutton - # sphinx-llms-txt - # sphinx-markdown-builder + # sphinx-last-updated-by-git + # sphinx-llm-friendly # sphinx-reredirects # sphinx-rtd-theme # sphinx-scrapy # sphinxcontrib-jquery sphinx-copybutton==0.5.2 # via sphinx-scrapy -sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 - # via sphinx-scrapy -sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@ac9f8babfe622e4300099ab44b96d9d9228e742e +sphinx-last-updated-by-git==0.3.8 + # via sphinx-sitemap +sphinx-llm-friendly==0.5.1 # via sphinx-scrapy sphinx-reredirects==1.1.0 # via -r requirements.in sphinx-rtd-theme==3.1.0 # via -r requirements.in -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@2b5f6c7de64c8317cb771fdeb2e5020d1c9c9dcf +sphinx-scrapy==0.12.0 # via -r requirements.in +sphinx-sitemap==2.9.0 + # via sphinx-scrapy sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -69,6 +76,8 @@ sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 # via sphinx tabulate==0.10.0 - # via sphinx-markdown-builder + # via sphinx-llm-friendly +tiktoken==0.14.0 + # via sphinx-llm-friendly urllib3==2.6.3 # via requests diff --git a/scrapinghub/client/items.py b/scrapinghub/client/items.py index e9be38c0..5469fe50 100644 --- a/scrapinghub/client/items.py +++ b/scrapinghub/client/items.py @@ -17,6 +17,11 @@ class Items(_DownloadableProxyMixin, _ItemsResourceProxy): :meth:`iter` method (all params and available filters are same for both methods). + Besides the parameters of the :ref:`zyte:api-items`, :meth:`iter` + supports a *filter* parameter: a list of ``(field, operator, values)`` + tuples, all of which an item must match, as shown in the last example + below. + Usage: - retrieve all scraped items from a job:: @@ -92,7 +97,7 @@ def _modify_iter_params(self, params): def list_iter(self, chunksize=1000, *args, **kwargs): """An alternative interface for reading items by returning them - as a generator which yields lists of items sized as `chunksize`. + as a generator which yields lists of up to *chunksize* items. This is a convenient method for cases when processing a large amount of items from a job isn't ideal in one go due to the large memory needed. @@ -101,10 +106,9 @@ def list_iter(self, chunksize=1000, *args, **kwargs): You can improve I/O overheads by increasing the chunk value but that would also increase the memory consumption. - :param chunksize: size of list to be returned per iteration - :param start: offset to specify the start of the item iteration - :param count: overall number of items to be returned, which is broken - down by `chunksize`. + *start* is the index of the first item to read, and *count* the + overall number of items to return. Other parameters, e.g. *filter*, + work as in :meth:`iter`. :return: an iterator over items, yielding lists of items. :rtype: :class:`collections.abc.Iterable` @@ -112,6 +116,12 @@ def list_iter(self, chunksize=1000, *args, **kwargs): start = kwargs.pop("start", 0) count = kwargs.pop("count", sys.maxsize) + meta = kwargs.pop("meta", None) or [] + if isinstance(meta, str): + meta = [meta] + drop_key = "_key" not in meta + if drop_key: + meta = [*meta, "_key"] processed = 0 while True: @@ -120,11 +130,18 @@ def list_iter(self, chunksize=1000, *args, **kwargs): chunksize = count - processed items = [ item for item in self.iter( - count=chunksize, start=next_key, *args, **kwargs) + count=chunksize, start=next_key, meta=meta, + *args, **kwargs) ] + if items: + # Filters may skip items, so resume right after the last + # item returned. + start = int(items[-1]["_key"].rsplit("/", 1)[1]) + 1 + if drop_key: + for item in items: + del item["_key"] yield items processed += len(items) - start += len(items) if processed >= count: break if len(items) < chunksize: diff --git a/scrapinghub/client/proxy.py b/scrapinghub/client/proxy.py index f7d10b4d..9928c8c4 100644 --- a/scrapinghub/client/proxy.py +++ b/scrapinghub/client/proxy.py @@ -102,9 +102,16 @@ def close(self, block=True): class _DownloadableProxyMixin(object): def iter(self, _path=None, count=None, requests_params=None, **apiparams): - """A general method to iterate through elements. + """Iterate through elements. + + *count* limits the number of elements returned. It is best to always + set it, or to set other pagination parameters, to avoid timeouts. + + *apiparams* are sent to the :ref:`Scrapy Cloud HTTP API + ` as query parameters, e.g. ``startts`` or + ``meta``, and *requests_params* are passed as keyword arguments to + :func:`requests.request`, e.g. ``{'timeout': 60}``. - :param count: limit amount of elements. :return: an iterator over elements list. :rtype: :class:`collections.abc.Iterable` """ diff --git a/tests/client/test_items.py b/tests/client/test_items.py index 79f68c33..7c6905e9 100644 --- a/tests/client/test_items.py +++ b/tests/client/test_items.py @@ -1,6 +1,9 @@ +import mock import pytest from six.moves import range +from scrapinghub.client.items import Items + from .utils import normalize_job_for_tests @@ -103,3 +106,22 @@ def test_items_list_iter_with_start_and_count_2(spider, json_and_msgpack): ] with pytest.raises(StopIteration): next(o) + + +def test_items_list_iter_with_skipped_items(): + """Items skipped by the server, e.g. due to a filter, must not cause + list_iter to return the same items more than once.""" + matching = [2, 5, 6, 9] + + def iter_(count, start, meta): + assert '_key' in meta + offset = int(start.rsplit('/', 1)[1]) + for index in [i for i in matching if i >= offset][:count]: + yield {'_key': '1/2/3/{}'.format(index), 'id': index} + + items = Items.__new__(Items) + items.key = '1/2/3' + with mock.patch.object(Items, 'iter', side_effect=iter_): + assert list(items.list_iter(chunksize=1)) == [ + [{'id': 2}], [{'id': 5}], [{'id': 6}], [{'id': 9}], [], + ] diff --git a/tox.ini b/tox.ini index 6f0dbb36..dcb25253 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.1 + sphinx-scrapy[tox]==0.12.0 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885 envlist = py{310,311,312}-{json,msgpack},docs From 492a3e6b0b2db7493513c513fe4fd9bad8fdf316 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 28 Sep 2026 10:14:55 +0200 Subject: [PATCH 2/2] Cover string meta in list_iter test --- tests/client/test_items.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/client/test_items.py b/tests/client/test_items.py index 7c6905e9..732e55bf 100644 --- a/tests/client/test_items.py +++ b/tests/client/test_items.py @@ -108,7 +108,11 @@ def test_items_list_iter_with_start_and_count_2(spider, json_and_msgpack): next(o) -def test_items_list_iter_with_skipped_items(): +@pytest.mark.parametrize('meta, keep_key', [ + ({}, False), + ({'meta': '_key'}, True), +]) +def test_items_list_iter_with_skipped_items(meta, keep_key): """Items skipped by the server, e.g. due to a filter, must not cause list_iter to return the same items more than once.""" matching = [2, 5, 6, 9] @@ -119,9 +123,14 @@ def iter_(count, start, meta): for index in [i for i in matching if i >= offset][:count]: yield {'_key': '1/2/3/{}'.format(index), 'id': index} + def item(index): + if keep_key: + return {'_key': '1/2/3/{}'.format(index), 'id': index} + return {'id': index} + items = Items.__new__(Items) items.key = '1/2/3' with mock.patch.object(Items, 'iter', side_effect=iter_): - assert list(items.list_iter(chunksize=1)) == [ - [{'id': 2}], [{'id': 5}], [{'id': 6}], [{'id': 9}], [], - ] + assert list(items.list_iter(chunksize=1, **meta)) == [ + [item(i)] for i in matching + ] + [[]]