Make msgpack a required dependency

.github/workflows/main.yml

@@ -15,28 +15,16 @@ jobs:
         include:
           - python-version: "3.10"
             env:
-              TOXENV: "msgpack"
-          - python-version: "3.10"
-            env:
-              TOXENV: "json"
+              TOXENV: "py"
           - python-version: "3.11"
             env:
-              TOXENV: "msgpack"
-          - python-version: "3.11"
-            env:
-              TOXENV: "json"
-          - python-version: "3.12"
-            env:
-              TOXENV: "msgpack"
+              TOXENV: "py"
           - python-version: "3.12"
             env:
-              TOXENV: "json"
-          - python-version: "3.13"
-            env:
-              TOXENV: "msgpack"
+              TOXENV: "py"
           - python-version: "3.13"
             env:
-              TOXENV: "json"
+              TOXENV: "py"
           # Keep in sync with pyproject.toml's
           # tool.sphinx-scrapy.python-version.
           - python-version: "3.14"
@@ -46,10 +34,7 @@ jobs:
           # https://github.com/kevin1024/vcrpy/issues/885
           # - python-version: "pypy3.11-v7.3.20"
           #   env:
-          #     TOXENV: "msgpack"
-          # - python-version: "pypy3.11-v7.3.20"
-          #   env:
-          #     TOXENV: "json"
+          #     TOXENV: "py"
 
     steps:
     - uses: actions/checkout@v2

README.rst

@@ -25,11 +25,6 @@ The quick way::
 
     pip install scrapinghub
 
-You can also install the library with MessagePack support, it provides better
-response time and improved bandwidth usage::
-
-    pip install scrapinghub[msgpack]
-
 
 Documentation
 -------------

docs/intro/install.rst

@@ -13,11 +13,3 @@ Installing scrapinghub
 The quick way::
 
     pip install scrapinghub
-
-It is recommended to install the library with `MessagePack`_ support,
-it provides better response time and improved bandwidth usage::
-
-    pip install scrapinghub[msgpack]
-
-
-.. _MessagePack: https://en.wikipedia.org/wiki/MessagePack

scrapinghub/hubstorage/client.py

@@ -12,7 +12,6 @@
 from .jobq import JobQ
 from .batchuploader import BatchUploader
 from .resourcetype import ResourceType
-from .serialization import MSGPACK_AVAILABLE
 
 
 __all__ = ["HubstorageClient"]
@@ -108,10 +107,7 @@ def __init__(self, auth=None, endpoint=None, connection_timeout=None,
         self.projects = Projects(self, None)
         self.root = ResourceType(self, None)
         self._batchuploader = None
-        self.use_msgpack = MSGPACK_AVAILABLE and use_msgpack
-        if use_msgpack != self.use_msgpack:
-            logger.warning('Messagepack is not available, please ensure that '
-                           'msgpack library is properly installed.')
+        self.use_msgpack = use_msgpack
 
     def request(self, is_idempotent=False, **kwargs):
         """

scrapinghub/hubstorage/serialization.py

@@ -2,19 +2,12 @@
 from json import dumps, loads
 
 import six
+from msgpack import Unpacker
 
 EPOCH = datetime(1970, 1, 1, 0, 0)
 ADAYINSECONDS = 24 * 3600
 
 
-try:
-    from msgpack import Unpacker
-
-    MSGPACK_AVAILABLE = True
-except ImportError:
-    MSGPACK_AVAILABLE = False
-
-
 def jlencode(iterable):
     if isinstance(iterable, (dict, six.string_types)):
         iterable = [iterable]

setup.py

@@ -1,4 +1,3 @@
-import sys
 from os.path import dirname, join
 
 try:
@@ -10,11 +9,6 @@
 with open(join(dirname(__file__), 'scrapinghub/VERSION'), 'rb') as f:
     version = f.read().decode('ascii').strip()
 
-is_pypy = '__pypy__' in sys.builtin_module_names
-mpack_required = ['msgpack>=1.0.0']
-if is_pypy:
-    mpack_required.append('msgpack-pypy>=0.0.2')
-
 setup(
     name='scrapinghub',
     version=version,
@@ -26,9 +20,8 @@
     platforms=['Any'],
     packages=['scrapinghub', 'scrapinghub.client', 'scrapinghub.hubstorage'],
     package_data={'scrapinghub': ['VERSION']},
-    install_requires=['python-dotenv>=1.0.0', 'requests>=1.0',
-                      'retrying>=1.3.3', 'six>=1.10.0'],
-    extras_require={'msgpack': mpack_required},
+    install_requires=['msgpack>=1.0.0', 'python-dotenv>=1.0.0',
+                      'requests>=1.0', 'retrying>=1.3.3', 'six>=1.10.0'],
     python_requires='>=3.10',
     classifiers=[
         'Development Status :: 5 - Production/Stable',

tests/client/conftest.py

@@ -6,7 +6,6 @@
 
 from scrapinghub import ScrapinghubClient
 from scrapinghub.client.exceptions import NotFound
-from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE
 
 from ..conftest import request_accept_header_matcher
 from ..conftest import VCRGzipSerializer
@@ -107,7 +106,7 @@ def setup_session(client, project, collection, request):
 def json_and_msgpack(client, monkeypatch, request):
     if request.param == 'json':
         monkeypatch.setattr(client._hsclient, 'use_msgpack', False)
-    elif not MSGPACK_AVAILABLE or request.config.getoption("--disable-msgpack"):
+    elif request.config.getoption("--disable-msgpack"):
         pytest.skip("messagepack-based tests are disabled")
     return request.param
 

tests/conftest.py

@@ -6,7 +6,6 @@
 import sys
 import zlib
 
-from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE
 from scrapinghub import HubstorageClient
 from scrapinghub.legacy import Connection
 

tests/hubstorage/conftest.py

@@ -8,7 +8,6 @@
 
 from scrapinghub import HubstorageClient
 from scrapinghub.hubstorage.utils import urlpathjoin
-from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE
 
 from ..conftest import request_accept_header_matcher
 from ..conftest import VCRGzipSerializer
@@ -89,7 +88,7 @@ def setup_session(hsclient, hsproject, hscollection, request):
 def json_and_msgpack(hsclient, monkeypatch, request):
     if request.param == 'json':
         monkeypatch.setattr(hsclient, 'use_msgpack', False)
-    elif not MSGPACK_AVAILABLE or request.config.getoption("--disable-msgpack"):
+    elif request.config.getoption("--disable-msgpack"):
         pytest.skip("messagepack-based tests are disabled")
     return request.param
 

tox.ini

@@ -4,14 +4,14 @@
 # and then run "tox" from this directory.
 
 [tox]
+min_version = 4.25.0
 requires =
     sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.1
 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885
-envlist = py{310,311,312}-{json,msgpack},docs
+envlist = py3{10-13},docs
 
 [testenv]
 deps =
     -r tests/requirements.txt
-    msgpack: -r tests/requirements-msgpack.txt
 commands = py.test --cov=scrapinghub --cov-report=xml {posargs: scrapinghub tests}