diff --git a/docs/concepts/search-and-indexing.md b/docs/concepts/search-and-indexing.md index 5cab8733..613d9a5f 100644 --- a/docs/concepts/search-and-indexing.md +++ b/docs/concepts/search-and-indexing.md @@ -97,6 +97,15 @@ This is a short indexing delay, not data loss. A later query sees the documents Applications that require strict search read-after-write behavior can start Redis with `--search-workers 0`. This is a Redis server setting, not a RedisVL option: it keeps indexing in the foreground but gives up the throughput benefits of multithreaded background indexing. +## Index Commands After a Cluster Failover + +On Redis Cluster, index commands carry no key, so RedisVL sends them to the client's default node: a primary that redis-py picks when it connects. That covers `create()`, `delete()`, `exists()`, `info()`, `listall()` and queries. redis-py learns that a failover has happened only when a keyed command is redirected with `MOVED`, or when a connection fails. Until then, a client that has sent only index commands since the failover keeps sending them to the demoted node, which is now a replica: + +- `create()` and `delete()` raise `RedisSearchError`, with the redis-py `ReadOnlyError` as its `__cause__`. +- `exists()`, `info()`, `listall()` and queries are answered by the replica. Replication is asynchronous, so its copy can lag behind the new primary: a new index can be reported missing, or a dropped one still present. + +This lasts until the client sends any keyed command, such as `load()` or `fetch()`, or reconnects. Most applications send keyed commands constantly. A process that only manages indexes can recover: if `create()` or `delete()` raises `RedisSearchError` caused by a `ReadOnlyError`, build a new cluster client and retry the call. + ## Bulk Delete and Update Redis has no server-side "delete/update by query", so mutating many documents traditionally means scanning for keys and issuing writes yourself. RedisVL wraps that pattern behind two filter-driven methods (available on both `SearchIndex` and `AsyncSearchIndex`). diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 8cd23da8..f253149e 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -2324,6 +2324,13 @@ def listall(self) -> list[str]: Returns: List[str]: The list of indices in the database. """ + if isinstance(self._redis_client, SyncRedisCluster): + # redis-py 7+ cannot route a keyless FT._LIST on a cluster, and a + # replica may lag behind, so ask the default node, as in _info(). + node = self._redis_client.get_default_node() + return convert_bytes( + self._redis_client.execute_command("FT._LIST", target_nodes=node) + ) return convert_bytes(self._redis_client.execute_command("FT._LIST")) def exists(self) -> bool: @@ -2354,11 +2361,14 @@ def _info(name: str, redis_client: SyncRedisClient) -> dict[str, Any]: """Run FT.INFO to fetch information about the index.""" try: if isinstance(redis_client, SyncRedisCluster): - node = redis_client.get_random_node() + # Ask the node FT.CREATE and FT.DROPINDEX go to. A replica + # learns of a new index only once replication catches up. + node = redis_client.get_default_node() values = redis_client.execute_command( "FT.INFO", name, target_nodes=node ) - info = make_dict(values) + # A raw reply is a flat list on RESP2 and a map on RESP3. + info = values if isinstance(values, dict) else make_dict(values) else: info = redis_client.ft(name).info() return convert_bytes(info) @@ -2627,17 +2637,28 @@ async def _get_client(self) -> AsyncRedisClient: self._lib_name, ) self._validated_client = True + if isinstance(self._redis_client, AsyncRedisCluster): + # The async cluster client discovers its nodes on first use, so + # get_default_node() returns None on a fresh client. This is a + # no-op once the client is initialised. + await self._redis_client.initialize() return self._redis_client @staticmethod async def _info(name: str, redis_client: AsyncRedisClient) -> dict[str, Any]: try: if isinstance(redis_client, AsyncRedisCluster): - node = redis_client.get_random_node() + # Callers such as from_existing() pass a client that has not + # been through _get_client(), so initialise it here too. + await redis_client.initialize() + # Ask the node FT.CREATE and FT.DROPINDEX go to. A replica + # learns of a new index only once replication catches up. + node = redis_client.get_default_node() values = await redis_client.execute_command( "FT.INFO", name, target_nodes=node ) - info = make_dict(values) + # A raw reply is a flat list on RESP2 and a map on RESP3. + info = values if isinstance(values, dict) else make_dict(values) else: info = await redis_client.ft(name).info() return convert_bytes(info) @@ -3693,8 +3714,9 @@ async def listall(self) -> list[str]: """ client = await self._get_client() if isinstance(client, AsyncRedisCluster): - target_nodes = client.get_random_node() - return convert_bytes(await target_nodes.execute_command("FT._LIST")) + # The default node, as in _info(): a replica may lag behind. + node = client.get_default_node() + return convert_bytes(await node.execute_command("FT._LIST")) else: return convert_bytes(await client.execute_command("FT._LIST")) diff --git a/tests/unit/test_cluster_index_info.py b/tests/unit/test_cluster_index_info.py new file mode 100644 index 00000000..c8031e5c --- /dev/null +++ b/tests/unit/test_cluster_index_info.py @@ -0,0 +1,169 @@ +""" +Unit tests for the Redis Cluster branch of SearchIndex._info() and +AsyncSearchIndex._info(). + +On a cluster, _info() sends a raw FT.INFO to one node, so redis-py's search +parser never sees the reply and its shape depends on the wire protocol: a flat +list on RESP2, a map on RESP3 (the redis-py 8 default). Both must produce the +same index information. + +_info() must also ask the node that FT.CREATE went to, the default node: a +replica learns of a new index only once replication catches up, so a replica +can answer "no such index" straight after create(). The stand-in clients below +model that with a replica that never has the index. + +The async cluster client also discovers its nodes lazily, on its first command, +so a freshly built client has no node to pick. The live cluster suite covers +these cases but only runs with --run-cluster-tests, which CI never passes. +""" + +from unittest.mock import create_autospec + +import pytest +from redis.asyncio.cluster import ClusterNode as AsyncClusterNode +from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster +from redis.cluster import ClusterNode, RedisCluster +from redis.exceptions import ResponseError + +from redisvl.index import AsyncSearchIndex, SearchIndex +from redisvl.redis.connection import convert_index_info_to_schema +from redisvl.schema import IndexSchema + +INDEX_NAME = "my_index" + +# The same index as FT.INFO returns it on each protocol, trimmed to the +# sections from_existing() reads. +RESP2_REPLY = [ + b"index_name", + INDEX_NAME.encode(), + b"index_definition", + [b"key_type", b"HASH", b"prefixes", [b"doc"]], + b"attributes", + [[b"identifier", b"title", b"attribute", b"title", b"type", b"TEXT"]], + b"num_docs", + b"0", +] +RESP3_REPLY = { + b"index_name": INDEX_NAME.encode(), + b"index_definition": {b"key_type": b"HASH", b"prefixes": [b"doc"]}, + b"attributes": [ + { + b"identifier": b"title", + b"attribute": b"title", + b"type": b"TEXT", + b"flags": [], + } + ], + b"num_docs": 0, +} +REPLIES = pytest.mark.parametrize( + "reply", [RESP2_REPLY, RESP3_REPLY], ids=["resp2", "resp3"] +) + + +def _schema(): + return IndexSchema.from_dict( + { + "index": {"name": INDEX_NAME, "prefix": "doc"}, + "fields": [{"name": "title", "type": "text"}], + } + ) + + +def _answer_from_primary(primary, reply): + """FT.INFO that only the primary can answer: the replica has not caught up.""" + + def execute_command(*args, target_nodes=None): + if target_nodes not in (primary, [primary]): + raise ResponseError(f"{INDEX_NAME}: no such index") + return reply + + return execute_command + + +def _sync_cluster(reply): + client = create_autospec(RedisCluster, instance=True) + primary = create_autospec(ClusterNode, instance=True) + client.get_default_node.return_value = primary + client.get_random_node.return_value = create_autospec(ClusterNode, instance=True) + client.execute_command.side_effect = _answer_from_primary(primary, reply) + return client + + +def _fresh_async_cluster(reply=None): + """An async cluster client that, like a real one, knows no nodes until + initialize() has run.""" + client = create_autospec(AsyncRedisCluster, instance=True) + primary = create_autospec(AsyncClusterNode, instance=True) + replica = create_autospec(AsyncClusterNode, instance=True) + initialized = False + + async def initialize(): + nonlocal initialized + initialized = True + return client + + def get_random_node(): + if not initialized: + raise IndexError("Cannot choose from an empty sequence") + return replica + + client.initialize.side_effect = initialize + client.get_random_node.side_effect = get_random_node + client.get_default_node.side_effect = lambda: primary if initialized else None + client.execute_command.side_effect = _answer_from_primary(primary, reply) + primary.execute_command.return_value = [INDEX_NAME.encode()] + replica.execute_command.return_value = [] + return client, primary + + +class TestClusterInfoReplyShape: + @REPLIES + def test_sync_info_reads_either_protocol(self, reply): + info = SearchIndex._info(INDEX_NAME, _sync_cluster(reply)) + + assert info["index_name"] == INDEX_NAME + assert convert_index_info_to_schema(info) == convert_index_info_to_schema( + SearchIndex._info(INDEX_NAME, _sync_cluster(RESP2_REPLY)) + ) + + @REPLIES + @pytest.mark.asyncio + async def test_async_info_reads_either_protocol(self, reply): + client, _ = _fresh_async_cluster(reply) + reference, _ = _fresh_async_cluster(RESP2_REPLY) + + info = await AsyncSearchIndex._info(INDEX_NAME, client) + + assert info["index_name"] == INDEX_NAME + assert convert_index_info_to_schema(info) == convert_index_info_to_schema( + await AsyncSearchIndex._info(INDEX_NAME, reference) + ) + + +class TestFreshAsyncClusterClient: + @pytest.mark.asyncio + async def test_delete_targets_a_node_not_none(self): + client, node = _fresh_async_cluster() + index = AsyncSearchIndex(schema=_schema(), redis_client=client) + + await index.delete(drop=False) + + client.execute_command.assert_awaited_once_with( + "FT.DROPINDEX", INDEX_NAME, target_nodes=[node] + ) + + +class TestListallOnCluster: + @pytest.mark.asyncio + async def test_listall_sees_a_new_index(self): + client, _ = _fresh_async_cluster() + index = AsyncSearchIndex(schema=_schema(), redis_client=client) + + assert await index.listall() == [INDEX_NAME] + + def test_sync_listall_sees_a_new_index(self): + client = _sync_cluster([INDEX_NAME.encode()]) + index = SearchIndex(schema=_schema(), redis_client=client) + + assert index.listall() == [INDEX_NAME] diff --git a/tests/unit/test_error_handling.py b/tests/unit/test_error_handling.py index 09b768a0..a1dc3da5 100644 --- a/tests/unit/test_error_handling.py +++ b/tests/unit/test_error_handling.py @@ -414,6 +414,7 @@ async def test_async_drop_documents_cluster_validation(self, mock_validate): # Create a mock that properly inherits from RedisCluster for isinstance check mock_cluster_client = Mock() mock_cluster_client.__class__ = AsyncRedisCluster + mock_cluster_client.initialize = AsyncMock() mock_cluster_client.delete = AsyncMock(return_value=2) index = AsyncSearchIndex(schema=schema, redis_client=mock_cluster_client) diff --git a/tests/unit/test_index_exists.py b/tests/unit/test_index_exists.py index b7bf3019..fae2b306 100644 --- a/tests/unit/test_index_exists.py +++ b/tests/unit/test_index_exists.py @@ -147,8 +147,8 @@ async def test_async_index_name_resolving_elsewhere_returns_false(self): class TestExistsOnCluster: """On a cluster, exists() must go through _info()'s node-targeted path. - Index metadata is cluster-wide, so any single node can answer -- but the - command has to carry target_nodes to get there. These are the only cluster + The command has to carry target_nodes, and it must name the default node, + where FT.CREATE goes: a replica may not have the new index yet. These are the only cluster checks that run by default; the live cluster suite needs --run-cluster-tests. Sync and async _info() each have their own cluster branch, and exists() reaches both for the first time with this change. @@ -157,7 +157,7 @@ class TestExistsOnCluster: def test_ft_info_is_routed_to_a_node(self): node = Mock() client = MagicMock(spec=RedisCluster) - client.get_random_node.return_value = node + client.get_default_node.return_value = node client.execute_command.return_value = ["index_name", INDEX_NAME] index = SearchIndex(schema=_schema(), redis_client=client) @@ -171,7 +171,8 @@ async def test_async_ft_info_is_routed_to_a_node(self): node = Mock() client = MagicMock() client.__class__ = AsyncRedisCluster # isinstance drives the branch - client.get_random_node.return_value = node + client.initialize = AsyncMock() + client.get_default_node.return_value = node client.execute_command = AsyncMock(return_value=["index_name", INDEX_NAME]) index = AsyncSearchIndex(schema=_schema(), redis_client=client)