Repository navigation
fix(index): read cluster FT.INFO on RESP3 and initialise fresh async cluster clients #795
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
e175877
fix(index): read cluster FT.INFO on RESP3 and initialise fresh async …
vishal-bala d477904
test(index): drop a cluster info test the parametrised case already c…
vishal-bala efb00ec
fix(index): read cluster FT.INFO and FT._LIST from the default node
vishal-bala c688b1d
fix(index): send sync cluster listall() to the default node
vishal-bala d5d2bef
docs(concepts): describe index commands after a Redis Cluster failover
vishal-bala 1e5b7ab
Merge main into fix/cluster-ft-info-resp3-async-init
vishal-bala a5098cb
docs(concepts): name the exception create() and delete() raise after …
vishal-bala File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,169 @@ | ||
| """ | ||
| Unit tests for the Redis Cluster branch of SearchIndex._info() and | ||
| AsyncSearchIndex._info(). | ||
|
|
||
| On a cluster, _info() sends a raw FT.INFO to one node, so redis-py's search | ||
| parser never sees the reply and its shape depends on the wire protocol: a flat | ||
| list on RESP2, a map on RESP3 (the redis-py 8 default). Both must produce the | ||
| same index information. | ||
|
|
||
| _info() must also ask the node that FT.CREATE went to, the default node: a | ||
| replica learns of a new index only once replication catches up, so a replica | ||
| can answer "no such index" straight after create(). The stand-in clients below | ||
| model that with a replica that never has the index. | ||
|
|
||
| The async cluster client also discovers its nodes lazily, on its first command, | ||
| so a freshly built client has no node to pick. The live cluster suite covers | ||
| these cases but only runs with --run-cluster-tests, which CI never passes. | ||
| """ | ||
|
|
||
| from unittest.mock import create_autospec | ||
|
|
||
| import pytest | ||
| from redis.asyncio.cluster import ClusterNode as AsyncClusterNode | ||
| from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster | ||
| from redis.cluster import ClusterNode, RedisCluster | ||
| from redis.exceptions import ResponseError | ||
|
|
||
| from redisvl.index import AsyncSearchIndex, SearchIndex | ||
| from redisvl.redis.connection import convert_index_info_to_schema | ||
| from redisvl.schema import IndexSchema | ||
|
|
||
| INDEX_NAME = "my_index" | ||
|
|
||
| # The same index as FT.INFO returns it on each protocol, trimmed to the | ||
| # sections from_existing() reads. | ||
| RESP2_REPLY = [ | ||
| b"index_name", | ||
| INDEX_NAME.encode(), | ||
| b"index_definition", | ||
| [b"key_type", b"HASH", b"prefixes", [b"doc"]], | ||
| b"attributes", | ||
| [[b"identifier", b"title", b"attribute", b"title", b"type", b"TEXT"]], | ||
| b"num_docs", | ||
| b"0", | ||
| ] | ||
| RESP3_REPLY = { | ||
| b"index_name": INDEX_NAME.encode(), | ||
| b"index_definition": {b"key_type": b"HASH", b"prefixes": [b"doc"]}, | ||
| b"attributes": [ | ||
| { | ||
| b"identifier": b"title", | ||
| b"attribute": b"title", | ||
| b"type": b"TEXT", | ||
| b"flags": [], | ||
| } | ||
| ], | ||
| b"num_docs": 0, | ||
| } | ||
| REPLIES = pytest.mark.parametrize( | ||
| "reply", [RESP2_REPLY, RESP3_REPLY], ids=["resp2", "resp3"] | ||
| ) | ||
|
|
||
|
|
||
| def _schema(): | ||
| return IndexSchema.from_dict( | ||
| { | ||
| "index": {"name": INDEX_NAME, "prefix": "doc"}, | ||
| "fields": [{"name": "title", "type": "text"}], | ||
| } | ||
| ) | ||
|
|
||
|
|
||
| def _answer_from_primary(primary, reply): | ||
| """FT.INFO that only the primary can answer: the replica has not caught up.""" | ||
|
|
||
| def execute_command(*args, target_nodes=None): | ||
| if target_nodes not in (primary, [primary]): | ||
| raise ResponseError(f"{INDEX_NAME}: no such index") | ||
| return reply | ||
|
|
||
| return execute_command | ||
|
|
||
|
|
||
| def _sync_cluster(reply): | ||
| client = create_autospec(RedisCluster, instance=True) | ||
| primary = create_autospec(ClusterNode, instance=True) | ||
| client.get_default_node.return_value = primary | ||
| client.get_random_node.return_value = create_autospec(ClusterNode, instance=True) | ||
| client.execute_command.side_effect = _answer_from_primary(primary, reply) | ||
| return client | ||
|
|
||
|
|
||
| def _fresh_async_cluster(reply=None): | ||
| """An async cluster client that, like a real one, knows no nodes until | ||
| initialize() has run.""" | ||
| client = create_autospec(AsyncRedisCluster, instance=True) | ||
| primary = create_autospec(AsyncClusterNode, instance=True) | ||
| replica = create_autospec(AsyncClusterNode, instance=True) | ||
| initialized = False | ||
|
|
||
| async def initialize(): | ||
| nonlocal initialized | ||
| initialized = True | ||
| return client | ||
|
|
||
| def get_random_node(): | ||
| if not initialized: | ||
| raise IndexError("Cannot choose from an empty sequence") | ||
| return replica | ||
|
|
||
| client.initialize.side_effect = initialize | ||
| client.get_random_node.side_effect = get_random_node | ||
| client.get_default_node.side_effect = lambda: primary if initialized else None | ||
| client.execute_command.side_effect = _answer_from_primary(primary, reply) | ||
| primary.execute_command.return_value = [INDEX_NAME.encode()] | ||
| replica.execute_command.return_value = [] | ||
| return client, primary | ||
|
|
||
|
|
||
| class TestClusterInfoReplyShape: | ||
| @REPLIES | ||
| def test_sync_info_reads_either_protocol(self, reply): | ||
| info = SearchIndex._info(INDEX_NAME, _sync_cluster(reply)) | ||
|
|
||
| assert info["index_name"] == INDEX_NAME | ||
| assert convert_index_info_to_schema(info) == convert_index_info_to_schema( | ||
| SearchIndex._info(INDEX_NAME, _sync_cluster(RESP2_REPLY)) | ||
| ) | ||
|
|
||
| @REPLIES | ||
| @pytest.mark.asyncio | ||
| async def test_async_info_reads_either_protocol(self, reply): | ||
| client, _ = _fresh_async_cluster(reply) | ||
| reference, _ = _fresh_async_cluster(RESP2_REPLY) | ||
|
|
||
| info = await AsyncSearchIndex._info(INDEX_NAME, client) | ||
|
|
||
| assert info["index_name"] == INDEX_NAME | ||
| assert convert_index_info_to_schema(info) == convert_index_info_to_schema( | ||
| await AsyncSearchIndex._info(INDEX_NAME, reference) | ||
| ) | ||
|
|
||
|
|
||
| class TestFreshAsyncClusterClient: | ||
| @pytest.mark.asyncio | ||
| async def test_delete_targets_a_node_not_none(self): | ||
| client, node = _fresh_async_cluster() | ||
| index = AsyncSearchIndex(schema=_schema(), redis_client=client) | ||
|
|
||
| await index.delete(drop=False) | ||
|
|
||
| client.execute_command.assert_awaited_once_with( | ||
| "FT.DROPINDEX", INDEX_NAME, target_nodes=[node] | ||
| ) | ||
|
|
||
|
|
||
| class TestListallOnCluster: | ||
| @pytest.mark.asyncio | ||
| async def test_listall_sees_a_new_index(self): | ||
| client, _ = _fresh_async_cluster() | ||
| index = AsyncSearchIndex(schema=_schema(), redis_client=client) | ||
|
|
||
| assert await index.listall() == [INDEX_NAME] | ||
|
|
||
| def test_sync_listall_sees_a_new_index(self): | ||
| client = _sync_cluster([INDEX_NAME.encode()]) | ||
| index = SearchIndex(schema=_schema(), redis_client=client) | ||
|
|
||
| assert index.listall() == [INDEX_NAME] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.