Skip to content
9 changes: 9 additions & 0 deletions docs/concepts/search-and-indexing.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,15 @@ This is a short indexing delay, not data loss. A later query sees the documents

Applications that require strict search read-after-write behavior can start Redis with `--search-workers 0`. This is a Redis server setting, not a RedisVL option: it keeps indexing in the foreground but gives up the throughput benefits of multithreaded background indexing.

## Index Commands After a Cluster Failover

On Redis Cluster, index commands carry no key, so RedisVL sends them to the client's default node: a primary that redis-py picks when it connects. That covers `create()`, `delete()`, `exists()`, `info()`, `listall()` and queries. redis-py learns that a failover has happened only when a keyed command is redirected with `MOVED`, or when a connection fails. Until then, a client that has sent only index commands since the failover keeps sending them to the demoted node, which is now a replica:

- `create()` and `delete()` raise `RedisSearchError`, with the redis-py `ReadOnlyError` as its `__cause__`.
- `exists()`, `info()`, `listall()` and queries are answered by the replica. Replication is asynchronous, so its copy can lag behind the new primary: a new index can be reported missing, or a dropped one still present.

This lasts until the client sends any keyed command, such as `load()` or `fetch()`, or reconnects. Most applications send keyed commands constantly. A process that only manages indexes can recover: if `create()` or `delete()` raises `RedisSearchError` caused by a `ReadOnlyError`, build a new cluster client and retry the call.

## Bulk Delete and Update

Redis has no server-side "delete/update by query", so mutating many documents traditionally means scanning for keys and issuing writes yourself. RedisVL wraps that pattern behind two filter-driven methods (available on both `SearchIndex` and `AsyncSearchIndex`).
Expand Down
34 changes: 28 additions & 6 deletions redisvl/index/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -2324,6 +2324,13 @@ def listall(self) -> list[str]:
Returns:
List[str]: The list of indices in the database.
"""
if isinstance(self._redis_client, SyncRedisCluster):
# redis-py 7+ cannot route a keyless FT._LIST on a cluster, and a
# replica may lag behind, so ask the default node, as in _info().
node = self._redis_client.get_default_node()
return convert_bytes(
self._redis_client.execute_command("FT._LIST", target_nodes=node)
)
return convert_bytes(self._redis_client.execute_command("FT._LIST"))

def exists(self) -> bool:
Expand Down Expand Up @@ -2354,11 +2361,14 @@ def _info(name: str, redis_client: SyncRedisClient) -> dict[str, Any]:
"""Run FT.INFO to fetch information about the index."""
try:
if isinstance(redis_client, SyncRedisCluster):
node = redis_client.get_random_node()
# Ask the node FT.CREATE and FT.DROPINDEX go to. A replica
# learns of a new index only once replication catches up.
node = redis_client.get_default_node()
values = redis_client.execute_command(
"FT.INFO", name, target_nodes=node
)
info = make_dict(values)
# A raw reply is a flat list on RESP2 and a map on RESP3.
info = values if isinstance(values, dict) else make_dict(values)
else:
info = redis_client.ft(name).info()
return convert_bytes(info)
Expand Down Expand Up @@ -2627,17 +2637,28 @@ async def _get_client(self) -> AsyncRedisClient:
self._lib_name,
)
self._validated_client = True
if isinstance(self._redis_client, AsyncRedisCluster):
# The async cluster client discovers its nodes on first use, so
# get_default_node() returns None on a fresh client. This is a
# no-op once the client is initialised.
await self._redis_client.initialize()
return self._redis_client

@staticmethod
async def _info(name: str, redis_client: AsyncRedisClient) -> dict[str, Any]:
try:
if isinstance(redis_client, AsyncRedisCluster):
node = redis_client.get_random_node()
# Callers such as from_existing() pass a client that has not
# been through _get_client(), so initialise it here too.
await redis_client.initialize()
# Ask the node FT.CREATE and FT.DROPINDEX go to. A replica
# learns of a new index only once replication catches up.
node = redis_client.get_default_node()
Comment thread
vishal-bala marked this conversation as resolved.
Resolved
values = await redis_client.execute_command(
"FT.INFO", name, target_nodes=node
)
info = make_dict(values)
# A raw reply is a flat list on RESP2 and a map on RESP3.
info = values if isinstance(values, dict) else make_dict(values)
else:
info = await redis_client.ft(name).info()
return convert_bytes(info)
Expand Down Expand Up @@ -3693,8 +3714,9 @@ async def listall(self) -> list[str]:
"""
client = await self._get_client()
if isinstance(client, AsyncRedisCluster):
target_nodes = client.get_random_node()
return convert_bytes(await target_nodes.execute_command("FT._LIST"))
# The default node, as in _info(): a replica may lag behind.
node = client.get_default_node()
return convert_bytes(await node.execute_command("FT._LIST"))
else:
return convert_bytes(await client.execute_command("FT._LIST"))

Expand Down
169 changes: 169 additions & 0 deletions tests/unit/test_cluster_index_info.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
"""
Unit tests for the Redis Cluster branch of SearchIndex._info() and
AsyncSearchIndex._info().

On a cluster, _info() sends a raw FT.INFO to one node, so redis-py's search
parser never sees the reply and its shape depends on the wire protocol: a flat
list on RESP2, a map on RESP3 (the redis-py 8 default). Both must produce the
same index information.

_info() must also ask the node that FT.CREATE went to, the default node: a
replica learns of a new index only once replication catches up, so a replica
can answer "no such index" straight after create(). The stand-in clients below
model that with a replica that never has the index.

The async cluster client also discovers its nodes lazily, on its first command,
so a freshly built client has no node to pick. The live cluster suite covers
these cases but only runs with --run-cluster-tests, which CI never passes.
"""

from unittest.mock import create_autospec

import pytest
from redis.asyncio.cluster import ClusterNode as AsyncClusterNode
from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster
from redis.cluster import ClusterNode, RedisCluster
from redis.exceptions import ResponseError

from redisvl.index import AsyncSearchIndex, SearchIndex
from redisvl.redis.connection import convert_index_info_to_schema
from redisvl.schema import IndexSchema

INDEX_NAME = "my_index"

# The same index as FT.INFO returns it on each protocol, trimmed to the
# sections from_existing() reads.
RESP2_REPLY = [
b"index_name",
INDEX_NAME.encode(),
b"index_definition",
[b"key_type", b"HASH", b"prefixes", [b"doc"]],
b"attributes",
[[b"identifier", b"title", b"attribute", b"title", b"type", b"TEXT"]],
b"num_docs",
b"0",
]
RESP3_REPLY = {
b"index_name": INDEX_NAME.encode(),
b"index_definition": {b"key_type": b"HASH", b"prefixes": [b"doc"]},
b"attributes": [
{
b"identifier": b"title",
b"attribute": b"title",
b"type": b"TEXT",
b"flags": [],
}
],
b"num_docs": 0,
}
REPLIES = pytest.mark.parametrize(
"reply", [RESP2_REPLY, RESP3_REPLY], ids=["resp2", "resp3"]
)


def _schema():
return IndexSchema.from_dict(
{
"index": {"name": INDEX_NAME, "prefix": "doc"},
"fields": [{"name": "title", "type": "text"}],
}
)


def _answer_from_primary(primary, reply):
"""FT.INFO that only the primary can answer: the replica has not caught up."""

def execute_command(*args, target_nodes=None):
if target_nodes not in (primary, [primary]):
raise ResponseError(f"{INDEX_NAME}: no such index")
return reply

return execute_command


def _sync_cluster(reply):
client = create_autospec(RedisCluster, instance=True)
primary = create_autospec(ClusterNode, instance=True)
client.get_default_node.return_value = primary
client.get_random_node.return_value = create_autospec(ClusterNode, instance=True)
client.execute_command.side_effect = _answer_from_primary(primary, reply)
return client


def _fresh_async_cluster(reply=None):
"""An async cluster client that, like a real one, knows no nodes until
initialize() has run."""
client = create_autospec(AsyncRedisCluster, instance=True)
primary = create_autospec(AsyncClusterNode, instance=True)
replica = create_autospec(AsyncClusterNode, instance=True)
initialized = False

async def initialize():
nonlocal initialized
initialized = True
return client

def get_random_node():
if not initialized:
raise IndexError("Cannot choose from an empty sequence")
return replica

client.initialize.side_effect = initialize
client.get_random_node.side_effect = get_random_node
client.get_default_node.side_effect = lambda: primary if initialized else None
client.execute_command.side_effect = _answer_from_primary(primary, reply)
primary.execute_command.return_value = [INDEX_NAME.encode()]
replica.execute_command.return_value = []
return client, primary


class TestClusterInfoReplyShape:
@REPLIES
def test_sync_info_reads_either_protocol(self, reply):
info = SearchIndex._info(INDEX_NAME, _sync_cluster(reply))

assert info["index_name"] == INDEX_NAME
assert convert_index_info_to_schema(info) == convert_index_info_to_schema(
SearchIndex._info(INDEX_NAME, _sync_cluster(RESP2_REPLY))
)

@REPLIES
@pytest.mark.asyncio
async def test_async_info_reads_either_protocol(self, reply):
client, _ = _fresh_async_cluster(reply)
reference, _ = _fresh_async_cluster(RESP2_REPLY)

info = await AsyncSearchIndex._info(INDEX_NAME, client)

assert info["index_name"] == INDEX_NAME
assert convert_index_info_to_schema(info) == convert_index_info_to_schema(
await AsyncSearchIndex._info(INDEX_NAME, reference)
)


class TestFreshAsyncClusterClient:
@pytest.mark.asyncio
async def test_delete_targets_a_node_not_none(self):
client, node = _fresh_async_cluster()
index = AsyncSearchIndex(schema=_schema(), redis_client=client)

await index.delete(drop=False)

client.execute_command.assert_awaited_once_with(
"FT.DROPINDEX", INDEX_NAME, target_nodes=[node]
)


class TestListallOnCluster:
@pytest.mark.asyncio
async def test_listall_sees_a_new_index(self):
client, _ = _fresh_async_cluster()
index = AsyncSearchIndex(schema=_schema(), redis_client=client)

assert await index.listall() == [INDEX_NAME]

def test_sync_listall_sees_a_new_index(self):
client = _sync_cluster([INDEX_NAME.encode()])
index = SearchIndex(schema=_schema(), redis_client=client)

assert index.listall() == [INDEX_NAME]
1 change: 1 addition & 0 deletions tests/unit/test_error_handling.py
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,7 @@ async def test_async_drop_documents_cluster_validation(self, mock_validate):
# Create a mock that properly inherits from RedisCluster for isinstance check
mock_cluster_client = Mock()
mock_cluster_client.__class__ = AsyncRedisCluster
mock_cluster_client.initialize = AsyncMock()
mock_cluster_client.delete = AsyncMock(return_value=2)

index = AsyncSearchIndex(schema=schema, redis_client=mock_cluster_client)
Expand Down
9 changes: 5 additions & 4 deletions tests/unit/test_index_exists.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ async def test_async_index_name_resolving_elsewhere_returns_false(self):
class TestExistsOnCluster:
"""On a cluster, exists() must go through _info()'s node-targeted path.

Index metadata is cluster-wide, so any single node can answer -- but the
command has to carry target_nodes to get there. These are the only cluster
The command has to carry target_nodes, and it must name the default node,
where FT.CREATE goes: a replica may not have the new index yet. These are the only cluster
checks that run by default; the live cluster suite needs
--run-cluster-tests. Sync and async _info() each have their own cluster
branch, and exists() reaches both for the first time with this change.
Expand All @@ -157,7 +157,7 @@ class TestExistsOnCluster:
def test_ft_info_is_routed_to_a_node(self):
node = Mock()
client = MagicMock(spec=RedisCluster)
client.get_random_node.return_value = node
client.get_default_node.return_value = node
client.execute_command.return_value = ["index_name", INDEX_NAME]
index = SearchIndex(schema=_schema(), redis_client=client)

Expand All @@ -171,7 +171,8 @@ async def test_async_ft_info_is_routed_to_a_node(self):
node = Mock()
client = MagicMock()
client.__class__ = AsyncRedisCluster # isinstance drives the branch
client.get_random_node.return_value = node
client.initialize = AsyncMock()
client.get_default_node.return_value = node
client.execute_command = AsyncMock(return_value=["index_name", INDEX_NAME])
index = AsyncSearchIndex(schema=_schema(), redis_client=client)

Expand Down
Loading