diff --git a/README.md b/README.md index 56e795e5..dd41ed18 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,13 @@ This runs Redis 8+ with built-in vector search capabilities. +
+Flex databases - Datasets that extend beyond RAM + +Flex databases use both RAM and flash storage. [Redis Search on Flex](https://redis.io/docs/latest/operate/rs/flex/search/) is in preview and can store HNSW vector indexes on flash. In your RedisVL schema, set `rerank` explicitly on each disk HNSW vector field. + +
+
Redis Sentinel - High availability with automatic failover diff --git a/docs/api/schema.rst b/docs/api/schema.rst index aab08fc3..5f3eafd3 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -218,6 +218,22 @@ HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with - ``ef_runtime``: Controls search accuracy (higher = better recall, slower search). Default: 10 - ``epsilon``: Range search approximation factor for VectorRangeQuery. Default: 0.01 +For disk HNSW indexes in Flex databases, set ``rerank`` explicitly on each +vector field. For example, use the following YAML field definition. RedisVL +supplies the HNSW defaults for ``m``, ``ef_construction``, and ``ef_runtime``. +Omitting ``rerank`` leaves the option out of the Redis command. + +.. code-block:: yaml + + - name: embedding + type: vector + attrs: + algorithm: hnsw + dims: 768 + rerank: true + +See `Index and search data in Flex databases `_. + .. autoclass:: HNSWVectorField :members: :show-inheritance: diff --git a/docs/concepts/field-attributes.md b/docs/concepts/field-attributes.md index 65e05fca..009b7a76 100644 --- a/docs/concepts/field-attributes.md +++ b/docs/concepts/field-attributes.md @@ -376,6 +376,7 @@ The migration wizard (`rvl migrate wizard`) supports updating field attributes o | Attribute | Notes | |-----------|-------| | `withsuffixtrie` | Suffix/contains search optimization | +| `rerank` | Set to `true` or `false` for disk HNSW fields in Flex databases | *Note: `phonetic_matcher` is supported by the wizard for text fields.* diff --git a/redisvl/migration/utils.py b/redisvl/migration/utils.py index 73c23c06..68411a8c 100644 --- a/redisvl/migration/utils.py +++ b/redisvl/migration/utils.py @@ -152,11 +152,9 @@ def get_schema_field_path(schema: Dict[str, Any], field_name: str) -> Optional[s return None -# Attributes excluded from schema validation comparison. -# These are query-time or creation-hint parameters that FT.INFO does not return -# and are not relevant for index structure validation (confirmed by RediSearch team). -# - ef_runtime, epsilon: query-time tuning knobs, not index definition attributes -# - initial_cap: creation-time memory pre-allocation hint, diverges after indexing +# Compare vector index structure independently of query defaults and creation hints. +# Flex can report ef_runtime in FT.INFO. Queries can override this default. +# epsilon tunes range queries and initial_cap hints at initial allocation. EXCLUDED_VECTOR_ATTRS = {"ef_runtime", "epsilon", "initial_cap"} # phonetic_matcher: the matcher string (e.g. "dm:en") is not stored server-side, # only a boolean flag is kept, so it cannot be read back. diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 2b635210..f4fe41cd 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -456,6 +456,7 @@ def normalize_index_definition(index_info: dict[str, Any]) -> dict[str, Any]: "m", "ef_construction", "ef_runtime", + "rerank", "epsilon", "compression", "reduce", @@ -583,7 +584,7 @@ def parse_vector_attrs(attrs): try: normalized["dims"] = int(raw_dims) except (ValueError, TypeError): - # Unparseable dims: fall through to the required-dims check + # Invalid dims: fall through to the required-dims check # below, which skips the field with a warning. pass else: @@ -624,17 +625,15 @@ def parse_vector_attrs(attrs): normalized["datatype"] = "float32" # Handle HNSW-specific parameters - if "m" in vector_attrs: - try: - normalized["m"] = int(vector_attrs["m"]) - except (ValueError, TypeError): - pass - - if "ef_construction" in vector_attrs: - try: - normalized["ef_construction"] = int(vector_attrs["ef_construction"]) - except (ValueError, TypeError): - pass + for key in ("m", "ef_construction", "ef_runtime"): + if key in vector_attrs: + try: + normalized[key] = int(vector_attrs[key]) + except (ValueError, TypeError): + pass + + if "rerank" in vector_attrs: + normalized["rerank"] = vector_attrs["rerank"] # Handle SVS-VAMANA specific parameters # Compression - Redis uses different internal names, so we need to map them diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index e5fd805e..7eeddf98 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -24,7 +24,7 @@ Algorithm-Specific Parameters: - FLAT: block_size (memory management for dynamic indices) - - HNSW: m, ef_construction, ef_runtime, epsilon (graph tuning) + - HNSW: m, ef_construction, ef_runtime, epsilon, rerank - SVS-VAMANA: graph_max_degree, construction_window_size, search_window_size, compression, reduce, training_threshold (VAMANA graph algorithm with Intel hardware optimization and vector compression) @@ -265,6 +265,9 @@ class HNSWVectorFieldAttributes(BaseVectorFieldAttributes): ef_runtime: int = Field(default=10) """Max top candidates during search (default: 10) - primary tuning parameter""" + rerank: bool | None = None + """Enable or disable disk HNSW reranking in Flex databases. Omit to leave RERANK unset.""" + epsilon: float = Field(default=0.01) """Range search boundary factor (default: 0.01)""" @@ -594,6 +597,8 @@ def as_redis_field(self) -> RedisField: "EPSILON": self.attrs.epsilon, } ) + if self.attrs.rerank is not None: + field_data["RERANK"] = "TRUE" if self.attrs.rerank else "FALSE" return RedisVectorField( name, self.attrs.algorithm, diff --git a/tests/unit/test_convert_index_info.py b/tests/unit/test_convert_index_info.py index 6c924e4a..1f230cd6 100644 --- a/tests/unit/test_convert_index_info.py +++ b/tests/unit/test_convert_index_info.py @@ -328,6 +328,8 @@ def test_convert_index_info_resp2_unified_vector_flags(): 24, "ef_construction", 300, + "ef_runtime", + "invalid", # Skip this value while recovering the rest of the field. "INDEXMISSING", ], } @@ -421,3 +423,33 @@ def test_convert_index_info_skips_vector_field_with_unparseable_dims(caplog): assert result["fields"] == [] assert "'emb'" in caplog.text + + +@pytest.mark.parametrize( + "shape, rerank", [("pairs", None), ("mapping", "TRUE"), ("flags", "FALSE")] +) +def test_hnsw_info_round_trip(shape, rerank): + field = {"identifier": "emb", "attribute": "emb", "type": "VECTOR"} + params = {"algorithm": "HNSW", "dim": 4, "ef_runtime": 47} + if rerank is not None: + params["rerank"] = rerank + pairs = [item for pair in params.items() for item in pair] + if shape == "flags": + field["flags"] = pairs + elif shape == "mapping": + field.update(params) + else: + field = [item for pair in field.items() for item in pair] + pairs + + info = { + "index_name": "idx", + "index_definition": ["key_type", "HASH", "prefixes", ["idx"]], + "attributes": [field], + } + schema = IndexSchema.from_dict(convert_index_info_to_schema(info)) + args = schema.redis_fields[0].redis_args() + assert args[args.index("EF_RUNTIME") + 1] == 47 + if rerank is None: + assert "RERANK" not in args + else: + assert args[args.index("RERANK") + 1] == rerank diff --git a/tests/unit/test_fields.py b/tests/unit/test_fields.py index fb3c1dfe..6a0e9b79 100644 --- a/tests/unit/test_fields.py +++ b/tests/unit/test_fields.py @@ -164,6 +164,7 @@ def test_hnsw_vector_field_optional_params_not_set(): assert field_exported.args[field_exported.args.index("EF_CONSTRUCTION") + 1] == 200 assert field_exported.args[field_exported.args.index("EF_RUNTIME") + 1] == 10 assert field_exported.args[field_exported.args.index("EPSILON") + 1] == 0.01 + assert "RERANK" not in field_exported.args def test_flat_vector_field_block_size_not_set(): @@ -950,3 +951,12 @@ def test_multiple_warnings(caplog): assert any("LeanVec compression" in msg for msg in messages) assert any("graph_max_degree=20" in msg for msg in messages) assert any("search_window_size=120" in msg for msg in messages) + + +@pytest.mark.parametrize("rerank, expected", [(True, "TRUE"), (False, "FALSE")]) +def test_hnsw_rerank(rerank, expected): + field = HNSWVectorField( + name="emb", attrs={"algorithm": "hnsw", "dims": 4, "rerank": rerank} + ) + args = field.as_redis_field().redis_args() + assert args[args.index("RERANK") + 1] == expected