Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ public SortClause buildClauseFromDefinition(ObjectNode sortNode) {
* @return {@link SortExpression} for the regular sort
*/
private SortExpression buildRegularSortExpression(String path, JsonNode innerValue) {
return SortExpression.sort(path, parseAscending(path, innerValue));
}

public static boolean parseAscending(String path, JsonNode innerValue) {
if (!innerValue.isInt()) {
// Special checking for String and ArrayNode to give less confusing error messages
if (innerValue.isArray()) {
Expand Down Expand Up @@ -172,8 +176,7 @@ private SortExpression buildRegularSortExpression(String path, JsonNode innerVal
.formatted(innerValue)));
}

boolean ascending = innerValue.intValue() == 1;
return SortExpression.sort(path, ascending);
return innerValue.intValue() == 1;
}

private void validateSortExpressionPaths(ObjectNode sortNode) {
Expand All @@ -183,7 +186,7 @@ private void validateSortExpressionPaths(ObjectNode sortNode) {
}
}

private void validateSortExpressionPath(String path) {
public static void validateSortExpressionPath(String path) {
if (!NamingRules.FIELD.apply(path)) {
// This is only called after handling "special" sort expressions so simple validation
if (path.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public SortClause build(JsonNode node) {
*/
protected abstract SortClause buildClauseFromDefinition(ObjectNode sortNode);

protected float[] tryDecodeBinaryVector(String path, JsonNode innerValue) {
public static float[] tryDecodeBinaryVector(String path, JsonNode innerValue) {
if (innerValue instanceof ObjectNode innerObject) {
var ejsonWrapped = EJSONWrapper.maybeFrom(innerObject);
if (ejsonWrapped == null || ejsonWrapped.type() != EJSONWrapper.EJSONType.BINARY) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,161 +1,130 @@
package io.stargate.sgv2.jsonapi.api.model.command.clause.sort;

import static io.stargate.sgv2.jsonapi.config.constants.DocumentConstants.Fields.*;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import io.stargate.sgv2.jsonapi.metrics.CommandFeatures;
import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortExpression2.FieldSortExpression;
import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortExpression2.LexicalSortExpression;
import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortExpression2.VectorLegSortExpression;
import io.stargate.sgv2.jsonapi.util.recordable.Recordable;
import java.util.*;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import org.eclipse.microprofile.openapi.annotations.media.SchemaProperty;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;

/**
* The sort clause of a findAndRerank command, see {@link FindAndRerankSortClauseDeserializer} for
* the forms it can take.
* The parsed sort of a findAndRerank command, with one record for each form of sort.
*
* @param rerankLegMode {@link RerankLegMode#MULTI_LEG} when the sort clause uses $hybrid, otherwise
* {@link RerankLegMode#SINGLE_LEG}.
* @param hybridSort If <code>null</code> then the $hybrid field was not text, OR was blank text,
* otherwise it is the trimmed text.
* @param vectorizeSort If <code>null</code> then the $vectorize field was not present in the sort
* clause, OR was present but set to JSON null OR it was blank string, otherwise it is the value
* provided.
* @param lexicalSort If <code>null</code> then the $lexical field was not present in the sort
* clause, OR was present but set to JSON null OR it was blank string, otherwise it is the value
* provided.
* @param vectorSort If <code>null</code> then the vector field was not present in the sort clause,
* or present and JSON null, otherwise it is the value provided.
* <ul>
* <li>{@link MultiLeg}: a sort with $hybrid.
* <li>{@link SingleLeg}: any other sort.
* </ul>
*/
@JsonDeserialize(using = FindAndRerankSortClauseDeserializer.class)
@Schema(
name = "FindAndRerankSort",
type = SchemaType.OBJECT,
implementation = Map.class,
nullable = true,
additionalProperties = Schema.False.class,
properties =
@SchemaProperty(
name = HYBRID_FIELD,
anyOf = {String.class, FindAndRerankSort.HybridSortSchema.class}),
description =
"Use $hybrid with a query string or an object containing $vectorize or $vector, and optionally $lexical."
+ " $vector and $vectorize cannot be used together.",
examples = {
"""
{"$hybrid": "Same query for vectorize and BM25 sorting"}
""",
"""
{"$hybrid": {"$vectorize": "vectorize sort query", "$lexical": "lexical sort"}}
""",
"""
{"$hybrid": {"$vector": [1, 2, 3], "$lexical": "lexical sort"}}
""",
"""
{"$hybrid": {"$vector": {"$binary": "P4AAAEAAAABAQAAA"}, "$lexical": "lexical query"}}
"""
})
public record FindAndRerankSort(
@JsonIgnore RerankLegMode rerankLegMode,
@JsonIgnore String hybridSort,
@JsonIgnore String vectorizeSort,
@JsonIgnore String lexicalSort,
@JsonIgnore float[] vectorSort,
@JsonIgnore CommandFeatures commandFeatures)
implements Recordable {

// Schema-only classes. @SchemaProperty can point at a nested schema only through a class, so
// these empty classes carry the @Schema for each nested part of the sort. Nothing creates them.

/** The object form of $hybrid, with $vector or $vectorize and an optional $lexical. */
@Schema(
name = "HybridSort",
type = SchemaType.OBJECT,
implementation = Object.class,
additionalProperties = Schema.False.class,
properties = {
@SchemaProperty(
name = VECTOR_EMBEDDING_TEXT_FIELD,
type = SchemaType.STRING,
nullable = true),
@SchemaProperty(name = LEXICAL_CONTENT_FIELD, type = SchemaType.STRING, nullable = true),
@SchemaProperty(
name = VECTOR_EMBEDDING_FIELD,
nullable = true,
anyOf = {float[].class, BinaryVectorSchema.class})
},
not = ConflictingVectorSortSchema.class)
public static class HybridSortSchema {}

/** A binary encoded vector such as {"$binary": "P4AAAEAAAABAQAAA"}. */
@Schema(
name = "BinaryVector",
type = SchemaType.OBJECT,
implementation = Object.class,
nullable = true,
additionalProperties = Schema.False.class,
requiredProperties = BINARY_VECTOR_TEXT_FIELD,
properties = @SchemaProperty(name = BINARY_VECTOR_TEXT_FIELD, type = SchemaType.STRING))
public static class BinaryVectorSchema {}
public sealed interface FindAndRerankSort extends Recordable
permits FindAndRerankSort.MultiLeg, FindAndRerankSort.SingleLeg {

/** A sort with $hybrid. It always has a value for the vector leg. */
sealed interface MultiLeg extends FindAndRerankSort permits HybridText, HybridObject {}

/**
* Matches an object that has both $vector and $vectorize. HybridSort uses it as its {@code not}
* schema, so a sort that sets both fails validation.
* <code>{"$hybrid": "cheese"}</code>: the same text for the vector leg and the lexical leg.
*
* @param text The trimmed text. It is never blank.
*/
@Schema(
name = "ConflictingVectorSort",
type = SchemaType.OBJECT,
implementation = Object.class,
requiredProperties = {VECTOR_EMBEDDING_FIELD, VECTOR_EMBEDDING_TEXT_FIELD})
public static class ConflictingVectorSortSchema {}

public static final FindAndRerankSort NO_ARG_SORT =
new FindAndRerankSort(
RerankLegMode.SINGLE_LEG, null, null, null, null, CommandFeatures.EMPTY);

public FindAndRerankSort {
Objects.requireNonNull(rerankLegMode, "rerankLegMode must not be null");
Objects.requireNonNull(commandFeatures, "commandFeatures must not be null");
}
record HybridText(String text) implements MultiLeg {

public HybridText {
Objects.requireNonNull(text, "text must not be null");
if (text.isBlank()) {
throw new IllegalArgumentException("text must not be blank");
}
}

@Override
public DataRecorder recordTo(DataRecorder dataRecorder) {
return dataRecorder
.append("rerankLegMode", rerankLegMode)
.append("hybridSort", hybridSort)
.append("vectorizeSort", vectorizeSort)
.append("lexicalSort", lexicalSort)
.append("vectorSort", Arrays.toString(vectorSort))
.append("commandFeatures", commandFeatures);
@Override
public DataRecorder recordTo(DataRecorder dataRecorder) {
return dataRecorder.append("text", text);
}
}

/**
* Override to do a value equality check on the vector
* <code>{"$hybrid": {"$vectorize": "cheese", "$lexical": "cows"}}</code>: a separate value for
* the vector leg and the lexical leg. The vector leg can use $vector instead of $vectorize.
*
* @param obj the reference object with which to compare.
* @return
* <p>At least one of $vectorize and $vector is set. $lexical can be <code>null</code>.
*/
record HybridObject(String vectorize, String lexical, float[] vector) implements MultiLeg {

public HybridObject {
if (vectorize == null && vector == null) {
throw new IllegalArgumentException("vectorize or vector must not be null");
}
}

@Override
public DataRecorder recordTo(DataRecorder dataRecorder) {
return dataRecorder
.append("vectorize", vectorize)
.append("lexical", lexical)
.append("vector", Arrays.toString(vector));
}

/** Override to do a value equality check on the vector. */
@Override
public boolean equals(Object obj) {
return (obj instanceof HybridObject other)
&& Objects.equals(vectorize, other.vectorize)
&& Objects.equals(lexical, other.lexical)
&& Arrays.equals(vector, other.vector);
}

/** Override to do a value equality hash on the vector. */
@Override
public int hashCode() {
return Objects.hash(vectorize, lexical, Arrays.hashCode(vector));
}
}

/** A sort without $hybrid, the command reads with one leg. */
sealed interface SingleLeg extends FindAndRerankSort permits VectorLeg, LexicalLeg, FilterLeg {}

/**
* <code>{"$vectorize": "cheese"}</code> or <code>{"$vector": [0.1, 0.2]}</code>: reads with the
* vector leg.
*/
@Override
public boolean equals(Object obj) {
return (obj instanceof FindAndRerankSort other)
&& rerankLegMode == other.rerankLegMode
&& Objects.equals(hybridSort, other.hybridSort)
&& Objects.equals(vectorizeSort, other.vectorizeSort)
&& Objects.equals(lexicalSort, other.lexicalSort)
&& Arrays.equals(vectorSort, other.vectorSort)
&& Objects.equals(commandFeatures, other.commandFeatures);
record VectorLeg(VectorLegSortExpression sort) implements SingleLeg {

public VectorLeg {
Objects.requireNonNull(sort, "sort must not be null");
}

@Override
public DataRecorder recordTo(DataRecorder dataRecorder) {
return dataRecorder.append("sort", sort);
}
}

/** <code>{"$lexical": "cows"}</code>: reads with the lexical leg. */
record LexicalLeg(LexicalSortExpression sort) implements SingleLeg {

public LexicalLeg {
Objects.requireNonNull(sort, "sort must not be null");
}

@Override
public DataRecorder recordTo(DataRecorder dataRecorder) {
return dataRecorder.append("sort", sort);
}
}

/** Override to do a value equality hash on the vector */
@Override
public int hashCode() {
return Objects.hash(
rerankLegMode,
hybridSort,
vectorizeSort,
lexicalSort,
Arrays.hashCode(vectorSort),
commandFeatures);
/**
* <code>{"age": -1, "name": 1}</code>: reads the documents the filter matches with the filter
* leg, sorted by the fields.
*/
record FilterLeg(List<FieldSortExpression> sorts) implements SingleLeg {

public FilterLeg {
sorts = List.copyOf(Objects.requireNonNull(sorts, "sorts must not be null"));
}

@Override
public DataRecorder recordTo(DataRecorder dataRecorder) {
return dataRecorder.append("sorts", Recordable.copyOf(sorts));
}
}
}
Loading
Loading