diff --git a/backend/app/api/docs/config/delete.md b/backend/app/api/docs/config/delete.md index 93303c2e0..091cfc33a 100644 --- a/backend/app/api/docs/config/delete.md +++ b/backend/app/api/docs/config/delete.md @@ -3,3 +3,7 @@ Delete a configuration and all its versions. This operation performs a delete, marking the configuration and all associated versions as deleted in the database while retaining records for audit purposes. + +The lookup is scoped by `tag`. It defaults to `default`; pass +`ASSESSMENT` to delete an assessment config. A config that exists under a +different tag is reported as not found. diff --git a/backend/app/api/docs/config/get.md b/backend/app/api/docs/config/get.md index 3421c3be9..035b563b4 100644 --- a/backend/app/api/docs/config/get.md +++ b/backend/app/api/docs/config/get.md @@ -3,3 +3,7 @@ Retrieve a specific configuration by its ID. Returns the configuration metadata including name, description, and timestamps. This endpoint provides configuration-level details but does not include version information. + +The lookup is scoped by `tag`. It defaults to `default`; pass +`ASSESSMENT` to fetch an assessment config. A config that exists under a +different tag is reported as not found. diff --git a/backend/app/api/routes/config/config.py b/backend/app/api/routes/config/config.py index 020a91d16..46227ef8f 100644 --- a/backend/app/api/routes/config/config.py +++ b/backend/app/api/routes/config/config.py @@ -90,12 +90,19 @@ def get_config( config_id: UUID, current_user: AuthContextDep, session: SessionDep, + tag: ConfigTag = Query( + ConfigTag.DEFAULT, + description=( + "Config scope. Use 'default' for general configs or 'ASSESSMENT' " + "for assessment configs. Supported values: 'default', 'ASSESSMENT'." + ), + ), ): """ Get a specific configuration by its ID. """ config_crud = ConfigCrud(session=session, project_id=current_user.project_.id) - config = config_crud.exists_or_raise(config_id=config_id) + config = config_crud.exists_in_tag_scope_or_raise(config_id=config_id, tag=tag) return APIResponse.success_response( data=config, ) @@ -138,12 +145,19 @@ def delete_config( config_id: UUID, current_user: AuthContextDep, session: SessionDep, + tag: ConfigTag = Query( + ConfigTag.DEFAULT, + description=( + "Config scope. Use 'default' for general configs or 'ASSESSMENT' " + "for assessment configs. Supported values: 'default', 'ASSESSMENT'." + ), + ), ): """ Delete a specific configuration. """ config_crud = ConfigCrud(session=session, project_id=current_user.project_.id) - config_crud.delete_or_raise(config_id=config_id) + config_crud.delete_or_raise(config_id=config_id, tag=tag) return APIResponse.success_response( data=Message(message="Config deleted successfully"), diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 3bc408eae..c285fea51 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -40,9 +40,9 @@ class Settings(BaseSettings): ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 1 # 60 minutes * 24 hours * 7 days = 7 days REFRESH_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 7 - ENVIRONMENT: Literal[ - "development", "testing", "staging", "production" - ] = "development" + ENVIRONMENT: Literal["development", "testing", "staging", "production"] = ( + "development" + ) PROJECT_NAME: str API_VERSION: str = "0.5.0" diff --git a/backend/app/crud/config/config.py b/backend/app/crud/config/config.py index fdfd055f0..5345e96c2 100644 --- a/backend/app/crud/config/config.py +++ b/backend/app/crud/config/config.py @@ -5,7 +5,7 @@ from sqlmodel import Session, and_, select from app.core.util import now -from app.crud.model_config import validate_blob_model_or_raise +from app.crud.model_config import validate_blob_completion_models from app.models import ( Config, ConfigCreate, @@ -34,7 +34,8 @@ def create_or_raise( """ self._check_unique_name_or_raise(config_create.name) - validate_blob_model_or_raise(self.session, config_create.config_blob) + # validate that the completion models in the config blob are valid + validate_blob_completion_models(self.session, config_create.config_blob) try: config = Config( @@ -144,8 +145,10 @@ def update_or_raise(self, config_id: UUID, config_update: ConfigUpdate) -> Confi ) return config - def delete_or_raise(self, config_id: UUID) -> None: - config = self.exists_or_raise(config_id) + def delete_or_raise( + self, config_id: UUID, tag: ConfigTag = ConfigTag.DEFAULT + ) -> None: + config = self.exists_in_tag_scope_or_raise(config_id, tag) config.deleted_at = now() self.session.add(config) diff --git a/backend/app/crud/config/version.py b/backend/app/crud/config/version.py index a0a312c93..56a0fcd70 100644 --- a/backend/app/crud/config/version.py +++ b/backend/app/crud/config/version.py @@ -3,12 +3,11 @@ from uuid import UUID from fastapi import HTTPException -from pydantic import ValidationError from sqlalchemy.orm import defer from sqlmodel import Session, and_, select from app.core.util import now -from app.crud.model_config import is_reasoning_model, validate_blob_model_or_raise +from app.crud.model_config import is_reasoning_model, validate_config_blob_for_tag from app.models import ( Config, ConfigVersion, @@ -17,7 +16,6 @@ ConfigVersionUpdate, ) from app.models.config.config import ConfigTag -from app.models.llm.request import ConfigBlob from .config import ConfigCrud @@ -67,24 +65,15 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: updates=version_create.config_blob, ) - self._strip_unsupported_params(merged_config) + # These operate on a top-level `completion` block, which assessment + # blobs don't have — skip them for the ASSESSMENT tag. + if self.tag != ConfigTag.ASSESSMENT: + self._strip_unsupported_params(merged_config) + self._validate_immutable_fields(latest_version.config_blob, merged_config) - # Validate that provider and type haven't been changed - self._validate_immutable_fields(latest_version.config_blob, merged_config) - - # Validate the merged config as ConfigBlob - try: - validated_blob = ConfigBlob.model_validate(merged_config) - except ValidationError as e: - validation_errors = e.errors() - logger.warning( - f"[ConfigVersionCrud.create_or_raise] Validation failed | " - f"{{'config_id': '{self.config_id}', 'error_count': {len(validation_errors)}, " - f"'fields': {['.'.join(str(part) for part in err['loc']) for err in validation_errors]}}}" - ) - raise HTTPException(status_code=400, detail=validation_errors) - - validate_blob_model_or_raise(self.session, validated_blob) + validated_blob = validate_config_blob_for_tag( + self.session, self.tag, merged_config + ) try: next_version = self._get_next_version(self.config_id) diff --git a/backend/app/crud/model_config.py b/backend/app/crud/model_config.py index 2cc2b3ef8..65542feb8 100644 --- a/backend/app/crud/model_config.py +++ b/backend/app/crud/model_config.py @@ -3,12 +3,16 @@ from typing import Any, Literal, get_args from fastapi import HTTPException +from pydantic import JsonValue, ValidationError from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select from app.models import ModelConfig -from app.models.llm.constants import CompletionType, Provider as ProviderEnum -from app.models.llm.request import ConfigBlob +from app.models.config.assessment_blob import AssessmentConfigBlob +from app.models.config.config import ConfigTag +from app.models.llm.constants import CompletionType +from app.models.llm.constants import Provider as ProviderEnum +from app.models.llm.request import CompletionConfig, ConfigBlob from app.models.model_config import ( ModelConfigBulkUpdateItem, ModelConfigCreate, @@ -126,7 +130,43 @@ def is_model_supported( def validate_blob_model_or_raise(session: Session, blob: ConfigBlob) -> None: - """Reject ConfigBlob whose completion.params.model is not in model_config. + """Reject ConfigBlob whose completion.params.model is not in model_config.""" + _validate_completion_model_or_raise(session, blob.completion) + + +def validate_config_blob_for_tag( + session: Session, tag: ConfigTag, raw_blob: dict[str, JsonValue] +) -> ConfigBlob | AssessmentConfigBlob: + """Validate a raw (or merged partial) blob against the type its tag dictates. + + DEFAULT → ConfigBlob, ASSESSMENT → AssessmentConfigBlob. Shape errors surface + as 422; the model-existence check stays liberal (warn, don't raise). + """ + blob_type: type[ConfigBlob] | type[AssessmentConfigBlob] = ( + AssessmentConfigBlob if tag == ConfigTag.ASSESSMENT else ConfigBlob + ) + try: + blob = blob_type.model_validate(raw_blob) + except ValidationError as e: + raise HTTPException(status_code=422, detail=e.errors()) from e + validate_blob_completion_models(session, blob) + return blob + + +def validate_blob_completion_models( + session: Session, blob: ConfigBlob | AssessmentConfigBlob +) -> None: + """Run the model-existence check on an already-parsed blob (create path).""" + completion = ( + blob.assessment if isinstance(blob, AssessmentConfigBlob) else blob.completion + ) + _validate_completion_model_or_raise(session, completion) + + +def _validate_completion_model_or_raise( + session: Session, completion: CompletionConfig +) -> None: + """Reject a completion whose params.model is not in model_config. model_config is the source of truth — all providers/types validated. Native configs are exempt (they forward raw params to the provider). @@ -134,7 +174,6 @@ def validate_blob_model_or_raise(session: Session, blob: ConfigBlob) -> None: # As of now - this whole validation is liberal # change this if we want to be more strict about unsupported models/providers or missing model configs. """ - completion = blob.completion raw_provider = completion.provider completion_type = completion.type @@ -165,7 +204,7 @@ def validate_blob_model_or_raise(session: Session, blob: ConfigBlob) -> None: ) if model_row is None: logger.warning( - f"[validate_blob_model_or_raise] Model '{model_name}' not found for provider='{provider}'." + f"[_validate_completion_model_or_raise] Model '{model_name}' not found for provider='{provider}'." "Kaapi does not yet support this model, but will forward as long as the `model` field has no typos and the model is not deprecated by the provider" ) @@ -181,7 +220,7 @@ def validate_blob_model_or_raise(session: Session, blob: ConfigBlob) -> None: ) if voice and allowed_voices and voice not in allowed_voices: logger.warning( - f"[validate_blob_model_or_raise] Voice '{voice}' is not supported for provider='{provider}' " + f"[_validate_completion_model_or_raise] Voice '{voice}' is not supported for provider='{provider}' " f"model='{model_name}'. Allowed: {allowed_voices}." ) diff --git a/backend/app/models/config/assessment_blob.py b/backend/app/models/config/assessment_blob.py new file mode 100644 index 000000000..879ce9a86 --- /dev/null +++ b/backend/app/models/config/assessment_blob.py @@ -0,0 +1,82 @@ +from pydantic import JsonValue, field_validator, model_validator +from sqlmodel import Field, SQLModel +from typing import Literal + +from app.models.llm.constants import TextProvider +from app.models.llm.request import KaapiCompletionConfig, CompletionType, TextLLMParams + +# json_schema is validated shallowly at config time: it must be a non-empty +# object-typed dict. Provider strict-mode normalisation is a run-mode concern. +JSON_SCHEMA_OBJECT_TYPE = "object" + + +class TopicRelevanceFilter(SQLModel): + """Pre-filter that scores each item's relevance to the assessment topic.""" + + prompt: str = Field( + ..., + description=( + "Relevance-scoring prompt. May embed {{col}} placeholders that the " + "run mode substitutes per dataset row (batch) or pre-fills (response)." + ), + ) + + +class DuplicateDetectionFilter(SQLModel): + """Pre-filter that flags items duplicating prior corpus content.""" + + content: str | None = Field( + default=None, + description=( + "Duplicate-comparison template. May embed {{col}} placeholders " + "resolved by the run mode." + ), + ) + knowledge_base_id: str | None = Field( + default=None, + description="Vector store to compare against; defaults to the platform corpus when unset.", + ) + + +class AssessmentPreFilters(SQLModel): + """Optional pre-filters applied before the main assessment call.""" + + topic_relevance: TopicRelevanceFilter | None = None + duplicate_detection: DuplicateDetectionFilter | None = None + + +class AssessmentTextParams(TextLLMParams): + """Text params + structured-output schema, scoped to assessment.""" + + json_schema: dict[str, JsonValue] | None = Field( + default=None, + description="Object-typed JSON schema for structured output. Omit for free-form text.", + ) + + +class AssessmentCompletionConfig(KaapiCompletionConfig): + provider: TextProvider = Field( + ..., description="Provider to use for the assessment completion call." + ) + type: Literal[CompletionType.TEXT] = CompletionType.TEXT + + @model_validator(mode="after") + def validate_params(self): # overrides KaapiCompletionConfig.validate_params + user_set_temp = "temperature" in self.params + validated = AssessmentTextParams.model_validate(self.params) + self.params = validated.model_dump(exclude_none=True) + if not user_set_temp: + self.params.pop("temperature", None) + return self + + +class AssessmentConfigBlob(SQLModel): + """config_blob shape for a config tagged ASSESSMENT. + + De-associated from ConfigBlob: no inference_mode (the run mode is chosen by + the caller, not the config), no columns/attachment_columns (mode-agnosticism + comes from {{col}} prompt interpolation), and no post_processing. + """ + + pre_filters: AssessmentPreFilters | None = None + assessment: AssessmentCompletionConfig diff --git a/backend/app/models/config/config.py b/backend/app/models/config/config.py index 8ee56cdb4..5027497f9 100644 --- a/backend/app/models/config/config.py +++ b/backend/app/models/config/config.py @@ -3,13 +3,14 @@ from uuid import UUID, uuid4 import sqlalchemy as sa -from pydantic import field_validator +from pydantic import model_validator from sqlalchemy.dialects import postgresql from sqlmodel import Field, Index, SQLModel, text from app.core.util import now from app.models.llm.request import ConfigBlob +from .assessment_blob import AssessmentConfigBlob from .version import ConfigVersionPublic @@ -121,8 +122,10 @@ class Config(ConfigBase, table=True): class ConfigCreate(ConfigBase): """Create new configuration""" - # Initial version data - config_blob: ConfigBlob = Field(description="Provider-specific parameters") + # Shape picked by `tag`; `_check_blob_matches_tag` enforces the pairing. + config_blob: ConfigBlob | AssessmentConfigBlob = Field( + description="Provider-specific parameters; shape must match `tag`" + ) commit_message: str | None = Field( default=None, max_length=512, @@ -136,11 +139,16 @@ class ConfigCreate(ConfigBase): ), ) - @field_validator("config_blob") - def validate_blob_not_empty(cls, value): - if not value: - raise ValueError("config_blob cannot be empty") - return value + @model_validator(mode="after") + def _check_blob_matches_tag(self) -> "ConfigCreate": + expected = ( + AssessmentConfigBlob if self.tag == ConfigTag.ASSESSMENT else ConfigBlob + ) + if not isinstance(self.config_blob, expected): + raise ValueError( + f"config_blob shape does not match tag '{self.tag.value}'" + ) + return self class ConfigUpdate(SQLModel): diff --git a/backend/app/models/llm/constants.py b/backend/app/models/llm/constants.py index 31ac8352a..7cb1ea721 100644 --- a/backend/app/models/llm/constants.py +++ b/backend/app/models/llm/constants.py @@ -38,6 +38,8 @@ class Provider(StrEnum): Provider.GOOGLE_AISTUDIO, ] +TextProvider = Literal[Provider.OPENAI, Provider.GOOGLE, Provider.ANTHROPIC] + # Native provider names are the Kaapi providers with a "-native" suffix. # Kept as explicit strings since there's no corresponding enum member. NativeProvider = Literal[