From 42d5bb364ac9c37ed21d31267fbdfdf644acb935 Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:54:13 +0530 Subject: [PATCH 1/2] feat(assessment): add assessment config handling and validation --- backend/app/api/docs/config/delete.md | 4 + backend/app/api/docs/config/get.md | 4 + backend/app/api/routes/config/config.py | 18 +++- backend/app/core/config.py | 6 +- backend/app/crud/config/config.py | 12 ++- backend/app/crud/config/version.py | 29 ++--- backend/app/crud/model_config.py | 53 ++++++++-- backend/app/models/config/assessment_blob.py | 106 +++++++++++++++++++ backend/app/models/config/config.py | 24 +++-- 9 files changed, 213 insertions(+), 43 deletions(-) create mode 100644 backend/app/models/config/assessment_blob.py diff --git a/backend/app/api/docs/config/delete.md b/backend/app/api/docs/config/delete.md index 93303c2e0..091cfc33a 100644 --- a/backend/app/api/docs/config/delete.md +++ b/backend/app/api/docs/config/delete.md @@ -3,3 +3,7 @@ Delete a configuration and all its versions. This operation performs a delete, marking the configuration and all associated versions as deleted in the database while retaining records for audit purposes. + +The lookup is scoped by `tag`. It defaults to `default`; pass +`ASSESSMENT` to delete an assessment config. A config that exists under a +different tag is reported as not found. diff --git a/backend/app/api/docs/config/get.md b/backend/app/api/docs/config/get.md index 3421c3be9..035b563b4 100644 --- a/backend/app/api/docs/config/get.md +++ b/backend/app/api/docs/config/get.md @@ -3,3 +3,7 @@ Retrieve a specific configuration by its ID. Returns the configuration metadata including name, description, and timestamps. This endpoint provides configuration-level details but does not include version information. + +The lookup is scoped by `tag`. It defaults to `default`; pass +`ASSESSMENT` to fetch an assessment config. A config that exists under a +different tag is reported as not found. diff --git a/backend/app/api/routes/config/config.py b/backend/app/api/routes/config/config.py index 020a91d16..46227ef8f 100644 --- a/backend/app/api/routes/config/config.py +++ b/backend/app/api/routes/config/config.py @@ -90,12 +90,19 @@ def get_config( config_id: UUID, current_user: AuthContextDep, session: SessionDep, + tag: ConfigTag = Query( + ConfigTag.DEFAULT, + description=( + "Config scope. Use 'default' for general configs or 'ASSESSMENT' " + "for assessment configs. Supported values: 'default', 'ASSESSMENT'." + ), + ), ): """ Get a specific configuration by its ID. """ config_crud = ConfigCrud(session=session, project_id=current_user.project_.id) - config = config_crud.exists_or_raise(config_id=config_id) + config = config_crud.exists_in_tag_scope_or_raise(config_id=config_id, tag=tag) return APIResponse.success_response( data=config, ) @@ -138,12 +145,19 @@ def delete_config( config_id: UUID, current_user: AuthContextDep, session: SessionDep, + tag: ConfigTag = Query( + ConfigTag.DEFAULT, + description=( + "Config scope. Use 'default' for general configs or 'ASSESSMENT' " + "for assessment configs. Supported values: 'default', 'ASSESSMENT'." + ), + ), ): """ Delete a specific configuration. """ config_crud = ConfigCrud(session=session, project_id=current_user.project_.id) - config_crud.delete_or_raise(config_id=config_id) + config_crud.delete_or_raise(config_id=config_id, tag=tag) return APIResponse.success_response( data=Message(message="Config deleted successfully"), diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 3bc408eae..c285fea51 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -40,9 +40,9 @@ class Settings(BaseSettings): ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 1 # 60 minutes * 24 hours * 7 days = 7 days REFRESH_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 7 - ENVIRONMENT: Literal[ - "development", "testing", "staging", "production" - ] = "development" + ENVIRONMENT: Literal["development", "testing", "staging", "production"] = ( + "development" + ) PROJECT_NAME: str API_VERSION: str = "0.5.0" diff --git a/backend/app/crud/config/config.py b/backend/app/crud/config/config.py index fdfd055f0..0b3950af0 100644 --- a/backend/app/crud/config/config.py +++ b/backend/app/crud/config/config.py @@ -5,7 +5,7 @@ from sqlmodel import Session, and_, select from app.core.util import now -from app.crud.model_config import validate_blob_model_or_raise +from app.crud.model_config import validate_blob_completion_models from app.models import ( Config, ConfigCreate, @@ -34,7 +34,9 @@ def create_or_raise( """ self._check_unique_name_or_raise(config_create.name) - validate_blob_model_or_raise(self.session, config_create.config_blob) + # config_create.config_blob is already parsed to the tag-matching model at the + # request boundary; only the model-existence check remains. + validate_blob_completion_models(self.session, config_create.config_blob) try: config = Config( @@ -144,8 +146,10 @@ def update_or_raise(self, config_id: UUID, config_update: ConfigUpdate) -> Confi ) return config - def delete_or_raise(self, config_id: UUID) -> None: - config = self.exists_or_raise(config_id) + def delete_or_raise( + self, config_id: UUID, tag: ConfigTag = ConfigTag.DEFAULT + ) -> None: + config = self.exists_in_tag_scope_or_raise(config_id, tag) config.deleted_at = now() self.session.add(config) diff --git a/backend/app/crud/config/version.py b/backend/app/crud/config/version.py index a0a312c93..56a0fcd70 100644 --- a/backend/app/crud/config/version.py +++ b/backend/app/crud/config/version.py @@ -3,12 +3,11 @@ from uuid import UUID from fastapi import HTTPException -from pydantic import ValidationError from sqlalchemy.orm import defer from sqlmodel import Session, and_, select from app.core.util import now -from app.crud.model_config import is_reasoning_model, validate_blob_model_or_raise +from app.crud.model_config import is_reasoning_model, validate_config_blob_for_tag from app.models import ( Config, ConfigVersion, @@ -17,7 +16,6 @@ ConfigVersionUpdate, ) from app.models.config.config import ConfigTag -from app.models.llm.request import ConfigBlob from .config import ConfigCrud @@ -67,24 +65,15 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: updates=version_create.config_blob, ) - self._strip_unsupported_params(merged_config) + # These operate on a top-level `completion` block, which assessment + # blobs don't have — skip them for the ASSESSMENT tag. + if self.tag != ConfigTag.ASSESSMENT: + self._strip_unsupported_params(merged_config) + self._validate_immutable_fields(latest_version.config_blob, merged_config) - # Validate that provider and type haven't been changed - self._validate_immutable_fields(latest_version.config_blob, merged_config) - - # Validate the merged config as ConfigBlob - try: - validated_blob = ConfigBlob.model_validate(merged_config) - except ValidationError as e: - validation_errors = e.errors() - logger.warning( - f"[ConfigVersionCrud.create_or_raise] Validation failed | " - f"{{'config_id': '{self.config_id}', 'error_count': {len(validation_errors)}, " - f"'fields': {['.'.join(str(part) for part in err['loc']) for err in validation_errors]}}}" - ) - raise HTTPException(status_code=400, detail=validation_errors) - - validate_blob_model_or_raise(self.session, validated_blob) + validated_blob = validate_config_blob_for_tag( + self.session, self.tag, merged_config + ) try: next_version = self._get_next_version(self.config_id) diff --git a/backend/app/crud/model_config.py b/backend/app/crud/model_config.py index 2cc2b3ef8..eaec6f576 100644 --- a/backend/app/crud/model_config.py +++ b/backend/app/crud/model_config.py @@ -3,12 +3,16 @@ from typing import Any, Literal, get_args from fastapi import HTTPException +from pydantic import JsonValue, ValidationError from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select from app.models import ModelConfig -from app.models.llm.constants import CompletionType, Provider as ProviderEnum -from app.models.llm.request import ConfigBlob +from app.models.config.assessment_blob import AssessmentConfigBlob +from app.models.config.config import ConfigTag +from app.models.llm.constants import CompletionType +from app.models.llm.constants import Provider as ProviderEnum +from app.models.llm.request import CompletionConfig, ConfigBlob from app.models.model_config import ( ModelConfigBulkUpdateItem, ModelConfigCreate, @@ -126,7 +130,45 @@ def is_model_supported( def validate_blob_model_or_raise(session: Session, blob: ConfigBlob) -> None: - """Reject ConfigBlob whose completion.params.model is not in model_config. + """Reject ConfigBlob whose completion.params.model is not in model_config.""" + _validate_completion_model_or_raise(session, blob.completion) + + +def validate_config_blob_for_tag( + session: Session, tag: ConfigTag, raw_blob: dict[str, JsonValue] +) -> ConfigBlob | AssessmentConfigBlob: + """Validate a raw (or merged partial) blob against the type its tag dictates. + + DEFAULT → ConfigBlob, ASSESSMENT → AssessmentConfigBlob. Shape errors surface + as 422; the model-existence check stays liberal (warn, don't raise). + """ + blob_type: type[ConfigBlob] | type[AssessmentConfigBlob] = ( + AssessmentConfigBlob if tag == ConfigTag.ASSESSMENT else ConfigBlob + ) + try: + blob = blob_type.model_validate(raw_blob) + except ValidationError as e: + raise HTTPException(status_code=422, detail=e.errors()) from e + validate_blob_completion_models(session, blob) + return blob + + +def validate_blob_completion_models( + session: Session, blob: ConfigBlob | AssessmentConfigBlob +) -> None: + """Run the model-existence check on an already-parsed blob (create path).""" + completion = ( + blob.assessment.model + if isinstance(blob, AssessmentConfigBlob) + else blob.completion + ) + _validate_completion_model_or_raise(session, completion) + + +def _validate_completion_model_or_raise( + session: Session, completion: CompletionConfig +) -> None: + """Reject a completion whose params.model is not in model_config. model_config is the source of truth — all providers/types validated. Native configs are exempt (they forward raw params to the provider). @@ -134,7 +176,6 @@ def validate_blob_model_or_raise(session: Session, blob: ConfigBlob) -> None: # As of now - this whole validation is liberal # change this if we want to be more strict about unsupported models/providers or missing model configs. """ - completion = blob.completion raw_provider = completion.provider completion_type = completion.type @@ -165,7 +206,7 @@ def validate_blob_model_or_raise(session: Session, blob: ConfigBlob) -> None: ) if model_row is None: logger.warning( - f"[validate_blob_model_or_raise] Model '{model_name}' not found for provider='{provider}'." + f"[_validate_completion_model_or_raise] Model '{model_name}' not found for provider='{provider}'." "Kaapi does not yet support this model, but will forward as long as the `model` field has no typos and the model is not deprecated by the provider" ) @@ -181,7 +222,7 @@ def validate_blob_model_or_raise(session: Session, blob: ConfigBlob) -> None: ) if voice and allowed_voices and voice not in allowed_voices: logger.warning( - f"[validate_blob_model_or_raise] Voice '{voice}' is not supported for provider='{provider}' " + f"[_validate_completion_model_or_raise] Voice '{voice}' is not supported for provider='{provider}' " f"model='{model_name}'. Allowed: {allowed_voices}." ) diff --git a/backend/app/models/config/assessment_blob.py b/backend/app/models/config/assessment_blob.py new file mode 100644 index 000000000..409b8f341 --- /dev/null +++ b/backend/app/models/config/assessment_blob.py @@ -0,0 +1,106 @@ +from pydantic import JsonValue, field_validator, model_validator +from sqlmodel import Field, SQLModel + +from app.models.llm.constants import CompletionType +from app.models.llm.request import CompletionConfig + +# json_schema is validated shallowly at config time: it must be a non-empty +# object-typed dict. Provider strict-mode normalisation is a run-mode concern. +JSON_SCHEMA_OBJECT_TYPE = "object" + + +class TopicRelevanceFilter(SQLModel): + """Pre-filter that scores each item's relevance to the assessment topic.""" + + prompt: str = Field( + ..., + description=( + "Relevance-scoring prompt. May embed {{col}} placeholders that the " + "run mode substitutes per dataset row (batch) or pre-fills (response)." + ), + ) + + +class DuplicateDetectionFilter(SQLModel): + """Pre-filter that flags items duplicating prior corpus content.""" + + content: str | None = Field( + default=None, + description=( + "Duplicate-comparison template. May embed {{col}} placeholders " + "resolved by the run mode." + ), + ) + knowledge_base_id: str | None = Field( + default=None, + description="Vector store to compare against; defaults to the platform corpus when unset.", + ) + + +class AssessmentPreFilters(SQLModel): + """Optional pre-filters applied before the main assessment call.""" + + topic_relevance: TopicRelevanceFilter | None = None + duplicate_detection: DuplicateDetectionFilter | None = None + + +class AssessmentBlock(SQLModel): + """The core assessment call: system prompt, output schema, and model.""" + + system_prompt: str = Field( + ..., + description=( + "System prompt for the assessment. May embed {{col}} placeholders " + "resolved by the run mode." + ), + ) + json_schema: dict[str, JsonValue] = Field( + ..., + description="Object-typed JSON schema describing the structured assessment output.", + ) + model: CompletionConfig = Field( + ..., description="Shared LLM completion config used to run the assessment." + ) + + @model_validator(mode="before") + @classmethod + def _lift_flat_model_config(cls, data: object) -> object: + """Accept the contract's flat model shape and lift it into a CompletionConfig. + + The contract sends `{provider, model, temperature, max_output_tokens, ...}`; + the stored/validated type is the shared CompletionConfig union, whose Kaapi + variant nests those params under `params`. A payload that already carries + `type`/`params` is passed through untouched. + """ + if not isinstance(data, dict): + return data + model = data.get("model") + if not isinstance(model, dict) or "type" in model or "params" in model: + return data + data["model"] = { + "provider": model.get("provider"), + "type": CompletionType.TEXT.value, + "params": {k: v for k, v in model.items() if k != "provider"}, + } + return data + + @field_validator("json_schema") + @classmethod + def _validate_json_schema(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]: + if not value: + raise ValueError("json_schema must be a non-empty object") + if value.get("type") != JSON_SCHEMA_OBJECT_TYPE: + raise ValueError(f"json_schema.type must be '{JSON_SCHEMA_OBJECT_TYPE}'") + return value + + +class AssessmentConfigBlob(SQLModel): + """config_blob shape for a config tagged ASSESSMENT. + + De-associated from ConfigBlob: no inference_mode (the run mode is chosen by + the caller, not the config), no columns/attachment_columns (mode-agnosticism + comes from {{col}} prompt interpolation), and no post_processing. + """ + + pre_filters: AssessmentPreFilters | None = None + assessment: AssessmentBlock diff --git a/backend/app/models/config/config.py b/backend/app/models/config/config.py index 8ee56cdb4..5027497f9 100644 --- a/backend/app/models/config/config.py +++ b/backend/app/models/config/config.py @@ -3,13 +3,14 @@ from uuid import UUID, uuid4 import sqlalchemy as sa -from pydantic import field_validator +from pydantic import model_validator from sqlalchemy.dialects import postgresql from sqlmodel import Field, Index, SQLModel, text from app.core.util import now from app.models.llm.request import ConfigBlob +from .assessment_blob import AssessmentConfigBlob from .version import ConfigVersionPublic @@ -121,8 +122,10 @@ class Config(ConfigBase, table=True): class ConfigCreate(ConfigBase): """Create new configuration""" - # Initial version data - config_blob: ConfigBlob = Field(description="Provider-specific parameters") + # Shape picked by `tag`; `_check_blob_matches_tag` enforces the pairing. + config_blob: ConfigBlob | AssessmentConfigBlob = Field( + description="Provider-specific parameters; shape must match `tag`" + ) commit_message: str | None = Field( default=None, max_length=512, @@ -136,11 +139,16 @@ class ConfigCreate(ConfigBase): ), ) - @field_validator("config_blob") - def validate_blob_not_empty(cls, value): - if not value: - raise ValueError("config_blob cannot be empty") - return value + @model_validator(mode="after") + def _check_blob_matches_tag(self) -> "ConfigCreate": + expected = ( + AssessmentConfigBlob if self.tag == ConfigTag.ASSESSMENT else ConfigBlob + ) + if not isinstance(self.config_blob, expected): + raise ValueError( + f"config_blob shape does not match tag '{self.tag.value}'" + ) + return self class ConfigUpdate(SQLModel): From e2234db0c0256fced2995bae31d2e878e6eccf9b Mon Sep 17 00:00:00 2001 From: Prashant Vasudevan <71649489+vprashrex@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:11:49 +0530 Subject: [PATCH 2/2] feat(assessment): enhance validation and structure of assessment configuration --- backend/app/crud/config/config.py | 3 +- backend/app/crud/model_config.py | 4 +- backend/app/models/config/assessment_blob.py | 70 +++++++------------- backend/app/models/llm/constants.py | 2 + 4 files changed, 27 insertions(+), 52 deletions(-) diff --git a/backend/app/crud/config/config.py b/backend/app/crud/config/config.py index 0b3950af0..5345e96c2 100644 --- a/backend/app/crud/config/config.py +++ b/backend/app/crud/config/config.py @@ -34,8 +34,7 @@ def create_or_raise( """ self._check_unique_name_or_raise(config_create.name) - # config_create.config_blob is already parsed to the tag-matching model at the - # request boundary; only the model-existence check remains. + # validate that the completion models in the config blob are valid validate_blob_completion_models(self.session, config_create.config_blob) try: diff --git a/backend/app/crud/model_config.py b/backend/app/crud/model_config.py index eaec6f576..65542feb8 100644 --- a/backend/app/crud/model_config.py +++ b/backend/app/crud/model_config.py @@ -158,9 +158,7 @@ def validate_blob_completion_models( ) -> None: """Run the model-existence check on an already-parsed blob (create path).""" completion = ( - blob.assessment.model - if isinstance(blob, AssessmentConfigBlob) - else blob.completion + blob.assessment if isinstance(blob, AssessmentConfigBlob) else blob.completion ) _validate_completion_model_or_raise(session, completion) diff --git a/backend/app/models/config/assessment_blob.py b/backend/app/models/config/assessment_blob.py index 409b8f341..879ce9a86 100644 --- a/backend/app/models/config/assessment_blob.py +++ b/backend/app/models/config/assessment_blob.py @@ -1,8 +1,9 @@ from pydantic import JsonValue, field_validator, model_validator from sqlmodel import Field, SQLModel +from typing import Literal -from app.models.llm.constants import CompletionType -from app.models.llm.request import CompletionConfig +from app.models.llm.constants import TextProvider +from app.models.llm.request import KaapiCompletionConfig, CompletionType, TextLLMParams # json_schema is validated shallowly at config time: it must be a non-empty # object-typed dict. Provider strict-mode normalisation is a run-mode concern. @@ -44,54 +45,29 @@ class AssessmentPreFilters(SQLModel): duplicate_detection: DuplicateDetectionFilter | None = None -class AssessmentBlock(SQLModel): - """The core assessment call: system prompt, output schema, and model.""" +class AssessmentTextParams(TextLLMParams): + """Text params + structured-output schema, scoped to assessment.""" - system_prompt: str = Field( - ..., - description=( - "System prompt for the assessment. May embed {{col}} placeholders " - "resolved by the run mode." - ), - ) - json_schema: dict[str, JsonValue] = Field( - ..., - description="Object-typed JSON schema describing the structured assessment output.", + json_schema: dict[str, JsonValue] | None = Field( + default=None, + description="Object-typed JSON schema for structured output. Omit for free-form text.", ) - model: CompletionConfig = Field( - ..., description="Shared LLM completion config used to run the assessment." + + +class AssessmentCompletionConfig(KaapiCompletionConfig): + provider: TextProvider = Field( + ..., description="Provider to use for the assessment completion call." ) + type: Literal[CompletionType.TEXT] = CompletionType.TEXT - @model_validator(mode="before") - @classmethod - def _lift_flat_model_config(cls, data: object) -> object: - """Accept the contract's flat model shape and lift it into a CompletionConfig. - - The contract sends `{provider, model, temperature, max_output_tokens, ...}`; - the stored/validated type is the shared CompletionConfig union, whose Kaapi - variant nests those params under `params`. A payload that already carries - `type`/`params` is passed through untouched. - """ - if not isinstance(data, dict): - return data - model = data.get("model") - if not isinstance(model, dict) or "type" in model or "params" in model: - return data - data["model"] = { - "provider": model.get("provider"), - "type": CompletionType.TEXT.value, - "params": {k: v for k, v in model.items() if k != "provider"}, - } - return data - - @field_validator("json_schema") - @classmethod - def _validate_json_schema(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]: - if not value: - raise ValueError("json_schema must be a non-empty object") - if value.get("type") != JSON_SCHEMA_OBJECT_TYPE: - raise ValueError(f"json_schema.type must be '{JSON_SCHEMA_OBJECT_TYPE}'") - return value + @model_validator(mode="after") + def validate_params(self): # overrides KaapiCompletionConfig.validate_params + user_set_temp = "temperature" in self.params + validated = AssessmentTextParams.model_validate(self.params) + self.params = validated.model_dump(exclude_none=True) + if not user_set_temp: + self.params.pop("temperature", None) + return self class AssessmentConfigBlob(SQLModel): @@ -103,4 +79,4 @@ class AssessmentConfigBlob(SQLModel): """ pre_filters: AssessmentPreFilters | None = None - assessment: AssessmentBlock + assessment: AssessmentCompletionConfig diff --git a/backend/app/models/llm/constants.py b/backend/app/models/llm/constants.py index 31ac8352a..7cb1ea721 100644 --- a/backend/app/models/llm/constants.py +++ b/backend/app/models/llm/constants.py @@ -38,6 +38,8 @@ class Provider(StrEnum): Provider.GOOGLE_AISTUDIO, ] +TextProvider = Literal[Provider.OPENAI, Provider.GOOGLE, Provider.ANTHROPIC] + # Native provider names are the Kaapi providers with a "-native" suffix. # Kept as explicit strings since there's no corresponding enum member. NativeProvider = Literal[