diff --git a/backend/app/api/routes/collections.py b/backend/app/api/routes/collections.py index 6134855ea..4487fcd3a 100644 --- a/backend/app/api/routes/collections.py +++ b/backend/app/api/routes/collections.py @@ -6,6 +6,7 @@ from fastapi import Path as FastPath from app.api.deps import SessionDep, AuthContextDep +from app.crud.collection.collection import CollectionNameConflictError from app.api.permissions import Permission, require_permission from app.core.telemetry import log_context from app.core.rate_monitor import monitor_rate @@ -14,7 +15,6 @@ CollectionJobCrud, DocumentCollectionCrud, ) -from app.crud.collection.collection import CollectionNameConflictError from app.core.cloud import get_cloud_storage from app.models import ( CollectionJobStatus, diff --git a/backend/app/api/routes/stt_evaluations/evaluation.py b/backend/app/api/routes/stt_evaluations/evaluation.py index 6b96f4414..c5dc34283 100644 --- a/backend/app/api/routes/stt_evaluations/evaluation.py +++ b/backend/app/api/routes/stt_evaluations/evaluation.py @@ -62,7 +62,7 @@ def start_stt_evaluation( sample_count = (dataset.dataset_metadata or {}).get("sample_count", 0) if sample_count == 0: - raise HTTPException(status_code=400, detail="Dataset has no samples") + raise HTTPException(status_code=422, detail="Dataset has no samples") # Use language_id from the dataset language_id = dataset.language_id @@ -97,7 +97,8 @@ def start_stt_evaluation( except Exception as e: logger.error( f"[start_stt_evaluation] Failed to queue batch submission | " - f"run_id: {run.id}, error: {str(e)}" + f"run_id: {run.id}, error: {str(e)}", + exc_info=True, ) update_stt_run( session=session, @@ -106,8 +107,8 @@ def start_stt_evaluation( error_message=f"Failed to queue batch submission: {str(e)}", ) raise HTTPException( - status_code=500, - detail=f"Failed to queue batch submission: {e}", + status_code=503, + detail="Could not queue the batch submission. Retry shortly.", ) return APIResponse.success_response( diff --git a/backend/app/api/routes/tts_evaluations/evaluation.py b/backend/app/api/routes/tts_evaluations/evaluation.py index 6751d8219..e27d39666 100644 --- a/backend/app/api/routes/tts_evaluations/evaluation.py +++ b/backend/app/api/routes/tts_evaluations/evaluation.py @@ -66,7 +66,7 @@ def start_tts_evaluation( sample_count = (dataset.dataset_metadata or {}).get("sample_count", 0) if sample_count == 0: - raise HTTPException(status_code=400, detail="Dataset has no samples") + raise HTTPException(status_code=422, detail="Dataset has no samples") language_id = dataset.language_id @@ -101,7 +101,8 @@ def start_tts_evaluation( except Exception as e: logger.error( f"[start_tts_evaluation] Failed to queue batch submission | " - f"run_id: {run.id}, error: {str(e)}" + f"run_id: {run.id}, error: {str(e)}", + exc_info=True, ) update_tts_run( session=session, @@ -110,8 +111,8 @@ def start_tts_evaluation( error_message=f"Failed to queue batch submission: {str(e)}", ) raise HTTPException( - status_code=500, - detail=f"Failed to queue batch submission: {e}", + status_code=503, + detail="Could not queue the batch submission. Retry shortly.", ) return APIResponse.success_response( diff --git a/backend/app/core/exception_handlers.py b/backend/app/core/exception_handlers.py index 4d1e4d747..c114e6803 100644 --- a/backend/app/core/exception_handlers.py +++ b/backend/app/core/exception_handlers.py @@ -1,6 +1,9 @@ +import logging import re from collections import defaultdict +import sentry_sdk +from asgi_correlation_id import correlation_id from fastapi import FastAPI, Request, HTTPException from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse @@ -11,6 +14,10 @@ from app.utils import APIResponse +logger = logging.getLogger(__name__) + +GENERIC_ERROR_DETAIL = "An unexpected error occurred." + _BRANCH_PATTERN = re.compile(r"^[A-Z]|[\[\]()]") @@ -102,6 +109,20 @@ async def http_exception_handler( detail = exc.detail if isinstance(detail, list): detail = _sanitize_validation_errors(detail) + + # Tag at the boundary: the status is only known here, and call-site logs + # fire before it. 5xx is a server fault (error event); 4xx is the caller's. + if sentry_sdk.get_client().is_active(): + sentry_sdk.set_tag("http.status_code", str(exc.status_code)) + sentry_sdk.set_tag( + "error.class", "server" if exc.status_code >= 500 else "client" + ) + if exc.status_code >= 500: + logger.error( + f"[http_exception] {request.method} {request.url.path} " + f"-> {exc.status_code}", + exc_info=exc, + ) return JSONResponse( status_code=exc.status_code, content=APIResponse.failure_response(detail).model_dump(), @@ -109,9 +130,13 @@ async def http_exception_handler( @app.exception_handler(Exception) async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse: + # Fixed detail: str(exc) here leaked provider bodies and DB text to callers. + logger.error( + f"[generic_error_handler] Unhandled exception | " + f"path: {request.url.path}, trace_id: {correlation_id.get() or 'N/A'}", + exc_info=True, + ) return JSONResponse( status_code=HTTP_500_INTERNAL_SERVER_ERROR, - content=APIResponse.failure_response( - str(exc) or "An unexpected error occurred." - ).model_dump(), + content=APIResponse.failure_response(GENERIC_ERROR_DETAIL).model_dump(), ) diff --git a/backend/app/core/security.py b/backend/app/core/security.py index 406f46ed8..ad1393da5 100644 --- a/backend/app/core/security.py +++ b/backend/app/core/security.py @@ -10,6 +10,7 @@ import base64 import json import logging +from fastapi import HTTPException import os import secrets from datetime import UTC, datetime, timedelta @@ -212,8 +213,12 @@ def encrypt_credentials(credentials: dict[str, Any]) -> str: return get_fernet().encrypt(credentials_str.encode()).decode() except Exception as e: # Log the real cause (may carry AWS ARNs); never surface it to callers. - logger.error(f"[encrypt_credentials] Encryption failed | error: {e}") - raise ValueError("Failed to encrypt credentials") + logger.error( + f"[encrypt_credentials] Encryption failed | error: {e}", exc_info=True + ) + raise HTTPException( + status_code=502, detail="Failed to encrypt credentials. Retry shortly." + ) def decrypt_credentials(encrypted_credentials: str) -> dict[str, Any]: @@ -232,8 +237,12 @@ def decrypt_credentials(encrypted_credentials: str) -> dict[str, Any]: return json.loads(decrypted_str) except Exception as e: # Log the real cause (may carry AWS ARNs); never surface it to callers. - logger.error(f"[decrypt_credentials] Decryption failed | error: {e}") - raise ValueError("Failed to decrypt credentials") + logger.error( + f"[decrypt_credentials] Decryption failed | error: {e}", exc_info=True + ) + raise HTTPException( + status_code=502, detail="Failed to decrypt credentials. Retry shortly." + ) class APIKeyManager: diff --git a/backend/app/crud/collection/collection.py b/backend/app/crud/collection/collection.py index 96c4e82be..6de091548 100644 --- a/backend/app/crud/collection/collection.py +++ b/backend/app/crud/collection/collection.py @@ -111,7 +111,7 @@ def read_one_if_delete(self, collection_id: UUID) -> Collection: "[CollectionCrud.read_one_if_delete] Collection already deleted | " f"{{'project_id': '{self.project_id}', 'collection_id': '{collection_id}'}}" ) - raise HTTPException(status_code=400, detail="Collection already deleted") + raise HTTPException(status_code=409, detail="Collection already deleted") return collection diff --git a/backend/app/crud/config/config.py b/backend/app/crud/config/config.py index fdfd055f0..ca65a8151 100644 --- a/backend/app/crud/config/config.py +++ b/backend/app/crud/config/config.py @@ -2,6 +2,7 @@ from uuid import UUID from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError from sqlmodel import Session, and_, select from app.core.util import now @@ -67,6 +68,21 @@ def create_or_raise( return config, version + except HTTPException: + self.session.rollback() + raise + + except IntegrityError as e: + self.session.rollback() + logger.warning( + f"[ConfigCrud.create] Duplicate configuration name | " + f"{{'name': '{config_create.name}', 'project_id': {self.project_id}, 'error': '{str(e)}'}}", + ) + raise HTTPException( + status_code=409, + detail=f"Config with name '{config_create.name}' already exists in this project.", + ) + except Exception as e: self.session.rollback() logger.error( diff --git a/backend/app/crud/config/version.py b/backend/app/crud/config/version.py index a0a312c93..45dabcfdc 100644 --- a/backend/app/crud/config/version.py +++ b/backend/app/crud/config/version.py @@ -4,6 +4,7 @@ from fastapi import HTTPException from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import defer from sqlmodel import Session, and_, select @@ -82,7 +83,7 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: f"{{'config_id': '{self.config_id}', 'error_count': {len(validation_errors)}, " f"'fields': {['.'.join(str(part) for part in err['loc']) for err in validation_errors]}}}" ) - raise HTTPException(status_code=400, detail=validation_errors) + raise HTTPException(status_code=422, detail=validation_errors) validate_blob_model_or_raise(self.session, validated_blob) @@ -101,16 +102,31 @@ def create_or_raise(self, version_create: ConfigVersionUpdate) -> ConfigVersion: self.session.refresh(version) logger.info( - f"[ConfigVersionCrud.create_from_partial] Version created successfully | " + f"[ConfigVersionCrud.create_or_raise] Version created successfully | " f"{{'config_id': '{self.config_id}', 'version_id': '{version.id}'}}" ) return version + except HTTPException: + self.session.rollback() + raise + + except IntegrityError as e: + self.session.rollback() + logger.warning( + f"[ConfigVersionCrud.create_or_raise] Version conflict | " + f"{{'config_id': '{self.config_id}', 'error': '{str(e)}'}}", + ) + raise HTTPException( + status_code=409, + detail="A version with this number already exists for this config.", + ) + except Exception as e: self.session.rollback() logger.error( - f"[ConfigVersionCrud.create_from_partial] Failed to create version | " + f"[ConfigVersionCrud.create_or_raise] Failed to create version | " f"{{'config_id': '{self.config_id}', 'error': '{str(e)}'}}", exc_info=True, ) diff --git a/backend/app/crud/credentials.py b/backend/app/crud/credentials.py index cc070852b..38c106601 100644 --- a/backend/app/crud/credentials.py +++ b/backend/app/crud/credentials.py @@ -5,7 +5,8 @@ from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select -from app.core.exception_handlers import HTTPException +from fastapi import HTTPException + from app.core.providers import validate_provider, validate_provider_credentials from app.core.security import decrypt_credentials, encrypt_credentials from app.core.util import now @@ -54,21 +55,23 @@ def set_creds_for_org( created_credentials.append(credential) except IntegrityError as e: session.rollback() - logger.error( + is_duplicate = ( + "uq_credential_org_project_provider" in str(e) + or "unique constraint" in str(e).lower() + ) + log = logger.warning if is_duplicate else logger.error + log( f"[set_creds_for_org] Integrity error while adding credentials | organization_id {organization_id}, project_id {project_id}, provider {provider}: {str(e)}", exc_info=True, ) - # Check if it's a duplicate constraint violation - if ( - "uq_credential_org_project_provider" in str(e) - or "unique constraint" in str(e).lower() - ): + if is_duplicate: raise HTTPException( - status_code=400, + status_code=409, detail=f"Credentials for provider '{provider}' already exist for this organization and project combination", ) - raise ValueError( - f"Error while adding credentials for provider {provider}: {str(e)}" + raise HTTPException( + status_code=500, + detail=f"Could not store credentials for provider '{provider}'. Contact Kaapi if it persists.", ) logger.info( f"[set_creds_for_org] Successfully created credentials | organization_id {organization_id}, project_id {project_id}" @@ -187,7 +190,9 @@ def update_creds_for_org( ) -> list[Credential]: """Updates credentials for a specific provider of an organization.""" if not creds_in.provider or not creds_in.credential: - raise ValueError("Provider and credential must be provided") + raise HTTPException( + status_code=400, detail="Provider and credential must be provided" + ) # Auto-unwrap nested format: {"google": {"api_key": "..."}} -> {"api_key": "..."} # so the same payload shape works for both create and update. @@ -280,12 +285,11 @@ def remove_provider_credential( rows_deleted = result.rowcount if rows_deleted == 0: session.rollback() - logger.error( - f"[remove_provider_credential] Failed to delete credential | organization_id {org_id}, provider {provider}, project_id {project_id}" + logger.warning( + f"[remove_provider_credential] Credential not found | organization_id {org_id}, provider {provider}, project_id {project_id}" ) raise HTTPException( - status_code=500, - detail="Failed to delete provider credential", + status_code=404, detail=f"No credentials found for provider '{provider}'." ) session.commit() logger.info( @@ -318,6 +322,16 @@ def remove_creds_for_org(*, session: Session, org_id: int, project_id: int) -> N rows_deleted = result.rowcount + if rows_deleted == 0: + session.rollback() + logger.warning( + f"[remove_creds_for_org] No credentials to delete | organization_id {org_id}, project_id {project_id}" + ) + raise HTTPException( + status_code=404, + detail="No credentials found for this organization and project.", + ) + if rows_deleted < expected_count: logger.error( f"[remove_creds_for_org] Failed to delete all credentials | organization_id {org_id}, project_id {project_id}, expected {expected_count}, deleted {rows_deleted}" diff --git a/backend/app/crud/document/doc_transformation_job.py b/backend/app/crud/document/doc_transformation_job.py index 3e329df70..35b29cc1e 100644 --- a/backend/app/crud/document/doc_transformation_job.py +++ b/backend/app/crud/document/doc_transformation_job.py @@ -2,6 +2,7 @@ from uuid import UUID from typing import List, Optional +from fastapi import HTTPException from sqlmodel import Session, select, and_ from app.crud import DocumentCrud @@ -13,7 +14,6 @@ ) from app.models.document import Document from app.core.util import now -from app.core.exception_handlers import HTTPException logger = logging.getLogger(__name__) diff --git a/backend/app/crud/document/document.py b/backend/app/crud/document/document.py index ef2cddb91..7634d0a67 100644 --- a/backend/app/crud/document/document.py +++ b/backend/app/crud/document/document.py @@ -1,11 +1,11 @@ import logging from uuid import UUID +from fastapi import HTTPException from sqlmodel import Session, select, and_ from app.models import Document from app.core.util import now -from app.core.exception_handlers import HTTPException logger = logging.getLogger(__name__) diff --git a/backend/app/crud/evaluations/batch.py b/backend/app/crud/evaluations/batch.py index 7b8892644..a1c659139 100644 --- a/backend/app/crud/evaluations/batch.py +++ b/backend/app/crud/evaluations/batch.py @@ -8,9 +8,12 @@ """ import logging +from fastapi import HTTPException from typing import Any from langfuse import Langfuse +from langfuse.api import NotFoundError as LangfuseNotFoundError +from langfuse.api.core import ApiError as LangfuseApiError from sqlmodel import Session from app.core.batch import ( @@ -44,18 +47,46 @@ def fetch_dataset_items(langfuse: Langfuse, dataset_name: str) -> list[dict[str, List of dataset items with input and expected_output Raises: - ValueError: If dataset not found or empty + HTTPException: 404 if the dataset is missing, 502 if Langfuse errored + or was unreachable, 422 if the dataset is empty. """ try: dataset = langfuse.get_dataset(dataset_name) - except Exception as e: + except LangfuseNotFoundError as e: logger.warning( - f"[fetch_dataset_items] Failed to fetch dataset | dataset={dataset_name} | {e}" + f"[fetch_dataset_items] Dataset not found | dataset={dataset_name} | {e}" + ) + raise HTTPException( + status_code=404, detail=f"Dataset '{dataset_name}' not found in Langfuse." + ) + except LangfuseApiError as e: + logger.error( + f"[fetch_dataset_items] Langfuse rejected the request | " + f"dataset={dataset_name}, code={getattr(e, 'status_code', 'unknown')}", + exc_info=True, + ) + raise HTTPException( + status_code=502, + detail=f"Langfuse could not return dataset '{dataset_name}' " + f"(code: {getattr(e, 'status_code', 'unknown')}). Retry shortly.", + ) + except Exception: + logger.error( + f"[fetch_dataset_items] Failed to reach Langfuse | dataset={dataset_name}", + exc_info=True, + ) + raise HTTPException( + status_code=502, + detail=f"Could not reach Langfuse to fetch dataset '{dataset_name}'. Retry shortly.", ) - raise ValueError(f"Dataset '{dataset_name}' not found: {e}") if not dataset.items: - raise ValueError(f"Dataset '{dataset_name}' is empty") + logger.warning( + f"[fetch_dataset_items] Dataset is empty | dataset={dataset_name}" + ) + raise HTTPException( + status_code=422, detail=f"Dataset '{dataset_name}' is empty." + ) items = [] for item in dataset.items: @@ -200,8 +231,9 @@ def start_evaluation_batch( dataset_items=dataset_items, openai_params=mapped_params ) if not jsonl_data: - raise ValueError( - "Evaluation dataset did not produce any JSONL entries (missing questions?)." + raise HTTPException( + status_code=422, + detail="Evaluation dataset did not produce any JSONL entries (missing questions?).", ) openai_client = get_openai_client( @@ -240,8 +272,9 @@ def start_evaluation_batch( dataset_items=dataset_items, google_params=mapped_params ) if not jsonl_data: - raise ValueError( - "Evaluation dataset did not produce any JSONL entries (missing questions?)." + raise HTTPException( + status_code=422, + detail="Evaluation dataset did not produce any JSONL entries (missing questions?).", ) gemini_client = GeminiClient.from_credentials( @@ -271,7 +304,10 @@ def start_evaluation_batch( ) else: - raise ValueError(f"Unsupported provider for evaluation batches: {provider}") + raise HTTPException( + status_code=400, + detail=f"Unsupported provider for evaluation batches: {provider}", + ) eval_run.batch_job_id = batch_job.id eval_run.status = "processing" diff --git a/backend/app/crud/evaluations/core.py b/backend/app/crud/evaluations/core.py index ed15f720e..864d80a7c 100644 --- a/backend/app/crud/evaluations/core.py +++ b/backend/app/crud/evaluations/core.py @@ -103,7 +103,10 @@ def create_evaluation_run( session.refresh(eval_run) except Exception as e: session.rollback() - logger.error(f"Failed to create EvaluationRun: {e}", exc_info=True) + logger.error( + f"[create_evaluation_run] Failed to create EvaluationRun: {e}", + exc_info=True, + ) raise logger.info( @@ -146,7 +149,7 @@ def list_evaluation_runs( runs = session.exec(statement).all() logger.info( - f"Found {len(runs)} evaluation runs for org_id={organization_id}, " + f"[list_evaluation_runs] Found {len(runs)} evaluation runs for org_id={organization_id}, " f"project_id={project_id}" ) @@ -183,12 +186,12 @@ def get_evaluation_run_by_id( if eval_run: logger.info( - f"Found evaluation run {evaluation_id}: status={eval_run.status}, " + f"[get_evaluation_run_by_id] Found evaluation run {evaluation_id}: status={eval_run.status}, " f"batch_job_id={eval_run.batch_job_id}" ) else: logger.warning( - f"Evaluation run {evaluation_id} not found or not accessible " + f"[get_evaluation_run_by_id] Evaluation run {evaluation_id} not found or not accessible " f"for org_id={organization_id}, project_id={project_id}" ) @@ -236,7 +239,10 @@ def update_evaluation_run( session.refresh(eval_run) except Exception as e: session.rollback() - logger.error(f"Failed to update EvaluationRun: {e}", exc_info=True) + logger.error( + f"[update_evaluation_run] Failed to update EvaluationRun: {e}", + exc_info=True, + ) raise if should_notify: diff --git a/backend/app/crud/evaluations/cron_utils.py b/backend/app/crud/evaluations/cron_utils.py index 6cfc06a0d..52758db77 100644 --- a/backend/app/crud/evaluations/cron_utils.py +++ b/backend/app/crud/evaluations/cron_utils.py @@ -324,7 +324,8 @@ async def poll_all_pending_evaluations_by_type( except Exception as client_err: logger.error( f"[{fn_name}] Failed to get Gemini client | " - f"org_id={org_id} | project_id={project_id} | error={client_err}" + f"org_id={org_id} | project_id={project_id} | error={client_err}", + exc_info=True, ) for run in project_runs: update_run_fn( diff --git a/backend/app/crud/evaluations/embeddings.py b/backend/app/crud/evaluations/embeddings.py index 5d1458271..552235ab1 100644 --- a/backend/app/crud/evaluations/embeddings.py +++ b/backend/app/crud/evaluations/embeddings.py @@ -102,13 +102,15 @@ def build_embedding_jsonl( ground_truth = result.get("ground_truth", "") if not item_id: - logger.warning("Skipping result with no item_id") + logger.warning("[build_embedding_jsonl] Skipping result with no item_id") continue # Get trace_id from mapping trace_id = trace_id_mapping.get(item_id) if not trace_id: - logger.warning(f"Skipping item {item_id} - no trace_id found") + logger.warning( + f"[build_embedding_jsonl] Skipping item {item_id} - no trace_id found" + ) skipped.append( {"item_id": item_id, "trace_id": None, "reason": "missing_trace_id"} ) @@ -117,7 +119,9 @@ def build_embedding_jsonl( # Empty output/ground_truth can't be embedded; record the reason. if not generated_output or not ground_truth: reason = "empty_output" if not generated_output else "empty_ground_truth" - logger.warning(f"Skipping item {item_id} - {reason}") + logger.warning( + f"[build_embedding_jsonl] Skipping item {item_id} - {reason}" + ) skipped.append({"item_id": item_id, "trace_id": trace_id, "reason": reason}) continue @@ -140,7 +144,7 @@ def build_embedding_jsonl( jsonl_data.append(batch_request) logger.info( - f"Built {len(jsonl_data)} embedding JSONL lines | skipped={len(skipped)}" + f"[build_embedding_jsonl] Built {len(jsonl_data)} embedding JSONL lines | skipped={len(skipped)}" ) return jsonl_data, skipped @@ -160,7 +164,9 @@ def parse_embedding_results( - failed_trace_ids: trace_ids whose embedding could not be parsed (flagged embedding_failed downstream). Lines with no trace_id are dropped silently. """ - logger.info(f"Parsing embedding results from {len(raw_results)} lines") + logger.info( + f"[parse_embedding_results] Parsing embedding results from {len(raw_results)} lines" + ) embedding_pairs = [] failed_trace_ids: list[str] = [] @@ -171,7 +177,9 @@ def parse_embedding_results( # Extract BATCH_KEY (which is now the Langfuse trace_id) trace_id = response.get(BATCH_KEY) if not trace_id: - logger.warning(f"Line {line_num}: No {BATCH_KEY} found, skipping") + logger.warning( + f"[parse_embedding_results] Line {line_num}: No {BATCH_KEY} found, skipping" + ) continue # Handle errors in batch processing @@ -189,7 +197,7 @@ def parse_embedding_results( if len(embedding_data) < 2: logger.warning( - f"Trace {trace_id}: Expected 2 embeddings, got {len(embedding_data)}" + f"[parse_embedding_results] Trace {trace_id}: Expected 2 embeddings, got {len(embedding_data)}" ) failed_trace_ids.append(trace_id) continue @@ -214,7 +222,7 @@ def parse_embedding_results( if output_embedding is None or ground_truth_embedding is None: logger.warning( - f"Trace {trace_id}: Missing embeddings (output={output_embedding is not None}, " + f"[parse_embedding_results] Trace {trace_id}: Missing embeddings (output={output_embedding is not None}, " f"ground_truth={ground_truth_embedding is not None})" ) failed_trace_ids.append(trace_id) @@ -229,13 +237,16 @@ def parse_embedding_results( ) except Exception as e: - logger.error(f"Line {line_num}: Unexpected error: {e}", exc_info=True) + logger.error( + f"[parse_embedding_results] Line {line_num}: Unexpected error: {e}", + exc_info=True, + ) if trace_id: failed_trace_ids.append(trace_id) continue logger.info( - f"Parsed {len(embedding_pairs)} embedding pairs from {len(raw_results)} lines " + f"[parse_embedding_results] Parsed {len(embedding_pairs)} embedding pairs from {len(raw_results)} lines " f"| failed={len(failed_trace_ids)}" ) return embedding_pairs, failed_trace_ids @@ -293,7 +304,9 @@ def calculate_average_similarity( "per_item_scores": [...] # Individual scores with trace_ids } """ - logger.info(f"Calculating similarity for {len(embedding_pairs)} pairs") + logger.info( + f"[calculate_average_similarity] Calculating similarity for {len(embedding_pairs)} pairs" + ) if not embedding_pairs: return { @@ -323,12 +336,15 @@ def calculate_average_similarity( except Exception as e: logger.error( - f"Error calculating similarity for trace {pair.get('trace_id')}: {e}" + f"[calculate_average_similarity] Error calculating similarity for trace {pair.get('trace_id')}: {e}", + exc_info=True, ) continue if not similarities: - logger.warning("No valid similarities calculated") + logger.warning( + "[calculate_average_similarity] No valid similarities calculated" + ) return { "cosine_similarity_avg": 0.0, "cosine_similarity_std": 0.0, @@ -347,7 +363,7 @@ def calculate_average_similarity( } logger.info( - f"Calculated similarity stats: avg={stats['cosine_similarity_avg']:.3f}, " + f"[calculate_average_similarity] Calculated similarity stats: avg={stats['cosine_similarity_avg']:.3f}, " f"std={stats['cosine_similarity_std']:.3f}" ) @@ -384,7 +400,9 @@ def start_embedding_batch( Exception: If any step fails """ try: - logger.info(f"Starting embedding batch for evaluation run {eval_run.id}") + logger.info( + f"[start_embedding_batch] Starting embedding batch for evaluation run {eval_run.id}" + ) # Use default embedding model embedding_model = EMBEDDING_MODEL @@ -440,7 +458,7 @@ def start_embedding_batch( session.refresh(eval_run) logger.info( - f"Successfully started embedding batch: batch_job_id={batch_job.id}, " + f"[start_embedding_batch] Successfully started embedding batch: batch_job_id={batch_job.id}, " f"provider_batch_id={batch_job.provider_batch_id} " f"for evaluation run {eval_run.id} with {batch_job.total_items} items" ) @@ -448,6 +466,9 @@ def start_embedding_batch( return eval_run except Exception as e: - logger.error(f"Failed to start embedding batch: {e}", exc_info=True) + logger.error( + f"[start_embedding_batch] Failed to start embedding batch: {e}", + exc_info=True, + ) # Don't update eval_run status here - let caller decide raise diff --git a/backend/app/crud/evaluations/langfuse.py b/backend/app/crud/evaluations/langfuse.py index 53f6f2473..e5f3e01dd 100644 --- a/backend/app/crud/evaluations/langfuse.py +++ b/backend/app/crud/evaluations/langfuse.py @@ -205,7 +205,8 @@ def _create_single_trace(result: dict[str, Any]) -> tuple[str, str] | None: if getattr(e, "status_code", None) == 429: logger.error( f"[create_langfuse_dataset_run] Langfuse rate limit (429) | " - f"item_id={item_id}" + f"item_id={item_id}", + exc_info=True, ) else: logger.error( @@ -609,7 +610,8 @@ def _fetch_single_trace(trace_id: str) -> TraceData | None: f"[fetch_trace_scores_from_langfuse] Circuit breaker triggered | " f"consecutive_failures={consecutive_failures} | " f"total_failures={total_failures} | " - f"total_traces={len(trace_ids)}" + f"total_traces={len(trace_ids)}", + exc_info=True, ) raise RuntimeError( f"Langfuse API unavailable: {consecutive_failures} consecutive " diff --git a/backend/app/crud/evaluations/processing.py b/backend/app/crud/evaluations/processing.py index 331a9b66f..407c46b36 100644 --- a/backend/app/crud/evaluations/processing.py +++ b/backend/app/crud/evaluations/processing.py @@ -326,7 +326,8 @@ def parse_evaluation_output( except Exception as e: logger.error( - f"[parse_evaluation_output] Unexpected error | line={line_num} | {e}" + f"[parse_evaluation_output] Unexpected error | line={line_num} | {e}", + exc_info=True, ) continue @@ -1199,7 +1200,8 @@ async def poll_all_pending_evaluations(session: Session) -> dict[str, Any]: ) except HTTPException as http_exc: logger.error( - f"[poll_all_pending_evaluations] Failed to get API clients | org_id={org_id} | project_id={project_id} | error={http_exc.detail}" + f"[poll_all_pending_evaluations] Failed to get API clients | org_id={org_id} | project_id={project_id} | error={http_exc.detail}", + exc_info=True, ) # Mark all runs in this project as failed due to client configuration error for eval_run in project_runs: diff --git a/backend/app/crud/onboarding.py b/backend/app/crud/onboarding.py index e754414d0..23a48cfd7 100644 --- a/backend/app/crud/onboarding.py +++ b/backend/app/crud/onboarding.py @@ -1,5 +1,6 @@ import logging from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError from sqlmodel import Session from app.core.security import encrypt_credentials, get_password_hash @@ -136,7 +137,21 @@ def onboard_project( created_credentials.append(cred_row) - session.commit() + # Pre-checks are read-then-write; a concurrent onboard only collides here. + try: + session.commit() + except IntegrityError as e: + session.rollback() + logger.warning( + f"[onboard_project] Conflicting concurrent onboarding | " + f"org={onboard_in.organization_name}, project={onboard_in.project_name} | {e}" + ) + raise HTTPException( + status_code=409, + detail=f"Project '{onboard_in.project_name}' already exists for " + f"organization '{onboard_in.organization_name}'", + ) + cred_ids = [c.id for c in created_credentials] logger.info( diff --git a/backend/app/crud/stt_evaluations/batch.py b/backend/app/crud/stt_evaluations/batch.py index c9ffe61fa..0a5aaf013 100644 --- a/backend/app/crud/stt_evaluations/batch.py +++ b/backend/app/crud/stt_evaluations/batch.py @@ -227,7 +227,8 @@ def _upload_to_gemini(sample: STTSample) -> _UploadResult: except Exception as e: logger.error( f"[start_stt_evaluation_batch] Failed to submit batch | " - f"model: {model}, error: {str(e)}" + f"model: {model}, error: {str(e)}", + exc_info=True, ) if not batch_jobs: diff --git a/backend/app/crud/tts_evaluations/batch.py b/backend/app/crud/tts_evaluations/batch.py index 3a9abb543..fff0757a6 100644 --- a/backend/app/crud/tts_evaluations/batch.py +++ b/backend/app/crud/tts_evaluations/batch.py @@ -136,7 +136,8 @@ def start_tts_evaluation_batch( except Exception as e: logger.error( f"[start_tts_evaluation_batch] Failed to submit batch | " - f"model: {model}, error: {str(e)}" + f"model: {model}, error: {str(e)}", + exc_info=True, ) pending = get_pending_results_for_run( session=session, run_id=run.id, provider=model diff --git a/backend/app/services/collections/providers/gemini.py b/backend/app/services/collections/providers/gemini.py index cdf2e50c2..ab880e513 100644 --- a/backend/app/services/collections/providers/gemini.py +++ b/backend/app/services/collections/providers/gemini.py @@ -151,6 +151,7 @@ def upload_files( doc.id, uploaded.name, str(delete_err), + exc_info=True, ) doc.file_id.pop(GOOGLE_AISTUDIO_PROVIDER, None) raise diff --git a/backend/app/services/documents/helpers.py b/backend/app/services/documents/helpers.py index 7d78b6160..db86e49d3 100644 --- a/backend/app/services/documents/helpers.py +++ b/backend/app/services/documents/helpers.py @@ -56,12 +56,13 @@ def pre_transform_validation( - resolve actual transformer (or None if no target_format) Returns: (source_format, actual_transformer_or_none) - Raises: HTTPException(400) on client errors. + Raises: HTTPException(422) on an unreadable source format, 400 on an + unsupported transformation or transformer. """ try: source_format = get_file_format(src_filename) except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) + raise HTTPException(status_code=422, detail=str(e)) actual_transformer: Optional[str] = None if target_format: diff --git a/backend/app/services/evaluations/dataset.py b/backend/app/services/evaluations/dataset.py index e1603a605..1eec2458c 100644 --- a/backend/app/services/evaluations/dataset.py +++ b/backend/app/services/evaluations/dataset.py @@ -134,13 +134,14 @@ def upload_dataset( f"dataset={dataset_name} | id={langfuse_dataset_id}" ) - except Exception as e: + except Exception: logger.error( - f"[upload_dataset] Failed to upload dataset to Langfuse | {e}", + f"[upload_dataset] Failed to upload dataset to Langfuse | dataset={dataset_name}", exc_info=True, ) raise HTTPException( - status_code=500, detail=f"Failed to upload dataset to Langfuse: {e}" + status_code=502, + detail=f"Langfuse could not accept dataset '{dataset_name}'. Retry shortly.", ) # Step 5: Store metadata in database diff --git a/backend/app/services/evaluations/fast.py b/backend/app/services/evaluations/fast.py index 20b0e94c6..459da6e0a 100644 --- a/backend/app/services/evaluations/fast.py +++ b/backend/app/services/evaluations/fast.py @@ -55,6 +55,17 @@ ERR_DATASET_TOO_LARGE_FOR_FAST = "dataset_too_large_for_fast" +def _fail_run(*, session: Session, eval_run: EvaluationRun, reason: str) -> None: + update_evaluation_run( + session=session, + eval_run=eval_run, + update=EvaluationRunUpdate( + status="failed", + error_message=f"Failed to start fast eval: {reason}", + ), + ) + + def is_dataset_fast_eligible(*, original_items_count: int) -> bool: """A dataset is eligible for fast mode when its unique-row count is within cap.""" return original_items_count <= settings.EVAL_FAST_MAX_UNIQUE_ROWS @@ -287,7 +298,9 @@ def validate_and_start_fast_evaluation( ) total_items = len(dataset_items) if total_items == 0: - raise ValueError(f"Dataset '{dataset.name}' returned no items") + raise HTTPException( + status_code=422, detail=f"Dataset '{dataset.name}' returned no items" + ) n_chunks = math.ceil(total_items / settings.EVAL_FAST_CHUNK_SIZE) # total_items isn't on EvaluationRunUpdate; set it directly, then flip to @@ -310,20 +323,22 @@ def validate_and_start_fast_evaluation( f"eval_run_id={eval_run.id} | total_items={total_items} | " f"n_chunks={n_chunks}" ) + except HTTPException as exc: + log = logger.error if exc.status_code >= 500 else logger.warning + log( + f"[validate_and_start_fast_evaluation] Failed to start run | " + f"eval_run_id={eval_run.id} | code={exc.status_code}", + exc_info=True, + ) + _fail_run(session=session, eval_run=eval_run, reason=exc.detail) + raise except Exception as exc: logger.error( f"[validate_and_start_fast_evaluation] Failed to start run | " f"eval_run_id={eval_run.id} | error={exc}", exc_info=True, ) - update_evaluation_run( - session=session, - eval_run=eval_run, - update=EvaluationRunUpdate( - status="failed", - error_message=f"Failed to start fast eval: {exc}", - ), - ) + _fail_run(session=session, eval_run=eval_run, reason=str(exc)) raise HTTPException( status_code=500, detail="Failed to start fast evaluation", diff --git a/backend/app/services/guardrails/jobs.py b/backend/app/services/guardrails/jobs.py index 95f97c923..1ddf26c11 100644 --- a/backend/app/services/guardrails/jobs.py +++ b/backend/app/services/guardrails/jobs.py @@ -1,9 +1,9 @@ import logging +from fastapi import HTTPException from typing import Any from uuid import UUID, uuid4 from asgi_correlation_id import correlation_id -from fastapi import HTTPException from opentelemetry import trace from sqlmodel import Session @@ -85,8 +85,8 @@ def start_job( job_update=JobUpdate(status=JobStatus.FAILED, error_message=str(e)), ) raise HTTPException( - status_code=500, - detail="Internal server error while scheduling guardrails job", + status_code=503, + detail="Could not queue the guardrails job. Retry shortly.", ) logger.info( diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index e95ec9cb1..cface6301 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -179,7 +179,7 @@ def start_job( job_update = JobUpdate(status=JobStatus.FAILED, error_message=str(e)) job_crud.update(job_id=job.id, job_update=job_update) raise HTTPException( - status_code=500, detail="Internal server error while executing LLM call" + status_code=503, detail="Could not queue the LLM call. Retry shortly." ) _set_traceability_attributes(span, task_id=str(task_id)) @@ -239,8 +239,8 @@ def start_chain_job( job_update = JobUpdate(status=JobStatus.FAILED, error_message=str(e)) job_crud.update(job_id=job.id, job_update=job_update) raise HTTPException( - status_code=500, - detail="Internal server error while executing LLM chain job", + status_code=503, + detail="Could not queue the LLM chain job. Retry shortly.", ) _set_traceability_attributes(span, task_id=str(task_id)) diff --git a/backend/app/services/stt_evaluations/audio.py b/backend/app/services/stt_evaluations/audio.py index 8f890d7e9..1246a4a4c 100644 --- a/backend/app/services/stt_evaluations/audio.py +++ b/backend/app/services/stt_evaluations/audio.py @@ -8,7 +8,7 @@ from sqlmodel import Session from app.core.cloud.storage import get_cloud_storage -from app.core.exception_handlers import HTTPException +from fastapi import HTTPException from app.crud.file import create_file from app.models.file import FileType, AudioUploadResponse from app.services.stt_evaluations.constants import ( @@ -121,9 +121,10 @@ def upload_audio_file( except Exception as e: logger.error( f"[upload_audio_file] Failed to upload audio | " - f"project_id: {project_id}, error: {str(e)}" + f"project_id: {project_id}, error: {str(e)}", + exc_info=True, ) raise HTTPException( - status_code=500, - detail="Failed to upload audio file. Please try again later.", + status_code=502, + detail="Object storage could not accept the audio file. Retry shortly.", ) diff --git a/backend/app/tests/api/routes/documents/test_route_document_upload.py b/backend/app/tests/api/routes/documents/test_route_document_upload.py index 909249a42..2eca7ac5c 100644 --- a/backend/app/tests/api/routes/documents/test_route_document_upload.py +++ b/backend/app/tests/api/routes/documents/test_route_document_upload.py @@ -320,7 +320,7 @@ def test_upload_with_unsupported_file_extension( try: response = uploader.put(route, unsupported_file, target_format="markdown") - assert response.status_code == 400 + assert response.status_code == 422 error_data = response.json() assert "Unsupported file extension: .xyz" in error_data["error"] finally: diff --git a/backend/app/tests/api/routes/test_creds.py b/backend/app/tests/api/routes/test_creds.py index c5fd9c1ea..f171fbf1a 100644 --- a/backend/app/tests/api/routes/test_creds.py +++ b/backend/app/tests/api/routes/test_creds.py @@ -237,7 +237,7 @@ def test_duplicate_credential_creation_fails( client: TestClient, user_api_key: TestAuthContext, ) -> None: - """Test that creating duplicate credentials fails with 400.""" + """Test that creating duplicate credentials fails with 409.""" api_key = "sk-" + generate_random_string(10) duplicate_credential = { "organization_id": user_api_key.organization_id, @@ -257,7 +257,7 @@ def test_duplicate_credential_creation_fails( headers={"X-API-KEY": user_api_key.key}, ) - assert response.status_code == 400 + assert response.status_code == 409 assert "already exist" in response.json()["error"] diff --git a/backend/app/tests/api/routes/test_evaluation.py b/backend/app/tests/api/routes/test_evaluation.py index 18f0dfa72..629021836 100644 --- a/backend/app/tests/api/routes/test_evaluation.py +++ b/backend/app/tests/api/routes/test_evaluation.py @@ -4,6 +4,7 @@ from uuid import uuid4 import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient from sqlmodel import Session, select @@ -439,13 +440,13 @@ def test_upload_with_duplication_factor_boundary_minimum( class TestDatasetUploadErrors: """Test error handling.""" - def test_upload_returns_500_when_langfuse_upload_raises( + def test_upload_returns_502_when_langfuse_upload_raises( self, client: TestClient, user_api_key_header: dict[str, str], valid_csv_content: str, ) -> None: - """Tracing enabled but the Langfuse upload fails → 500.""" + """Tracing enabled but the Langfuse upload fails → 502.""" with ( patch("app.core.cloud.get_cloud_storage") as _mock_storage, patch( @@ -474,12 +475,12 @@ def test_upload_returns_500_when_langfuse_upload_raises( headers=user_api_key_header, ) - assert response.status_code == 500, response.text + assert response.status_code == 502, response.text response_data = response.json() error_str = response_data.get( "detail", response_data.get("error", str(response_data)) ) - assert "Failed to upload dataset to Langfuse" in str(error_str) + assert "Langfuse could not accept dataset" in str(error_str) def test_upload_invalid_csv_format( self, client: TestClient, user_api_key_header: dict[str, str] @@ -1133,7 +1134,7 @@ def test_unsupported_provider_marks_failed( ) -> None: mock_fetch.return_value = _BATCH_DATASET_ITEMS - with pytest.raises(ValueError, match="Unsupported provider"): + with pytest.raises(HTTPException, match="Unsupported provider"): start_evaluation_batch( langfuse=MagicMock(), session=db, @@ -1157,7 +1158,7 @@ def test_empty_jsonl_marks_failed( ] mock_map.return_value = ({"model": "gpt-4o"}, []) - with pytest.raises(ValueError, match="did not produce any JSONL"): + with pytest.raises(HTTPException, match="did not produce any JSONL"): start_evaluation_batch( langfuse=MagicMock(), session=db, diff --git a/backend/app/tests/api/routes/test_tts_evaluation.py b/backend/app/tests/api/routes/test_tts_evaluation.py index ae9ade1c8..0e3957c53 100644 --- a/backend/app/tests/api/routes/test_tts_evaluation.py +++ b/backend/app/tests/api/routes/test_tts_evaluation.py @@ -626,7 +626,7 @@ def test_start_evaluation_celery_failure( headers=user_api_key_header, ) - assert response.status_code == 500 + assert response.status_code == 503 assert "queue" in response.json()["error"].lower() def test_start_evaluation_invalid_dataset( @@ -674,7 +674,7 @@ def test_start_evaluation_empty_dataset( headers=user_api_key_header, ) - assert response.status_code == 400 + assert response.status_code == 422 assert "no samples" in response.json()["error"].lower() def test_start_evaluation_unsupported_model( diff --git a/backend/app/tests/core/test_exception_handlers.py b/backend/app/tests/core/test_exception_handlers.py index 4955136be..53afbef66 100644 --- a/backend/app/tests/core/test_exception_handlers.py +++ b/backend/app/tests/core/test_exception_handlers.py @@ -1,7 +1,12 @@ +from fastapi import FastAPI from fastapi.testclient import TestClient from app.core.config import settings -from app.core.exception_handlers import _sanitize_validation_errors +from app.core.exception_handlers import ( + GENERIC_ERROR_DETAIL, + _sanitize_validation_errors, + register_exception_handlers, +) from app.tests.utils.auth import TestAuthContext @@ -138,3 +143,23 @@ def test_union_noise_filtered( for error in response.json()["errors"]: assert "openai-native" not in error["message"] assert "NativeCompletionConfig" not in error["field"] + + +def _app_raising(exc: Exception) -> TestClient: + app = FastAPI() + register_exception_handlers(app) + + @app.get("/boom") + def boom() -> None: + raise exc + + return TestClient(app, raise_server_exceptions=False) + + +class TestGenericErrorHandler: + def test_does_not_leak_exception_text(self) -> None: + response = _app_raising(ValueError("secret arn:aws:kms:key/abc")).get("/boom") + + assert response.status_code == 500 + assert response.json()["error"] == GENERIC_ERROR_DETAIL + assert "arn:aws:kms" not in response.text diff --git a/backend/app/tests/core/test_security.py b/backend/app/tests/core/test_security.py index 412c48c25..8683bb1db 100644 --- a/backend/app/tests/core/test_security.py +++ b/backend/app/tests/core/test_security.py @@ -1,8 +1,10 @@ from datetime import timedelta +from unittest.mock import MagicMock import boto3 import jwt import pytest +from fastapi import HTTPException from moto import mock_aws from sqlmodel import Session @@ -78,6 +80,31 @@ def test_dual_read_fernet_row_with_kms_active(self, monkeypatch, kms_key): monkeypatch.setattr(settings, "ENVIRONMENT", "staging") assert decrypt_credentials(fernet_encrypted) == creds + def test_kms_encrypt_failure_raises_502_without_leaking_cause( + self, monkeypatch, kms_key + ): + broken = MagicMock() + broken.encrypt.side_effect = Exception("arn:aws:kms:ap-south-1:secret") + monkeypatch.setattr(security, "_kms_client", broken) + + with pytest.raises(HTTPException) as exc: + encrypt_credentials({"api_key": "sk-1"}) + + assert exc.value.status_code == 502 + assert "arn:aws:kms" not in exc.value.detail + + def test_kms_decrypt_failure_raises_502(self, monkeypatch, kms_key): + encrypted = encrypt_credentials({"api_key": "sk-1"}) + broken = MagicMock() + broken.decrypt.side_effect = Exception("arn:aws:kms:ap-south-1:secret") + monkeypatch.setattr(security, "_kms_client", broken) + + with pytest.raises(HTTPException) as exc: + decrypt_credentials(encrypted) + + assert exc.value.status_code == 502 + assert "arn:aws:kms" not in exc.value.detail + class TestAPIKeyManager: """Test suite for APIKeyManager class.""" diff --git a/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py b/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py new file mode 100644 index 000000000..1b044d51b --- /dev/null +++ b/backend/app/tests/crud/collections/collection/test_crud_collection_read_one_if_delete.py @@ -0,0 +1,57 @@ +"""Deleting an already-deleted collection is a conflict, not bad input.""" + +import uuid + +import pytest +from sqlmodel import Session + +from fastapi import HTTPException + +from app.core.util import now +from app.crud import CollectionCrud +from app.models import Collection, ProviderType +from app.tests.utils.utils import get_project + + +def _make_collection(db: Session, project_id: int, deleted: bool) -> Collection: + collection = Collection( + project_id=project_id, + knowledge_base_id="vs_test", + knowledge_base_provider="openai vector store", + provider=ProviderType.openai, + deleted_at=now() if deleted else None, + ) + db.add(collection) + db.commit() + db.refresh(collection) + return collection + + +def test_already_deleted_raises_409(db: Session) -> None: + project = get_project(db) + collection = _make_collection(db, project.id, deleted=True) + + with pytest.raises(HTTPException) as exc: + CollectionCrud(db, project.id).read_one_if_delete(collection.id) + + assert exc.value.status_code == 409 + assert "already deleted" in exc.value.detail + + +def test_active_collection_is_returned(db: Session) -> None: + project = get_project(db) + collection = _make_collection(db, project.id, deleted=False) + + assert ( + CollectionCrud(db, project.id).read_one_if_delete(collection.id).id + == collection.id + ) + + +def test_missing_collection_still_raises_404(db: Session) -> None: + project = get_project(db) + + with pytest.raises(HTTPException) as exc: + CollectionCrud(db, project.id).read_one_if_delete(uuid.uuid4()) + + assert exc.value.status_code == 404 diff --git a/backend/app/tests/crud/evaluations/test_batch_errors.py b/backend/app/tests/crud/evaluations/test_batch_errors.py new file mode 100644 index 000000000..262ed9f93 --- /dev/null +++ b/backend/app/tests/crud/evaluations/test_batch_errors.py @@ -0,0 +1,53 @@ +"""Langfuse failures map to distinct status codes rather than a flat 500.""" + +from unittest.mock import MagicMock + +import pytest +from fastapi import HTTPException +from langfuse.api import NotFoundError as LangfuseNotFoundError +from langfuse.api.core import ApiError as LangfuseApiError + +from app.crud.evaluations.batch import fetch_dataset_items + + +def test_dataset_missing_raises_404() -> None: + langfuse = MagicMock() + langfuse.get_dataset.side_effect = LangfuseNotFoundError( + body={"message": "Dataset not found"} + ) + + with pytest.raises(HTTPException) as exc: + fetch_dataset_items(langfuse=langfuse, dataset_name="missing") + + assert exc.value.status_code == 404 + assert "missing" in exc.value.detail + + +def test_langfuse_api_error_raises_502() -> None: + langfuse = MagicMock() + langfuse.get_dataset.side_effect = LangfuseApiError(status_code=500, body="boom") + + with pytest.raises(HTTPException) as exc: + fetch_dataset_items(langfuse=langfuse, dataset_name="ds") + + assert exc.value.status_code == 502 + + +def test_langfuse_unreachable_raises_502() -> None: + langfuse = MagicMock() + langfuse.get_dataset.side_effect = ConnectionError("dns failure") + + with pytest.raises(HTTPException) as exc: + fetch_dataset_items(langfuse=langfuse, dataset_name="ds") + + assert exc.value.status_code == 502 + + +def test_empty_dataset_raises_422() -> None: + langfuse = MagicMock() + langfuse.get_dataset.return_value = MagicMock(items=[]) + + with pytest.raises(HTTPException) as exc: + fetch_dataset_items(langfuse=langfuse, dataset_name="ds") + + assert exc.value.status_code == 422 diff --git a/backend/app/tests/crud/test_credentials.py b/backend/app/tests/crud/test_credentials.py index 8b3269ca6..90a7770b5 100644 --- a/backend/app/tests/crud/test_credentials.py +++ b/backend/app/tests/crud/test_credentials.py @@ -1,4 +1,7 @@ +from unittest.mock import MagicMock + import pytest +from fastapi import HTTPException from sqlmodel import Session from app.crud import ( @@ -353,3 +356,73 @@ def test_langfuse_credential_validation(db: Session) -> None: ) assert len(created_credentials) == 1 assert created_credentials[0].provider == "langfuse" + + +def test_remove_provider_credential_missing_raises_404(db: Session) -> None: + """Deleting a credential that was never stored is a 404, not a 500.""" + project = create_test_project(db) + + with pytest.raises(HTTPException) as exc: + remove_provider_credential( + session=db, + org_id=project.organization_id, + provider="openai", + project_id=project.id, + ) + + assert exc.value.status_code == 404 + + +def test_remove_provider_credential_race_deletes_nothing_raises_404( + db: Session, monkeypatch +) -> None: + """Row vanishes between the existence check and the delete: 404, not 500.""" + _, project = create_test_credential(db) + monkeypatch.setattr( + "app.crud.credentials.get_provider_credential", lambda **k: MagicMock() + ) + monkeypatch.setattr(db, "exec", lambda *a, **k: MagicMock(rowcount=0)) + + with pytest.raises(HTTPException) as exc: + remove_provider_credential( + session=db, + org_id=project.organization_id, + provider="openai", + project_id=project.id, + ) + + assert exc.value.status_code == 404 + + +def test_remove_creds_for_org_when_none_exist_raises_404(db: Session) -> None: + project = create_test_project(db) + + with pytest.raises(HTTPException) as exc: + remove_creds_for_org( + session=db, + org_id=project.organization_id, + project_id=project.id, + ) + + assert exc.value.status_code == 404 + + +def test_duplicate_provider_raises_409(db: Session) -> None: + """Unique-constraint violation is a conflict, not bad input.""" + _, project = create_test_credential(db) + + creds_create = CredsCreate( + is_active=True, + credential={"openai": {"api_key": "another-key"}}, + ) + + with pytest.raises(HTTPException) as exc: + set_creds_for_org( + session=db, + creds_add=creds_create, + organization_id=project.organization_id, + project_id=project.id, + ) + + assert exc.value.status_code == 409 + assert "already exist" in exc.value.detail diff --git a/backend/app/tests/crud/test_onboarding.py b/backend/app/tests/crud/test_onboarding.py index 13f652bfa..f46ec8eae 100644 --- a/backend/app/tests/crud/test_onboarding.py +++ b/backend/app/tests/crud/test_onboarding.py @@ -1,7 +1,9 @@ import pytest from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select + from app.crud.onboarding import onboard_project from app.crud import ( get_organization_by_name, @@ -289,3 +291,53 @@ def test_onboard_project_response_data_integrity(db: Session) -> None: assert project.name == response.project_name assert project.organization_id == response.organization_id assert user.email == response.user_email + + +def test_onboard_kms_failure_raises_502(db: Session, monkeypatch) -> None: + """A KMS outage while encrypting credentials is upstream, not our bug.""" + monkeypatch.setattr( + "app.crud.onboarding.encrypt_credentials", + lambda _: (_ for _ in ()).throw( + HTTPException(status_code=502, detail="kms down") + ), + ) + + with pytest.raises(HTTPException) as exc: + onboard_project( + session=db, + onboard_in=OnboardingRequest( + organization_name=random_lower_string(), + project_name=random_lower_string(), + email=random_email(), + user_name=random_lower_string(), + password=random_lower_string(), + credentials=[{"openai": {"api_key": "sk-test"}}], + ), + ) + + assert exc.value.status_code == 502 + + +def test_onboard_concurrent_commit_conflict_raises_409( + db: Session, monkeypatch +) -> None: + """The pre-checks are racy; a collision at commit is a 409, not a 500.""" + + def _raise_integrity_error() -> None: + raise IntegrityError("INSERT", {}, Exception("duplicate key")) + + monkeypatch.setattr(db, "commit", _raise_integrity_error) + + with pytest.raises(HTTPException) as exc: + onboard_project( + session=db, + onboard_in=OnboardingRequest( + organization_name=random_lower_string(), + project_name=random_lower_string(), + email=random_email(), + user_name=random_lower_string(), + password=random_lower_string(), + ), + ) + + assert exc.value.status_code == 409 diff --git a/backend/app/tests/services/guardrails/test_jobs.py b/backend/app/tests/services/guardrails/test_jobs.py index 84bca8290..a09da6c2b 100644 --- a/backend/app/tests/services/guardrails/test_jobs.py +++ b/backend/app/tests/services/guardrails/test_jobs.py @@ -4,6 +4,7 @@ import pytest from fastapi import HTTPException + from sqlmodel import Session, select from app.models import Job, JobStatus, JobType @@ -143,7 +144,7 @@ def test_start_job_happy_path_creates_row_and_enqueues( assert kwargs["job_id"] == str(job.id) -def test_start_job_celery_failure_marks_failed_and_raises_500( +def test_start_job_celery_failure_marks_failed_and_raises_503( db: Session, user_api_key: TestAuthContext ) -> None: with patch("app.services.guardrails.jobs.start_guardrails_job") as mock_enqueue: @@ -156,7 +157,7 @@ def test_start_job_celery_failure_marks_failed_and_raises_500( organization_id=user_api_key.organization_id, ) - assert exc.value.status_code == 500 + assert exc.value.status_code == 503 rows = db.exec(select(Job).where(Job.project_id == user_api_key.project_id)).all() failed = [r for r in rows if r.status == JobStatus.FAILED] diff --git a/backend/app/tests/services/llm/test_jobs.py b/backend/app/tests/services/llm/test_jobs.py index 2f56b14a0..40007e140 100644 --- a/backend/app/tests/services/llm/test_jobs.py +++ b/backend/app/tests/services/llm/test_jobs.py @@ -1,11 +1,12 @@ import pytest +from fastapi import HTTPException from unittest.mock import patch, MagicMock from uuid import UUID, uuid4 from celery.exceptions import SoftTimeLimitExceeded from gevent import Timeout -from fastapi import HTTPException + from sqlmodel import Session, select from app.crud import JobCrud @@ -98,10 +99,8 @@ def test_start_job_celery_scheduling_fails( with pytest.raises(HTTPException) as exc_info: start_job(db, llm_call_request, project.id, project.organization_id) - assert exc_info.value.status_code == 500 - assert "Internal server error while executing LLM call" in str( - exc_info.value.detail - ) + assert exc_info.value.status_code == 503 + assert "Could not queue the LLM call" in str(exc_info.value.detail) def test_start_job_exception_during_job_creation( self, db: Session, llm_call_request: LLMCallRequest @@ -2009,10 +2008,8 @@ def test_start_chain_job_celery_failure(self, db: Session, chain_request): with pytest.raises(HTTPException) as exc_info: start_chain_job(db, chain_request, project.id, project.organization_id) - assert exc_info.value.status_code == 500 - assert "Internal server error while executing LLM chain job" in str( - exc_info.value.detail - ) + assert exc_info.value.status_code == 503 + assert "Could not queue the LLM chain job" in str(exc_info.value.detail) class TestExecuteChainJob: diff --git a/backend/app/tests/services/stt_evaluations/test_audio.py b/backend/app/tests/services/stt_evaluations/test_audio.py index a1c1d2342..48a6ab6b7 100644 --- a/backend/app/tests/services/stt_evaluations/test_audio.py +++ b/backend/app/tests/services/stt_evaluations/test_audio.py @@ -5,7 +5,8 @@ import pytest from fastapi import UploadFile -from app.core.exception_handlers import HTTPException +from fastapi import HTTPException + from app.services.stt_evaluations.audio import ( _resolve_extension, _validate_audio_file, @@ -185,8 +186,8 @@ def test_upload_storage_error(self, mock_get_storage) -> None: organization_id=1, project_id=1, ) - assert exc_info.value.status_code == 500 - assert "Failed to upload audio file" in str(exc_info.value.detail) + assert exc_info.value.status_code == 502 + assert "Object storage could not accept" in str(exc_info.value.detail) @patch("app.services.stt_evaluations.audio.create_file") @patch("app.services.stt_evaluations.audio.get_cloud_storage") diff --git a/docs/wiki/cross-cutting/exceptions.md b/docs/wiki/cross-cutting/exceptions.md index 99da15713..0b4368980 100644 --- a/docs/wiki/cross-cutting/exceptions.md +++ b/docs/wiki/cross-cutting/exceptions.md @@ -3,9 +3,24 @@ All paths relative to `backend/app/`. ## Global handlers -- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. +- `core/exception_handlers.py` — registered on the FastAPI app; maps exception types to HTTP responses. The `Exception` catch-all returns a fixed message, never `str(exc)`, so provider bodies and DB text don't reach callers. - `core/middleware.py` — request-level middleware. +## Status-code selection +Raise `HTTPException` directly with the right status code from routes, crud, and service. Never raise a bare `ValueError` for a caller-facing failure: nothing catches it, so it becomes a 500 with a generic body. + +| Condition | Code | +|---|---| +| Resource absent (ours, or one we reference upstream) | 404 | +| State conflict (already exists, already deleted) | 409 | +| Payload unparseable or wrong shape | 422 | +| Valid shape, unacceptable value | 400 | +| Upstream errored or unreachable (Langfuse, OpenAI, Gemini, KMS, object storage) | 502 | +| Our async infra could not accept the work (Celery broker) | 503 | +| Our bug (DB write failed, unexpected exception) | 500 | + +- A broad `except Exception` around a DB write must re-raise `HTTPException` first, otherwise a typed 4xx/5xx gets flattened to 500. + ## Provider/SDK error convention - `.claude/conventions/error-handling.md` — the standardized pattern for wrapping provider/SDK exceptions at service/crud call sites. Follow it for any new OpenAI/Gemini/Anthropic call site. diff --git a/features/status-code-consistency/PLAN.md b/features/status-code-consistency/PLAN.md new file mode 100644 index 000000000..8b7674f5a --- /dev/null +++ b/features/status-code-consistency/PLAN.md @@ -0,0 +1,195 @@ +# Status Code and Error Message Consistency (issue #834): Implementation Plan + +Source spec: [GitHub issue #834](https://github.com/ProjectTech4DevAI/kaapi-backend/issues/834), "Observability: Improve status codes consistency" + +## Summary + +Kaapi returns HTTP 500 for conditions that are not server bugs: a missing Langfuse dataset, a provider or storage failure, a KMS encrypt that failed, a broker that could not accept a job. Callers cannot tell "you asked for something that does not exist" from "we broke", and ops alerting on 5xx rate is polluted by caller errors. This change introduces a central domain exception taxonomy in `app/core/exceptions.py` with registered handlers, then converts the mis-coded raise sites across eight domains, and completes the log-level and log-prefix audit on the same lines. No schema change, so no Alembic migration. + +Domains in scope: the five the issue names (LLM Call, LLM Chain, Collections, Documents, Evaluations including STT/TTS), plus Onboarding, Config Management, and Credentials added by later scope decision. + +### Note on the issue's two cited line numbers + +Both citations are stale against `main`; the paths also moved under `backend/`. + +- `app/crud/assistants.py:68` is already correct. [backend/app/crud/assistants.py:64-73](backend/app/crud/assistants.py#L64-L73) raises 404 on `openai.NotFoundError` and 502 on `openai.OpenAIError`. No change needed, and `assistants` belongs to the Responses module, which is not among the in-scope domains. +- `app/crud/evaluations/batch.py:47` is now a docstring line, and no `success` boolean exists in that file. The underlying defect survives in a different shape: [backend/app/crud/evaluations/batch.py:49-58](backend/app/crud/evaluations/batch.py#L49-L58) raises a bare `ValueError` when Langfuse cannot return the dataset, which nothing catches on the fast-eval path, so the generic handler turns it into a 500. Production evidence: `app/logs/app.log.3:9975` logs `status_code: 404, body: {'message': 'Dataset not found', 'error': 'LangfuseNotFoundError'}` on a request that returned 500. Step 3 fixes it. + +### The taxonomy this plan applies + +Every code change below is an application of this table. It is derived from `.claude/conventions/route.md` ("Status codes (the ones to get right)") and `.claude/conventions/error-handling.md`. + +| Condition | Code | Log level | +|---|---|---| +| Resource absent, ours or an upstream-held resource we reference | 404 | `warning` | +| State conflict (already exists, already deleted) | 409 | `warning` | +| Payload unparseable or wrong shape (bad CSV, unreadable file) | 422 | `warning` | +| Valid shape, unacceptable value (unsupported transform pair) | 400 | `warning` | +| Upstream returned an error or was unreachable (Langfuse, OpenAI, Gemini, object storage) | 502 | `error` | +| Our async infrastructure could not accept the work (Celery broker, RabbitMQ) | 503 | `error` | +| Genuinely our bug (DB write failed, unexpected exception) | 500 | `error` | + +## Blast Radius + +Primary entities: this is a cross-cutting change to the HTTP error contract, not to any table. Entities whose read/write paths change their failure responses: `EvaluationDataset`, `EvaluationRun`, `STTSample`/`STTResult`, `TTSResult`, `Collection`, `CollectionJob`, `Document`, `DocTransformationJob`, `Job`, `LlmCall`, `LlmChain`, `Config`, `ConfigVersion`, `Credential`, `Organization`, `Project`, `User`, `APIKey`. + +| Surface | Hop | Impact | Decision | +|---|---|---|---| +| `app/core/exception_handlers.py` | 1 | New typed handler for `KaapiError`; `generic_error_handler` stops echoing `str(exc)` | in scope | +| Evaluations routes/services/crud (text, STT, TTS) | 1 | 500 becomes 404 / 422 / 502 / 503 at named sites | in scope | +| Collections crud/routes | 1 | "already deleted" 400 becomes 409; `CollectionNameConflictError` re-based on `ConflictError` | in scope | +| Documents services/crud | 1 | Unsupported file format 400 becomes 422; wrong-module `HTTPException` import corrected | in scope | +| LLM Call, LLM Chain, Guardrails job start | 1 | Celery enqueue failure 500 becomes 503 | in scope | +| Config Management (`crud/config/`) | 1 | Catch-alls swallow `IntegrityError` into 500; `ConfigBlob` validation returns 400 where the request boundary returns 422 | in scope | +| Credentials (`crud/credentials.py`, `routes/credentials.py`) | 1 | Unique-constraint 400 becomes 409; delete-miss 500 becomes 404; bare `ValueError` sites typed | in scope | +| `app/core/security.py` KMS encrypt/decrypt | 1 | `ValueError` on AWS KMS failure surfaces as 500; becomes 502 | in scope | +| Onboarding (`crud/onboarding.py`) | 1 | Unguarded `encrypt_credentials` and unguarded final `session.commit()` both surface as 500 | in scope | +| `app/tests/` status-code assertions | 1 | 8 assertions encode the current wrong codes and must flip | in scope | +| `docs/wiki/cross-cutting/exceptions.md` | 1 | Must document the new taxonomy module (wiki maintenance rule) | in scope | +| `app/api/docs/**/*.md` swagger copy | 2 | 3 files in the touched domains mention status codes and may go stale | in scope | +| Sentry (`app/core/sentry_filters.py`) | 2 | Reclassifying caller errors off 5xx reduces event volume; no filter code change needed | in scope, verify only | +| kaapi-frontend console | 2 | Breaking if it branches on status code; `APIResponse` body shape is unchanged | **out of scope** (user decision) | +| Langfuse | 2 | Trace and score shape unchanged | unaffected | +| Provider Batch APIs (OpenAI, Gemini, Anthropic) | 2 | Batch payload shape unchanged | unaffected | +| Object storage | 2 | Upload failures resurface as 502 instead of 500; no storage code change | in scope, read path only | +| auth, fine_tuning, assessment, response, api_key | 2 | Same defect class present but outside the eight in-scope domains | **out of scope**, follow-up issue (step 13) | + +## Steps + +### 1. Core: add the domain exception taxonomy +- Files: `backend/app/core/exceptions.py` (new) +- Define `KaapiError(Exception)` carrying `status_code: int` and `detail: str`, plus subclasses `NotFoundError` (404), `ConflictError` (409), `InvalidPayloadError` (422), `InvalidValueError` (400), `UpstreamError` (502), `ServiceUnavailableError` (503). +- `UpstreamError` takes a `provider: str` so the message can name who failed. +- Precedent to follow, not duplicate: the four existing ad-hoc exceptions (`CloudStorageError` in `core/cloud/storage.py:34`, `GeminiClientError` in `core/batch/client.py:15`, `CollectionNameConflictError` in `crud/collection/collection.py:18`, `TransformationError` in `services/doctransform/registry.py:9`) stay where they are; only `CollectionNameConflictError` is re-based in step 6. +- Depends on: nothing + +### 2. Core: register the handler and stop the 500 body leak +- Files: `backend/app/core/exception_handlers.py` (change) +- Add `@app.exception_handler(KaapiError)` inside `register_exception_handlers`, returning `JSONResponse(status_code=exc.status_code, content=APIResponse.failure_response(exc.detail).model_dump())`, matching the existing `http_exception_handler` shape at lines 98-108. +- Change `generic_error_handler` (lines 110-117) to log the exception with `exc_info=True` and the correlation id, and return a fixed message instead of `str(exc)`. This is what currently exposes raw `ValueError` text (including Langfuse response bodies) to API clients. +- Handler registration order matters: register the `KaapiError` handler before the bare `Exception` handler. +- Depends on: step 1 + +### 3. Evaluations crud: dataset fetch raises typed errors +- Files: `backend/app/crud/evaluations/batch.py` (change) +- `fetch_dataset_items` (lines 35-70): in the `except Exception` at line 51, branch on the Langfuse error. Not-found raises `NotFoundError(f"Dataset '{dataset_name}' not found")`; any other Langfuse failure raises `UpstreamError(provider="langfuse", ...)`. Log level follows the taxonomy table (`warning` for not-found, `error` with `exc_info=True` for upstream). +- Line 58 empty-dataset `ValueError` becomes `InvalidPayloadError`. +- The three other `raise ValueError` sites in this file get the same treatment; the file has 5 total. +- Depends on: step 1 + +### 4. Evaluations services: upstream and broker failures +- Files: `backend/app/services/evaluations/dataset.py`, `backend/app/services/evaluations/fast.py`, `backend/app/services/evaluations/prompt_improvement.py` (change) +- `dataset.py:131-138`: Langfuse upload failure becomes `UpstreamError` (502) instead of `HTTPException(500)`. +- `fast.py:211-215`: the broad `except` currently swallows step 3's typed errors into a flat 500. Re-raise `KaapiError` untouched before the catch-all, so a missing dataset reaches the caller as 404. The run is still marked `failed` in both branches. +- `prompt_improvement.py:207-210`: Celery enqueue failure becomes `ServiceUnavailableError` (503). +- Leave `crud/evaluations/dataset.py:102-105` at 500. A failed DB write is genuinely our fault and already logs at `error` with `exc_info=True`. +- Depends on: steps 1, 3 + +### 5. STT/TTS evaluations: queue, storage, and empty-dataset codes +- Files: `backend/app/api/routes/stt_evaluations/evaluation.py`, `backend/app/api/routes/tts_evaluations/evaluation.py`, `backend/app/services/stt_evaluations/audio.py` (change) +- `stt_evaluations/evaluation.py:108-111` and `tts_evaluations/evaluation.py:112-115`: "Failed to queue batch submission" becomes 503. +- `stt_evaluations/evaluation.py:65` and `tts_evaluations/evaluation.py:69`: "Dataset has no samples" moves 400 to 422, matching the empty-dataset code chosen in step 3. +- `audio.py:126-129`: object storage upload failure becomes `UpstreamError` (502). The `except HTTPException: raise` guard at line 119 becomes `except (HTTPException, KaapiError): raise`. +- Depends on: steps 1, 3 + +### 6. Collections: conflict semantics +- Files: `backend/app/crud/collection/collection.py`, `backend/app/api/routes/collections.py` (change) +- `collection.py:114`: "Collection already deleted" moves 400 to 409. It is a state conflict, not bad input. +- `collection.py:18`: `CollectionNameConflictError` subclasses `ConflictError` and carries its own detail, so the manual translation at `collections.py:220-224` can be deleted and the handler from step 2 covers it. Keep the `name` attribute; it is read to build the message. +- Depends on: steps 1, 2 + +### 7. Documents: format errors and the wrong-module import +- Files: `backend/app/services/documents/helpers.py`, `backend/app/crud/document/document.py`, `backend/app/crud/document/doc_transformation_job.py` (change) +- `helpers.py:62-64`: unsupported or unreadable source file format moves 400 to 422, matching `services/evaluations/validators.py:84-97`, which already returns 422 for the same class of problem. See Open Questions. +- `helpers.py:67-72` and `helpers.py:76-83` stay at 400. An unsupported transform pair and an unknown transformer are valid-shape, unacceptable-value cases. +- `helpers.py:59`: the docstring says `Raises: HTTPException(400)` and goes stale; update it. +- `document.py:8` and `doc_transformation_job.py:16` import `HTTPException` from `app.core.exception_handlers` rather than `fastapi`. Correct both imports. Behaviour is identical today, but it couples crud to the handler module. +- Depends on: step 1 + +### 8. LLM Call, LLM Chain, Guardrails: broker failures +- Files: `backend/app/services/llm/jobs.py`, `backend/app/services/guardrails/jobs.py` (change) +- `llm/jobs.py:181-183` (`start_job`), `llm/jobs.py:241-244` (`start_chain_job`), `guardrails/jobs.py:87-90` (`start_job`): all three catch a failed Celery enqueue and return 500. All three become `ServiceUnavailableError` (503). The job is still marked `FAILED` first, unchanged. +- Guardrails is included because `docs/wiki/modules/llm-call.md` places `api/routes/guardrails.py` in the LLM Call module. +- Depends on: step 1 + +### 9. Config Management: catch-all scope and validation code +- Files: `backend/app/crud/config/config.py`, `backend/app/crud/config/version.py` (change) +- `config.py:70-80` (`ConfigCrud.create`) and `version.py:110-120` (`ConfigVersionCrud.create_from_partial`) both wrap their DB writes in a bare `except Exception` that returns 500. A duplicate config name raises `IntegrityError` inside that block and is flattened to 500, even though `config.py:191` already returns 409 for the same conflict on another path. Add an `except IntegrityError` branch returning `ConflictError` ahead of the catch-all, and a `except (HTTPException, KaapiError): raise` guard so typed errors raised inside are not swallowed. +- `version.py:78-85`: a `ConfigBlob` Pydantic `ValidationError` returns `HTTPException(400, detail=validation_errors)`. The same `ConfigBlob` failing at the request boundary returns 422 through `validation_error_handler`. Move this to 422 so one config blob has one rejection code regardless of entry path. The list `detail` still routes through `_sanitize_validation_errors` at `exception_handlers.py:103-104`, so the body shape is unchanged. +- Leave the immutability checks at `version.py:201`, `version.py:330`, and `version.py:340` at 400 for now. See Open Questions. +- Depends on: steps 1, 2 + +### 10. Credentials: conflict, not-found, and KMS failures +- Files: `backend/app/crud/credentials.py`, `backend/app/api/routes/credentials.py`, `backend/app/core/security.py` (change) +- `credentials.py:66-69`: the unique-constraint branch (`uq_credential_org_project_provider`) returns 400. `.claude/conventions/route.md` names 409 for exactly this case. Move to `ConflictError`. +- `credentials.py:70-72`: the non-duplicate `IntegrityError` path raises a bare `ValueError` carrying the raw DB error, which reaches the client as a 500 body. Raise a typed error with a sanitized message; the DB text stays in the log only. +- `credentials.py:190`: `raise ValueError("Provider and credential must be provided")` surfaces as 500, while `routes/credentials.py:122` and `routes/credentials.py:263` raise 400 for the identical condition. Make the crud site 400 to match. +- `credentials.py:285-289` (`remove_provider_credential`): `rowcount == 0` means nothing matched, so it is a 404, not the current 500. Change the code and drop the log from `error` to `warning`, since a caller deleting a non-existent credential is not an outage. +- `credentials.py:322-329` (`remove_creds_for_org`): split the branch. Zero rows deleted returns 404; a genuine partial delete (`0 < rows_deleted < expected_count`) keeps 500 and stays at `logger.error`, because it is a real anomaly. +- `security.py:213-216` and `security.py:233-236`: `encrypt_credentials` and `decrypt_credentials` raise `ValueError` when AWS KMS fails. KMS is upstream, so raise `UpstreamError(provider="kms")` (502). Keep the existing behaviour of never surfacing the underlying message, which may carry AWS ARNs, and add the missing `exc_info=True` to both `logger.error` calls. +- Leave `routes/credentials.py:49` at 500. An empty result from `set_creds_for_org` after no exception is a genuine internal anomaly. +- Leave the `validate_provider_credentials` 400s at `credentials.py:36`, `credentials.py:160`, and `credentials.py:208` unchanged. See Open Questions. +- Depends on: steps 1, 2 + +### 11. Onboarding: KMS and the unguarded commit +- Files: `backend/app/crud/onboarding.py` (change) +- `onboarding.py:126`: `encrypt_credentials(values)` is called with no guard, so a KMS failure becomes a 500. Step 10 makes it raise `UpstreamError`, so this site only needs to stop being the reason a 500 escapes; verify no local `except` re-flattens it. +- `onboarding.py:139` (`session.commit()`): the whole multi-step onboarding has no `try`. The pre-checks at `onboarding.py:67-71` and `onboarding.py:100-105` are read-then-write races, so two concurrent onboards of the same org and project name hit an `IntegrityError` at commit and return 500 with raw DB text. Wrap the commit in `except IntegrityError` returning `ConflictError` with the same wording the pre-check uses, and roll back. +- The two existing 409s (`onboarding.py:68`, `onboarding.py:102`) are already correct and stay as they are. +- Depends on: steps 1, 2, 10 + +### 12. Observability: log-level and prefix audit in the in-scope domains +- Files (change): `backend/app/api/routes/stt_evaluations/evaluation.py:98`, `backend/app/api/routes/tts_evaluations/evaluation.py:102`, `backend/app/crud/evaluations/cron_utils.py:325`, `backend/app/crud/evaluations/embeddings.py:325`, `backend/app/crud/evaluations/langfuse.py:197,599`, `backend/app/crud/evaluations/processing.py:328,1027,1078,1201`, `backend/app/crud/stt_evaluations/batch.py:228`, `backend/app/crud/tts_evaluations/batch.py:137`, `backend/app/services/collections/providers/gemini.py:148,173` +- Add `exc_info=True` to these 14 `logger.error` calls that caught a real exception without it, per `.claude/conventions/error-handling.md` ("`exc_info=True` on any path that caught a real exception"). +- Files (change): `backend/app/crud/evaluations/core.py:106,239` and `backend/app/crud/evaluations/embeddings.py:105,111,120,163,174,232,296,331,387` +- Add the missing `[function_name]` bracket prefix required by CLAUDE.md. 12 lines. +- Log levels already match the fault table at the status-code sites this plan touches; verify rather than assume when editing each one. The two credentials delete sites are the exception and are handled in step 10. +- Depends on: steps 3 through 11 (same lines, edit once) + +### 13. Docs: wiki, swagger, and the follow-up issue +- Files: `docs/wiki/cross-cutting/exceptions.md` (change), `backend/app/api/docs/evaluation/improve_prompt.md`, `backend/app/api/docs/evaluation/create_evaluation.md`, `backend/app/api/docs/collections/update.md`, `backend/app/api/docs/onboarding/onboarding.md` (change if their stated codes went stale) +- Add an "Domain exceptions" section to `exceptions.md` pointing at `core/exceptions.py`, listing the taxonomy table above as the source of truth, and noting that the `KaapiError` handler is registered ahead of the generic one. +- The touched module pages (`llm-call.md`, `evaluations.md`, `knowledge-base.md`, `tenancy.md`, `platform.md`) hold no status codes, so they need no edit. +- Open a follow-up issue listing the out-of-scope sites found during the survey: `app/api/routes/auth.py:65,306,357`, `app/api/routes/fine_tuning.py:269`, `app/crud/api_key.py:97`, `app/crud/assessment/dataset.py:72`, `app/services/assessment/dataset.py:293`, `app/services/assessment/utils/export.py:329`, `app/services/response/jobs.py:42`. +- Depends on: steps 1 through 12 + +## Tests + +Existing assertions that encode the current wrong codes and must flip: + +| Test | Now | After | +|---|---|---| +| `app/tests/api/routes/test_evaluation.py:477` | 500 | per step 3/4 | +| `app/tests/api/routes/test_evaluation_fast.py:1271` | 500 | 404 (missing dataset) | +| `app/tests/api/routes/test_tts_evaluation.py:629` | 500 | 503 | +| `app/tests/api/routes/test_improve_prompt.py:517` | 500 | 503 | +| `app/tests/services/llm/test_jobs.py:101,2012` | 500 | 503 | +| `app/tests/services/stt_evaluations/test_audio.py:188` | 500 | 502 | +| `app/tests/services/guardrails/test_jobs.py:159` | 500 | 503 | +| `app/tests/api/routes/test_creds.py:260` (plus its docstring at line 240, which says "fails with 400") | 400 | 409 | + +Out of scope and untouched: `app/tests/api/test_auth.py:49,313,412` and `app/tests/assessment/test_dataset.py:304`. + +Assertions deliberately left alone: `app/tests/crud/test_credentials.py:273` ("Unsupported provider") and `:332` ("Missing required fields for langfuse") stay at 400, matching the Open Questions decision below. + +New coverage: +- `app/tests/core/test_exceptions.py` (new): each `KaapiError` subclass maps to its status code through the registered handler, and `generic_error_handler` no longer returns `str(exc)` in the body. +- Evaluations: Langfuse dataset-not-found returns 404 and Langfuse unreachable returns 502, on both the batch path and the fast path. Mock the Langfuse client boundary (existing patch target `app.services.evaluations.fast.fetch_dataset_items` and `app.crud.evaluations.batch.fetch_dataset_items`, already used at `test_evaluation_fast.py:322` and `test_evaluation.py:1036`). +- Collections: deleting an already-deleted collection returns 409; duplicate name still returns 409 after the handler takes over from the manual translation. +- Documents: unsupported source format returns 422; unsupported transform pair still returns 400. +- Config: creating a config whose name already exists returns 409 rather than 500; a `ConfigBlob` that fails validation during partial-version merge returns 422 with the same body shape the request boundary produces. +- Credentials: duplicate provider credential returns 409; deleting a credential that does not exist returns 404; a KMS failure during create returns 502 and the response body does not contain the AWS error text. +- Onboarding: a KMS failure during onboarding returns 502; a concurrent onboard hitting `IntegrityError` at commit returns 409, and no partial org/project/user rows survive. + +HTTP boundaries mocked: Langfuse client, OpenAI and Gemini SDK clients, object storage (`core/cloud/storage.py`), the AWS KMS client (`get_kms_client` in `core/security.py`), and the Celery `.delay`/`apply_async` call for the 503 paths. + +`app/tests/services/credentials/` already exists and is the home for the KMS-failure cases. + +## Open Questions + +- **Documents 422 versus 400 for an unsupported file format (step 7).** Two live conventions disagree: `services/documents/helpers.py:64` returns 400, `services/evaluations/validators.py:84-97` returns 422 for the same class of problem. This plan picks 422 on the reading that the uploaded file is the unparseable payload, which also makes evaluations the larger surface that does not have to change. Reverse it if the reviewer reads an unsupported extension as a value problem rather than a shape problem, in which case `validators.py` moves to 400 instead and evaluations tests change. +- **503 versus 500 for a failed Celery enqueue (steps 4, 5, 8).** RabbitMQ being unable to accept a job is an availability problem the caller can retry, which 503 signals and 500 does not. Assumed, not stated in the issue. +- **Config immutability violations: 400 or 409 (step 9).** `version.py:201`, `version.py:330`, and `version.py:340` reject a change to an immutable field (`type`, `provider`) with 400. `.claude/conventions/error-handling.md` describes 409 as "request conflicts with current resource state", which is literally what this is. This plan leaves them at 400 to keep the config diff to the two defects that have no defence. Flip all three to 409 if the reviewer wants the taxonomy applied strictly. +- **Credentials validation: 400 or 422 (step 10).** `validate_provider_credentials` rejects both an unknown provider ("Unsupported provider", a value problem, 400) and a well-formed request missing provider-specific keys ("Missing required fields for langfuse", arguably a shape problem, 422) with the same 400. This plan keeps both at 400 rather than splitting one helper's output across two codes. Splitting them is defensible and would change `app/tests/crud/test_credentials.py:332`. +- **Onboarding bypasses credential validation.** `crud/onboarding.py:118-133` builds `Credential` rows inline instead of calling `set_creds_for_org`, so it never runs `validate_provider_credentials`. The same bad payload gets 400 through `POST /credentials` and is silently stored through `POST /onboard`. This is an error-message consistency defect, but fixing it changes onboarding behaviour rather than its status codes, so it is called out here rather than planned. Confirm whether it belongs in this PR or its own. +- **Frontend coordination is out of scope by decision.** Eleven status codes now change on endpoints the console calls, including `POST /credentials` and `POST /onboard`. The `APIResponse` body shape is untouched, so only code-branching logic can break. Worth a line in the PR description even though no investigation is planned.