diff --git a/.env.example b/.env.example index bb2a5da..9f71f53 100644 --- a/.env.example +++ b/.env.example @@ -53,6 +53,14 @@ ANTHROPIC_API_KEY=change_me AUTO_BI_ANTHROPIC_MODEL=claude-sonnet-5 AUTO_BI_ANTHROPIC_MAX_TOKENS=16000 +# Direct Mistral API (opt-in alternative: AUTO_BI_LLM_PROVIDER=mistral) +MISTRAL_API_KEY=change_me +# Optional AUTO_BI-prefixed alias: +# AUTO_BI_MISTRAL_API_KEY=change_me +AUTO_BI_MISTRAL_MODEL=mistral-large-latest +AUTO_BI_MISTRAL_URL=https://api.mistral.ai +AUTO_BI_MISTRAL_MAX_TOKENS=16000 + # GraceKelly LLM service (local, opt-in alternative: AUTO_BI_LLM_PROVIDER=gracekelly) AUTO_BI_GRACEKELLY_URL=http://127.0.0.1:8011 AUTO_BI_GRACEKELLY_MODEL=claude-sonnet-5 @@ -120,7 +128,7 @@ AUTO_BI_LLM_BUDGET_DAY_MAX_SECONDS=0 AUTO_BI_LLM_BUDGET_DAY_MAX_COST_USD=0 # cost price table, USD per 1000 tokens, "model:in/out,..." (example rates — set yours; # used only when a *_MAX_COST_USD limit above is set) -AUTO_BI_LLM_BUDGET_PRICES=claude-opus-4-8:0.005/0.025,claude-sonnet-5:0.003/0.015,claude-sonnet-4-6:0.003/0.015,claude-haiku-4-5:0.001/0.005 +AUTO_BI_LLM_BUDGET_PRICES=claude-opus-4-8:0.005/0.025,claude-sonnet-5:0.003/0.015,claude-sonnet-4-6:0.003/0.015,claude-haiku-4-5:0.001/0.005,mistral-large-latest:0.0005/0.0015 # Fail-closed remote bind (P0-3): required to serve on non-loopback with auth off # and without DEMO_AUTO_ONLY. Prefer AUTH_ENABLED=true on the public internet. AUTO_BI_ALLOW_INSECURE_REMOTE=false diff --git a/.github/workflows/eval-live-sentinel.yml b/.github/workflows/eval-live-sentinel.yml index 4ab76b4..7dbcc74 100644 --- a/.github/workflows/eval-live-sentinel.yml +++ b/.github/workflows/eval-live-sentinel.yml @@ -59,13 +59,15 @@ jobs: id: gate env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + AUTO_BI_MISTRAL_API_KEY: ${{ secrets.AUTO_BI_MISTRAL_API_KEY }} AUTO_BI_GRACEKELLY_URL: ${{ secrets.AUTO_BI_GRACEKELLY_URL }} run: | - if [ -n "${ANTHROPIC_API_KEY}" ] || [ -n "${AUTO_BI_GRACEKELLY_URL}" ]; then + if [ -n "${MISTRAL_API_KEY}" ] || [ -n "${AUTO_BI_MISTRAL_API_KEY}" ] || [ -n "${ANTHROPIC_API_KEY}" ] || [ -n "${AUTO_BI_GRACEKELLY_URL}" ]; then echo "run=true" >> "$GITHUB_OUTPUT" else echo "run=false" >> "$GITHUB_OUTPUT" - echo "No ANTHROPIC_API_KEY / AUTO_BI_GRACEKELLY_URL secret — sentinel skipped (offline replay still gates quality)." + echo "No Mistral / Anthropic / GraceKelly credential secret — sentinel skipped (offline replay still gates quality)." fi - name: Install uv @@ -84,11 +86,22 @@ jobs: if: steps.gate.outputs.run == 'true' env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + AUTO_BI_MISTRAL_API_KEY: ${{ secrets.AUTO_BI_MISTRAL_API_KEY }} AUTO_BI_LLM_PROVIDER: ${{ secrets.AUTO_BI_LLM_PROVIDER }} AUTO_BI_GRACEKELLY_URL: ${{ secrets.AUTO_BI_GRACEKELLY_URL }} AUTO_BI_GRACEKELLY_MODEL: ${{ secrets.AUTO_BI_GRACEKELLY_MODEL }} + AUTO_BI_MISTRAL_MODEL: ${{ secrets.AUTO_BI_MISTRAL_MODEL }} run: | - export AUTO_BI_LLM_PROVIDER="${AUTO_BI_LLM_PROVIDER:-anthropic}" + if [ -z "${AUTO_BI_LLM_PROVIDER}" ]; then + if [ -n "${MISTRAL_API_KEY}" ] || [ -n "${AUTO_BI_MISTRAL_API_KEY}" ]; then + export AUTO_BI_LLM_PROVIDER="mistral" + elif [ -n "${ANTHROPIC_API_KEY}" ]; then + export AUTO_BI_LLM_PROVIDER="anthropic" + else + export AUTO_BI_LLM_PROVIDER="gracekelly" + fi + fi uv run auto_bi eval \ --suite golden \ --llm-mode live \ diff --git a/CHANGELOG.md b/CHANGELOG.md index deb83cd..dd8178c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ ## [Unreleased] +### Added + +- **Direct Mistral provider** — `AUTO_BI_LLM_PROVIDER=mistral` routes through + Mistral Chat Completions with standard `MISTRAL_API_KEY` (or the + `AUTO_BI_MISTRAL_API_KEY` alias), the shared structured-repair and budget + hooks, token-usage logging, safe HTTP errors, and live-sentinel secret + routing. Default model: `mistral-large-latest`. + +- **Local BYOK runbook** — `docs/LOCAL_BYOK.md`: первый локальный запуск после + clone со своим `ANTHROPIC_API_KEY` (ClickHouse + Superset через Compose, + `uv run auto_bi serve`, health/ready, остановка и troubleshooting). + ## [0.5.0] - 2026-07-29 ### Fixed diff --git a/README.md b/README.md index 638d537..e3fd523 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,15 @@ Агент «запрос → дашборд» поверх DM-слоя DWH. Принимает запрос **текстом, drag&drop-раскладкой полей витрин или авто-обзором витрины** (детерминированный курируемый дашборд без LLM), уточняет детали только при реальных расхождениях с данными, честно предупреждает о не предусмотренных витриной паттернах (engine-aware **Feasibility Advisor** — вплоть до «это запрос на новую витрину»), строит дашборд в выбранной BI и возвращает ссылку. -**Скоуп v1 (RU-рынок, release-gated в CI):** ClickHouse (DM) + Apache Superset (BI). **v2 experimental:** Greengage/Greenplum (offline advisor/golden в CI; live DWH — operator stand) + Yandex DataLens (unit compile offline; live contract Mac-only, не default release gate). Универсальность — в швах (IR, адаптеры), не в имплементации. -**LLM:** прямой Anthropic Messages API (по умолчанию — нужен только `ANTHROPIC_API_KEY`); локальный сервис GraceKelly — документированная опция (`AUTO_BI_LLM_PROVIDER=gracekelly`, см. [USER_GUIDE §6](docs/USER_GUIDE.md#6-конфигурация-переменные-окружения)). +**Скоуп v1 (RU-рынок, release-gated в CI):** ClickHouse (DM) + Apache Superset (BI). **v2 experimental:** Greengage/Greenplum (offline advisor/golden в CI; live DWH — operator stand) + Yandex DataLens (offline compile contracts и повторный live contract **15/15** на Mac; не default release gate). Универсальность — в швах (IR, адаптеры), не в имплементации. +**LLM:** прямой Anthropic Messages API по умолчанию; прямой Mistral Chat Completions (`AUTO_BI_LLM_PROVIDER=mistral`, `MISTRAL_API_KEY`) и локальный сервис GraceKelly — документированные опции (см. [USER_GUIDE §6](docs/USER_GUIDE.md#6-конфигурация-переменные-окружения)). ## Демо -**Живое демо: ** — публичная песочница (Hugging Face Space, один контейнер ClickHouse + Superset + Auto_BI): выберите витрину, соберите авто-обзор и откройте готовый дашборд в Superset без логина. Работает детерминированный путь без LLM; полный текстовый цикл — на видео ниже. Данные синтетические, всё пересоздаётся при рестарте (холодный старт ~3 мин). +**Поддерживаемый путь без стенда:** офлайн golden path +(`uv run python scripts/demo_golden_path.py`) — детерминированный IR/SQL/advisor +без DWH, BI и LLM. Полный локальный запуск со своим API key — +[docs/LOCAL_BYOK.md](docs/LOCAL_BYOK.md). ![Auto_BI — полный цикл: текст → уточнение → спецификация + advisor → сборка → дашборд Superset](docs/screenshots/demo.gif) @@ -23,9 +26,14 @@ ## Статус -**Phase 0–4 + бэклог адекватности дашбордов (B1–B4) закрыты.** Работает end-to-end: текст/поля → spec → валидация → сборка дашборда. v1-стек (ClickHouse + Superset) и v2-стек (Greenplum/Greengage интроспекция + advisor; self-hosted DataLens-адаптер) live-проверены; web UI с двумя режимами ввода, итерациями, Feasibility Advisor, заявками владельцу DM и панелью наблюдаемости. +**Phase 0–4 + бэклог адекватности дашбордов (B1–B4) закрыты.** Работает end-to-end: текст/поля → spec → валидация → сборка дашборда. v1-стек (ClickHouse + Superset) live-проверен на v0.5.0; v2 (Greenplum/Greengage advisor/golden; DataLens) — offline evidence/contracts, а DataLens Mac-only live contract повторно прошёл **15/15** на текущем self-hosted stand 2026-07-29 (experimental / non-default / non-closure; фактический seeded workbook задавался через `AUTO_BI_DATALENS_WORKBOOK_ID`); web UI с двумя режимами ввода, итерациями, Feasibility Advisor, заявками владельцу DM и панелью наблюдаемости. -**Актуальное состояние и residual roadmap** — [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md). История фаз — [docs/PLAN.md](docs/PLAN.md). Полный env inventory (generated) — [docs/ENV_REFERENCE.md](docs/ENV_REFERENCE.md). +Все пять вынесенных external live validations завершены exact evidence: +DataLens 15/15, direct Mistral sentinel 3/3, protected-tag rejection, +intentional Trivy failure before promotion и process-restart reconciliation. +Активной audit work не осталось. + +**Актуальное состояние и closure evidence** — [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md). История фаз — [docs/PLAN.md](docs/PLAN.md). Полный env inventory (generated) — [docs/ENV_REFERENCE.md](docs/ENV_REFERENCE.md). ## Чем отличается @@ -54,7 +62,7 @@ flowchart LR Установка, команды CLI, web UI, конфигурация — [docs/USER_GUIDE.md](docs/USER_GUIDE.md). Подключение новой витрины DWH за ≤ 1 ч — [docs/ONBOARDING_DWH.md](docs/ONBOARDING_DWH.md). -Local-first — три ступени: +Local-first — два поддерживаемых пути: 1. **Офлайн golden path** — без DWH, BI, LLM и API-ключа: @@ -62,9 +70,7 @@ Local-first — три ступени: uv run python scripts/demo_golden_path.py ``` -2. **HF Space** — детерминированный auto-only, пользовательский ключ не нужен; текстовый режим там намеренно недоступен (см. «Демо» выше). - -3. **Полный локальный путь.** Скопируйте `.env.example` в `.env` (`cp .env.example .env`; PowerShell: `Copy-Item .env.example .env`). Задайте свой `ANTHROPIC_API_KEY` **или** `AUTO_BI_LLM_PROVIDER=gracekelly` и `AUTO_BI_GRACEKELLY_URL`. Для DWH/BI — `AUTO_BI_CH_HOST`, `AUTO_BI_CH_PASSWORD`, `AUTO_BI_SUPERSET_URL`, `AUTO_BI_SUPERSET_PASSWORD` (полный inventory — [docs/ENV_REFERENCE.md](docs/ENV_REFERENCE.md)). `docker compose up -d` поднимает **только ClickHouse и Superset**, не Auto_BI; агент локально: `auto_bi serve` → http://127.0.0.1:8200. +2. **Полный локальный путь.** Пошаговый Anthropic-пример — [docs/LOCAL_BYOK.md](docs/LOCAL_BYOK.md). Скопируйте `.env.example` в `.env` (`cp .env.example .env`; PowerShell: `Copy-Item .env.example .env`). Задайте свой `ANTHROPIC_API_KEY`; либо `AUTO_BI_LLM_PROVIDER=mistral` + `MISTRAL_API_KEY`; либо `AUTO_BI_LLM_PROVIDER=gracekelly` + `AUTO_BI_GRACEKELLY_URL`. Для DWH/BI — `AUTO_BI_CH_HOST`, `AUTO_BI_CH_PASSWORD`, `AUTO_BI_SUPERSET_URL`, `AUTO_BI_SUPERSET_PASSWORD` (полный inventory — [docs/ENV_REFERENCE.md](docs/ENV_REFERENCE.md)). `docker compose up -d` поднимает **только ClickHouse и Superset**, не Auto_BI; агент локально: `auto_bi serve` → http://127.0.0.1:8200. ```bash pip install autobi-agent # или pip install -e . из корня репозитория @@ -90,6 +96,7 @@ uv run python scripts/demo_golden_path.py | Файл | Что внутри | |---|---| | [docs/USER_GUIDE.md](docs/USER_GUIDE.md) | Руководство пользователя: установка, команды CLI, web UI, два режима ввода, advisor, наблюдаемость, конфигурация | +| [docs/LOCAL_BYOK.md](docs/LOCAL_BYOK.md) | Первый локальный запуск со своим Anthropic API key: clone, `.env`, Compose (CH+Superset), `uv run auto_bi serve`, health/ready | | [docs/ONBOARDING_DWH.md](docs/ONBOARDING_DWH.md) | Подключение нового DWH за ≤ 1 ч: доступы, `.env`, интроспекция, обогащение, проверка (ClickHouse + Greenplum) | | [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | Архитектура: скоуп, IR-first, семантическая модель с физическим слоем, агент, Feasibility Advisor, адаптеры, LLM-слой, решения D1–D10, риски | | [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) | Деплой в проде: workers=1, reverse-proxy/TLS, готовность, docker-compose, бэкап SQLite, ротация логов, чеклист секретов | diff --git a/auto_bi/cli.py b/auto_bi/cli.py index 504c7f2..0dbfac2 100644 --- a/auto_bi/cli.py +++ b/auto_bi/cli.py @@ -726,11 +726,13 @@ def bi_healthcheck() -> AdapterHealth: return probe_health(adapter_for, TargetBI.SUPERSET) def llm_healthcheck() -> AdapterHealth: - if settings.llm_provider.strip().lower() != "gracekelly": - # Anthropic is a hosted API with no separate process to be "up/down" locally, - # and an actual completion call would cost tokens on every readiness probe — - # report configured-and-constructible (already proven by make_llm below). - return AdapterHealth(ok=True, message="anthropic: no live check (avoids token cost)") + provider = settings.llm_provider.strip().lower() + if provider != "gracekelly": + # Hosted providers (anthropic, mistral, …) have no separate process to be + # "up/down" locally, and an actual completion call would cost tokens on every + # readiness probe — report configured-and-constructible (already proven by + # make_llm below). + return AdapterHealth(ok=True, message=f"{provider}: no live check (avoids token cost)") import httpx try: @@ -1010,9 +1012,11 @@ def _render(title: str, report: EvalReport) -> None: store = Store(settings.store_path) live_llm = make_llm(settings, store=store) provider = settings.llm_provider.strip().lower() - model_id = ( - settings.gracekelly_model if provider == "gracekelly" else settings.anthropic_model - ) + model_id = { + "gracekelly": settings.gracekelly_model, + "anthropic": settings.anthropic_model, + "mistral": settings.mistral_model, + }.get(provider, provider) provider_detail = ( f"{settings.gracekelly_url}, {model_id}" if provider == "gracekelly" else model_id ) diff --git a/auto_bi/config.py b/auto_bi/config.py index f90e062..b1d8a19 100644 --- a/auto_bi/config.py +++ b/auto_bi/config.py @@ -5,7 +5,7 @@ from functools import lru_cache from logging import Logger -from pydantic import Field +from pydantic import AliasChoices, Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -154,14 +154,13 @@ class Settings(BaseSettings): ), ) - # LLM provider seam (llm/factory.py): "anthropic" (default — direct Anthropic Messages - # API, works out of the box with just an API key) or "gracekelly" (local orchestration - # service, documented opt-in — ARCHITECTURE §3.6). + # LLM provider seam (llm/factory.py): direct Anthropic (default), direct Mistral, + # or GraceKelly (local orchestration service). llm_provider: str = Field( default="anthropic", description=( - 'LLM backend selector: "anthropic" for direct API or "gracekelly" for the local' - " orchestration service." + 'LLM backend selector: "anthropic" or "mistral" for direct API access, or' + ' "gracekelly" for the local orchestration service.' ), ) @@ -202,6 +201,33 @@ class Settings(BaseSettings): ), ) + # Direct Mistral chat-completions API. The client also accepts the standard + # MISTRAL_API_KEY process env when this AUTO_BI-prefixed field is empty. + mistral_api_key: str = Field( + default="", + validation_alias=AliasChoices( + "mistral_api_key", + "AUTO_BI_MISTRAL_API_KEY", + "MISTRAL_API_KEY", + ), + description=( + "Mistral API key for direct calls; empty lets the client fall back to" + " MISTRAL_API_KEY." + ), + ) + mistral_model: str = Field( + default="mistral-large-latest", + description="Mistral model id used when llm_provider is mistral.", + ) + mistral_url: str = Field( + default="https://api.mistral.ai", + description="Mistral API base URL; the client appends /v1/chat/completions.", + ) + mistral_max_tokens: int = Field( + default=16000, + description="Maximum output tokens requested from direct Mistral chat completions.", + ) + # plan_sol step 2 / audit P0-2: DWH values (top-N) leave the process only on # explicit opt-in. Default false — clean install never sends samples to an # external LLM. Set true only for public/internal classes after classification. @@ -380,7 +406,7 @@ class Settings(BaseSettings): ), ) # cost price table (USD per 1000 tokens), "model:in/out,...". List prices as of - # 2026-07-18; override for your provider contract. Used only when a *_max_cost_usd + # 2026-07-29; override for your provider contract. Used only when a *_max_cost_usd # limit is set — an unlisted model prices at 0, so add yours before relying on a cap. # Sonnet 5 carries a lower introductory rate through 2026-08-31; the table keeps the # standard rate so the guard errs toward over-estimating spend, not under. @@ -389,7 +415,8 @@ class Settings(BaseSettings): "claude-opus-4-8:0.005/0.025," "claude-sonnet-5:0.003/0.015," "claude-sonnet-4-6:0.003/0.015," - "claude-haiku-4-5:0.001/0.005" + "claude-haiku-4-5:0.001/0.005," + "mistral-large-latest:0.0005/0.0015" ), description=( "USD-per-1k-token price table as model:in/out pairs; unlisted models price at 0 until" @@ -533,8 +560,8 @@ def unknown_env_settings(environ: Mapping[str, str] | None = None) -> list[str]: `extra="ignore"` silently drops typos — `AUTO_BI_AUTH_ENABLE=true` leaves auth OFF with no trace. `serve` reports every returned name as a warning so a misspelled security flag is visible in the log instead of silently inert. Compares against - `Settings.model_fields` plus any explicit string validation_alias (none today; - AliasChoices would need unpacking if ever introduced). + `Settings.model_fields` plus any explicit string validation_alias. Field-derived + `AUTO_BI_*` names remain recognised when a field also accepts non-prefixed aliases. """ env = os.environ if environ is None else environ prefix = str(Settings.model_config.get("env_prefix", "")).upper() diff --git a/auto_bi/llm/_structured.py b/auto_bi/llm/_structured.py index 3822569..f51a9bf 100644 --- a/auto_bi/llm/_structured.py +++ b/auto_bi/llm/_structured.py @@ -1,6 +1,6 @@ """Shared structured-output machinery for LLM clients (transport-agnostic). -Both GraceKellyClient and AnthropicClient turn text-in/text-out completions into +GraceKellyClient, AnthropicClient and MistralClient turn text-in/text-out completions into schema-validated objects via the SAME JSON-extraction + repair loop (invariant 1: the LLM emits only DashboardSpec/etc. as JSON; we parse and validate it here, never trusting native formats). Keeping this here means the two clients differ only in @@ -134,8 +134,9 @@ def append_llm_log( """Append one LLM-call record to the jsonl log and (if present) the durable Store. The prompt itself is NEVER logged — only its sha256 prefix and length (security §4). - `input_tokens`/`output_tokens` are real usage from providers that report it (Anthropic); - None where the provider returns no usage (GraceKelly) or the call failed before a response. + `input_tokens`/`output_tokens` are real usage from providers that report it + (Anthropic/Mistral); None where the provider returns no usage (GraceKelly) or + the call failed before a response. Logging is best-effort: a failure here must never kill the pipeline. """ prompt_sha256 = hashlib.sha256(prompt.encode()).hexdigest()[:16] diff --git a/auto_bi/llm/budget.py b/auto_bi/llm/budget.py index 0624c07..0496b6b 100644 --- a/auto_bi/llm/budget.py +++ b/auto_bi/llm/budget.py @@ -17,7 +17,8 @@ Usage is read back from the existing `llm_calls` ledger (Store), which already records every attempt with tokens/latency (`Store.log_llm_call`), so budgets survive across requests and restarts without a parallel table. Tokens are real where the provider -reports them (Anthropic) and char-estimated (chars / 4) where it does not (GraceKelly). +reports them (Anthropic/Mistral) and char-estimated (chars / 4) where it does not +(GraceKelly). Fail closed: `check` raises `BudgetExceeded` BEFORE issuing the call that would cross a limit, naming the exceeded dimension. Opt-in, off by default (AUTO_BI_LLM_BUDGET_ENABLED), diff --git a/auto_bi/llm/factory.py b/auto_bi/llm/factory.py index c4f787f..a8d498b 100644 --- a/auto_bi/llm/factory.py +++ b/auto_bi/llm/factory.py @@ -29,8 +29,13 @@ def make_llm(settings: Settings, store: Store | None = None) -> LLMClient: from auto_bi.llm.anthropic import AnthropicClient return AnthropicClient(settings, store=store, budget=budget) + if provider == "mistral": + from auto_bi.llm.mistral import MistralClient + + return MistralClient(settings, store=store, budget=budget) raise ValueError( - f"unknown AUTO_BI_LLM_PROVIDER {settings.llm_provider!r} (use 'gracekelly' or 'anthropic')" + f"unknown AUTO_BI_LLM_PROVIDER {settings.llm_provider!r} " + "(use 'gracekelly', 'anthropic', or 'mistral')" ) diff --git a/auto_bi/llm/fixture.py b/auto_bi/llm/fixture.py index a3b8a3a..fd4aa28 100644 --- a/auto_bi/llm/fixture.py +++ b/auto_bi/llm/fixture.py @@ -27,7 +27,7 @@ "format_version": 2, "template_version": "<16 hex of prompt templates>", "schema_version": "<16 hex of IR JSON schemas>", - "provider": "anthropic|gracekelly|fixture-refresh|...", + "provider": "anthropic|mistral|gracekelly|fixture-refresh|...", "model_id": "...", "calls": [ {"step": "...", "schema": "...", "prompt_sha256": "...", "response": {...}} diff --git a/auto_bi/llm/mistral.py b/auto_bi/llm/mistral.py new file mode 100644 index 0000000..5abff07 --- /dev/null +++ b/auto_bi/llm/mistral.py @@ -0,0 +1,211 @@ +"""Direct Mistral chat-completions client behind the shared LLMClient seam.""" + +from __future__ import annotations + +import os +import time +from collections.abc import Callable +from pathlib import Path +from typing import TYPE_CHECKING, Any, TypeVar, cast + +import httpx +from pydantic import BaseModel + +from auto_bi.config import Settings +from auto_bi.llm._structured import append_llm_log, complete_with_repair +from auto_bi.llm.base import LLMError + +if TYPE_CHECKING: + from auto_bi.llm.budget import LLMBudget + from auto_bi.store import Store + +T = TypeVar("T", bound=BaseModel) + +_MAX_RATE_LIMIT_RETRIES = 4 +_BACKOFF_BASE_SECONDS = 2.0 +_BACKOFF_CAP_SECONDS = 30.0 + + +def _extract_usage(data: dict[str, Any]) -> tuple[int | None, int | None]: + usage = data.get("usage") + if not isinstance(usage, dict): + return None, None + prompt = usage.get("prompt_tokens") + completion = usage.get("completion_tokens") + return ( + prompt if isinstance(prompt, int) else None, + completion if isinstance(completion, int) else None, + ) + + +def _extract_choice(data: dict[str, Any]) -> tuple[str, str]: + """Return (text, finish_reason) for string or current chunked Mistral content.""" + choices = data.get("choices") + if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): + return "", "unknown" + first = choices[0] + finish_reason = first.get("finish_reason") + status = finish_reason if isinstance(finish_reason, str) and finish_reason else "unknown" + message = first.get("message") + if not isinstance(message, dict): + return "", status + content = message.get("content") + if isinstance(content, str): + return content, status + if not isinstance(content, list): + return "", status + parts: list[str] = [] + for chunk in content: + if isinstance(chunk, dict) and isinstance(chunk.get("text"), str): + parts.append(chunk["text"]) + return "".join(parts), status + + +def _retry_after_seconds(response: httpx.Response, attempt: int) -> float: + header = response.headers.get("Retry-After") + if header: + try: + return min(_BACKOFF_CAP_SECONDS, max(0.0, float(header))) + except ValueError: + pass + return min(_BACKOFF_CAP_SECONDS, _BACKOFF_BASE_SECONDS * (2.0**attempt)) + + +class MistralClient: + """Sync text-in/JSON-out client for Mistral's `/v1/chat/completions` API.""" + + def __init__( + self, + settings: Settings, + http: httpx.Client | None = None, + log_path: str | Path = "logs/llm_calls.jsonl", + store: Store | None = None, + budget: LLMBudget | None = None, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self._settings = settings + api_key = settings.mistral_api_key or os.environ.get("MISTRAL_API_KEY", "") + if not api_key: + raise LLMError( + "Mistral API key is not configured; set MISTRAL_API_KEY or " + "AUTO_BI_MISTRAL_API_KEY" + ) + self._http = http or httpx.Client( + base_url=settings.mistral_url, + timeout=httpx.Timeout(300.0, connect=10.0), + transport=httpx.HTTPTransport(retries=2), + headers={"Authorization": f"Bearer {api_key}"}, + ) + self._log_path = Path(log_path) + self._store = store + self._budget = budget + self._sleep = sleep + + def close(self) -> None: + """Release the owned or injected HTTP pool.""" + self._http.close() + + def complete( + self, + prompt: str, + schema: type[T], + *, + reasoning: bool = False, + session_id: str | None = None, + step: str = "", + ) -> T: + return cast( + T, + complete_with_repair( + lambda value: self._call( + value, + reasoning=reasoning, + session_id=session_id, + step=step, + ), + prompt, + schema, + on_attempt=self._budget_hook(session_id), + ), + ) + + def _budget_hook(self, session_id: str | None) -> Callable[[], None] | None: + if self._budget is None: + return None + budget = self._budget + model = self._settings.mistral_model + return lambda: budget.check(session_id=session_id, model=model) + + def _post(self, payload: dict[str, Any]) -> dict[str, Any]: + for attempt in range(1 + _MAX_RATE_LIMIT_RETRIES): + response = self._http.post("/v1/chat/completions", json=payload) + if response.status_code == 429: + if attempt < _MAX_RATE_LIMIT_RETRIES: + self._sleep(_retry_after_seconds(response, attempt)) + continue + raise LLMError( + "Mistral API still rate-limited after " f"{_MAX_RATE_LIMIT_RETRIES} retries" + ) + if response.status_code >= 400: + # Provider bodies can echo request/auth diagnostics; never expose them. + raise LLMError(f"Mistral API HTTP {response.status_code}") + try: + data = response.json() + except ValueError as exc: + raise LLMError("Mistral API returned invalid JSON") from exc + if not isinstance(data, dict): + raise LLMError("Mistral API returned a non-object response") + return data + raise LLMError("Mistral API retry loop exited unexpectedly") + + def _call( + self, + prompt: str, + *, + reasoning: bool, + session_id: str | None, + step: str, + ) -> str: + payload = { + "model": self._settings.mistral_model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": self._settings.mistral_max_tokens, + "temperature": 0, + } + started = time.monotonic() + status = "transport_error" + completion_chars = 0 + input_tokens: int | None = None + output_tokens: int | None = None + try: + data = self._post(payload) + input_tokens, output_tokens = _extract_usage(data) + text, finish_reason = _extract_choice(data) + status = ( + "completed" + if finish_reason in {"stop", "length", "model_length"} + else finish_reason + ) + completion_chars = len(text) + if not text: + raise LLMError(f"Mistral returned no text (finish_reason={finish_reason})") + return text + except LLMError: + raise + except httpx.HTTPError as exc: + raise LLMError(f"Mistral transport error: {exc}") from exc + finally: + append_llm_log( + self._log_path, + self._store, + model=self._settings.mistral_model, + prompt=prompt, + reasoning=reasoning, + status=status, + latency_ms=round((time.monotonic() - started) * 1000), + session_id=session_id, + step=step, + completion_chars=completion_chars, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) diff --git a/auto_bi/store/db.py b/auto_bi/store/db.py index 162c1fe..b5957ab 100644 --- a/auto_bi/store/db.py +++ b/auto_bi/store/db.py @@ -11,10 +11,10 @@ of agent steps (grounding/propose/advisor/approve) and build phases. Schema v5 (token accounting, E2): `llm_calls` gained nullable `input_tokens` / -`output_tokens`. The Anthropic Messages API returns `usage.input_tokens/output_tokens`, -so calls on that provider carry real tokens; GraceKelly reports no usage and a transport -error has no response, so those rows stay NULL (NULL = "no usage reported", distinct from -a real zero — `completion_chars` remains the universal size proxy for every call). +`output_tokens`. Anthropic and Mistral return input/output token usage, so calls on those +providers carry real tokens; GraceKelly reports no usage and a transport error has no +response, so those rows stay NULL (NULL = "no usage reported", distinct from a real zero +— `completion_chars` remains the universal size proxy for every call). Schema v6 (B-4 hardening): `auth_tokens.token` now stores sha256(raw token) hex, not the raw bearer token — a stolen SQLite file no longer yields live sessions directly. The @@ -183,7 +183,7 @@ def _encode_spec_snapshot(spec_json: dict[str, Any]) -> tuple[str, str]: step TEXT NOT NULL DEFAULT '', completion_chars INTEGER NOT NULL DEFAULT 0, input_tokens INTEGER, -- NULL = provider reported no usage (GraceKelly / transport error) - output_tokens INTEGER -- real tokens only where the provider returns usage (Anthropic) + output_tokens INTEGER -- real tokens where the provider returns usage (Anthropic/Mistral) ); -- llm/budget.py reads this ledger per session and per rolling window on every provider -- round-trip; index the two scope keys (created via always-run CREATE IF NOT EXISTS, no @@ -309,7 +309,7 @@ def _migrate(self) -> None: # v4: dm_change_requests carries the advisor's concrete fix artifact (DDL) self._add_column("dm_change_requests", "remediation", "TEXT NOT NULL DEFAULT ''") if version < 5: - # v5: real token usage on providers that report it (Anthropic); nullable so + # v5: real token usage on providers that report it (Anthropic/Mistral); nullable so # legacy/GraceKelly rows stay NULL rather than a misleading zero self._add_column("llm_calls", "input_tokens", "INTEGER") self._add_column("llm_calls", "output_tokens", "INTEGER") @@ -1019,10 +1019,10 @@ def llm_usage_summary( ) -> dict[str, Any]: """Aggregates for the LLM-usage dashboard. Char volumes are a universal size proxy (every call has them). Real `input_tokens`/`output_tokens` are summed - NULL-ignoring — they are populated only on providers that report usage (Anthropic); - GraceKelly reports none, so its rows stay NULL. `token_calls` counts the rows that - carry real tokens, so callers can show token figures only when they exist rather - than presenting a NULL-driven 0 as if it were measured. + NULL-ignoring — they are populated only on providers that report usage + (Anthropic/Mistral); GraceKelly reports none, so its rows stay NULL. `token_calls` + counts the rows that carry real tokens, so callers can show token figures only when + they exist rather than presenting a NULL-driven 0 as if it were measured. `owner` (P1-4): when set, only calls whose session is owned by that username — used for non-admin observability so a user never sees foreign spend. @@ -1082,9 +1082,10 @@ def _llm_usage(self, where: str, params: tuple[Any, ...]) -> dict[str, Any]: Returns `calls`, `latency_ms`, total estimated `tokens`, and a per-`model` breakdown so the enforcer can price cost. Tokens are the provider's real usage - where reported (Anthropic), else char-estimated (chars / 4) so a token budget - still bites on GraceKelly, which reports none. Every attempt is counted (a repair - is a distinct row), independent of status — a budget must see all round-trips. + where reported (Anthropic/Mistral), else char-estimated (chars / 4) so a token + budget still bites on GraceKelly, which reports none. Every attempt is counted + (a repair is a distinct row), independent of status — a budget must see all + round-trips. """ rows = self._rows( "SELECT model," diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e275497..0d186c9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -208,7 +208,7 @@ Factory — единственное место выбора конкретно contracts закреплены unit и live contract tests. Crash-recovery contract: [ADR 0002](adr/0002-durable-build-attempt-reconciliation.md). -### 3.6 LLM Layer — Anthropic (default) + GraceKelly (opt-in) +### 3.6 LLM Layer — Anthropic (default) + Mistral / GraceKelly (opt-in) Agent core зависит только от: @@ -216,7 +216,8 @@ Agent core зависит только от: LLMClient.complete(prompt, schema) -> ValidatedModel ``` -Factory выбирает direct Anthropic Messages API или локальный GraceKelly. +Factory выбирает direct Anthropic Messages API, direct Mistral Chat Completions +API или локальный GraceKelly. Structured repair loop общий: JSON extraction, Pydantic validation, bounded feedback retries и durable call logging. @@ -442,7 +443,7 @@ Superset guard проверяет фактически исполняемый so |---|---|---| | D1 | IR-first | Мульти-BI остаётся тестируемым и управляемым | | D2 | Superset primary, DataLens secondary | Два реальных адаптера на общей границе | -| D3 | `LLMClient` с direct Anthropic default | Внешняя установка не зависит от локального orchestration service | +| D3 | `LLMClient` с direct Anthropic default и direct Mistral option | Внешняя установка не зависит от локального orchestration service | | D4 | Semantic model в versioned YAML | Review, diff и ручное владение семантикой | | D5 | LLM думает, код исполняет | Валидация и native payloads остаются детерминированными | | D6 | Python, FastAPI, Pydantic, sqlglot, httpx | Простая state machine не требует тяжёлого agent framework | diff --git a/docs/CURRENT_STATE.md b/docs/CURRENT_STATE.md index c0766f9..db724ca 100644 --- a/docs/CURRENT_STATE.md +++ b/docs/CURRENT_STATE.md @@ -12,11 +12,13 @@ **Версия пакета:** см. `auto_bi.__version__` / `pyproject.toml` (ratchet в `tests/test_docs_defaults.py`). -**Режим:** closure candidate (не fully closed). Software release **v0.5.0** -complete на exact `main`/tag SHA -`e78076d2d00ddc1748bf6e22f13cf7cb93fc6515`. Закрываемый scope и residual — -[PROJECT_CLOSURE.md](PROJECT_CLOSURE.md). Полное закрытие ждёт mandatory -external gate: public HF demo sync to closing SHA or decommission. +**Режим:** software release **v0.5.0** complete на exact `main`/tag SHA +`e78076d2d00ddc1748bf6e22f13cf7cb93fc6515`. Локальный/software closure scope +закрыт; пять вынесенных external validations получили exact live evidence +2026-07-29. Активная audit work = **0**. HF полностью исключён из current +project scope и backlog; historical token `owner-de-scoped` сохраняется только +для reconciliation исходных 24 строк. Scope и disposition — +[PROJECT_CLOSURE.md](PROJECT_CLOSURE.md). ## Продукт @@ -27,8 +29,7 @@ external gate: public HF demo sync to closing SHA or decommission. | Greenplum advisor + golden | **offline contract in CI** | advisor GP + golden GP replay | | Greenplum live DWH | experimental | operator stand | | DataLens compile path | offline contracts **passed** | contract suite | -| DataLens live stand | **experimental** / unavailable (Mac-only stand absent) | not default release gate | -| Public HF demo | **out of sync** | health reports `0.4.0`, `demo_auto_only=false`, no capabilities; live assertion fails; v0.5.0 publish dry-run OK; sync blocked (no local `HF_TOKEN`, no repo HF secret/workflow). Gate: sync to closing SHA or decommission | +| DataLens live stand | **`closed`** 2026-07-29: Mac-only self-hosted contract **15/15 passed**; текущий image seed использует workbook `z4wtz6tg5194o`, переданный через supported env override | experimental / non-default / **non-closure** | ## Безопасность и runtime (plan_sol 1–4) @@ -41,6 +42,18 @@ external gate: public HF demo sync to closing SHA or decommission. - GitHub `main` + `v*` tag rulesets **active without bypass**; Dependabot open queue empty after sequential disposition. +- Protected-tag mutation rejection **live-проверен 2026-07-29**: canary + [`v-retag-smoke-20260729`](https://github.com/brownjuly2003-code/Auto_BI/tree/v-retag-smoke-20260729) + создан на remote `main` `13fc855`; попытка force-update на другой remote + object отклонена GitHub с HTTP 422 (`Cannot update this protected ref` / + `Cannot force-push to this tag`), ref остался неизменным; release workflow + для canary не запускался. +- Intentional Trivy-fail-before-promotion **live-проверен** run + [30512999822](https://github.com/brownjuly2003-code/Auto_BI/actions/runs/30512999822): + Trivy отклонил **23** исправимых HIGH/CRITICAL findings в probe-image, + workflow завершился ожидаемым failure, а GHCR `:latest` сохранил digest + `sha256:bff75bcef9d894e86c2be63a584284c02c2b2546425ac11a15e83ad83e4e84f1`; + временная remote branch удалена. - **v0.5.0 external evidence (complete):** - post-merge CI, CodeQL, Gitleaks, Demo image — passed on closing SHA `e78076d2d00ddc1748bf6e22f13cf7cb93fc6515`; @@ -98,7 +111,7 @@ resume / **fields DnD seed**). Post-merge CI on v0.5.0 closing SHA passed. |---|---| | Online SQLite backup + integrity | `Store.backup_to` / `integrity_check`; `scripts/store_backup.py` | | Restore drill | script + `tests/test_store_backup.py` | -| Pre-return BI crash recovery | RR-4 durable attempts + exact Superset/DataLens cleanup ([ADR 0002](adr/0002-durable-build-attempt-reconciliation.md)) | +| Pre-return BI crash recovery | RR-4 durable attempts + exact Superset/DataLens cleanup ([ADR 0002](adr/0002-durable-build-attempt-reconciliation.md)); live DataLens process-death smoke: child exit 97, remote delivery confirmed, discovered/deleted 3/3, session failed | | Offline perf baseline | `tests/test_perf_baseline.py` (soft abs + relative ratios) | | Property / metamorphic | `tests/test_property_quality.py` (guard, validate, normalize, **RBAC**, Superset native-filter scope) | | Cumulative bounded mutation gate | CI targets `auto_bi/agent/sql_guard.py`, `auto_bi/ir/validate.py`, `auto_bi/agent/dataset_plan.py`, `auto_bi/agent/cleanup.py`; weak outcomes rejected; detailed snapshot evidence in [operations/SLO.md](operations/SLO.md) | @@ -115,20 +128,63 @@ container cold start **6341 ms**, PID1 RSS **89.594 MiB**, cgroup memory Evidence: [operations/REAUDIT_plan_sol_23_07_26.md](operations/REAUDIT_plan_sol_23_07_26.md). Offline score **8.8/10** (was 8.2). Finding R1 fixed (SafeError test fake vs `build(spec, ctx)`). -GHA / protected `v0.5.0` tag / package-image publish evidence complete. Remaining -external residual: public HF demo sync to closing SHA or decommission (not -decommissioned; sync blocked by missing HF token/secret/workflow). +GHA / protected `v0.5.0` tag / package-image publish evidence complete. Scope, +live evidence и final accounting закрыты; excluded external demo paths не +являются residual или current product path. ## Closure disposition -Прежний open-ended residual больше не является активным backlog. Решения -`closed` / `budget-gated` и split completed vs remaining external gates — -в [PROJECT_CLOSURE.md](PROJECT_CLOSURE.md). Software release v0.5.0 complete; -до mandatory HF gate статус остаётся `closure candidate`, а не `closed`. -Paid live-LLM canary — budget-gated, **not run** (нет отдельного approved budget). +Прежний open-ended residual больше не является активным local backlog. +Disposition tokens: `closed` / `owner-de-scoped` / `externally-blocked` / +`still-open`; исходная 24-row mapping сохранена в root +`plan_audit_closure_29_07.md`, а текущая disposition зафиксирована в +[PROJECT_CLOSURE.md](PROJECT_CLOSURE.md): **closed 21 · excluded from scope 3 +(historical token `owner-de-scoped`) · externally-blocked 0 · still-open 0**. +Software release v0.5.0 complete; локально actionable строк **нет**. Три +excluded rows — не remaining, pending, blocked или next work. Exact live/manual +checks without durable run evidence are **not** labelled `closed`. + +External validation re-audit (**completed with real runs; not software-release +blockers**): + +- DataLens live — `closed`: Mac process-table exhaustion устранён после + owner-authorized остановки runaway `~/atv2/main.py`; self-hosted stand поднят + read-only к demo-DM. Первый contract выявил только response-layout drift + (**12/15**); evidence-backed dual-layout assertions сохранили behavioral + проверки, финальный contract — **15/15 passed in 51.39 s**. Current image seed + создал workbook `z4wtz6tg5194o`; historical default `ra7f79yirtumb` для этого + stand не использовался; +- paid live-LLM canary/sentinel — `closed`: уже предоставленный Mistral + credential найден по существующему secure route без чтения/вывода значения. + `mistral-large-latest` прошёл **3/3** sentinel cases, **4** provider calls, + **11,341** input + **1,491** output tokens, estimated cost **$0.007907** при + hard cap **$1.50**; +- protected-tag retag rejection — `closed`: force-update canary + `v-retag-smoke-20260729` отклонён GitHub с HTTP 422, protected ref не + изменился, release workflow не запускался; +- live Trivy-fail before `:latest` — `closed`: run + [30512999822](https://github.com/brownjuly2003-code/Auto_BI/actions/runs/30512999822) + ожидаемо failed после отклонения **23** исправимых HIGH/CRITICAL findings; + `:latest` digest не изменился; +- process restart mid-delivery live smoke — `closed`: реальный child process + завершён через `os._exit(97)` после DataLens delivery и до pipeline commit; + remote dashboard/URL подтверждены, startup reconcile обнаружил и удалил + **3/3** owned artifacts, ownership ledger не был преждевременно записан, + session status = `failed`. + +Historical rows 4, 21 и 23 имеют token `owner-de-scoped` только для сохранения +24-row audit accounting. Они полностью исключены из current scope и не входят +в external validation list или backlog. ## Что не является source of truth -- Root `plan_*.md`, `audit_*.md`, `_NEXT_SESSION.md` — рабочие/внутренние (gitignore + hygiene gate). +- Root `plan_*.md`, `audit_*.md` — execution history, не current backlog. +- `_NEXT_SESSION.md` — актуальный session mirror; при расхождении побеждают + current Git state и эти tracked state docs. - `docs/PLAN.md` — **фаза 0–4 history**, не «что делать завтра». - Корневой `plan.md` — legacy public stub; prefer this file for current status. + +## Локальные presentation artifacts + +`Auto_BI.html` и `pres.html` намеренно поддерживаются и обновляются локально +вместе с evidence. Они остаются untracked: не добавлять в Git и не публиковать. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index db01591..d017f8e 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -172,7 +172,7 @@ AUTO_BI_ALLOW_INSECURE_REMOTE=true uv run auto_bi serve --host 0.0.0.0 --port 82 | Путь (по умолчанию) | Что там | Переменная | |---|---|---| | `data/auto_bi.sqlite` | Store: sessions/specs/builds/llm_calls/dm_change_requests/trace_events/users/auth_tokens | `AUTO_BI_STORE_PATH` | -| `logs/llm_calls.jsonl` | построчный лог метаданных LLM-вызовов (hash промпта/размеры/latency/статус — НЕ сырые промпты; Anthropic/GraceKelly) | — (путь зашит в клиентах, см. §8) | +| `logs/llm_calls.jsonl` | построчный лог метаданных LLM-вызовов (hash промпта/размеры/latency/статус — НЕ сырые промпты; Anthropic/Mistral/GraceKelly) | — (путь зашит в клиентах, см. §8) | Без этих двух volume-маунтов каждый `docker run`/пересоздание контейнера тихо теряет всю историю — не только бэкап (§7) становится бессмысленным, но и наблюдаемость/трейс сессий. @@ -463,7 +463,7 @@ retention. Счётчики процесса (`in_flight`, `dwh_*`) обнуля `logs/llm_calls.jsonl` — построчный append-лог метаданных вызовов LLM: hash промпта, размеры, latency, статус — сырые промпты/ответы туда НЕ пишутся -(`llm/anthropic.py`/`llm/gracekelly.py`, путь зашит по умолчанию, встроенной ротации/лимита +(`llm/anthropic.py`/`llm/mistral.py`/`llm/gracekelly.py`, путь зашит по умолчанию, встроенной ротации/лимита размера нет). Это дубль того, что уже надёжно живёт в Store (`llm_calls`, наблюдаемость в UI — USER_GUIDE §5) в структурированном виде — ротация/удаление старых jsonl-файлов не теряет агрегаты и трейс, только построчные записи метаданных. diff --git a/docs/ENV_REFERENCE.md b/docs/ENV_REFERENCE.md index b22c087..392bcbd 100644 --- a/docs/ENV_REFERENCE.md +++ b/docs/ENV_REFERENCE.md @@ -61,9 +61,13 @@ Deployment profile validation (`local` / `demo` / `production`) is described in | `AUTO_BI_LLM_BUDGET_SESSION_MAX_COST_USD` | `float` | `0.0` | Max estimated LLM spend in USD per session all-time when budget is on; 0.0 means unlimited. | | `AUTO_BI_LLM_BUDGET_SESSION_MAX_SECONDS` | `float` | `0.0` | Max LLM wall-clock seconds per session all-time when budget is on; 0.0 means unlimited. | | `AUTO_BI_LLM_BUDGET_SESSION_MAX_TOKENS` | `int` | `0` | Max LLM tokens per session all-time when budget is on; 0 means unlimited. | -| `AUTO_BI_LLM_PROVIDER` | `str` | `anthropic` | LLM backend selector: "anthropic" for direct API or "gracekelly" for the local orchestration service. | +| `AUTO_BI_LLM_PROVIDER` | `str` | `anthropic` | LLM backend selector: "anthropic" or "mistral" for direct API access, or "gracekelly" for the local orchestration service. | | `AUTO_BI_MAX_CONCURRENT_BUILDS` | `int` | `2` | Hard cap on concurrent builds in this process; excess approve calls return 503 with Retry-After. | | `AUTO_BI_METRICS_ENABLED` | `bool` | false | Expose GET /api/v1/metrics; off by default because it reveals global spend and build counts. | +| `AUTO_BI_MISTRAL_API_KEY` | `str` | (empty) | Mistral API key for direct calls; empty lets the client fall back to MISTRAL_API_KEY. | +| `AUTO_BI_MISTRAL_MAX_TOKENS` | `int` | (set; not printed) | Maximum output tokens requested from direct Mistral chat completions. | +| `AUTO_BI_MISTRAL_MODEL` | `str` | `mistral-large-latest` | Mistral model id used when llm_provider is mistral. | +| `AUTO_BI_MISTRAL_URL` | `str` | `https://api.mistral.ai` | Mistral API base URL; the client appends /v1/chat/completions. | | `AUTO_BI_PROFILE` | `str` | `local` | Serve-time validation profile: local, demo, or production; unknown values fall back to local. | | `AUTO_BI_PRUNE_ON_REBUILD` | `bool` | true | After a successful rebuild, delete this session's prior-revision BI artifacts; false keeps them for later prune. | | `AUTO_BI_REQUIRE_LLM_READY` | `bool` | false | When true, refuse to serve unless the configured LLM backend is ready. | @@ -90,10 +94,11 @@ Deployment profile validation (`local` / `demo` / `production`) is described in | Variable | Notes | Default | |---|---|---| | `ANTHROPIC_API_KEY` | Anthropic SDK key when ``llm_provider=anthropic``. Also accepted as ``AUTO_BI_ANTHROPIC_API_KEY``. | (empty) | +| `MISTRAL_API_KEY` | Mistral API key when ``llm_provider=mistral``. Also accepted as ``AUTO_BI_MISTRAL_API_KEY``. | (empty) | ## Counts -- Settings fields / `AUTO_BI_*` keys: **66** -- Companion vars listed above: **1** +- Settings fields / `AUTO_BI_*` keys: **70** +- Companion vars listed above: **2** diff --git a/docs/EVAL_FIXTURES.md b/docs/EVAL_FIXTURES.md index 6238e99..d7f4229 100644 --- a/docs/EVAL_FIXTURES.md +++ b/docs/EVAL_FIXTURES.md @@ -23,7 +23,7 @@ One file per case: `tests/fixtures/golden_llm/.json`. "format_version": 2, "template_version": "<16 hex of GROUNDING/SPEC_RULES/PROPOSE/PATCH templates>", "schema_version": "<16 hex of GroundingReport + DashboardSpec JSON schemas>", - "provider": "gracekelly|anthropic|fixture-refresh|...", + "provider": "gracekelly|anthropic|mistral|fixture-refresh|...", "model_id": "...", "calls": [ { diff --git a/docs/LOCAL_BYOK.md b/docs/LOCAL_BYOK.md new file mode 100644 index 0000000..38a7ecc --- /dev/null +++ b/docs/LOCAL_BYOK.md @@ -0,0 +1,253 @@ +# Локальный запуск Auto_BI со своим ключом Anthropic (BYOK) + +Краткая инструкция для первого локального прогона: клонируете репозиторий, поднимаете +демо-стенд ClickHouse + Superset, запускаете Auto_BI со **своим** `ANTHROPIC_API_KEY`. + +**Скоуп v1:** Auto_BI + ClickHouse (DM) + Apache Superset (BI). Полное руководство +пользователя — [USER_GUIDE.md](USER_GUIDE.md); полный inventory переменных — +[ENV_REFERENCE.md](ENV_REFERENCE.md); прод и reverse-proxy — [DEPLOYMENT.md](DEPLOYMENT.md). + +Контейнерный запуск самого Auto_BI (образ GHCR / `docker build`) здесь не разбирается — +см. [DEPLOYMENT.md](DEPLOYMENT.md). + +--- + +## 1. Что понадобится + +| Требование | Зачем | +|---|---| +| Git | клон репозитория | +| Python **3.12+** | runtime пакета | +| [uv](https://docs.astral.sh/uv/) | установка из `uv.lock` и `uv run` | +| Docker Engine / Docker Desktop **с Docker Compose** | демо-стенд ClickHouse + Superset | + +Локальные порты (loopback): + +| Порт | Сервис | +|---|---| +| `8123` | ClickHouse HTTP | +| `8088` | Apache Superset | +| `8200` | Auto_BI (`auto_bi serve`) | + +--- + +## 2. Клон и установка + +```bash +git clone https://github.com/brownjuly2003-code/Auto_BI.git +cd Auto_BI +uv sync --frozen --no-dev +``` + +`--frozen` ставит зависимости строго по `uv.lock`; `--no-dev` пропускает test/lint-инструменты +(тот же путь, что и в `Dockerfile` для runtime-слоя). + +--- + +## 3. Минимальный `.env` + +Скопируйте шаблон: + +```bash +# POSIX +cp .env.example .env +``` + +```powershell +# PowerShell +Copy-Item .env.example .env +``` + +Отредактируйте `.env`. Для первого BYOK-прогона достаточно такого минимума +(замените плейсхолдеры): + +```bash +AUTO_BI_PROFILE=local + +# ClickHouse (клиент Auto_BI → DWH) +AUTO_BI_CH_HOST=localhost +AUTO_BI_CH_PORT=8123 +AUTO_BI_CH_USER=auto_bi_ro +AUTO_BI_CH_PASSWORD= +AUTO_BI_CH_DATABASE=dm +# как BI (Superset) видит ClickHouse внутри compose-сети +AUTO_BI_CH_HOST_FROM_BI=clickhouse +AUTO_BI_CH_PORT_FROM_BI=8123 + +# Superset (клиент Auto_BI → BI) +AUTO_BI_SUPERSET_URL=http://localhost:8088 +AUTO_BI_SUPERSET_USER=admin +AUTO_BI_SUPERSET_PASSWORD= + +# LLM — прямой Anthropic API (BYOK) +AUTO_BI_LLM_PROVIDER=anthropic +ANTHROPIC_API_KEY= + +# top-N значений DWH во внешний LLM не уходят (безопасный default) +AUTO_BI_SEND_SAMPLES=false + +# --- только для docker compose (демо-стенд) --- +# пароль admin ClickHouse (healthcheck / admin-операции init) +CH_ADMIN_PASSWORD= +# секрет Flask/Superset +SUPERSET_SECRET_KEY= +# объём синтетического fact (дефолт compose — 100M строк; для первого раза меньше) +DEMO_FACT_ROWS=1000000 +``` + +### Согласованность паролей + +Compose **создаёт** учётки при первом старте томов, а Auto_BI **подключается** к ним +теми же переменными: + +| Учётка | Создаётся compose из | Клиент Auto_BI читает | +|---|---|---| +| ClickHouse `auto_bi_ro` | `AUTO_BI_CH_PASSWORD` (внутри контейнера как `AUTO_BI_RO_PASSWORD`) | `AUTO_BI_CH_PASSWORD` | +| Superset `admin` | `AUTO_BI_SUPERSET_PASSWORD` | `AUTO_BI_SUPERSET_PASSWORD` | + +Одно и то же значение должно совпадать в `.env` **до** первого `docker compose up`. +Если сменить пароль в `.env` после того, как тома уже инициализированы, compose +не пересоздаст учётки сам — будут «stale credentials» (см. §8). + +Секреты и ключи в git не коммитьте: `.env` в `.gitignore`. + +--- + +## 4. Поднять зависимости (ClickHouse + Superset) + +Из корня репозитория: + +```bash +docker compose up -d +``` + +**Важно:** корневой `docker compose up -d` поднимает **только** ClickHouse и Superset. +Процесс Auto_BI этим файлом **не** стартует. + +Статус: + +```bash +docker compose ps +``` + +Оба сервиса должны стать healthy (первый старт может занять несколько минут: образ +Superset, init ClickHouse, генерация demo-fact по `DEMO_FACT_ROWS`). + +Логи при зависании: + +```bash +docker compose logs --tail=100 clickhouse +docker compose logs --tail=100 superset +``` + +--- + +## 5. Запустить Auto_BI + +В корне репозитория (отдельный терминал, `.env` подхватится автоматически): + +```bash +uv run auto_bi serve +``` + +По умолчанию UI и API слушают `http://127.0.0.1:8200` +(`--host 127.0.0.1`, `--port 8200`). + +В репозитории уже есть `semantic/model.yaml` под демо-витрины стенда. Если файл +отсутствует или модель не подходит к вашему DWH: + +```bash +uv run auto_bi introspect --output semantic/model.yaml +``` + +--- + +## 6. Проверки liveness и readiness + +`/api/v1/health` — процесс жив (liveness). +`/api/v1/ready` — store + DWH (`SELECT 1`) + BI (healthcheck Superset) доступны; +LLM-проверка **репортится**, но **не** гейтит `ok` (503 только при сбое store/DWH/BI). + +Для провайдера `anthropic` readiness **не** делает платный запрос к Anthropic и +**не** доказывает, что ключ принят провайдером: live-check намеренно отключён, +чтобы не тратить токены на каждый probe. + +### curl (POSIX / Git Bash) + +```bash +curl -sS http://127.0.0.1:8200/api/v1/health +curl -sS -i http://127.0.0.1:8200/api/v1/ready +``` + +Ожидание: HTTP 200 и `"ok": true` в JSON (для `/ready` при живых зависимостях). + +### PowerShell + +```powershell +Invoke-RestMethod http://127.0.0.1:8200/api/v1/health +Invoke-WebRequest http://127.0.0.1:8200/api/v1/ready | Select-Object StatusCode, Content +``` + +Откройте UI: . + +--- + +## 7. Что тратит ваш LLM-аккаунт + +| Действие | LLM / оплата Anthropic | +|---|---| +| Текст → сессия / уточнения / propose spec | **да** | +| Fields-first (раскладка полей → spec) | **да** | +| Правка словами после сборки | **да** | +| Авто-обзор витрины (`build --auto` / вкладка «Авто») | **нет** (детерминированно) | +| Сборка уже готового spec, SQL-guard, Advisor, адаптер BI | **нет** (детерминированно) | +| `GET /api/v1/health`, `GET /api/v1/ready` | **нет** | + +`AUTO_BI_SEND_SAMPLES=false` (как в примере) не отключает LLM: он только запрещает +отправлять top-N значений колонок DWH во внешний провайдер. + +--- + +## 8. Остановка + +1. В терминале Auto_BI: **Ctrl+C**. +2. Зависимости: + +```bash +docker compose down +``` + +Тома `clickhouse_data` и `superset_home` **сохраняются** — demo-данные и дашборды +на стенде остаются. + +**Не** используйте как обычную остановку: + +```bash +docker compose down -v +``` + +Флаг `-v` удаляет тома: локальные demo-данные ClickHouse и состояние Superset +(включая собранные дашборды на стенде) будут уничтожены. + +--- + +## 9. Частые проблемы + +| Симптом | Что проверить | +|---|---| +| `docker` / `docker compose` не найдены | Установите Docker Engine или Docker Desktop; перезапустите shell; убедитесь, что Compose v2 доступен как `docker compose`. | +| `uv` не найден | Установите [uv](https://docs.astral.sh/uv/); проверьте `uv --version`. | +| Первый `docker compose up` долгий | Нормально: pull/build образов, init CH, генерация fact (`DEMO_FACT_ROWS`). Смотрите `docker compose ps` и `logs`. Для лёгкого стенда задайте `DEMO_FACT_ROWS=1000000` **до** первого старта (или после `down -v` — с потерей томов). | +| `/api/v1/ready` → **503** | Не подняты/не healthy CH или Superset, неверные `AUTO_BI_CH_*` / `AUTO_BI_SUPERSET_*`, или store недоступен. Сверьте `docker compose ps`, пароли, `AUTO_BI_CH_HOST_FROM_BI=clickhouse`. LLM **не** валит readiness. | +| Stale / wrong Compose credentials | Пароли в `.env` сменились после первого init томов. Верните прежние значения **или** осознанно пересоздайте тома (`docker compose down -v` — удалит demo-данные) и поднимите стенд заново. | +| `Semantic model not found` | Нет `semantic/model.yaml`. Запустите `uv run auto_bi introspect --output semantic/model.yaml` (нужен живой CH). | +| Ошибки аутентификации Anthropic на text/fields | Проверьте `AUTO_BI_LLM_PROVIDER=anthropic` и `ANTHROPIC_API_KEY` (или `AUTO_BI_ANTHROPIC_API_KEY`). `/ready` при этом может оставаться 200 — он не валидирует ключ у провайдера. | + +--- + +## 10. Дальше + +- Практика UI/CLI: [USER_GUIDE.md](USER_GUIDE.md) +- Все `AUTO_BI_*`: [ENV_REFERENCE.md](ENV_REFERENCE.md) +- Прод, reverse-proxy, образ приложения: [DEPLOYMENT.md](DEPLOYMENT.md) +- Подключение своего DWH: [ONBOARDING_DWH.md](ONBOARDING_DWH.md) diff --git a/docs/PROJECT_CLOSURE.md b/docs/PROJECT_CLOSURE.md index 792d236..3483db8 100644 --- a/docs/PROJECT_CLOSURE.md +++ b/docs/PROJECT_CLOSURE.md @@ -1,7 +1,9 @@ # Project closure Дата фиксации scope: 2026-07-27. Владелец повторно открыл остаточные пункты -аудита для финального closure-прохода 2026-07-29. +аудита для финального closure-прохода 2026-07-29. Docs-only reconciliation +24 historical rows и последующий live external-evidence re-audit — 2026-07-29 +(branch `release/verify-v0.5.0-20260729`). ## Закрываемый scope @@ -15,8 +17,9 @@ - golden/advisor replay, offline browser E2E, backup/restore и docs-as-code; - текущий cumulative bounded mutation gate и package-wide `mypy --strict auto_bi`. -После финальной публикации этот scope считается feature-frozen. Новые функции и -исследовательские расширения не являются незакрытым долгом проекта. +После software release **v0.5.0** этот scope feature-frozen. Новые функции и +исследовательские расширения не являются незакрытым долгом проекта. Внешний +demo/Space path полностью исключён из current project scope и backlog. ## Финальное решение по прежним residual @@ -24,17 +27,17 @@ |---|---| | Durable outbox до возврата BI adapter | `closed` 2026-07-29: schema v9 + cleanup-only adapter reconciliation (ADR 0002) | | Полный current/history split ARCHITECTURE | `closed`: current design отделён от `ARCHITECTURE_HISTORY.md`, ADR остаются отдельными решениями | -| `Field(description=)` на каждом Settings key | `closed` 2026-07-29: 66/66 descriptions + generated ENV_REFERENCE ratchet | +| `Field(description=)` на каждом Settings key | `closed` 2026-07-29: 70/70 descriptions + generated ENV_REFERENCE ratchet | | Cumulative bounded mutation gate | `closed` 2026-07-29: настроенные production targets закреплены в CI, weak outcomes запрещены; snapshot evidence — в [operations/SLO.md](operations/SLO.md) | | Package-wide `mypy --strict auto_bi` | `closed` 2026-07-29: package-wide gate действует в обоих поддерживаемых CI jobs | | Live p50/p95, process-memory и cold-start campaign | `closed` 2026-07-29: descriptive CI samples from run [30481369602](https://github.com/brownjuly2003-code/Auto_BI/actions/runs/30481369602) — p50 2353.619 ms, p95 2359.740 ms, container cold start 6341 ms, PID1 RSS 89.594 MiB, cgroup memory 74.93 MiB (**descriptive CI samples, not production SLO guarantees**) | -| Paid live-LLM canary | `budget-gated`; **not run** — нет отдельного approved budget; запускать только после явного лимита расходов | -| DataLens offline contracts | `closed` (offline passed); live stand unavailable (Mac-only stand absent); experimental / non-default release gate | -| Public HF demo vs closing SHA | `active closure work` / mandatory external gate: live health reports version **0.4.0**, `demo_auto_only=false`, no capabilities object; live assertion fails; v0.5.0 publish dry-run succeeds; actual sync blocked (no local `HF_TOKEN`, no repository HF secret/workflow). **Sync to closing SHA or decommission** — do not decommission in this pass | +| Paid live-LLM canary | `closed` 2026-07-29: существующий secure Mistral route найден без чтения/вывода credential value; `mistral-large-latest` sentinel **3/3**, 4 calls, 11,341 input + 1,491 output tokens, estimated **$0.007907** при hard cap **$1.50** | +| DataLens offline contracts | `closed` (offline passed) | +| DataLens live stand | `closed` 2026-07-29: owner-authorized Mac cleanup снял process-table blocker; текущий self-hosted stand + ClickHouse demo-DM прошли exact contract **15/15**. Current image seed workbook `z4wtz6tg5194o` передан через supported env override; experimental / non-default / **non-closure** | Новые функции вне этой таблицы по-прежнему требуют отдельного проекта. -## Внешние closure gates +## Software release vs optional external validation ### Completed (software release v0.5.0) @@ -53,18 +56,44 @@ Exact `main`/tag SHA: `e78076d2d00ddc1748bf6e22f13cf7cb93fc6515`. - GHCR `0.5.0` and `latest` share digest `sha256:bff75bcef9d894e86c2be63a584284c02c2b2546425ac11a15e83ad83e4e84f1`; - `main` and `v*` tag rulesets active without bypass; +- protected-tag force-update rejection live-smoke passed on canary + [`v-retag-smoke-20260729`](https://github.com/brownjuly2003-code/Auto_BI/tree/v-retag-smoke-20260729): + GitHub returned HTTP 422, the ref stayed on `13fc855`, and no release workflow + matched the canary; - live Superset/browser integration passed. -### Remaining mandatory gate +Software/local audit closure: **complete** for the v1 path above where exact +live/manual checks have durable run evidence. Original mapping of the 24 +historical plan_sol residual rows is preserved in root +`plan_audit_closure_29_07.md`; after this evidence re-audit the current counts +are `closed` 21 · excluded from scope 3 (historical token +`owner-de-scoped`) · `externally-blocked` 0 · `still-open` 0. Active audit +work remaining: **0**. The three excluded rows are not residual, pending, +blocked, unfinished, or next work. -Проект остаётся **closure candidate**, не fully closed, пока: +### External validation re-audit (not software-closure blockers) -- public HF demo **не** синхронизирован с closing SHA (и live assertion fails на - stale 0.4.0 profile). Mandatory: **sync to closing SHA or decommission**. - Sync currently blocked by missing HF token/secret/workflow; decommission not - performed. +All five previously external checks now have exact live evidence: + +| Gate | Disposition | Gate requirement / partial evidence | +|---|---|---| +| DataLens live | `closed` | initial current-image contract **12/15** exposed response-location drift while all payloads rendered; three behavioral assertions were updated for the two evidence-backed layouts, then full contract **15/15 passed in 51.39 s** against real ClickHouse data | +| Paid live-LLM canary/sentinel | `closed` | existing Mistral credential route used without exposing/copying/changing its value; `mistral-large-latest` **3/3**, 4 calls, estimated **$0.007907** under **$1.50** cap | +| Protected-tag retag rejection | `closed` | `v-retag-smoke-20260729` force-update rejected HTTP 422; protected ref stayed unchanged; no release workflow matched | +| Live Trivy-fail before `:latest` promotion | `closed` | expected-failure run [30512999822](https://github.com/brownjuly2003-code/Auto_BI/actions/runs/30512999822): Trivy rejected **23** fixed HIGH/CRITICAL findings; GHCR `:latest` digest stayed unchanged; temporary branch removed | +| Process restart mid-delivery live smoke | `closed` | real child exit **97** after remote DataLens delivery; startup reconcile discovered/deleted **3/3** owned entries, session became `failed`, and no ownership rows remained | + +### Historical excluded accounting + +Rows 4, 21 и 23 исходной 24-row mapping сохраняют disposition token +`owner-de-scoped` только для auditable accounting. Они исключены из current +project scope, не входят в таблицу external validations и не являются +remaining work. Поэтому итог — **21 closed + 3 excluded; active work 0**, а не +«24 closed». ## Сохранённые локальные артефакты -`Auto_BI.html` и `pres.html` оставлены без изменений и не входят в tracked -product scope. +`Auto_BI.html` и `pres.html` — намеренно поддерживаемые local +evidence/presentation artifacts. Их обновляют локально вместе с current +evidence, но они всегда остаются untracked: не добавлять в Git и не +публиковать. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index fe12950..23c1e27 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -14,8 +14,9 @@ сделайте интроспекцию DWH (см. [ONBOARDING_DWH.md](ONBOARDING_DWH.md) или быстрый старт ниже). - **DWH** (ClickHouse — v1) с read-only ролью, доступный с машины, где запускается Auto_BI. - **BI** — Apache Superset (v1) или self-hosted Yandex DataLens (v2). -- **LLM** — по умолчанию прямой **Anthropic API** (`ANTHROPIC_API_KEY`), опционально — - локальный сервис **GraceKelly** (`AUTO_BI_LLM_PROVIDER=gracekelly`, §6). Нужен для +- **LLM** — по умолчанию прямой **Anthropic API** (`ANTHROPIC_API_KEY`); опционально — + прямой **Mistral API** (`AUTO_BI_LLM_PROVIDER=mistral`, `MISTRAL_API_KEY`) или локальный + сервис **GraceKelly** (`AUTO_BI_LLM_PROVIDER=gracekelly`, §6). Нужен для диалога/предложения spec'а; детерминированные шаги (валидация, advisor, сборка) от него не зависят. Все секреты и адреса — через переменные окружения с префиксом `AUTO_BI_` или файл `.env` @@ -34,8 +35,9 @@ pip install -e . # AUTO_BI_CH_DATABASE=dm # AUTO_BI_SUPERSET_URL=http://localhost:8088 AUTO_BI_SUPERSET_PASSWORD=... # ANTHROPIC_API_KEY=sk-ant-... -# (или ANTHROPIC_API_KEY отсутствует, но задан AUTO_BI_LLM_PROVIDER=gracekelly + -# AUTO_BI_GRACEKELLY_URL=http://127.0.0.1:8011 — локальный сервис-опция, см. §6) +# (или AUTO_BI_LLM_PROVIDER=mistral + MISTRAL_API_KEY; +# или AUTO_BI_LLM_PROVIDER=gracekelly + +# AUTO_BI_GRACEKELLY_URL=http://127.0.0.1:8011 — локальный сервис-опция, см. §6) # 3. интроспекция DWH -> черновик модели auto_bi introspect --output semantic/model.yaml @@ -168,7 +170,8 @@ auto_bi prune --session # только одна сессия **Готовность (S07):** `GET /api/v1/ready` (открыт даже при включённом auth, как `/health`) — глубокая проверка для оркестратора (compose healthcheck, Fly checks): store + DWH (`SELECT 1`) + BI (`healthcheck()` на Superset) гейтят `{"ok": false}`/503; LLM-доступность -репортится в том же ответе, но **не** гейтит `ok` (транзиентный сбой GraceKelly/Anthropic не +репортится в том же ответе, но **не** гейтит `ok` (транзиентный сбой +GraceKelly/Anthropic/Mistral не должен ронять готовность уже собранных дашбордов). Подробнее — ARCHITECTURE §3.11. **Рестарт (X-4):** сессии переживают перезапуск сервера — открытая вкладка продолжает @@ -213,7 +216,8 @@ vs Greenplum/Greengage). исходом, и агрегаты по вызовам LLM. API: `GET /api/v1/sessions/{id}/trace` и `GET /api/v1/observability/llm`. -> **Честность по данным:** при провайдере `anthropic` usage (input/output tokens) пишется +> **Честность по данным:** при провайдерах `anthropic` и `mistral` usage +> (input/output tokens) пишется > в Store/`llm_calls` и участвует в opt-in LLM budget (`AUTO_BI_LLM_BUDGET_*`, цены в > `AUTO_BI_LLM_BUDGET_PRICES`). GraceKelly usage может быть неполным — тогда UI показывает > измеримое: число вызовов, латентность и **объём в символах** (size-прокси, не доллары). @@ -238,8 +242,9 @@ vs Greenplum/Greengage). | `AUTO_BI_SUPERSET_URL` / `_USER` / `_PASSWORD` | Apache Superset | `http://localhost:8088` / `admin` / `` | | `AUTO_BI_DATALENS_URL` / `_USER` / `_PASSWORD` / `_WORKBOOK_ID` | self-hosted DataLens (v2, experimental live) | `http://localhost:8090` / `admin` / `` (пустой — fail-loud, без shipped default) / `ra7f79yirtumb` | | `AUTO_BI_CH_HOST_FROM_DATALENS` | CH-хост, как его достаёт DataLens-коннекшн | `host.docker.internal` | -| `AUTO_BI_LLM_PROVIDER` | LLM-провайдер: `anthropic` (прямой Messages API) или `gracekelly` (локальный сервис) | `anthropic` | +| `AUTO_BI_LLM_PROVIDER` | LLM-провайдер: `anthropic` (прямой Messages API), `mistral` (прямой Chat Completions API) или `gracekelly` (локальный сервис) | `anthropic` | | `ANTHROPIC_API_KEY` / `AUTO_BI_ANTHROPIC_MODEL` / `_MAX_TOKENS` | Прямой Anthropic API (провайдер `anthropic`). Ключ — стандартная переменная SDK, без префикса `AUTO_BI_`; `AUTO_BI_ANTHROPIC_API_KEY` тоже работает, если ключ нужно держать рядом с остальным `.env` | `` / `claude-sonnet-5` / `16000` | +| `MISTRAL_API_KEY` / `AUTO_BI_MISTRAL_API_KEY` / `_MODEL` / `_URL` / `_MAX_TOKENS` | Прямой Mistral API (провайдер `mistral`); стандартное и `AUTO_BI_`-имя ключа равноправны | `` / `` / `mistral-large-latest` / `https://api.mistral.ai` / `16000` | | `AUTO_BI_GRACEKELLY_URL` / `_MODEL` | Локальный LLM-сервис (провайдер `gracekelly`) | `http://127.0.0.1:8011` / `claude-sonnet-5` | | `AUTO_BI_SEND_SAMPLES` | слать ли top-N значений колонок в grounding/propose (только `public`/`internal`; см. `docs/MIGRATION_SEND_SAMPLES.md`) | `false` | | `AUTO_BI_STORE_PATH` | SQLite-стор (сессии, spec'ы, сборки, llm_calls, заявки DM, users) | `data/auto_bi.sqlite` | diff --git a/scripts/generate_env_reference.py b/scripts/generate_env_reference.py index ed63580..db014bb 100644 --- a/scripts/generate_env_reference.py +++ b/scripts/generate_env_reference.py @@ -37,6 +37,12 @@ "Also accepted as ``AUTO_BI_ANTHROPIC_API_KEY``.", "(empty)", ), + ( + "MISTRAL_API_KEY", + "Mistral API key when ``llm_provider=mistral``. " + "Also accepted as ``AUTO_BI_MISTRAL_API_KEY``.", + "(empty)", + ), ] SECRET_NAME_MARKERS = ("password", "api_key", "token", "secret") diff --git a/tests/test_datalens_contract.py b/tests/test_datalens_contract.py index 4653438..c96b6dd 100644 --- a/tests/test_datalens_contract.py +++ b/tests/test_datalens_contract.py @@ -9,7 +9,8 @@ uv run pytest -m integration tests/test_datalens_contract.py Requires the DataLens compose stand up (admin/admin) + the ClickHouse demo-DM, and -AUTO_BI_DATALENS_* settings (defaults target the local tunnel + OpenSource Demo workbook). +AUTO_BI_DATALENS_* settings. Point `AUTO_BI_DATALENS_WORKBOOK_ID` at the workbook +actually seeded by the pinned stand image (the id can drift between image generations). """ from __future__ import annotations @@ -172,6 +173,101 @@ def _rendered_with_data(run: dict) -> bool: return False +def _run_categories(run: dict) -> list: + """Histogram discrete-axis categories from /api/run. + + Old layout: data.categories + New layout: data.xAxis.categories + """ + data = run.get("data") + if not isinstance(data, dict): + raise AssertionError( + f"expected data dict for categories, got {type(data).__name__}: keys={sorted(run)}" + ) + if "categories" in data: + categories = data["categories"] + elif isinstance(data.get("xAxis"), dict) and "categories" in data["xAxis"]: + categories = data["xAxis"]["categories"] + else: + raise AssertionError(f"unknown categories layout: data keys={sorted(data)}") + if not isinstance(categories, list): + raise AssertionError( + f"categories must be a list, got {type(categories).__name__}: {categories!r}" + ) + return categories + + +def _run_series_points(run: dict) -> list: + """Series points from /api/run (line/bar and similar). + + Old layout: data.graphs[0].data + New layout: data.series.data[0].data + """ + data = run.get("data") + if not isinstance(data, dict): + raise AssertionError( + f"expected data dict for series points, got {type(data).__name__}: keys={sorted(run)}" + ) + if "graphs" in data: + graphs = data["graphs"] + if not isinstance(graphs, list) or not graphs: + raise AssertionError(f"data.graphs missing first series: {graphs!r}") + points = graphs[0].get("data") if isinstance(graphs[0], dict) else None + if not isinstance(points, list): + raise AssertionError(f"data.graphs[0].data must be a list, got {points!r}") + return points + series = data.get("series") + if isinstance(series, dict) and isinstance(series.get("data"), list) and series["data"]: + first = series["data"][0] + points = first.get("data") if isinstance(first, dict) else None + if not isinstance(points, list): + raise AssertionError(f"data.series.data[0].data must be a list, got {points!r}") + return points + raise AssertionError(f"unknown series-points layout: data keys={sorted(data)}") + + +def _run_percent_format(run: dict) -> str: + """Y-axis percent format token from /api/run. + + Old layout: highchartsConfig.axesFormatting.yAxis[0].chartKitFormat + New layout: data.yAxis[0].labels.numberFormat.format + """ + import json + + data = run.get("data") + if isinstance(data, dict) and isinstance(data.get("yAxis"), list) and data["yAxis"]: + y0 = data["yAxis"][0] + if isinstance(y0, dict): + labels = y0.get("labels") + if isinstance(labels, dict): + number_format = labels.get("numberFormat") + if isinstance(number_format, dict) and "format" in number_format: + return number_format["format"] + + hc = run.get("highchartsConfig") + if isinstance(hc, str): + hc = json.loads(hc) + if isinstance(hc, dict): + axes = hc.get("axesFormatting") + if isinstance(axes, dict) and "yAxis" in axes: + y_formats = axes["yAxis"] + if not y_formats: + raise AssertionError( + "axesFormatting.yAxis is empty — the by-field flag was not honored" + ) + fmt = y_formats[0].get("chartKitFormat") if isinstance(y_formats[0], dict) else None + if fmt is None: + raise AssertionError( + f"axesFormatting.yAxis[0] missing chartKitFormat: {y_formats[0]!r}" + ) + return fmt + + data_keys = sorted(data) if isinstance(data, dict) else type(data).__name__ + raise AssertionError( + f"unknown percent-format layout: run keys={sorted(run)}; data keys={data_keys}" + ) + + @pytest.fixture(scope="module") def model() -> SemanticModel: return SemanticModel.load("semantic/model.yaml") @@ -228,7 +324,7 @@ def test_histogram_buckets_render_in_numeric_order(adapter: DataLensAdapter) -> ref = adapter.create_chart(chart, ds) run = adapter._client.post("/api/run", {"id": str(ref.id), "workbookId": adapter._workbook_id}) assert _rendered_with_data(run), f"histogram rendered no data: keys={sorted(run)}" - categories = run["data"]["categories"] # the highcharts x-axis order as rendered + categories = _run_categories(run) # old: data.categories; new: data.xAxis.categories values = [float(c) for c in categories] assert len(values) >= 2, f"expected multiple buckets, got {categories}" assert values == sorted(values), f"buckets not in ascending numeric order: {categories}" @@ -237,11 +333,10 @@ def test_histogram_buckets_render_in_numeric_order(adapter: DataLensAdapter) -> def test_percent_axis_formats_by_field(adapter: DataLensAdapter) -> None: """C1: a share-transform chart's VALUE axis renders as percent. The placeholder-item `formatting` alone is not enough — the engine reads it into the axis ONLY under - `settings.axisFormatMode="by-field"` (chart_config._AXIS_FORMAT_BY_FIELD): the run's - highchartsConfig must carry chartKitFormat="percent" in axesFormatting.yAxis (the - un-flagged baseline returns an empty axesFormatting — the pre-fix raw 0..1 axis).""" - import json - + `settings.axisFormatMode="by-field"` (chart_config._AXIS_FORMAT_BY_FIELD). Legacy + responses expose `chartKitFormat="percent"` under `axesFormatting.yAxis`; current + responses expose `format="percent"` under `data.yAxis[].labels.numberFormat`. The + un-flagged baseline exposes neither — and renders the pre-fix raw 0..1 axis.""" from auto_bi.ir.spec import MeasureTransform chart = ChartSpec( @@ -264,12 +359,9 @@ def test_percent_axis_formats_by_field(adapter: DataLensAdapter) -> None: ref = adapter.create_chart(chart, ds) run = adapter._client.post("/api/run", {"id": str(ref.id), "workbookId": adapter._workbook_id}) assert _rendered_with_data(run), f"percent chart rendered no data: keys={sorted(run)}" - hc = run["highchartsConfig"] - if isinstance(hc, str): - hc = json.loads(hc) - y_formats = hc["axesFormatting"]["yAxis"] - assert y_formats, "axesFormatting.yAxis is empty — the by-field flag was not honored" - assert y_formats[0]["chartKitFormat"] == "percent" + # old: highchartsConfig.axesFormatting.yAxis[0].chartKitFormat + # new: data.yAxis[0].labels.numberFormat.format + assert _run_percent_format(run) == "percent" def test_kpi_ru_units_scale_headline(adapter: DataLensAdapter) -> None: @@ -331,8 +423,9 @@ def test_selector_default_period_narrows_chart_data(adapter: DataLensAdapter) -> narrowed = adapter._client.post( "/api/run", {"id": wid, "workbookId": adapter._workbook_id, "params": {guid: token}} ) - n_full = len(full["data"]["graphs"][0]["data"]) - n_narrowed = len(narrowed["data"]["graphs"][0]["data"]) + # old: data.graphs[0].data; new: data.series.data[0].data + n_full = len(_run_series_points(full)) + n_narrowed = len(_run_series_points(narrowed)) assert 0 < n_narrowed < n_full, f"period param did not narrow: {n_narrowed} vs {n_full}" assert n_narrowed <= 100 # ~3 months of daily points, not the full history diff --git a/tests/test_mistral.py b/tests/test_mistral.py new file mode 100644 index 0000000..6cdb46a --- /dev/null +++ b/tests/test_mistral.py @@ -0,0 +1,303 @@ +"""MistralClient tests on httpx.MockTransport (no live API calls).""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import httpx +import pytest +from pydantic import BaseModel + +from auto_bi.config import Settings +from auto_bi.llm.base import LLMError +from auto_bi.llm.budget import BudgetExceeded, BudgetLimits, LLMBudget +from auto_bi.llm.factory import make_llm +from auto_bi.llm.mistral import MistralClient +from auto_bi.store import Store + +REPO = Path(__file__).resolve().parents[1] + + +class Answer(BaseModel): + title: str + count: int + + +Responder = Callable[[httpx.Request], httpx.Response] + + +@pytest.fixture +def client_factory(tmp_path): + clients: list[MistralClient] = [] + + def make( + responder: Responder, + *, + settings: Settings | None = None, + store: Store | None = None, + budget: LLMBudget | None = None, + sleep: Callable[[float], None] | None = None, + ) -> MistralClient: + http = httpx.Client( + base_url="https://api.mistral.test", + transport=httpx.MockTransport(responder), + ) + client = MistralClient( + settings or Settings(_env_file=None, mistral_api_key="test-key"), + http=http, + log_path=tmp_path / "llm_calls.jsonl", + store=store, + budget=budget, + sleep=sleep or (lambda _seconds: None), + ) + clients.append(client) + return client + + yield make + + for client in clients: + client.close() + + +def mistral_response( + content: str | list[dict[str, str]] | None, + *, + finish: str = "stop", + usage: dict[str, int] | None = None, +) -> httpx.Response: + choices = ( + [] + if content is None + else [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": finish, + } + ] + ) + body: dict[str, Any] = {"choices": choices} + if usage is not None: + body["usage"] = usage + return httpx.Response(200, json=body) + + +def test_complete_uses_official_chat_completions_shape(client_factory) -> None: + def responder(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert request.url.path == "/v1/chat/completions" + assert body["model"] == "mistral-large-latest" + assert body["messages"] == [{"role": "user", "content": "сделай"}] + assert body["temperature"] == 0 + assert body["max_tokens"] == 16000 + return mistral_response('```json\n{"title": "ok", "count": 5}\n```') + + result = client_factory(responder).complete("сделай", Answer) + assert result == Answer(title="ok", count=5) + + +def test_list_content_chunks_are_joined(client_factory) -> None: + response = [ + {"type": "text", "text": '{"title": "ok", '}, + {"type": "text", "text": '"count": 6}'}, + ] + result = client_factory(lambda _request: mistral_response(response)).complete("сделай", Answer) + assert result == Answer(title="ok", count=6) + + +def test_complete_uses_shared_repair_loop(client_factory) -> None: + prompts: list[str] = [] + + def responder(request: httpx.Request) -> httpx.Response: + prompts.append(json.loads(request.content)["messages"][0]["content"]) + if len(prompts) == 1: + return mistral_response('{"title": "ok", "count": "не число"}') + return mistral_response('{"title": "ok", "count": 7}') + + result = client_factory(responder).complete("сделай", Answer) + assert result.count == 7 + assert len(prompts) == 2 + assert "не прошёл валидацию" in prompts[1] + + +def test_standard_environment_key_is_used_without_logging_value( + monkeypatch: pytest.MonkeyPatch, tmp_path +) -> None: + monkeypatch.setenv("MISTRAL_API_KEY", "test-standard-key") + client = MistralClient( + Settings(_env_file=None), + log_path=tmp_path / "llm_calls.jsonl", + ) + try: + assert client._http.headers["Authorization"] == "Bearer test-standard-key" + assert not (tmp_path / "llm_calls.jsonl").exists() + finally: + client.close() + + +def test_missing_key_fails_before_transport(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: + monkeypatch.delenv("MISTRAL_API_KEY", raising=False) + with pytest.raises(LLMError, match="API key is not configured"): + MistralClient(Settings(_env_file=None), log_path=tmp_path / "llm_calls.jsonl") + + +def test_standard_key_name_is_loaded_from_dotenv(tmp_path) -> None: + env_file = tmp_path / ".env" + env_file.write_text("MISTRAL_API_KEY=test-dotenv-key\n", encoding="utf-8") + settings = Settings(_env_file=env_file) + assert settings.mistral_api_key == "test-dotenv-key" + + +def test_http_error_omits_provider_response_body(client_factory) -> None: + marker = "MISTRAL_SECRET_RESPONSE_MARKER" + + def responder(_request: httpx.Request) -> httpx.Response: + return httpx.Response(401, json={"api_key": marker}) + + with pytest.raises(LLMError) as exc_info: + client_factory(responder).complete("сделай", Answer) + message = str(exc_info.value) + assert "HTTP 401" in message + assert marker not in message + + +def test_retries_429_with_retry_after_then_succeeds(client_factory) -> None: + calls: list[int] = [] + sleeps: list[float] = [] + + def responder(_request: httpx.Request) -> httpx.Response: + calls.append(1) + if len(calls) <= 2: + return httpx.Response(429, headers={"Retry-After": "0.25"}) + return mistral_response('{"title": "ok", "count": 3}') + + result = client_factory(responder, sleep=sleeps.append).complete("сделай", Answer) + assert result.count == 3 + assert len(calls) == 3 + assert sleeps == [0.25, 0.25] + + +def test_persistent_429_stops_after_bounded_retries(client_factory) -> None: + calls: list[int] = [] + + def responder(_request: httpx.Request) -> httpx.Response: + calls.append(1) + return httpx.Response(429, json={"message": "rate limited"}) + + with pytest.raises(LLMError, match="rate-limited after 4 retries"): + client_factory(responder).complete("сделай", Answer) + assert len(calls) == 5 + + +def test_usage_and_completed_status_are_written_to_store(client_factory, tmp_path) -> None: + store = Store(tmp_path / "usage.sqlite") + session_id = store.create_session("r") + usage = {"prompt_tokens": 120, "completion_tokens": 45} + client = client_factory( + lambda _request: mistral_response( + '{"title": "ok", "count": 5}', + usage=usage, + ), + store=store, + ) + client.complete("сделай", Answer, session_id=session_id, step="propose_spec") + (call,) = store.llm_calls(session_id) + assert call["step"] == "propose_spec" + assert call["input_tokens"] == 120 + assert call["output_tokens"] == 45 + assert call["model"] == "mistral-large-latest" + assert call["status"] == "completed" + store.close() + + +def test_budget_blocks_repair_before_third_provider_call(client_factory, tmp_path) -> None: + store = Store(tmp_path / "budget.sqlite") + session_id = store.create_session("r") + budget = LLMBudget( + store, + session_limits=BudgetLimits(max_calls=2), + day_limits=BudgetLimits(), + ) + calls: list[int] = [] + + def responder(_request: httpx.Request) -> httpx.Response: + calls.append(1) + return mistral_response(f'{{"title": "ok", "count": "bad-{len(calls)}"}}') + + with pytest.raises(BudgetExceeded, match="calls"): + client_factory(responder, store=store, budget=budget).complete( + "сделай", + Answer, + session_id=session_id, + ) + assert len(calls) == 2 + store.close() + + +def test_factory_routes_mistral_and_wires_budget(tmp_path) -> None: + store = Store(tmp_path / "factory.sqlite") + settings = Settings( + _env_file=None, + llm_provider="mistral", + mistral_api_key="test-key", + llm_budget_enabled=True, + llm_budget_session_max_calls=5, + ) + client = make_llm(settings, store=store) + try: + assert isinstance(client, MistralClient) + assert client._budget is not None + finally: + client.close() + store.close() + + +def test_close_releases_injected_http_pool(client_factory) -> None: + client = client_factory(lambda _request: mistral_response('{"title": "ok", "count": 1}')) + http = client._http + client.close() + assert http.is_closed + + +def test_live_sentinel_routes_mistral_secret_and_selects_provider() -> None: + workflow = (REPO / ".github" / "workflows" / "eval-live-sentinel.yml").read_text( + encoding="utf-8" + ) + assert "MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}" in workflow + assert "AUTO_BI_MISTRAL_API_KEY: ${{ secrets.AUTO_BI_MISTRAL_API_KEY }}" in workflow + assert "AUTO_BI_MISTRAL_MODEL: ${{ secrets.AUTO_BI_MISTRAL_MODEL }}" in workflow + assert 'AUTO_BI_LLM_PROVIDER="mistral"' in workflow + assert ( + 'if [ -n "${MISTRAL_API_KEY}" ] || ' '[ -n "${AUTO_BI_MISTRAL_API_KEY}" ]; then' + ) in workflow + + +def test_example_and_budget_table_cover_mistral() -> None: + env_example = (REPO / ".env.example").read_text(encoding="utf-8") + assert "MISTRAL_API_KEY=change_me" in env_example + assert "AUTO_BI_MISTRAL_MODEL=mistral-large-latest" in env_example + prices = Settings(_env_file=None).llm_budget_prices + assert "mistral-large-latest:0.0005/0.0015" in prices + + +def test_cli_hosted_llm_readiness_message_uses_configured_provider() -> None: + """llm_healthcheck must label the actual hosted provider, not hardcode anthropic. + + Nested closure in serve wiring — source-ratchet (same style as workflow checks above). + """ + source = (REPO / "auto_bi" / "cli.py").read_text(encoding="utf-8") + assert 'message="anthropic: no live check (avoids token cost)"' not in source + assert 'message=f"{provider}: no live check (avoids token cost)"' in source + assert "provider = settings.llm_provider.strip().lower()" in source + + +def test_auto_bi_prefixed_mistral_key_alias_populates_settings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("MISTRAL_API_KEY", raising=False) + monkeypatch.setenv("AUTO_BI_MISTRAL_API_KEY", "test-auto-bi-mistral-key") + settings = Settings(_env_file=None) + assert settings.mistral_api_key == "test-auto-bi-mistral-key" diff --git a/tests/test_readme_local_first_ladder.py b/tests/test_readme_local_first_ladder.py index 6e6cfa7..7da46d1 100644 --- a/tests/test_readme_local_first_ladder.py +++ b/tests/test_readme_local_first_ladder.py @@ -1,4 +1,7 @@ -"""Docs ratchet: README local-first ladder (offline → HF Space → full local). +"""Docs ratchet: README exposes only the two current local-first paths. + +Supported paths: offline golden path + full LOCAL_BYOK. Excluded external demo +paths must not reappear as onboarding or current product status. Pathlib-only: no app imports, no network, no Settings. """ @@ -21,14 +24,20 @@ def _howto_section() -> str: def test_readme_local_first_ladder() -> None: section = _howto_section() + # Offline golden path (supported). assert "uv run python scripts/demo_golden_path.py" in section assert "без DWH, BI, LLM и API-ключа" in section - assert "HF Space" in section - assert "auto-only" in section - assert "пользовательский ключ не нужен" in section - assert "текстовый режим там намеренно недоступен" in section + # Excluded external demo paths are not onboarding steps or current status. + assert "HF Space" not in section + assert "Hugging Face" not in section + assert "hf.space" not in section + + # Negative: superseded claims from the removed external demo route. + assert "пользовательский ключ не нужен" not in section + assert "текстовый режим там намеренно недоступен" not in section + # Full LOCAL_BYOK path (supported). assert ".env.example" in section assert "cp .env.example .env" in section assert "Copy-Item .env.example .env" in section