From ba37199199f18b9a3396d88e5deb38be0eedf09e Mon Sep 17 00:00:00 2001 From: Muhammad Abdullah Farooqui Date: Thu, 27 Aug 2026 18:54:14 +0200 Subject: [PATCH 1/5] feat(exasol): add Exasol database provider Add Exasol support via pyexasol, Exasol's native WebSocket client. Since pyexasol is not DB-API 2.0, ExasolAdapter subclasses DatabaseAdapter directly rather than CursorBasedAdapter, matching the ClickHouse provider. Installed through a new "exasol" extra, also folded into "all". The connection schema covers on-prem username/password auth and Exasol SaaS token auth (access token / refresh token), modeled on the existing Snowflake schema. Default port is 8563. Unit tests run against mocked connections and import no driver, so they work in the driver-free CI job; pyexasol is imported lazily inside the fixtures to keep the conftest star-import safe when the extra is not installed. Integration tests run against exasol/docker-db, wired into the existing enterprise compose profile and a new test-exasol CI job. Re-locking also corrects pre-existing drift: pyproject.toml declares mariadb = ["PyMySQL>=1.1.0"] but the committed lock still pinned the mariadb C-extension at 1.1.14. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 44 ++- CONTRIBUTING.md | 14 +- README.md | 3 +- infra/docker/docker-compose.test.yml | 15 ++ pyproject.toml | 4 + sqlit/domains/connections/domain/config.py | 2 + .../connections/providers/exasol/__init__.py | 1 + .../connections/providers/exasol/adapter.py | 242 +++++++++++++++++ .../connections/providers/exasol/provider.py | 56 ++++ .../connections/providers/exasol/schema.py | 93 +++++++ tests/conftest.py | 1 + .../connections/providers/exasol/__init__.py | 0 .../providers/exasol/test_adapter.py | 255 ++++++++++++++++++ .../providers/exasol/test_connect.py | 218 +++++++++++++++ .../providers/exasol/test_schema.py | 84 ++++++ tests/fixtures/exasol.py | 202 ++++++++++++++ tests/test_exasol.py | 170 ++++++++++++ uv.lock | 58 ++-- 18 files changed, 1433 insertions(+), 29 deletions(-) create mode 100644 sqlit/domains/connections/providers/exasol/__init__.py create mode 100644 sqlit/domains/connections/providers/exasol/adapter.py create mode 100644 sqlit/domains/connections/providers/exasol/provider.py create mode 100644 sqlit/domains/connections/providers/exasol/schema.py create mode 100644 tests/connections/providers/exasol/__init__.py create mode 100644 tests/connections/providers/exasol/test_adapter.py create mode 100644 tests/connections/providers/exasol/test_connect.py create mode 100644 tests/connections/providers/exasol/test_schema.py create mode 100644 tests/fixtures/exasol.py create mode 100644 tests/test_exasol.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50d6b86d..bea79733 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,8 @@ jobs: --ignore=tests/test_turso.py \ --ignore=tests/test_firebird.py \ --ignore=tests/test_ssh.py \ - --ignore=tests/test_clickhouse.py + --ignore=tests/test_clickhouse.py \ + --ignore=tests/test_exasol.py test-sqlite: runs-on: ubuntu-latest @@ -480,6 +481,47 @@ jobs: CLICKHOUSE_DATABASE: test_sqlit run: uv run pytest tests/test_clickhouse.py -v --timeout=120 + test-exasol: + runs-on: ubuntu-latest + needs: build + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: uv sync --group test --no-dev --extra exasol + + - name: Start Exasol + run: | + docker run -d --name exasol --privileged \ + -p 8563:8563 \ + exasol/docker-db:latest-8 + for i in {1..60}; do + if nc -z localhost 8563 > /dev/null 2>&1; then + echo "Exasol port is open" + break + fi + echo "Waiting for Exasol... ($i/60)" + sleep 10 + done + + - name: Run Exasol integration tests + env: + EXASOL_HOST: localhost + EXASOL_PORT: 8563 + EXASOL_USER: sys + EXASOL_PASSWORD: exasol + EXASOL_SCHEMA: TEST_SQLIT + run: uv run pytest tests/test_exasol.py -v --timeout=300 + test-ssh: runs-on: ubuntu-latest needs: build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a66f78ca..78a5a265 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ To run the complete test suite including SQL Server, PostgreSQL, MySQL, MariaDB, ```bash docker compose -f infra/docker/docker-compose.test.yml up -d ``` - To include the enterprise test containers (Db2, Trino, Presto, Oracle 11g): + To include the enterprise test containers (Db2, Trino, Presto, Oracle 11g, Exasol): ```bash docker compose -f infra/docker/docker-compose.test.yml --profile enterprise up -d ``` @@ -176,6 +176,18 @@ The database tests can be configured with these environment variables: | `ORACLE11G_CLIENT_MODE` | `thick` | Oracle client mode | | `ORACLE11G_CLIENT_LIB_DIR` | `` | Oracle Instant Client library directory | +**Exasol:** +| Variable | Default | Description | +|----------|---------|-------------| +| `EXASOL_HOST` | `localhost` | Exasol hostname | +| `EXASOL_PORT` | `8563` | Exasol port | +| `EXASOL_USER` | `sys` | Exasol username | +| `EXASOL_PASSWORD` | `exasol` | Exasol password | +| `EXASOL_SCHEMA` | `TEST_SQLIT` | Schema the Exasol fixtures create and drop | +| `EXASOL_READY_TIMEOUT` | `300` | Seconds to wait for Exasol to accept a login | + +**Note:** Exasol runs in the `enterprise` profile and needs minutes, not seconds, before it accepts connections. `exasol/docker-db` binds port 8563 long before it will authenticate, so an open port is not yet a database that accepts a login. The fixtures retry a real connect until `EXASOL_READY_TIMEOUT` elapses; raise that value on slower hardware or a cold image pull. + **Flight SQL:** | Variable | Default | Description | |----------|---------|-------------| diff --git a/README.md b/README.md index 985f1fa1..b76fd3fd 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ --- ### Connect -Supports all major databases: SQL Server, PostgreSQL, MySQL, SQLite, MariaDB, FirebirdSQL, Oracle, DuckDB, CockroachDB, ClickHouse, Snowflake, Supabase, CloudFlare D1, Turso, Athena, BigQuery, Spanner, RedShift, IBM Db2, SAP HANA, Teradata, Trino, Presto, Apache Flight SQL, Apache Impala, SurrealDB and osquery. +Supports all major databases: SQL Server, PostgreSQL, MySQL, SQLite, MariaDB, FirebirdSQL, Oracle, DuckDB, CockroachDB, ClickHouse, Snowflake, Supabase, CloudFlare D1, Turso, Athena, BigQuery, Spanner, RedShift, IBM Db2, SAP HANA, Teradata, Exasol, Trino, Presto, Apache Flight SQL, Apache Impala, SurrealDB and osquery. ![Database Providers](docs/demos/demo-providers.gif) @@ -291,6 +291,7 @@ Most of the time you can just run `sqlit` and connect. If a Python driver is mis | Athena | `pyathena` | `pipx inject sqlit-tui pyathena` | `python -m pip install pyathena` | | BigQuery | `google-cloud-bigquery` | `pipx inject sqlit-tui google-cloud-bigquery` | `python -m pip install google-cloud-bigquery` | | Spanner | `google-cloud-spanner` | `pipx inject sqlit-tui google-cloud-spanner` | `python -m pip install google-cloud-spanner` | +| Exasol | `pyexasol` | `pipx inject sqlit-tui pyexasol` | `python -m pip install pyexasol` | | Apache Arrow Flight SQL | `adbc-driver-flightsql` | `pipx inject sqlit-tui adbc-driver-flightsql` | `python -m pip install adbc-driver-flightsql` | | Apache Impala | `impyla` | `pipx inject sqlit-tui impyla` | `python -m pip install impyla` | | SurrealDB | `surrealdb` | `pipx inject sqlit-tui surrealdb` | `python -m pip install surrealdb` | diff --git a/infra/docker/docker-compose.test.yml b/infra/docker/docker-compose.test.yml index b14db063..89d3a16c 100644 --- a/infra/docker/docker-compose.test.yml +++ b/infra/docker/docker-compose.test.yml @@ -143,6 +143,21 @@ services: profiles: - enterprise + # Exasol ships a single-container database image, but it manages its own + # storage volumes and kernel parameters, so it needs privileged mode and a + # long stop grace period - an abrupt kill leaves the data volume dirty. + # No healthcheck: the image ships no lightweight probe, and an open port is + # not readiness. tests/fixtures/exasol.py gates on a real connect instead. + exasol: + image: exasol/docker-db:latest-8 + container_name: sqlit-test-exasol + privileged: true + stop_grace_period: 120s + ports: + - "${EXASOL_PORT:-8563}:8563" + profiles: + - enterprise + trino: image: trinodb/trino:latest container_name: sqlit-test-trino diff --git a/pyproject.toml b/pyproject.toml index 8ff17711..8119392c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ all = [ "ibm_db>=3.2.0", "hdbcli>=2.20.0", "teradatasql>=20.0.0", + "pyexasol>=2.0.0", "trino>=0.329.0", "presto-python-client>=0.8.4", "google-cloud-bigquery", @@ -72,6 +73,7 @@ oracle = ["oracledb>=2.0.0"] db2 = ["ibm_db>=3.2.0"] hana = ["hdbcli>=2.20.0"] teradata = ["teradatasql>=20.0.0"] +exasol = ["pyexasol>=2.0.0"] trino = ["trino>=0.329.0"] presto = ["presto-python-client>=0.8.4"] bigquery = ["google-cloud-bigquery"] @@ -196,6 +198,7 @@ markers = [ "clickhouse: ClickHouse database tests", "flight: Apache Arrow Flight SQL database tests", "spanner: Google Cloud Spanner database tests", + "exasol: Exasol database tests", "asyncio: async tests", "integration: integration tests (may require external services)", ] @@ -238,6 +241,7 @@ module = [ "hdbcli", "hdbcli.dbapi", "teradatasql", + "pyexasol", "trino", "trino.dbapi", "trino.auth", diff --git a/sqlit/domains/connections/domain/config.py b/sqlit/domains/connections/domain/config.py index 8ad839de..ed56d447 100644 --- a/sqlit/domains/connections/domain/config.py +++ b/sqlit/domains/connections/domain/config.py @@ -16,6 +16,7 @@ class DatabaseType(str, Enum): D1 = "d1" DUCKDB = "duckdb" DB2 = "db2" + EXASOL = "exasol" FIREBIRD = "firebird" FLIGHT = "flight" HANA = "hana" @@ -52,6 +53,7 @@ class DatabaseType(str, Enum): DatabaseType.DB2, DatabaseType.HANA, DatabaseType.TERADATA, + DatabaseType.EXASOL, DatabaseType.SNOWFLAKE, DatabaseType.BIGQUERY, DatabaseType.SPANNER, diff --git a/sqlit/domains/connections/providers/exasol/__init__.py b/sqlit/domains/connections/providers/exasol/__init__.py new file mode 100644 index 00000000..3bbc4837 --- /dev/null +++ b/sqlit/domains/connections/providers/exasol/__init__.py @@ -0,0 +1 @@ +"""Provider package.""" diff --git a/sqlit/domains/connections/providers/exasol/adapter.py b/sqlit/domains/connections/providers/exasol/adapter.py new file mode 100644 index 00000000..b7b453b0 --- /dev/null +++ b/sqlit/domains/connections/providers/exasol/adapter.py @@ -0,0 +1,242 @@ +"""Exasol adapter using pyexasol.""" + +from __future__ import annotations + +import ssl +from typing import TYPE_CHECKING, Any + +from sqlit.domains.connections.providers.adapters.base import ( + ColumnInfo, + DatabaseAdapter, + IndexInfo, + SequenceInfo, + TableInfo, + TriggerInfo, +) +from sqlit.domains.connections.providers.registry import get_default_port +from sqlit.domains.connections.providers.tls import ( + TLS_MODE_DISABLE, + TLS_MODE_REQUIRE, + get_tls_files, + get_tls_mode, + tls_mode_verifies_cert, +) + +if TYPE_CHECKING: + from sqlit.domains.connections.domain.config import ConnectionConfig + + +class ExasolAdapter(DatabaseAdapter): + """Adapter for Exasol using pyexasol. + + pyexasol is a native WebSocket client rather than a DB-API 2.0 driver: it has + no ``cursor()``, so this subclasses ``DatabaseAdapter`` directly and implements + query execution against the ``ExaStatement`` returned by ``conn.execute()``. + """ + + @property + def name(self) -> str: + return "Exasol" + + @property + def install_extra(self) -> str: + return "exasol" + + @property + def install_package(self) -> str: + return "pyexasol" + + @property + def driver_import_names(self) -> tuple[str, ...]: + return ("pyexasol",) + + @property + def supports_multiple_databases(self) -> bool: + # Exasol has no database layer above schemas. + return False + + @property + def supports_cross_database_queries(self) -> bool: + return False + + @property + def supports_stored_procedures(self) -> bool: + # Exposed from EXA_ALL_SCRIPTS. + return True + + @property + def supports_indexes(self) -> bool: + # Exasol indexes are auto-managed and unnamed. + return False + + @property + def supports_triggers(self) -> bool: + # Exasol has no triggers. + return False + + @property + def supports_sequences(self) -> bool: + # Exasol uses IDENTITY columns instead of sequences. + return False + + @property + def default_schema(self) -> str: + # No universal default; every table stays schema-qualified. + return "" + + def _tls_args(self, config: ConnectionConfig) -> dict[str, Any]: + """Map the shared tls_mode option onto pyexasol encryption kwargs. + + pyexasol defaults to encryption=True, but exasol/docker-db and most + on-premise installations present a self-signed certificate, so an + unmapped connect fails certificate validation. + """ + tls_mode = get_tls_mode(config) + if tls_mode == TLS_MODE_DISABLE: + return {"encryption": False} + if tls_mode == TLS_MODE_REQUIRE: + return {"encryption": True, "websocket_sslopt": {"cert_reqs": ssl.CERT_NONE}} + if not tls_mode_verifies_cert(tls_mode): + return {"encryption": True} + + sslopt: dict[str, Any] = {"cert_reqs": ssl.CERT_REQUIRED} + tls_ca, tls_cert, tls_key, _ = get_tls_files(config) + if tls_ca: + sslopt["ca_certs"] = tls_ca + if tls_cert: + sslopt["certfile"] = tls_cert + if tls_key: + sslopt["keyfile"] = tls_key + return {"encryption": True, "websocket_sslopt": sslopt} + + def connect(self, config: ConnectionConfig) -> Any: + endpoint = config.tcp_endpoint + if endpoint is None: + raise ValueError("Exasol connections require a TCP-style endpoint.") + + pyexasol = self._import_driver_module( + "pyexasol", + driver_name=self.name, + extra_name=self.install_extra, + package_name=self.install_package, + ) + + port = int(endpoint.port or get_default_port("exasol")) + connect_args: dict[str, Any] = { + "dsn": f"{endpoint.host}:{port}", + "schema": config.get_option("schema", ""), + "autocommit": True, + } + + # Add only the selected method's credentials: pyexasol's login branches on + # token presence, so a stray access_token/refresh_token key changes the + # auth path rather than being ignored. + authenticator = config.get_option("authenticator", "password") + if authenticator == "access_token": + connect_args["access_token"] = config.get_option("access_token", "") + elif authenticator == "refresh_token": + connect_args["refresh_token"] = config.get_option("refresh_token", "") + else: + connect_args["user"] = endpoint.username + connect_args["password"] = endpoint.password + + connect_args.update(self._tls_args(config)) + connect_args.update(config.extra_options) + return pyexasol.connect(**connect_args) + + def get_databases(self, conn: Any) -> list[str]: + # Exasol has no database layer above schemas. + return [] + + def get_tables(self, conn: Any, database: str | None = None) -> list[TableInfo]: + # conn.meta.* wraps every query in Exasol's snapshot-execution hint, so it + # cannot be blocked by metadata locks. The list_* helpers return already + # fetched lists of dicts with UPPERCASE keys - pyexasol enforces + # fetch_dict=True there - so rows must be read by key, never by index. + return [(row["TABLE_SCHEMA"], row["TABLE_NAME"]) for row in conn.meta.list_tables()] + + def get_views(self, conn: Any, database: str | None = None) -> list[TableInfo]: + return [(row["VIEW_SCHEMA"], row["VIEW_NAME"]) for row in conn.meta.list_views()] + + def get_columns( + self, conn: Any, table: str, database: str | None = None, schema: str | None = None + ) -> list[ColumnInfo]: + schema = schema or "" + + # Unlike the list_* helpers, execute_snapshot returns an ExaStatement, + # so it needs an explicit fetchall(). + pk_rows = conn.meta.execute_snapshot( + "SELECT COLUMN_NAME FROM SYS.EXA_ALL_CONSTRAINT_COLUMNS " + "WHERE CONSTRAINT_TYPE = 'PRIMARY KEY' " + "AND CONSTRAINT_SCHEMA = {schema!s} AND CONSTRAINT_TABLE = {table!s}", + {"schema": schema, "table": table}, + ).fetchall() + pk_columns = {row["COLUMN_NAME"] for row in pk_rows} + + return [ + ColumnInfo( + name=row["COLUMN_NAME"], + data_type=row["COLUMN_TYPE"], + is_primary_key=row["COLUMN_NAME"] in pk_columns, + ) + for row in conn.meta.list_columns(schema, table) + ] + + def get_procedures(self, conn: Any, database: str | None = None) -> list[str]: + # SCRIPTING is Exasol's scripting-program type, as opposed to UDF, + # ADAPTER and PREPROCESSOR. + rows = conn.meta.execute_snapshot( + "SELECT SCRIPT_SCHEMA, SCRIPT_NAME FROM SYS.EXA_ALL_SCRIPTS " + "WHERE SCRIPT_TYPE = 'SCRIPTING' " + "ORDER BY SCRIPT_SCHEMA, SCRIPT_NAME" + ).fetchall() + return [row["SCRIPT_NAME"] for row in rows] + + def get_indexes(self, conn: Any, database: str | None = None) -> list[IndexInfo]: + # Exasol indexes are auto-managed and unnamed. + return [] + + def get_triggers(self, conn: Any, database: str | None = None) -> list[TriggerInfo]: + # Exasol has no triggers. + return [] + + def get_sequences(self, conn: Any, database: str | None = None) -> list[SequenceInfo]: + # Exasol uses IDENTITY columns instead of sequences. + return [] + + def quote_identifier(self, name: str) -> str: + escaped = name.replace('"', '""') + return f'"{escaped}"' + + def build_select_query(self, table: str, limit: int, database: str | None = None, schema: str | None = None) -> str: + quoted_table = self.quote_identifier(table) + if schema: + return f"SELECT * FROM {self.quote_identifier(schema)}.{quoted_table} LIMIT {limit}" + return f"SELECT * FROM {quoted_table} LIMIT {limit}" + + def execute_test_query(self, conn: Any) -> None: + # The inherited implementation calls conn.cursor(), which pyexasol lacks. + conn.execute(self.test_query).fetchval() + + def execute_query(self, conn: Any, query: str, max_rows: int | None = None) -> tuple[list[str], list[tuple], bool]: + stmt = conn.execute(query) + + # This guard must precede any fetch: ExaStatement.__next__ raises + # ExaRuntimeError ("Attempt to fetch from statement without result set") + # for a rowCount statement, and fetchmany() iterates. + if stmt.result_type != "resultSet": + return [], [], False + + columns = list(stmt.column_names()) + if max_rows is None: + return columns, [tuple(row) for row in stmt.fetchall()], False + + # Fetch one row beyond the limit to detect truncation, then trim. + rows = stmt.fetchmany(max_rows + 1) + truncated = len(rows) > max_rows + return columns, [tuple(row) for row in rows[:max_rows]], truncated + + def execute_non_query(self, conn: Any, query: str) -> int: + # rowcount is a method on ExaStatement, not a property. No explicit + # commit: the connection is opened with autocommit=True. + return int(conn.execute(query).rowcount()) diff --git a/sqlit/domains/connections/providers/exasol/provider.py b/sqlit/domains/connections/providers/exasol/provider.py new file mode 100644 index 00000000..c49af9ab --- /dev/null +++ b/sqlit/domains/connections/providers/exasol/provider.py @@ -0,0 +1,56 @@ +"""Provider registration.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlit.domains.connections.providers.adapter_provider import build_adapter_provider +from sqlit.domains.connections.providers.catalog import register_provider +from sqlit.domains.connections.providers.docker import DockerDetector +from sqlit.domains.connections.providers.exasol.schema import SCHEMA +from sqlit.domains.connections.providers.model import DatabaseProvider, ProviderSpec + +if TYPE_CHECKING: + from sqlit.domains.connections.domain.config import ConnectionConfig + + +def _provider_factory(spec: ProviderSpec) -> DatabaseProvider: + from sqlit.domains.connections.providers.exasol.adapter import ExasolAdapter + + return build_adapter_provider(spec, SCHEMA, ExasolAdapter()) + + +def _display_info(config: ConnectionConfig) -> str: + """Display host:port/SCHEMA — Exasol has no database layer, so schema is the scope.""" + endpoint = config.tcp_endpoint + if not endpoint: + return config.name + + port_part = f":{endpoint.port}" if endpoint.port else "" + schema = config.get_option("schema", "") + schema_part = f"/{schema}" if schema else "" + info = f"{endpoint.host}{port_part}{schema_part}".strip() + return info or config.name + + +SPEC = ProviderSpec( + db_type="exasol", + display_name="Exasol", + schema_path=("sqlit.domains.connections.providers.exasol.schema", "SCHEMA"), + supports_ssh=True, + is_file_based=False, + has_advanced_auth=True, + default_port="8563", + requires_auth=True, + badge_label="Exasol", + url_schemes=("exasol", "exa"), + docker_detector=DockerDetector( + image_patterns=("exasol/docker-db",), + env_vars={}, + default_user="sys", + ), + display_info=_display_info, + provider_factory=_provider_factory, +) + +register_provider(SPEC) diff --git a/sqlit/domains/connections/providers/exasol/schema.py b/sqlit/domains/connections/providers/exasol/schema.py new file mode 100644 index 00000000..4b9443af --- /dev/null +++ b/sqlit/domains/connections/providers/exasol/schema.py @@ -0,0 +1,93 @@ +"""Connection schema for Exasol.""" + +from sqlit.domains.connections.providers.schema_helpers import ( + SSH_FIELDS, + TLS_FIELDS, + ConnectionSchema, + FieldType, + SchemaField, + SelectOption, + _port_field, + _server_field, +) + + +def _get_exasol_auth_options() -> tuple[SelectOption, ...]: + return ( + SelectOption("password", "Username & Password"), + SelectOption("access_token", "OpenID Access Token"), + SelectOption("refresh_token", "OpenID Refresh Token"), + ) + + +def _auth_is_password(v: dict) -> bool: + return str(v.get("authenticator", "password")) == "password" + + +def _auth_is_access_token(v: dict) -> bool: + return str(v.get("authenticator", "password")) == "access_token" + + +def _auth_is_refresh_token(v: dict) -> bool: + return str(v.get("authenticator", "password")) == "refresh_token" + + +SCHEMA = ConnectionSchema( + db_type="exasol", + display_name="Exasol", + fields=( + _server_field(), + _port_field("8563"), + SchemaField( + name="authenticator", + label="Authentication", + field_type=FieldType.DROPDOWN, + options=_get_exasol_auth_options(), + default="password", + ), + SchemaField( + name="username", + label="Username", + placeholder="sys", + required=True, + group="credentials", + visible_when=_auth_is_password, + ), + SchemaField( + name="password", + label="Password", + field_type=FieldType.PASSWORD, + placeholder="(empty = ask every connect)", + required=False, + group="credentials", + visible_when=_auth_is_password, + ), + SchemaField( + name="access_token", + label="Access Token", + field_type=FieldType.PASSWORD, + placeholder="OpenID access token", + required=False, + visible_when=_auth_is_access_token, + ), + SchemaField( + name="refresh_token", + label="Refresh Token", + field_type=FieldType.PASSWORD, + placeholder="OpenID refresh token", + required=False, + visible_when=_auth_is_refresh_token, + ), + SchemaField( + name="schema", + label="Schema", + placeholder="(empty = browse all)", + required=False, + description="Initial Schema", + ), + ) + + SSH_FIELDS + + TLS_FIELDS, + has_advanced_auth=True, + default_port="8563", +) diff --git a/tests/conftest.py b/tests/conftest.py index 52016bc5..309cb6ed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,7 @@ from tests.fixtures.db2 import * from tests.fixtures.d1 import * from tests.fixtures.duckdb import * +from tests.fixtures.exasol import * from tests.fixtures.firebird import * from tests.fixtures.flight import * from tests.fixtures.impala import * diff --git a/tests/connections/providers/exasol/__init__.py b/tests/connections/providers/exasol/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/connections/providers/exasol/test_adapter.py b/tests/connections/providers/exasol/test_adapter.py new file mode 100644 index 00000000..35c4dd02 --- /dev/null +++ b/tests/connections/providers/exasol/test_adapter.py @@ -0,0 +1,255 @@ +"""Tests for ExasolAdapter introspection and query execution. + +Everything runs against a mocked pyexasol connection - no driver is imported. +Two shapes matter and are pinned deliberately: + +* ``conn.meta.list_*`` return already-fetched lists of dicts with UPPERCASE keys + (pyexasol hard-codes ``fetch_dict=True`` there), while ``execute_snapshot`` + returns an ``ExaStatement`` that still needs an explicit ``fetchall()``. + Feeding tuples here would pass against an index-based implementation and so + would pin nothing at all. +* Every statement mock sets ``result_type`` explicitly. On a bare ``MagicMock`` + the ``!= "resultSet"`` comparison is trivially true, which would make a + result-set test go green while asserting nothing. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from sqlit.domains.connections.providers.adapters.base import ColumnInfo +from sqlit.domains.connections.providers.exasol.adapter import ExasolAdapter + + +@pytest.fixture +def adapter() -> ExasolAdapter: + return ExasolAdapter() + + +@pytest.fixture +def mock_conn() -> MagicMock: + conn = MagicMock() + conn.meta.list_tables.return_value = [] + conn.meta.list_views.return_value = [] + conn.meta.list_columns.return_value = [] + conn.meta.execute_snapshot.return_value.fetchall.return_value = [] + return conn + + +def _statement(*, result_type: str, columns: tuple[str, ...] = (), rows: list[Any] | None = None) -> MagicMock: + """Build a statement mock with result_type ALWAYS set explicitly.""" + available = list(rows or []) + stmt = MagicMock() + stmt.result_type = result_type + stmt.column_names.return_value = list(columns) + stmt.fetchall.return_value = available + stmt.fetchmany.side_effect = lambda size: available[:size] + return stmt + + +# --- Introspection ---------------------------------------------------------- + + +def test_get_tables_reads_schema_and_name_by_key(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + mock_conn.meta.list_tables.return_value = [ + {"TABLE_SCHEMA": "SALES", "TABLE_NAME": "ORDERS"}, + {"TABLE_SCHEMA": "SALES", "TABLE_NAME": "CUSTOMERS"}, + ] + + assert adapter.get_tables(mock_conn) == [("SALES", "ORDERS"), ("SALES", "CUSTOMERS")] + + +def test_get_views_reads_schema_and_name_by_key(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + mock_conn.meta.list_views.return_value = [ + {"VIEW_SCHEMA": "SALES", "VIEW_NAME": "V_ORDERS"}, + {"VIEW_SCHEMA": "REPORTING", "VIEW_NAME": "V_TOTALS"}, + ] + + assert adapter.get_views(mock_conn) == [("SALES", "V_ORDERS"), ("REPORTING", "V_TOTALS")] + + +def test_get_columns_combines_primary_key_and_column_information(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + mock_conn.meta.execute_snapshot.return_value.fetchall.return_value = [{"COLUMN_NAME": "ID"}] + mock_conn.meta.list_columns.return_value = [ + {"COLUMN_NAME": "ID", "COLUMN_TYPE": "DECIMAL(18,0)"}, + {"COLUMN_NAME": "NAME", "COLUMN_TYPE": "VARCHAR(200) UTF8"}, + ] + + result = adapter.get_columns(mock_conn, "ORDERS", schema="SALES") + + assert result == [ + ColumnInfo(name="ID", data_type="DECIMAL(18,0)", is_primary_key=True), + ColumnInfo(name="NAME", data_type="VARCHAR(200) UTF8", is_primary_key=False), + ] + + +def test_primary_key_lookup_is_snapshot_executed_and_parameterised(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + adapter.get_columns(mock_conn, "ORDERS", schema="SALES") + + # conn.meta.* wraps the query in Exasol's snapshot-execution hint, so it + # cannot be blocked by a metadata lock; conn.execute would not be. + mock_conn.execute.assert_not_called() + mock_conn.meta.execute_snapshot.assert_called_once() + + sql, params = mock_conn.meta.execute_snapshot.call_args.args + assert "SYS.EXA_ALL_CONSTRAINT_COLUMNS" in sql + assert "CONSTRAINT_TYPE = 'PRIMARY KEY'" in sql + # Placeholders, not interpolated values. + assert "{schema!s}" in sql + assert "{table!s}" in sql + assert params == {"schema": "SALES", "table": "ORDERS"} + + +def test_table_without_a_primary_key_flags_no_column(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + mock_conn.meta.execute_snapshot.return_value.fetchall.return_value = [] + mock_conn.meta.list_columns.return_value = [ + {"COLUMN_NAME": "A", "COLUMN_TYPE": "BOOLEAN"}, + {"COLUMN_NAME": "B", "COLUMN_TYPE": "DATE"}, + ] + + result = adapter.get_columns(mock_conn, "LOG", schema="SALES") + + assert [column.is_primary_key for column in result] == [False, False] + + +def test_get_columns_without_a_schema_passes_an_empty_pattern(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + # Design D8: default_schema is "", so an unset schema reaches list_columns as + # "" and matches nothing. The path is unreachable from the explorer - every + # Exasol table arrives from get_tables() as a populated (schema, name) pair - + # so it was left spec-faithful rather than given a fallback. This pins that + # deliberate choice; it does not endorse it. + adapter.get_columns(mock_conn, "ORDERS") + + assert adapter.default_schema == "" + assert mock_conn.meta.list_columns.call_args.args == ("", "ORDERS") + + +def test_get_procedures_returns_scripting_script_names(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + mock_conn.meta.execute_snapshot.return_value.fetchall.return_value = [ + {"SCRIPT_SCHEMA": "SALES", "SCRIPT_NAME": "REBUILD"}, + {"SCRIPT_SCHEMA": "SALES", "SCRIPT_NAME": "PURGE"}, + ] + + assert adapter.get_procedures(mock_conn) == ["REBUILD", "PURGE"] + + sql = mock_conn.meta.execute_snapshot.call_args.args[0] + assert "SYS.EXA_ALL_SCRIPTS" in sql + # SCRIPTING is Exasol's scripting-program type, as opposed to UDF, ADAPTER + # and PREPROCESSOR. + assert "SCRIPT_TYPE = 'SCRIPTING'" in sql + mock_conn.execute.assert_not_called() + + +@pytest.mark.parametrize("method_name", ["get_databases", "get_indexes", "get_triggers", "get_sequences"]) +def test_unsupported_object_kinds_return_empty_without_querying(adapter: ExasolAdapter, method_name: str) -> None: + conn = MagicMock() + + assert getattr(adapter, method_name)(conn) == [] + assert conn.mock_calls == [] + + +# --- Query execution -------------------------------------------------------- + + +def test_row_count_statement_returns_empty_without_fetching(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + stmt = _statement(result_type="rowCount") + mock_conn.execute.return_value = stmt + + assert adapter.execute_query(mock_conn, "INSERT INTO SALES.ORDERS VALUES (1)") == ([], [], False) + + # The guard has to precede any fetch: ExaStatement.__next__ raises + # ExaRuntimeError for a rowCount statement, and fetchmany() iterates. + stmt.fetchall.assert_not_called() + stmt.fetchmany.assert_not_called() + + +def test_result_set_statement_returns_every_row_as_a_tuple(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + # Lists in, tuples out - this pins the conversion rather than the input type. + stmt = _statement(result_type="resultSet", columns=("ID", "NAME"), rows=[[1, "a"], [2, "b"]]) + mock_conn.execute.return_value = stmt + + columns, rows, truncated = adapter.execute_query(mock_conn, "SELECT * FROM SALES.ORDERS") + + assert columns == ["ID", "NAME"] + assert rows == [(1, "a"), (2, "b")] + assert all(isinstance(row, tuple) for row in rows) + assert truncated is False + stmt.fetchmany.assert_not_called() + + +@pytest.mark.parametrize(("available", "expected_truncated"), [(2, False), (3, True)]) +def test_truncation_flag_at_the_max_rows_boundary( + adapter: ExasolAdapter, mock_conn: MagicMock, available: int, expected_truncated: bool +) -> None: + stmt = _statement(result_type="resultSet", columns=("N",), rows=[(n,) for n in range(available)]) + mock_conn.execute.return_value = stmt + + _, rows, truncated = adapter.execute_query(mock_conn, "SELECT N FROM SALES.ORDERS", max_rows=2) + + assert len(rows) == 2 + assert truncated is expected_truncated + + +def test_one_row_beyond_the_limit_is_requested(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + stmt = _statement(result_type="resultSet", columns=("N",), rows=[(1,)]) + mock_conn.execute.return_value = stmt + + adapter.execute_query(mock_conn, "SELECT N FROM SALES.ORDERS", max_rows=2) + + # One extra row is what makes truncation detectable. + stmt.fetchmany.assert_called_once_with(3) + stmt.fetchall.assert_not_called() + + +def test_execute_non_query_calls_rowcount_as_a_method(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + stmt = _statement(result_type="rowCount") + stmt.rowcount.return_value = 7 + mock_conn.execute.return_value = stmt + + result = adapter.execute_non_query(mock_conn, "DELETE FROM SALES.ORDERS") + + # rowcount is a method on ExaStatement, not a property: reading it would hand + # int() a bound method. + stmt.rowcount.assert_called_once_with() + assert result == 7 + assert isinstance(result, int) + # No explicit commit - the connection is opened with autocommit=True. + mock_conn.commit.assert_not_called() + + +def test_execute_test_query_does_not_use_cursor(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: + # The inherited implementation calls conn.cursor(), which pyexasol lacks. + # Deleting the attribute makes any access raise AttributeError. + del mock_conn.cursor + + adapter.execute_test_query(mock_conn) + + assert adapter.test_query == "SELECT 1" + mock_conn.execute.assert_called_once_with("SELECT 1") + mock_conn.execute.return_value.fetchval.assert_called_once_with() + + +# --- Identifier quoting and select building --------------------------------- + + +def test_quote_identifier_wraps_in_double_quotes(adapter: ExasolAdapter) -> None: + assert adapter.quote_identifier("MY_TABLE") == '"MY_TABLE"' + + +def test_quote_identifier_doubles_an_embedded_double_quote(adapter: ExasolAdapter) -> None: + assert adapter.quote_identifier('WEIRD"NAME') == '"WEIRD""NAME"' + + +def test_build_select_query_qualifies_with_the_schema(adapter: ExasolAdapter) -> None: + assert adapter.build_select_query("T", 10, schema="S") == 'SELECT * FROM "S"."T" LIMIT 10' + + +def test_build_select_query_without_a_schema_omits_the_segment(adapter: ExasolAdapter) -> None: + query = adapter.build_select_query("T", 10) + + assert query == 'SELECT * FROM "T" LIMIT 10' + # No leading dot from an empty schema segment. + assert '."T"' not in query diff --git a/tests/connections/providers/exasol/test_connect.py b/tests/connections/providers/exasol/test_connect.py new file mode 100644 index 00000000..e6dba62e --- /dev/null +++ b/tests/connections/providers/exasol/test_connect.py @@ -0,0 +1,218 @@ +"""Tests for the kwargs ExasolAdapter.connect() passes to pyexasol. + +The driver is faked by seeding sys.modules, which importlib.import_module - and +so _import_driver_module - returns without touching the filesystem. That keeps +these tests passing with the exasol extra absent, which the default CI unit job +requires. _import_driver_module is deliberately NOT patched: the driver_name / +extra_name / package_name plumbing that produces sqlit's install prompt is part +of what is under test, and patching it would let a mistyped module name pass. + +Every kwarg name is spelled as a literal string. MagicMock accepts any keyword, +so a pyexasol rename cannot fail these tests - it has to surface as a visible +diff in this file instead. Only the Docker integration test can validate the +driver's actual contract. +""" + +from __future__ import annotations + +import ssl +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from sqlit.domains.connections.domain.config import ConnectionConfig, FileEndpoint, TcpEndpoint +from sqlit.domains.connections.providers.exasol.adapter import ExasolAdapter +from sqlit.domains.connections.providers.registry import get_default_port + +AUTHENTICATORS = ("password", "access_token", "refresh_token") + + +def _config( + *, + host: str = "db.example.com", + port: str = "1234", + username: str = "sys", + password: str | None = "exasol", + options: dict[str, Any] | None = None, + extra_options: dict[str, str] | None = None, +) -> ConnectionConfig: + return ConnectionConfig( + name="test_exasol", + db_type="exasol", + endpoint=TcpEndpoint(host=host, port=port, username=username, password=password), + options=dict(options or {}), + extra_options=dict(extra_options or {}), + ) + + +def _connect_kwargs(config: ConnectionConfig) -> dict[str, Any]: + """Run connect() against a faked pyexasol and return the recorded kwargs.""" + fake_driver = MagicMock() + with patch.dict("sys.modules", {"pyexasol": fake_driver}): + ExasolAdapter().connect(config) + return dict(fake_driver.connect.call_args.kwargs) + + +# --- Credentials ------------------------------------------------------------ +# The unused methods' keys must be ABSENT, not empty: pyexasol's _login() +# branches on token truthiness, so a present-but-empty access_token is falsy and +# silently falls back to password login - exactly the bug an emptiness-tolerant +# assertion would let through. + + +@pytest.mark.parametrize("options", [{}, {"authenticator": "password"}], ids=["unset", "explicit"]) +def test_password_auth_sends_endpoint_credentials(options: dict[str, Any]) -> None: + kwargs = _connect_kwargs(_config(options=options)) + + assert kwargs["user"] == "sys" + assert kwargs["password"] == "exasol" + assert "access_token" not in kwargs + assert "refresh_token" not in kwargs + + +def test_access_token_auth_sends_only_the_access_token() -> None: + kwargs = _connect_kwargs(_config(options={"authenticator": "access_token", "access_token": "acc-tok"})) + + assert kwargs["access_token"] == "acc-tok" + assert "user" not in kwargs + assert "password" not in kwargs + assert "refresh_token" not in kwargs + + +def test_refresh_token_auth_sends_only_the_refresh_token() -> None: + kwargs = _connect_kwargs(_config(options={"authenticator": "refresh_token", "refresh_token": "ref-tok"})) + + assert kwargs["refresh_token"] == "ref-tok" + assert "user" not in kwargs + assert "password" not in kwargs + assert "access_token" not in kwargs + + +# --- Endpoint, schema, autocommit ------------------------------------------- + + +def test_dsn_joins_host_and_port_with_a_colon() -> None: + assert _connect_kwargs(_config())["dsn"] == "db.example.com:1234" + + +def test_absent_port_falls_back_to_the_registered_default() -> None: + kwargs = _connect_kwargs(_config(port="")) + + assert get_default_port("exasol") == "8563" + assert kwargs["dsn"] == "db.example.com:8563" + + +def test_schema_option_is_forwarded_verbatim() -> None: + assert _connect_kwargs(_config(options={"schema": "TEST_SQLIT"}))["schema"] == "TEST_SQLIT" + + +def test_unset_schema_is_sent_as_an_empty_string() -> None: + kwargs = _connect_kwargs(_config()) + + # Present-but-empty, not omitted: pyexasol reads "" as "no initial schema". + assert "schema" in kwargs + assert kwargs["schema"] == "" + + +@pytest.mark.parametrize("authenticator", AUTHENTICATORS) +def test_autocommit_is_enabled_for_every_authenticator(authenticator: str) -> None: + assert _connect_kwargs(_config(options={"authenticator": authenticator}))["autocommit"] is True + + +def test_non_tcp_config_is_rejected_before_the_driver_is_called() -> None: + config = ConnectionConfig(name="test_exasol", db_type="exasol", endpoint=FileEndpoint(path="exasol.db")) + fake_driver = MagicMock() + + with patch.dict("sys.modules", {"pyexasol": fake_driver}), pytest.raises(ValueError, match="TCP-style endpoint"): + ExasolAdapter().connect(config) + + fake_driver.connect.assert_not_called() + + +# --- TLS mode mapping ------------------------------------------------------- + + +def test_tls_disable_turns_encryption_off() -> None: + kwargs = _connect_kwargs(_config(options={"tls_mode": "disable"})) + + assert kwargs["encryption"] is False + assert "websocket_sslopt" not in kwargs + + +@pytest.mark.parametrize("options", [{}, {"tls_mode": "default"}], ids=["unset", "explicit"]) +def test_tls_default_encrypts_and_leaves_ssl_options_to_the_driver(options: dict[str, Any]) -> None: + kwargs = _connect_kwargs(_config(options=options)) + + assert kwargs["encryption"] is True + assert "websocket_sslopt" not in kwargs + + +def test_tls_require_encrypts_without_verifying_the_certificate() -> None: + kwargs = _connect_kwargs(_config(options={"tls_mode": "require"})) + + assert kwargs["encryption"] is True + assert kwargs["websocket_sslopt"] == {"cert_reqs": ssl.CERT_NONE} + + +@pytest.mark.parametrize("tls_mode", ["verify-ca", "verify-full"]) +def test_verifying_modes_request_certificate_validation(tls_mode: str) -> None: + kwargs = _connect_kwargs(_config(options={"tls_mode": tls_mode})) + + assert kwargs["encryption"] is True + assert kwargs["websocket_sslopt"]["cert_reqs"] == ssl.CERT_REQUIRED + + +@pytest.mark.parametrize("tls_mode", ["verify-ca", "verify-full"]) +def test_verifying_modes_forward_configured_certificate_files(tls_mode: str) -> None: + kwargs = _connect_kwargs( + _config( + options={ + "tls_mode": tls_mode, + "tls_ca": "/certs/ca.pem", + "tls_cert": "/certs/client.pem", + "tls_key": "/certs/client.key", + } + ) + ) + + assert kwargs["websocket_sslopt"] == { + "cert_reqs": ssl.CERT_REQUIRED, + "ca_certs": "/certs/ca.pem", + "certfile": "/certs/client.pem", + "keyfile": "/certs/client.key", + } + + +@pytest.mark.parametrize( + "certificate_options", + [{}, {"tls_ca": "", "tls_cert": " ", "tls_key": ""}], + ids=["unset", "whitespace"], +) +def test_unconfigured_certificate_files_are_omitted(certificate_options: dict[str, Any]) -> None: + kwargs = _connect_kwargs(_config(options={"tls_mode": "verify-full", **certificate_options})) + + assert kwargs["websocket_sslopt"] == {"cert_reqs": ssl.CERT_REQUIRED} + + +# --- extra_options ---------------------------------------------------------- + + +def test_extra_options_reach_the_driver_verbatim() -> None: + kwargs = _connect_kwargs(_config(extra_options={"connection_timeout": "30"})) + + assert kwargs["connection_timeout"] == "30" + + +def test_extra_options_override_computed_kwargs() -> None: + # extra_options is applied last, so it wins over the tls_mode mapping and + # over the schema option. + kwargs = _connect_kwargs( + _config( + options={"tls_mode": "disable", "schema": "FROM_OPTIONS"}, + extra_options={"encryption": "True", "schema": "FROM_EXTRA"}, + ) + ) + + assert kwargs["encryption"] == "True" + assert kwargs["schema"] == "FROM_EXTRA" diff --git a/tests/connections/providers/exasol/test_schema.py b/tests/connections/providers/exasol/test_schema.py new file mode 100644 index 00000000..b40c6cf6 --- /dev/null +++ b/tests/connections/providers/exasol/test_schema.py @@ -0,0 +1,84 @@ +"""Tests for the Exasol schema's conditional credential-field visibility. + +Pins the rule that only the selected authentication method's credential fields +are shown: pyexasol's login branches on token truthiness, so a form that can +collect a password and a token at the same time silently changes the auth path. +No driver is involved - these are pure predicate evaluations. +""" + +from __future__ import annotations + +import pytest + +from sqlit.domains.connections.providers.exasol.schema import SCHEMA + +CREDENTIAL_FIELDS = ("username", "password", "access_token", "refresh_token") +UNCONDITIONAL_FIELDS = ("server", "port", "authenticator", "schema") +AUTHENTICATORS = ("password", "access_token", "refresh_token") + + +def _visible_fields(values: dict) -> set[str]: + """Names the form shows for these values; no predicate means always visible.""" + return {field.name for field in SCHEMA.fields if field.visible_when is None or field.visible_when(values)} + + +def _credential_visibility(values: dict) -> dict[str, bool]: + visible = _visible_fields(values) + return {name: name in visible for name in CREDENTIAL_FIELDS} + + +def test_password_authenticator_shows_username_and_password() -> None: + assert _credential_visibility({"authenticator": "password"}) == { + "username": True, + "password": True, + "access_token": False, + "refresh_token": False, + } + + +def test_access_token_authenticator_shows_only_the_access_token() -> None: + assert _credential_visibility({"authenticator": "access_token"}) == { + "username": False, + "password": False, + "access_token": True, + "refresh_token": False, + } + + +def test_refresh_token_authenticator_shows_only_the_refresh_token() -> None: + assert _credential_visibility({"authenticator": "refresh_token"}) == { + "username": False, + "password": False, + "access_token": False, + "refresh_token": True, + } + + +def test_absent_authenticator_falls_back_to_password() -> None: + # Each predicate defaults its lookup to "password", so an empty form is + # indistinguishable from an explicit password selection. + assert _visible_fields({}) == _visible_fields({"authenticator": "password"}) + + +def test_unrecognised_authenticator_shows_no_credential_fields() -> None: + # The predicates compare for equality rather than negating each other, so an + # unknown value hides all three methods instead of leaking one of them. + assert _credential_visibility({"authenticator": "kerberos"}) == { + "username": False, + "password": False, + "access_token": False, + "refresh_token": False, + } + + +def test_unconditional_fields_carry_no_predicate() -> None: + fields = {field.name: field for field in SCHEMA.fields} + for name in UNCONDITIONAL_FIELDS: + assert fields[name].visible_when is None, f"{name} must not be conditional" + + +@pytest.mark.parametrize("authenticator", AUTHENTICATORS) +def test_unconditional_fields_stay_visible_under_every_authenticator(authenticator: str) -> None: + visible = _visible_fields({"authenticator": authenticator}) + for name in UNCONDITIONAL_FIELDS: + assert name in visible diff --git a/tests/fixtures/exasol.py b/tests/fixtures/exasol.py new file mode 100644 index 00000000..609eaf28 --- /dev/null +++ b/tests/fixtures/exasol.py @@ -0,0 +1,202 @@ +"""Exasol fixtures.""" + +from __future__ import annotations + +import os +import ssl +import time +from typing import Any + +import pytest + +from tests.fixtures.utils import cleanup_connection, is_port_open, run_cli + +# Exasol Fixtures +EXASOL_HOST = os.environ.get("EXASOL_HOST", "localhost") +EXASOL_PORT = int(os.environ.get("EXASOL_PORT", "8563")) +EXASOL_USER = os.environ.get("EXASOL_USER", "sys") +EXASOL_PASSWORD = os.environ.get("EXASOL_PASSWORD", "exasol") +EXASOL_SCHEMA = os.environ.get("EXASOL_SCHEMA", "TEST_SQLIT") + +# exasol/docker-db binds 8563 long before it accepts a login, so readiness is a +# real connect retried until this deadline rather than a bare open port. +EXASOL_READY_TIMEOUT = float(os.environ.get("EXASOL_READY_TIMEOUT", "300")) +_READY_INTERVAL = 5.0 + +# Set by exasol_server_ready when the deadline passes, so the skip message can +# name the driver error instead of only reporting "not available". +_ready_error: str | None = None + + +def exasol_available() -> bool: + """Check if Exasol is available.""" + return is_port_open(EXASOL_HOST, EXASOL_PORT) + + +def _connect() -> Any: + """Open a pyexasol connection to the test server. + + pyexasol is imported here rather than at module level: tests/conftest.py + star-imports this module and is loaded by the driver-free unit CI job. + docker-db presents a self-signed certificate, hence cert_reqs=CERT_NONE. + """ + import pyexasol + + return pyexasol.connect( + dsn=f"{EXASOL_HOST}:{EXASOL_PORT}", + user=EXASOL_USER, + password=EXASOL_PASSWORD, + encryption=True, + websocket_sslopt={"cert_reqs": ssl.CERT_NONE}, + autocommit=True, + ) + + +@pytest.fixture(scope="session") +def exasol_server_ready() -> bool: + """Check if Exasol is ready and return True/False.""" + global _ready_error + + if not exasol_available(): + return False + + try: + import pyexasol # noqa: F401 + except ImportError: + pytest.skip("pyexasol is not installed") + + deadline = time.time() + EXASOL_READY_TIMEOUT + while True: + try: + _connect().close() + return True + except Exception as e: + _ready_error = str(e) + if time.time() >= deadline: + return False + time.sleep(_READY_INTERVAL) + + +@pytest.fixture(scope="function") +def exasol_db(exasol_server_ready: bool) -> str: + """Set up Exasol test schema.""" + if not exasol_server_ready: + detail = f": {_ready_error}" if _ready_error else "" + pytest.skip(f"Exasol is not available{detail}") + + try: + import pyexasol # noqa: F401 + except ImportError: + pytest.skip("pyexasol is not installed") + + try: + conn = _connect() + + conn.execute(f"DROP SCHEMA IF EXISTS {EXASOL_SCHEMA} CASCADE") + conn.execute(f"CREATE SCHEMA {EXASOL_SCHEMA}") + conn.execute(f"OPEN SCHEMA {EXASOL_SCHEMA}") + + # Identifiers stay unquoted so Exasol's uppercase folding makes them + # resolve from the shared suite's own unquoted queries. + conn.execute(""" + CREATE TABLE test_users ( + id DECIMAL(18,0) PRIMARY KEY, + name VARCHAR(100), + email VARCHAR(200) + ) + """) + + conn.execute(""" + CREATE TABLE test_products ( + id DECIMAL(18,0), + name VARCHAR(100), + price DECIMAL(10,2), + stock DECIMAL(18,0) + ) + """) + + # Exasol treats the empty string as NULL, so IS NOT NULL is the + # non-empty test here; `email != ''` would match no row at all. + conn.execute(""" + CREATE VIEW test_user_emails AS + SELECT id, name, email FROM test_users WHERE email IS NOT NULL + """) + + # No index, trigger or sequence: ExasolAdapter reports all three + # capabilities as unsupported, so the matching base tests self-skip. + + conn.execute(""" + INSERT INTO test_users (id, name, email) VALUES + (1, 'Alice', 'alice@example.com'), + (2, 'Bob', 'bob@example.com'), + (3, 'Charlie', 'charlie@example.com') + """) + + conn.execute(""" + INSERT INTO test_products (id, name, price, stock) VALUES + (1, 'Widget', 9.99, 100), + (2, 'Gadget', 19.99, 50), + (3, 'Gizmo', 29.99, 25) + """) + + conn.close() + + except Exception as e: + pytest.skip(f"Failed to setup Exasol schema: {e}") + + yield EXASOL_SCHEMA + + try: + conn = _connect() + conn.execute(f"DROP SCHEMA IF EXISTS {EXASOL_SCHEMA} CASCADE") + conn.close() + except Exception: + pass + + +@pytest.fixture(scope="function") +def exasol_connection(exasol_db: str) -> str: + """Create a sqlit CLI connection for Exasol and clean up after test.""" + connection_name = f"test_exasol_{os.getpid()}" + + cleanup_connection(connection_name) + + # --tls-mode require: docker-db presents a self-signed certificate, so the + # connection has to encrypt without verifying the chain. + run_cli( + "connections", + "add", + "exasol", + "--name", + connection_name, + "--server", + EXASOL_HOST, + "--port", + str(EXASOL_PORT), + "--username", + EXASOL_USER, + "--password", + EXASOL_PASSWORD, + "--schema", + exasol_db, + "--tls-mode", + "require", + ) + + yield connection_name + + cleanup_connection(connection_name) + + +__all__ = [ + "EXASOL_HOST", + "EXASOL_PASSWORD", + "EXASOL_PORT", + "EXASOL_READY_TIMEOUT", + "EXASOL_SCHEMA", + "EXASOL_USER", + "exasol_available", + "exasol_connection", + "exasol_db", + "exasol_server_ready", +] diff --git a/tests/test_exasol.py b/tests/test_exasol.py new file mode 100644 index 00000000..5ebe5443 --- /dev/null +++ b/tests/test_exasol.py @@ -0,0 +1,170 @@ +"""Integration tests for Exasol database operations.""" + +from __future__ import annotations + +import pytest + +from .test_database_base import BaseDatabaseTestsWithLimit, DatabaseTestConfig + + +class TestExasolIntegration(BaseDatabaseTestsWithLimit): + """Integration tests for Exasol database operations via CLI. + + These tests require a running Exasol instance (via Docker). + Tests are skipped if Exasol is not available. + """ + + @property + def config(self) -> DatabaseTestConfig: + return DatabaseTestConfig( + db_type="exasol", + display_name="Exasol", + connection_fixture="exasol_connection", + db_fixture="exasol_db", + create_connection_args=lambda: [], # Uses fixtures + ) + + def test_docker_container_connection(self, request): + """Docker-discovered credentials cannot connect to exasol/docker-db. + + Two independent properties of the image, neither fixable from tests/: + exasol/docker-db publishes no credentials through environment variables + (SPEC.docker_detector has env_vars={}), so the discovered config carries + no password; and a discovery-built config carries no tls_mode, so the + adapter verifies TLS against the image's self-signed certificate. + """ + pytest.skip( + "exasol/docker-db publishes no credentials through environment " + "variables (docker_detector env_vars={}), and a discovery-built " + "config has no tls_mode, so it verifies TLS against the image's " + "self-signed certificate" + ) + + def test_primary_key_detection(self, request): + """Test that adapter correctly detects primary key columns. + + Overrides the base version, which calls get_columns with a lowercase + table name and no schema. Against a live server that returns nothing: + the adapter passes an empty schema to pyexasol as a LIKE pattern + (LIKE '' matches nothing), and pyexasol's meta patterns are + case-sensitive while EXA_ALL_COLUMNS stores the folded TEST_USERS. + The app never makes that call - schema_service and process_worker both + pass the name and schema straight through from get_tables(), which + returns them uppercase - so this asserts the same contract through the + call shape the application actually uses. + """ + from sqlit.domains.connections.app.session import ConnectionSession + from sqlit.domains.connections.providers.registry import get_adapter + from sqlit.domains.connections.store.connections import load_connections + + from .conftest import EXASOL_SCHEMA + + connection_name = request.getfixturevalue(self.config.connection_fixture) + connections = load_connections() + config = next((c for c in connections if c.name == connection_name), None) + assert config is not None, f"Connection {connection_name} not found" + + with ConnectionSession.create(config, get_adapter) as session: + columns = session.adapter.get_columns( + session.connection, + "TEST_USERS", + database=None, + schema=EXASOL_SCHEMA, + ) + + assert len(columns) >= 3, f"Expected at least 3 columns, got {len(columns)}" + + id_column = next( + (col for col in columns if col.name.lower() == "id"), + None, + ) + assert id_column is not None, f"Column 'id' not found. Columns: {[c.name for c in columns]}" + assert id_column.is_primary_key, "Column 'id' should be marked as primary key" + + non_pk_columns = [col for col in columns if col.name.lower() != "id"] + for col in non_pk_columns: + assert not col.is_primary_key, f"Column '{col.name}' should NOT be marked as primary key" + + def test_create_exasol_connection(self, exasol_db, cli_runner): + """Test creating an Exasol connection via CLI.""" + from .conftest import ( + EXASOL_HOST, + EXASOL_PASSWORD, + EXASOL_PORT, + EXASOL_USER, + ) + + connection_name = "test_create_exasol" + + try: + result = cli_runner( + "connections", + "add", + "exasol", + "--name", + connection_name, + "--server", + EXASOL_HOST, + "--port", + str(EXASOL_PORT), + "--username", + EXASOL_USER, + "--password", + EXASOL_PASSWORD, + "--schema", + exasol_db, + "--tls-mode", + "require", + ) + assert result.returncode == 0 + assert "created successfully" in result.stdout + + # Verify it appears in list + result = cli_runner("connection", "list") + assert connection_name in result.stdout + assert "Exasol" in result.stdout + + finally: + # Cleanup + cli_runner("connection", "delete", connection_name, check=False) + + def test_delete_exasol_connection(self, exasol_db, cli_runner): + """Test deleting an Exasol connection.""" + from .conftest import ( + EXASOL_HOST, + EXASOL_PASSWORD, + EXASOL_PORT, + EXASOL_USER, + ) + + connection_name = "test_delete_exasol" + + # Create connection first + cli_runner( + "connections", + "add", + "exasol", + "--name", + connection_name, + "--server", + EXASOL_HOST, + "--port", + str(EXASOL_PORT), + "--username", + EXASOL_USER, + "--password", + EXASOL_PASSWORD, + "--schema", + exasol_db, + "--tls-mode", + "require", + ) + + # Delete it + result = cli_runner("connection", "delete", connection_name) + assert result.returncode == 0 + assert "deleted successfully" in result.stdout + + # Verify it's gone + result = cli_runner("connection", "list") + assert connection_name not in result.stdout diff --git a/uv.lock b/uv.lock index 35f44b86..1740081b 100644 --- a/uv.lock +++ b/uv.lock @@ -1911,27 +1911,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/28/2635a8141c9a4f4bc23f5135a92bbcf48d928d8ca094088c962df1879d64/lz4-4.4.5-cp314-cp314-win_arm64.whl", hash = "sha256:d994b87abaa7a88ceb7a37c90f547b8284ff9da694e6afcfaa8568d739faf3f7", size = 93812, upload-time = "2025-11-03T13:02:26.133Z" }, ] -[[package]] -name = "mariadb" -version = "1.1.14" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/ba/cedef19833be88e07bfff11964441cda8a998f1628dd3b2fa3e7751d36e0/mariadb-1.1.14.tar.gz", hash = "sha256:e6d702a53eccf20922e47f2f45cfb5c7a0c2c6c0a46e4ee2d8a80d0ff4a52f34", size = 111715, upload-time = "2025-10-07T06:45:48.017Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/bf/5e1a3a5be297c3a679e08f5359165491508fbfb64faf854dc1d626cea9c0/mariadb-1.1.14-cp310-cp310-win32.whl", hash = "sha256:4c7f33578da163a1b79929aae241f5f981d7b9d5a94d89e589aad7ec58e313ea", size = 185064, upload-time = "2025-10-07T06:45:24.858Z" }, - { url = "https://files.pythonhosted.org/packages/31/30/3a61991c13cb8257f5db64aca12bafaa3d811d407e1fae019139fd17c99b/mariadb-1.1.14-cp310-cp310-win_amd64.whl", hash = "sha256:3f6fdc4ded5e0500a6a29bf0c8bf1be94189dcef5a8d5e0e154a4b3456f86bcc", size = 202020, upload-time = "2025-10-07T06:45:27.785Z" }, - { url = "https://files.pythonhosted.org/packages/56/aa/a7b3c66b2792e8319ec9157d63851ff2e0b26496a05044e22b50a012a05e/mariadb-1.1.14-cp311-cp311-win32.whl", hash = "sha256:932a95016b7e9b8d78893aa5ee608e74199e3c6dd607dbe5e4da2010a4f67b88", size = 185061, upload-time = "2025-10-07T06:45:29.964Z" }, - { url = "https://files.pythonhosted.org/packages/54/04/ea2374867756b4082764484bc8b82e1798d94f171bcc914e08c60d640f8f/mariadb-1.1.14-cp311-cp311-win_amd64.whl", hash = "sha256:55ddbe5272c292cbcb2968d87681b5d2b327e65646a015e324b8eeb804d14531", size = 202016, upload-time = "2025-10-07T06:45:32.151Z" }, - { url = "https://files.pythonhosted.org/packages/00/04/659a8d30513700b5921ec96bddc07f550016c045fcbeb199d8cd18476ecc/mariadb-1.1.14-cp312-cp312-win32.whl", hash = "sha256:98d552a8bb599eceaa88f65002ad00bd88aeed160592c273a7e5c1d79ab733dd", size = 185266, upload-time = "2025-10-07T06:45:34.164Z" }, - { url = "https://files.pythonhosted.org/packages/e4/a9/8f210291bc5fc044e20497454f40d35b3bab326e2cab6fccdc38121cb2c1/mariadb-1.1.14-cp312-cp312-win_amd64.whl", hash = "sha256:685a1ad2a24fd0aae1c4416fe0ac794adc84ab9209c8d0c57078f770d39731db", size = 202112, upload-time = "2025-10-07T06:45:35.824Z" }, - { url = "https://files.pythonhosted.org/packages/f3/42/51130048bcce038bb859978250515f1aad90e9c4d273630a704e0a8b1ae1/mariadb-1.1.14-cp313-cp313-win32.whl", hash = "sha256:3d2c795cde606f4e12c0d73282b062433f414cae035675b0d81f2d65c9b79ac5", size = 185221, upload-time = "2025-10-07T06:45:37.848Z" }, - { url = "https://files.pythonhosted.org/packages/af/23/e952a7e442913abd8079cd27b80b69474895c93b3727fad41c7642a80c62/mariadb-1.1.14-cp313-cp313-win_amd64.whl", hash = "sha256:7fd603c5cf23c47ef0d28fdc2b4b79919ee7f75d00ed070d3cd1054dcf816aeb", size = 202121, upload-time = "2025-10-07T06:45:39.453Z" }, - { url = "https://files.pythonhosted.org/packages/e7/f2/7059f83543a4264b98777d7cc8aa203e1ca6a13a461f730d1c97f29628d4/mariadb-1.1.14-cp314-cp314-win32.whl", hash = "sha256:1a50b4612c0dd5b69690cebb34cef552a7f64dcadeb5aa91d70cd99bf01bc5b3", size = 190620, upload-time = "2025-10-07T06:45:41.349Z" }, - { url = "https://files.pythonhosted.org/packages/f8/7c/7e094b0b396d742494f6346f2ffa9709e429970b0461aca50526f5f02f12/mariadb-1.1.14-cp314-cp314-win_amd64.whl", hash = "sha256:6659725837e48fa6af05e20128fb525029f706f1921d5dbf639a25b2f80b9f93", size = 206263, upload-time = "2025-10-07T06:45:43.227Z" }, -] - [[package]] name = "markdown-it-py" version = "4.0.0" @@ -3369,6 +3348,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/f3/0a7087e5f861d66ca64ce927230b397cc264c87b712156e6a93b26a459c8/pydantic_core-2.46.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:154dbfdfb11b8cbd8ff4d00d0b81e3d19f4cb4bedd5aa9f091060ba071474c6a", size = 2192159, upload-time = "2026-04-17T09:11:20.123Z" }, ] +[[package]] +name = "pyexasol" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "packaging" }, + { name = "websocket-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/5a/362f7b99b7ac888e03eae6454013f7c5b0640c07433bcef390d30d204d8d/pyexasol-2.3.2.tar.gz", hash = "sha256:7a60517432122d60a217909f3bfdfeba171f79146cc55e8f5a24f4a744d9ccba", size = 64409, upload-time = "2026-08-25T09:26:26.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/74/fa606a85e80255a6ee1269bca4e952b2eff8bdd883b8933f6c14f94e2821/pyexasol-2.3.2-py3-none-any.whl", hash = "sha256:7e0277577bcaf211bbe3b8676faf815af5c6ef337d2d6358b0a07bfd3ed74e0d", size = 76296, upload-time = "2026-08-25T09:26:27.001Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -3894,7 +3887,6 @@ all = [ { name = "ibm-db" }, { name = "impyla" }, { name = "libsql" }, - { name = "mariadb" }, { name = "mssql-python" }, { name = "oracledb" }, { name = "osquery" }, @@ -3902,6 +3894,7 @@ all = [ { name = "presto-python-client" }, { name = "psycopg2-binary" }, { name = "pyathena" }, + { name = "pyexasol" }, { name = "pymysql" }, { name = "redshift-connector" }, { name = "requests" }, @@ -3932,6 +3925,9 @@ db2 = [ duckdb = [ { name = "duckdb" }, ] +exasol = [ + { name = "pyexasol" }, +] firebird = [ { name = "firebirdsql" }, ] @@ -3945,7 +3941,7 @@ impala = [ { name = "impyla" }, ] mariadb = [ - { name = "mariadb" }, + { name = "pymysql" }, ] mssql = [ { name = "mssql-python" }, @@ -4037,8 +4033,6 @@ requires-dist = [ { name = "keyring", specifier = ">=24.0.0" }, { name = "libsql", marker = "extra == 'all'", specifier = ">=0.1.0" }, { name = "libsql", marker = "extra == 'turso'", specifier = ">=0.1.0" }, - { name = "mariadb", marker = "extra == 'all'", specifier = ">=1.1.0" }, - { name = "mariadb", marker = "extra == 'mariadb'", specifier = ">=1.1.0" }, { name = "mssql-python", marker = "extra == 'all'", specifier = ">=1.1.0" }, { name = "mssql-python", marker = "extra == 'mssql'", specifier = ">=1.1.0" }, { name = "oracledb", marker = "extra == 'all'", specifier = ">=2.0.0" }, @@ -4054,7 +4048,10 @@ requires-dist = [ { name = "psycopg2-binary", marker = "extra == 'postgres'", specifier = ">=2.9.0" }, { name = "pyathena", marker = "extra == 'all'", specifier = ">=3.22.0" }, { name = "pyathena", marker = "extra == 'athena'", specifier = ">=3.22.0" }, + { name = "pyexasol", marker = "extra == 'all'", specifier = ">=2.0.0" }, + { name = "pyexasol", marker = "extra == 'exasol'", specifier = ">=2.0.0" }, { name = "pymysql", marker = "extra == 'all'", specifier = ">=1.1.0" }, + { name = "pymysql", marker = "extra == 'mariadb'", specifier = ">=1.1.0" }, { name = "pymysql", marker = "extra == 'mysql'", specifier = ">=1.1.0" }, { name = "pyperclip", specifier = ">=1.8.2" }, { name = "redshift-connector", marker = "extra == 'all'" }, @@ -4075,7 +4072,7 @@ requires-dist = [ { name = "trino", marker = "extra == 'all'", specifier = ">=0.329.0" }, { name = "trino", marker = "extra == 'trino'", specifier = ">=0.329.0" }, ] -provides-extras = ["all", "athena", "bigquery", "clickhouse", "cockroachdb", "d1", "db2", "duckdb", "firebird", "flight", "hana", "impala", "mariadb", "mssql", "mysql", "oracle", "osquery", "postgres", "presto", "redshift", "snowflake", "spanner", "ssh", "surrealdb", "teradata", "trino", "turso"] +provides-extras = ["all", "athena", "bigquery", "clickhouse", "cockroachdb", "d1", "db2", "duckdb", "exasol", "firebird", "flight", "hana", "impala", "mariadb", "mssql", "mysql", "oracle", "osquery", "postgres", "presto", "redshift", "snowflake", "spanner", "ssh", "surrealdb", "teradata", "trino", "turso"] [package.metadata.requires-dev] dev = [ @@ -4654,6 +4651,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b", size = 6005095, upload-time = "2025-10-29T06:57:37.598Z" }, ] +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + [[package]] name = "websockets" version = "16.0" From d633fb148f09f5f5934b2cad587c004f12df9d12 Mon Sep 17 00:00:00 2001 From: Muhammad Abdullah Farooqui Date: Fri, 28 Aug 2026 12:21:11 +0200 Subject: [PATCH 2/5] pyexasol TLS changed to default --- .serena/.gitignore | 2 + .serena/project.yml | 141 +++++ .../2026-08-27-exasol-adapter/.openspec.yaml | 2 + .../2026-08-27-exasol-adapter/design.md | 230 +++++++ .../2026-08-27-exasol-adapter/proposal.md | 77 +++ .../specs/exasol-adapter/spec.md | 366 +++++++++++ .../2026-08-27-exasol-adapter/tasks.md | 121 ++++ .../2026-08-27-exasol-docs/.openspec.yaml | 2 + .../archive/2026-08-27-exasol-docs/design.md | 167 +++++ .../2026-08-27-exasol-docs/proposal.md | 68 +++ .../specs/exasol-documentation/spec.md | 134 ++++ .../archive/2026-08-27-exasol-docs/tasks.md | 61 ++ .../.openspec.yaml | 2 + .../design.md | 202 ++++++ .../findings.md | 176 ++++++ .../proposal.md | 79 +++ .../specs/exasol-integration-coverage/spec.md | 127 ++++ .../specs/exasol-integration-harness/spec.md | 146 +++++ .../tasks.md | 152 +++++ .../.openspec.yaml | 2 + .../design.md | 245 ++++++++ .../proposal.md | 92 +++ .../exasol-provider-registration/spec.md | 320 ++++++++++ .../tasks.md | 129 ++++ .../.openspec.yaml | 2 + .../2026-08-27-exasol-unit-tests/design.md | 191 ++++++ .../2026-08-27-exasol-unit-tests/proposal.md | 65 ++ .../specs/exasol-driver-packaging/spec.md | 85 +++ .../specs/exasol-unit-coverage/spec.md | 309 ++++++++++ .../2026-08-27-exasol-unit-tests/tasks.md | 158 +++++ openspec/config.yaml | 20 + openspec/specs/exasol-adapter/spec.md | 382 ++++++++++++ openspec/specs/exasol-documentation/spec.md | 155 +++++ .../specs/exasol-driver-packaging/spec.md | 100 +++ .../specs/exasol-integration-coverage/spec.md | 151 +++++ .../specs/exasol-integration-harness/spec.md | 168 +++++ .../exasol-provider-registration/spec.md | 324 ++++++++++ openspec/specs/exasol-unit-coverage/spec.md | 326 ++++++++++ plan.md | 578 ++++++++++++++++++ .../connections/providers/exasol/adapter.py | 13 +- .../providers/exasol/test_connect.py | 6 +- tests/test_exasol.py | 12 +- 42 files changed, 6072 insertions(+), 16 deletions(-) create mode 100644 .serena/.gitignore create mode 100644 .serena/project.yml create mode 100644 openspec/changes/archive/2026-08-27-exasol-adapter/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-27-exasol-adapter/design.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-adapter/proposal.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-adapter/specs/exasol-adapter/spec.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-adapter/tasks.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-docs/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-27-exasol-docs/design.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-docs/proposal.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-docs/specs/exasol-documentation/spec.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-docs/tasks.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-integration-tests/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-27-exasol-integration-tests/design.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-integration-tests/findings.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-integration-tests/proposal.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-integration-tests/specs/exasol-integration-coverage/spec.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-integration-tests/specs/exasol-integration-harness/spec.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-integration-tests/tasks.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-provider-registration/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-27-exasol-provider-registration/design.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-provider-registration/proposal.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-provider-registration/specs/exasol-provider-registration/spec.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-provider-registration/tasks.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-unit-tests/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-27-exasol-unit-tests/design.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-unit-tests/proposal.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-unit-tests/specs/exasol-driver-packaging/spec.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-unit-tests/specs/exasol-unit-coverage/spec.md create mode 100644 openspec/changes/archive/2026-08-27-exasol-unit-tests/tasks.md create mode 100644 openspec/config.yaml create mode 100644 openspec/specs/exasol-adapter/spec.md create mode 100644 openspec/specs/exasol-documentation/spec.md create mode 100644 openspec/specs/exasol-driver-packaging/spec.md create mode 100644 openspec/specs/exasol-integration-coverage/spec.md create mode 100644 openspec/specs/exasol-integration-harness/spec.md create mode 100644 openspec/specs/exasol-provider-registration/spec.md create mode 100644 openspec/specs/exasol-unit-coverage/spec.md create mode 100644 plan.md diff --git a/.serena/.gitignore b/.serena/.gitignore new file mode 100644 index 00000000..2e510aff --- /dev/null +++ b/.serena/.gitignore @@ -0,0 +1,2 @@ +/cache +/project.local.yml diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 00000000..9d07b7ae --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,141 @@ +# the name by which the project can be referenced within Serena +project_name: "sqlit" + + +# list of languages for which language servers are started; choose from: +# al angular ansible bash clojure +# cpp cpp_ccls crystal csharp csharp_omnisharp +# dart elixir elm erlang fortran +# fsharp go groovy haskell haxe +# hlsl html java json julia +# kotlin lean4 lua luau markdown +# matlab msl nix ocaml pascal +# perl php php_phpactor powershell python +# python_jedi python_ty r rego ruby +# ruby_solargraph rust scala scss solidity +# svelte swift systemverilog terraform toml +# typescript typescript_vts vue yaml zig +# (This list may be outdated. For the current list, see values of Language enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py +# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) +# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) +# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# Some languages require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers +# When using multiple languages, the first language server that supports a given file will be used for that file. +# The first language is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +languages: +- python + +# the encoding used by text files in the project +# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings +encoding: "utf-8" + +# line ending convention to use when writing source files. +# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default) +# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings. +line_ending: + +# The language backend to use for this project. +# If not set, the global setting from serena_config.yml is used. +# Valid values: LSP, JetBrains +# Note: the backend is fixed at startup. If a project with a different backend +# is activated post-init, an error will be returned. +language_backend: + +# whether to use project's .gitignore files to ignore files +ignore_all_files_in_gitignore: true + +# advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. +# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. +# No documentation on options means no options are available. +ls_specific_settings: {} + +# list of additional paths to ignore in this project. +# Same syntax as gitignore, so you can use * and **. +# Note: global ignored_paths from serena_config.yml are also applied additively. +ignored_paths: [] + +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + +# list of tool names to exclude. +# This extends the existing exclusions (e.g. from the global configuration) +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +excluded_tools: [] + +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). +# This extends the existing inclusions (e.g. from the global configuration). +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +fixed_tools: [] + +# list of mode names to that are always to be included in the set of active modes +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this setting overrides the global configuration. +# Set this to [] to disable base modes for this project. +# Set this to a list of mode names to always include the respective modes for this project. +base_modes: + +# list of mode names that are to be activated by default, overriding the setting in the global configuration. +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply +# for this project. +# This setting can, in turn, be overridden by CLI parameters (--mode). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +default_modes: + +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). +initial_prompt: "" + +# time budget (seconds) per tool call for the retrieval of additional symbol information +# such as docstrings or parameter information. +# This overrides the corresponding setting in the global configuration; see the documentation there. +# If null or missing, use the setting from the global configuration. +symbol_info_budget: + +# list of regex patterns which, when matched, mark a memory entry as read‑only. +# Extends the list from the global configuration, merging the two lists. +read_only_memory_patterns: [] + +# list of regex patterns for memories to completely ignore. +# Matching memories will not appear in list_memories or activate_project output +# and cannot be accessed via read_memory or write_memory. +# To access ignored memory files, use the read_file tool on the raw file path. +# Extends the list from the global configuration, merging the two lists. +# Example: ["_archive/.*", "_episodes/.*"] +ignored_memory_patterns: [] + +# list of mode names to be activated additionally for this project, e.g. ["query-projects"] +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +added_modes: + +# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos). +# Paths can be absolute or relative to the project root. +# Each folder is registered as an LSP workspace folder, enabling language servers to discover +# symbols and references across package boundaries. +# Currently supported for: TypeScript. +# Example: +# additional_workspace_folders: +# - ../sibling-package +# - ../shared-lib +additional_workspace_folders: [] diff --git a/openspec/changes/archive/2026-08-27-exasol-adapter/.openspec.yaml b/openspec/changes/archive/2026-08-27-exasol-adapter/.openspec.yaml new file mode 100644 index 00000000..f05b045c --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-adapter/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-27 diff --git a/openspec/changes/archive/2026-08-27-exasol-adapter/design.md b/openspec/changes/archive/2026-08-27-exasol-adapter/design.md new file mode 100644 index 00000000..93e7f586 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-adapter/design.md @@ -0,0 +1,230 @@ +## Context + +sqlit auto-discovers providers: `providers/catalog.py::_discover_providers` walks every +subpackage of `sqlit/domains/connections/providers/` and imports `/provider.py`. + +**Correction (found in implementation, see D8):** that import was unconditional, so a +subpackage *without* `provider.py` broke discovery entirely instead of being ignored. + +The moment `provider.py` exists, the provider is live — and `tests/test_schema_capabilities.py` +asserts `{t.value for t in DatabaseType} == set(get_supported_db_types())` as an **exact set +equality**. That makes registration an all-or-nothing step involving three files at once +(`schema.py`, the `DatabaseType` enum, `provider.py`). + +This change deliberately sits *below* that line. It adds only the adapter, which nothing +imports until registration lands, so the repo cannot enter a state where Exasol is selectable +but broken. It corresponds to plan.md steps 1-4. + +Exasol's driver, `pyexasol`, is not DB-API 2.0 — it is a WebSocket client with no `.cursor()`. +That single fact drives most of the decisions below. + +**Constraint: `pyexasol` is not installed.** The `exasol` extra is plan.md step 8, a separate +change. Everything here was therefore verified against the published pyexasol source and the +Exasol system-table documentation rather than a live interpreter. The gate for this change is +static only: `uv run ruff check sqlit && uv run mypy sqlit`. + +**Reference implementations read for house style:** `providers/hana/adapter.py` (single +database, many schemas, `get_databases()` returning empty, doubled-double-quote quoting) and +`providers/clickhouse/adapter.py` (a `DatabaseAdapter` subclass that hand-rolls +`execute_query` and consumes the shared `providers/tls.py` helpers). + +## Goals / Non-Goals + +**Goals:** + +- A complete, concrete `ExasolAdapter` — every abstract member of `DatabaseAdapter` + implemented, so `mypy` accepts instantiation. +- Correct mapping of Exasol's three auth methods and sqlit's five `tls_mode` values onto + pyexasol's kwargs. +- Introspection that cannot deadlock against metadata locks. +- Query execution that reports truncation and survives statements with no result set. +- Zero change in behaviour for every existing provider and test. + +**Non-Goals:** + +- Registering the provider (`provider.py`, `schema.py`, `DatabaseType.EXASOL`) — plan.md + steps 5-7, the atomic `ACT` group. +- Declaring the `exasol` extra, the mypy override, or the pytest marker — plan.md step 8. +- Any test file. Behaviour is covered by plan.md steps 9-11; this change is statically gated. +- Docker/integration plumbing and docs — plan.md steps 12-15. + +## Decisions + +### D1 — Subclass `DatabaseAdapter`, not `CursorBasedAdapter` + +`CursorBasedAdapter` implements `execute_query`/`execute_non_query` in terms of +`conn.cursor()`. pyexasol has no `cursor()`; it returns an `ExaStatement` from +`conn.execute()`. Inheriting the cursor base would mean overriding both methods anyway plus +inheriting a misleading contract. + +*Alternative considered:* write a thin DB-API shim over pyexasol so `CursorBasedAdapter` could +be reused. Rejected — an adapter layer over an adapter layer, to save two short methods, and +it would hide the truncation and result-type handling that D3 needs to be explicit about. +`ClickHouseAdapter` sets the precedent for subclassing `DatabaseAdapter` directly. + +### D2 — Introspect through `conn.meta.*`, and read rows **by key** + +pyexasol's `ExaMetaData` prefixes every query with Exasol's snapshot-execution hint, so +metadata reads cannot be blocked by metadata locks. This is the driver-recommended path and +strictly better than hand-rolled `SELECT`s against `SYS.*`. + +**This corrects plan.md step 3.** The plan's notation — `conn.meta.list_tables()` yielding +`(TABLE_SCHEMA, TABLE_NAME)` — reads as tuple indexing. It is not. pyexasol's +`execute_snapshot` hard-codes `options = {"fetch_dict": True}`, with the explicit rationale +*"fetch_dict=True is enforced to prevent users from relying on order of columns"*. So: + +| Call | Returns | +|---|---| +| `conn.meta.list_tables(schema_pat, name_pat)` | `list[dict]` — already `.fetchall()`-ed | +| `conn.meta.list_views(schema_pat, name_pat)` | `list[dict]` — already `.fetchall()`-ed | +| `conn.meta.list_columns(schema_pat, table_pat, ...)` | `list[dict]` — already `.fetchall()`-ed | +| `conn.meta.execute_snapshot(query, params)` | an `ExaStatement` — needs an explicit `.fetchall()` | + +Note the asymmetry in that last row: the three `list_*` helpers materialise their rows, while +`execute_snapshot` does not. Indexing a row positionally would raise `KeyError` at runtime and +would not be caught by `mypy`, so this is the highest-value correction in this document. + +Keys are UPPERCASE, since Exasol upper-cases unquoted identifiers and the `list_*` helpers all +issue `SELECT *`. Column names used, all verified against docs.exasol.com: + +- `EXA_ALL_TABLES` → `TABLE_SCHEMA`, `TABLE_NAME` +- `EXA_ALL_VIEWS` → `VIEW_SCHEMA`, `VIEW_NAME` +- `EXA_ALL_COLUMNS` → `COLUMN_NAME`, `COLUMN_TYPE`, `COLUMN_ORDINAL_POSITION` +- `EXA_ALL_CONSTRAINT_COLUMNS` → `CONSTRAINT_SCHEMA`, `CONSTRAINT_TABLE`, `CONSTRAINT_TYPE`, + `COLUMN_NAME`; `CONSTRAINT_TYPE` is the literal string `PRIMARY KEY` +- `EXA_ALL_SCRIPTS` → `SCRIPT_SCHEMA`, `SCRIPT_NAME`, `SCRIPT_TYPE` + +### D3 — Discriminate result sets on `stmt.result_type`, and check it *before* fetching + +**This resolves plan.md's open item for step 4** (`column_names()`-empty vs +`stmt.result_type`). Verified against `pyexasol/statement.py`: + +- `result_type` is a plain public attribute, assigned `self.result_type = res["resultType"]`, + with exactly two values: `"resultSet"` and `"rowCount"`. +- `column_names()` is a method returning `self.col_names`, which stays `[]` for a `rowCount` + statement. +- `rowcount()` is a **method**, not a property, returning `num_rows_total` for a result set and + `row_count` otherwise. + +Both discriminators work, but `result_type` is chosen: it is the driver's own explicit signal, +whereas an empty `column_names()` is a side effect of that signal. `result_type` also states +the intent in one readable comparison. + +The ordering matters more than the choice. `ExaStatement.__next__` raises: + +```python +if self.result_type != "resultSet": + raise ExaRuntimeError( + self.connection, "Attempt to fetch from statement without result set" + ) +``` + +`fetchmany()` iterates, so it inherits that raise. The guard must therefore come **before** any +fetch. The plan's snippet happens to satisfy this by returning early, but the reason was not +recorded — it is now, because reordering those two blocks during a later refactor would turn +every `INSERT` into an `ExaRuntimeError`. + +Truncation uses the house pattern from `CursorBasedAdapter.execute_query`: fetch `max_rows + 1`, +compare, trim. + +### D4 — Default port via `get_default_port("exasol")`, matching every other adapter + +Every TCP adapter in the repo writes `int(endpoint.port or get_default_port(""))` — +hana, db2, mysql, oracle, postgresql, presto, impala, cockroachdb, mariadb, surrealdb. This +adapter follows suit for consistency in an upstream PR. + +**Known wrinkle, accepted:** `metadata.py::get_default_port` returns `provider.metadata.default_port` +if the provider resolves and otherwise falls back to `"1433"` (MSSQL). Until plan.md step 7 +registers `SPEC` with `default_port="8563"`, an Exasol config with a blank port would resolve to +1433. This is inert in practice — `DatabaseType.EXASOL` does not exist yet, so no Exasol config +can be constructed — and it becomes correct the moment step 7 lands. + +*Alternative considered:* a module-level `DEFAULT_PORT = "8563"` in `adapter.py`, correct at +every point in time. Rejected: it deviates from a ten-adapter-strong convention and duplicates +a literal that `SPEC` must declare anyway, to fix a hazard that is unreachable. + +### D5 — Auth kwargs are added, never blanked + +pyexasol rejects `password` combined with `access_token` or `refresh_token`. So the three +branches must *add* their own key rather than set all three with empty defaults. Passing +`access_token=None` alongside a password is a connection failure, not a no-op — hence the +spec's insistence that the unused keys be *absent*, and the unit test in plan.md step 10 that +asserts absence rather than emptiness. + +### D6 — `extra_options` applied last + +`connect_args.update(config.extra_options)` goes after the TLS kwargs, so a user can override +`encryption` or `websocket_sslopt` wholesale for an installation this mapping does not +anticipate. Matches `hana/adapter.py` and `clickhouse/adapter.py`. + +### D8 — Make provider discovery tolerant of a subpackage with no `provider.py` + +The premise of splitting steps 1-4 from the `ACT` group is that an adapter-only package is +inert. It was not: `_discover_providers` called +`import_module(f"{__package__}.{name}.provider")` for every subpackage with no existence check +and no `except`, so adding `providers/exasol/` without `provider.py` raised +`ModuleNotFoundError` and took down discovery for all 29 other providers - +`get_supported_db_types()` raised, and `tests/test_schema_capabilities.py` went from 9 passed +to 4 failed. + +Fixed by skipping the subpackage when `importlib.util.find_spec(...)` finds no `provider` +module. `find_spec` is used rather than `try`/`except ImportError` deliberately: catching +`ImportError` would also swallow a genuine broken-import bug inside a real `provider.py`, +whereas `find_spec` distinguishes "not present" from "present but failing". + +*Alternatives considered:* absorbing the `ACT` group into this change (larger scope, and this +change's spec forbids `provider.py`); or parking the adapter outside `providers/` until +registration (unidiomatic, needs a second move). The five-line fix keeps the staging strategy +the whole plan is built on, and is a defensible robustness fix upstream. + +### D7 — Do not override `supports_process_worker` + +`process_worker.py` calls `provider.connection_factory.connect(...)` *inside* the child +process — it opens a fresh connection rather than pickling one — so the WebSocket never +crosses the process boundary, and pyexasol yields plain tuples, which pickle fine. +`SurrealDBAdapter` disables the worker, but its reasoning does not transfer. + +## Risks / Trade-offs + +- **Positional access to `conn.meta.*` rows** → `KeyError` at runtime, invisible to `mypy` + because the rows are `dict[str, Any]`. Mitigated by D2 stating the return shapes explicitly + and by the spec scenario "Metadata rows are read by key", which the step-11 unit tests + should implement with deliberately reordered dict keys. +- **Guard/fetch ordering in `execute_query`** → reversing them makes every non-`SELECT` raise + `ExaRuntimeError`. Mitigated by D3 recording the raise, and by the spec scenario "Statement + with no result set" asserting no fetch method is called. +- **No runtime verification in this change** → `pyexasol` is absent, so nothing here is + executed end-to-end; a wrong kwarg name would pass `ruff` and `mypy` silently. Mitigated by + verifying every kwarg and system-table column against the driver source and Exasol docs + (cited in D2/D3), and accepted because plan.md steps 10-11 add mocked unit tests and step 13 + adds a Docker integration test. +- **Facts sourced from published docs, not Context7** → Context7 MCP was not connected in this + session (it exposes no tools), so pyexasol and Exasol details were read from + `raw.githubusercontent.com/exasol/pyexasol` (driver source) and `docs.exasol.com` (system + tables). These are the same upstream sources Context7 indexes, but the verification path + differs from the repo's usual convention. +- **`DatabaseAdapter` gains a subclass that cannot be smoke-tested by the suite** → the package + is unreachable from the registry, so a broken adapter is caught only at step 7. Accepted: + that isolation is the entire point of splitting steps 1-4 from the `ACT` group. + +## Migration Plan + +No migration, no rollback plan needed. The change is purely additive and unreachable: two new +files in a package nothing imports. Reverting is deleting the directory. + +Sequencing note for whoever picks up the next change: this must land before plan.md step 7, +because `provider.py`'s `provider_factory` imports `ExasolAdapter`. + +## Open Questions + +None blocking. Both items plan.md flagged for this step are now resolved: + +1. **`ExaStatement` result detection** — resolved in D3: use `stmt.result_type`, checked before + fetching. +2. **pyexasol / system-table details unverified through Context7** — re-verified in D2 and D3 + against the driver source and Exasol docs. Context7 is unavailable in this session; if it + comes online, D2's column lists and D3's `result_type` values are the things worth + re-confirming. + +Deferred to their own steps, not this one: how to patch the lazy driver import in unit tests +(step 10) and whether `exasol/docker-db` is acceptable in CI (step 14). diff --git a/openspec/changes/archive/2026-08-27-exasol-adapter/proposal.md b/openspec/changes/archive/2026-08-27-exasol-adapter/proposal.md new file mode 100644 index 00000000..014c2ba7 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-adapter/proposal.md @@ -0,0 +1,77 @@ +## Why + +sqlit ships ~29 database providers but not Exasol. Exasol is a native WebSocket-protocol +analytics database with no DB-API 2.0 driver, so it cannot reuse the `CursorBasedAdapter` +path that most existing providers share — it needs a purpose-built adapter before it can +become a selectable provider. + +This change delivers that adapter (plan.md steps 1-4). It is deliberately scoped *below* +provider registration: `providers/catalog.py` discovers a subpackage only once it has a +`provider.py`, so shipping the adapter without one keeps Exasol invisible to the UI, the +connection picker, and `tests/test_schema_capabilities.py`. The repo therefore never passes +through a state where Exasol is selectable but crashes on connect. + +**Correction found during implementation:** `_discover_providers` did *not* behave that way. +It imported `/provider.py` for **every** subpackage unconditionally - no existence +check, no `except` - so an adapter-only package broke discovery app-wide rather than staying +inert (`get_supported_db_types()` raised `ModuleNotFoundError`, and +`tests/test_schema_capabilities.py` went from 9 passed to 4 failed). This change therefore +also makes discovery skip subpackages that have no `provider` module. + +## What Changes + +- New provider package `sqlit/domains/connections/providers/exasol/` containing `__init__.py` + and `adapter.py`. **No `provider.py`** — registration is a later change. +- `providers/catalog.py::_discover_providers` skips a subpackage when + `importlib.util.find_spec` finds no `provider` module in it, so a package staged below + registration cannot break discovery for the other 29 providers. +- New `ExasolAdapter(DatabaseAdapter)` — subclassing `DatabaseAdapter` directly rather than + `CursorBasedAdapter`, because pyexasol exposes no `.cursor()`. +- Capability properties declaring Exasol's shape: schema-only (no database layer), stored + procedures yes, indexes/triggers/sequences no. +- `connect()` mapping three auth methods (password / OpenID access token / OpenID refresh + token) and all five shared `tls_mode` values onto pyexasol's `encryption` + + `websocket_sslopt` kwargs. +- Introspection via `conn.meta.*`, which wraps each query in Exasol's + `/*snapshot execution*/` hint and so cannot be blocked by metadata locks. +- Query execution against pyexasol's `ExaStatement`, including `max_rows` truncation + detection and an `execute_test_query` override (the base implementation calls + `conn.cursor()`). +- Exasol-style identifier quoting (double quotes, `"` doubled) and `LIMIT`-based + `build_select_query`. + +Not breaking: nothing imports this package until a later change adds `provider.py`. + +## Capabilities + +### New Capabilities +- `exasol-adapter`: Connecting to an Exasol database (auth methods, TLS mapping, driver + import), introspecting its schemas/tables/views/columns/procedures, and executing queries + and statements through pyexasol's non-DB-API WebSocket interface. + +### Modified Capabilities + + +## Impact + +- **New files:** `sqlit/domains/connections/providers/exasol/__init__.py`, + `sqlit/domains/connections/providers/exasol/adapter.py`. +- **Modified files:** `sqlit/domains/connections/providers/catalog.py` - five lines making + provider discovery tolerant of a subpackage with no `provider.py` (see the correction + above). Behaviour is unchanged for all 29 registered providers. +- **Dependency:** takes a runtime dependency on `pyexasol` — but only lazily, through the + inherited `_import_driver_module()`, so a missing driver surfaces sqlit's normal install + prompt rather than an ImportError. Declaring the `exasol` extra in `pyproject.toml` is + plan.md step 8, a separate change; until it lands, `pyexasol` is not installed and the + adapter's `connect()` is untestable end-to-end (its behaviour is covered by plan.md + steps 10-11). +- **Reused, unmodified:** `providers/adapters/base.py` (`DatabaseAdapter`, `ColumnInfo`, + `IndexInfo`, `TriggerInfo`, `SequenceInfo`, `TableInfo`), `providers/tls.py` + (`get_tls_mode`, `tls_mode_verifies_cert`, `get_tls_files`), `providers/metadata.py` + (`get_default_port`), `providers/driver.py` (`import_driver_module`). +- **No test-suite impact:** the package is unreachable from the registry, so no existing + test changes behaviour. `tests/test_schema_capabilities.py` passes 9/9 and + `get_supported_db_types()` returns the same 29 providers, without `exasol`. +- **Gate:** `ruff check sqlit && mypy sqlit`. Note that both tools are dirty on `main` + (117 ruff findings, 430 mypy errors, and CI runs neither), so the gate is applied as + "zero findings attributable to the changed files, repo totals unchanged". diff --git a/openspec/changes/archive/2026-08-27-exasol-adapter/specs/exasol-adapter/spec.md b/openspec/changes/archive/2026-08-27-exasol-adapter/specs/exasol-adapter/spec.md new file mode 100644 index 00000000..129d0eed --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-adapter/specs/exasol-adapter/spec.md @@ -0,0 +1,366 @@ +## ADDED Requirements + +### Requirement: Provider package stays inert until registration +The Exasol provider package SHALL consist of `__init__.py` and `adapter.py` only. It MUST NOT +contain `provider.py`, because `providers/catalog.py::_discover_providers` walks every +subpackage of `providers/` and imports `/provider.py`, which would register Exasol as a +live provider before `DatabaseType.EXASOL` and its connection schema exist. + +`_discover_providers` SHALL skip a subpackage when it contains no `provider` module, so that a +package staged below registration cannot break discovery. Without this, the unconditional +import raises `ModuleNotFoundError` and no provider at all resolves. + +#### Scenario: A subpackage without provider.py does not break discovery +- **WHEN** `get_supported_db_types()` is called with the adapter-only exasol package present +- **THEN** it returns the 29 registered providers +- **AND** no `ModuleNotFoundError` is raised + +#### Scenario: Package contains no provider module +- **WHEN** the provider package directory is listed +- **THEN** it contains exactly `__init__.py` and `adapter.py` +- **AND** no `provider.py` is present + +#### Scenario: Catalog discovery is unaffected +- **WHEN** `get_supported_db_types()` is called after this change +- **THEN** `"exasol"` is absent from the result +- **AND** `tests/test_schema_capabilities.py` passes unchanged (9 passed) + +#### Scenario: Package docstring matches house style +- **WHEN** `__init__.py` is read +- **THEN** its entire content is the single docstring `"""Provider package."""` + +### Requirement: Adapter subclasses DatabaseAdapter directly +`ExasolAdapter` SHALL extend `DatabaseAdapter`, not `CursorBasedAdapter`, because pyexasol is +a native WebSocket client that exposes no `.cursor()` method. Every abstract member of +`DatabaseAdapter` MUST be implemented. + +#### Scenario: Class is importable and concrete +- **WHEN** `ExasolAdapter()` is instantiated +- **THEN** instantiation succeeds without `TypeError` for unimplemented abstract methods + +#### Scenario: Base class choice +- **WHEN** the class declaration is inspected +- **THEN** `DatabaseAdapter` is its direct base +- **AND** `CursorBasedAdapter` does not appear in its MRO + +### Requirement: Adapter declares Exasol's capability shape +The adapter SHALL expose the following capability properties, which +`build_adapter_provider` reads via `getattr`. + +| Property | Value | +|---|---| +| `name` | `"Exasol"` | +| `install_extra` | `"exasol"` | +| `install_package` | `"pyexasol"` | +| `driver_import_names` | `("pyexasol",)` | +| `supports_multiple_databases` | `False` | +| `supports_cross_database_queries` | `False` | +| `supports_stored_procedures` | `True` | +| `supports_indexes` | `False` | +| `supports_triggers` | `False` | +| `supports_sequences` | `False` | +| `default_schema` | `""` | + +The adapter MUST NOT override `supports_process_worker`: `process_worker.py` calls +`provider.connection_factory.connect(...)` inside the child process, so it opens its own +WebSocket rather than pickling one, and pyexasol returns plain picklable tuples. + +#### Scenario: Capability properties report Exasol's shape +- **WHEN** each property in the table above is read from an `ExasolAdapter` instance +- **THEN** it returns the listed value + +#### Scenario: Process worker support is inherited +- **WHEN** the class body is inspected +- **THEN** `supports_process_worker` is not defined on `ExasolAdapter` +- **AND** the inherited value is `True` + +#### Scenario: Schema-only table display +- **WHEN** `format_table_name("MYSCHEMA", "T")` is called +- **THEN** it returns `"MYSCHEMA.T"`, because `default_schema` is empty so no schema is elided + +### Requirement: Driver import is lazy +`connect()` SHALL obtain the driver through the inherited +`self._import_driver_module("pyexasol", ...)`, passing `driver_name=self.name`, +`extra_name=self.install_extra` and `package_name=self.install_package`. The module MUST NOT +be imported at module scope, so that a missing driver produces sqlit's normal install prompt +instead of an `ImportError` at collection time. + +#### Scenario: Module imports without the driver installed +- **WHEN** `adapter.py` is imported in an environment with no `pyexasol` +- **THEN** the import succeeds + +#### Scenario: Missing driver surfaces the install prompt +- **WHEN** `connect()` is called with no `pyexasol` installed +- **THEN** the error raised by `import_driver_module` names the `exasol` extra and the + `pyexasol` package + +### Requirement: Connection parameters are assembled from the endpoint and options +`connect()` SHALL require a TCP endpoint and raise `ValueError` otherwise. It SHALL build +`dsn` as host and port joined by a colon, where port is +`int(endpoint.port or get_default_port("exasol"))`, pass `schema` from +`config.get_option("schema", "")`, pass `autocommit=True`, then apply the TLS kwargs, then +apply `config.extra_options` last so callers can override anything. + +#### Scenario: Endpoint is not TCP +- **WHEN** `connect()` is called with a config whose `tcp_endpoint` is `None` +- **THEN** `ValueError` is raised +- **AND** no connection attempt is made + +#### Scenario: DSN combines host and port +- **WHEN** `connect()` runs for host `db.example.com` and port `8563` +- **THEN** `pyexasol.connect` receives `dsn` equal to `db.example.com:8563` + +#### Scenario: Schema is forwarded +- **WHEN** the config option `schema` is `ANALYTICS` +- **THEN** `pyexasol.connect` receives `schema` equal to `ANALYTICS` + +#### Scenario: Empty schema browses everything +- **WHEN** the config option `schema` is unset +- **THEN** `pyexasol.connect` receives `schema` equal to the empty string + +#### Scenario: Autocommit is enabled +- **WHEN** `connect()` runs +- **THEN** `pyexasol.connect` receives `autocommit=True` + +#### Scenario: Extra options are applied last +- **WHEN** `config.extra_options` sets `autocommit` to `False` +- **THEN** `pyexasol.connect` receives `autocommit=False` + +### Requirement: Authentication method selects mutually exclusive credentials +`connect()` SHALL read `config.get_option("authenticator", "password")` and pass only that +method's credentials. Credentials for the other two methods MUST be absent from the kwargs, +not present-and-empty, because pyexasol rejects combinations of `password`, `access_token` +and `refresh_token`. + +#### Scenario: Username and password +- **WHEN** `authenticator` is `password` +- **THEN** `pyexasol.connect` receives `user` and `password` from the endpoint +- **AND** `access_token` and `refresh_token` are absent from the kwargs + +#### Scenario: OpenID access token +- **WHEN** `authenticator` is `access_token` +- **THEN** `pyexasol.connect` receives `access_token` from the `access_token` option +- **AND** `user`, `password` and `refresh_token` are absent from the kwargs + +#### Scenario: OpenID refresh token +- **WHEN** `authenticator` is `refresh_token` +- **THEN** `pyexasol.connect` receives `refresh_token` from the `refresh_token` option +- **AND** `user`, `password` and `access_token` are absent from the kwargs + +#### Scenario: Unset authenticator defaults to password +- **WHEN** the `authenticator` option is absent +- **THEN** the password credentials are used + +### Requirement: TLS mode maps onto pyexasol encryption settings +A private `_tls_args(config)` helper SHALL translate the shared `tls_mode` option into +pyexasol kwargs using `get_tls_mode`, `tls_mode_verifies_cert` and `get_tls_files` from +`providers/tls.py`. This mapping matters because pyexasol defaults to `encryption=True` while +`exasol/docker-db` and most on-premise installations present a self-signed certificate, so an +unmapped connect fails certificate validation. + +| `tls_mode` | kwargs | +|---|---| +| `default` | `encryption=True` | +| `disable` | `encryption=False` | +| `require` | `encryption=True`, `websocket_sslopt` with `cert_reqs` of `ssl.CERT_NONE` | +| `verify-ca` | `encryption=True`, `websocket_sslopt` with `cert_reqs` of `ssl.CERT_REQUIRED`, plus any configured files | +| `verify-full` | same as `verify-ca` | + +Under the verifying modes, `ca_certs`, `certfile` and `keyfile` SHALL be included in +`websocket_sslopt` only when the corresponding path from `get_tls_files` is non-empty. + +#### Scenario: Default mode encrypts +- **WHEN** `tls_mode` is absent or `default` +- **THEN** the kwargs contain `encryption=True` +- **AND** no `websocket_sslopt` key is present + +#### Scenario: Disabled mode turns encryption off +- **WHEN** `tls_mode` is `disable` +- **THEN** the kwargs contain `encryption=False` +- **AND** no `websocket_sslopt` key is present + +#### Scenario: Require mode encrypts without verifying +- **WHEN** `tls_mode` is `require` +- **THEN** the kwargs contain `encryption=True` +- **AND** `cert_reqs` in `websocket_sslopt` is `ssl.CERT_NONE` + +#### Scenario: Verifying mode demands a valid certificate +- **WHEN** `tls_mode` is `verify-ca` or `verify-full` +- **THEN** the kwargs contain `encryption=True` +- **AND** `cert_reqs` in `websocket_sslopt` is `ssl.CERT_REQUIRED` + +#### Scenario: Certificate files are forwarded when configured +- **WHEN** `tls_mode` is `verify-full` and `tls_ca`, `tls_cert` and `tls_key` are all set +- **THEN** `websocket_sslopt` contains `ca_certs`, `certfile` and `keyfile` with those paths + +#### Scenario: Unconfigured certificate files are omitted +- **WHEN** `tls_mode` is `verify-ca` and only `tls_ca` is set +- **THEN** `websocket_sslopt` contains `ca_certs` +- **AND** `certfile` and `keyfile` are absent from `websocket_sslopt` + +### Requirement: Introspection uses snapshot metadata reads +All introspection SHALL go through `conn.meta.*`, which prefixes each query with Exasol's +snapshot-execution hint and therefore cannot be blocked by metadata locks. + +`conn.meta.list_tables()`, `list_views()` and `list_columns()` return **lists of dicts** with +UPPERCASE keys — pyexasol enforces `fetch_dict=True` on metadata reads specifically to stop +callers depending on column order — so results MUST be read by key, never by index. +`conn.meta.execute_snapshot()` returns an `ExaStatement`, so it requires an explicit +`.fetchall()`, which likewise yields dicts. + +#### Scenario: Databases are not a concept in Exasol +- **WHEN** `get_databases(conn)` is called +- **THEN** it returns an empty list without querying the connection + +#### Scenario: Tables are listed by schema and name +- **WHEN** `get_tables(conn)` is called +- **THEN** `conn.meta.list_tables()` is used +- **AND** each row yields a pair taken from the `TABLE_SCHEMA` and `TABLE_NAME` keys + +#### Scenario: Views are listed by schema and name +- **WHEN** `get_views(conn)` is called +- **THEN** `conn.meta.list_views()` is used +- **AND** each row yields a pair taken from the `VIEW_SCHEMA` and `VIEW_NAME` keys + +#### Scenario: Metadata rows are read by key +- **WHEN** a mocked `conn.meta.list_tables()` returns dicts whose keys are in a different + order than declared +- **THEN** `get_tables` still returns the correct schema and name pairs + +### Requirement: Column introspection reports primary keys +`get_columns(conn, table, database=None, schema=None)` SHALL read name and type from +`conn.meta.list_columns(schema, table)` using keys `COLUMN_NAME` and `COLUMN_TYPE`, and +determine the primary-key set with `conn.meta.execute_snapshot(...).fetchall()` against +`SYS.EXA_ALL_CONSTRAINT_COLUMNS` filtered on a `CONSTRAINT_TYPE` of `PRIMARY KEY` plus the +schema and table, reading `COLUMN_NAME` from each row. It returns `ColumnInfo` objects in the +order `list_columns` yields them. + +#### Scenario: Columns carry name and declared type +- **WHEN** `get_columns` runs for a table with a decimal `ID` and a varchar `NAME` +- **THEN** it returns `ColumnInfo` entries with those names and their `COLUMN_TYPE` strings + +#### Scenario: Primary key columns are flagged +- **WHEN** the constraint query reports `ID` as a primary-key column +- **THEN** the `ID` entry has `is_primary_key` set to `True` +- **AND** every other entry has `is_primary_key` set to `False` + +#### Scenario: Table without a primary key +- **WHEN** the constraint query returns no rows +- **THEN** every returned `ColumnInfo` has `is_primary_key` set to `False` + +#### Scenario: Composite primary key +- **WHEN** the constraint query reports both `A` and `B` as primary-key columns +- **THEN** both entries have `is_primary_key` set to `True` + +### Requirement: Stored procedures are read from EXA_ALL_SCRIPTS +`get_procedures(conn, database=None)` SHALL query `SYS.EXA_ALL_SCRIPTS` through +`conn.meta.execute_snapshot(...).fetchall()`, filtering on a `SCRIPT_TYPE` of `SCRIPTING` — +the value Exasol uses for scripting programs, as distinct from `UDF`, `ADAPTER` and +`PREPROCESSOR` — and return the script names. + +#### Scenario: Scripting programs are returned +- **WHEN** `EXA_ALL_SCRIPTS` contains a `SCRIPTING` entry named `MY_PROC` +- **THEN** `get_procedures` includes that name + +#### Scenario: UDFs are not stored procedures +- **WHEN** `EXA_ALL_SCRIPTS` contains a `UDF` entry +- **THEN** `get_procedures` excludes it + +### Requirement: Unsupported object types return empty lists +`get_indexes`, `get_triggers` and `get_sequences` SHALL each return an empty list without +querying the connection. All three MUST still be defined even though their capability flags +are `False`, because they are abstract on `DatabaseAdapter`. They are empty because Exasol's +indexes are auto-managed and unnamed, it has no triggers, and it uses IDENTITY columns rather +than sequences. + +#### Scenario: Indexes, triggers and sequences are empty +- **WHEN** `get_indexes(conn)`, `get_triggers(conn)` and `get_sequences(conn)` are called +- **THEN** each returns an empty list +- **AND** no method on `conn` is invoked + +### Requirement: Query execution returns columns, rows and a truncation flag +`execute_query(conn, query, max_rows=None)` SHALL execute via `conn.execute(query)` and +return a triple of columns, rows and a truncation flag. It MUST decide whether a result set +exists by testing `stmt.result_type` **before** fetching, because pyexasol raises +`ExaRuntimeError` with the message "Attempt to fetch from statement without result set" when +iterating a row-count statement. `result_type` is a public attribute whose values are exactly +`resultSet` and `rowCount`. + +When `max_rows` is set, the implementation SHALL fetch one row beyond the limit to detect +truncation, then trim to `max_rows`. Rows MUST be returned as tuples. + +#### Scenario: Unlimited fetch +- **WHEN** `execute_query` runs with `max_rows` of `None` on a statement returning 3 rows +- **THEN** it returns those 3 rows and the truncation flag is `False` + +#### Scenario: Fewer rows than the limit +- **WHEN** `max_rows` is 10 and the statement has 4 rows +- **THEN** all 4 rows are returned and the truncation flag is `False` + +#### Scenario: Exactly the limit is not truncation +- **WHEN** `max_rows` is 10 and the statement has exactly 10 rows +- **THEN** 10 rows are returned and the truncation flag is `False` + +#### Scenario: One row over the limit is truncation +- **WHEN** `max_rows` is 10 and the statement has 11 rows +- **THEN** 10 rows are returned and the truncation flag is `True` + +#### Scenario: Statement with no result set +- **WHEN** `execute_query` runs a statement whose `result_type` is `rowCount` +- **THEN** it returns empty columns, no rows, and a truncation flag of `False` +- **AND** no fetch method is called on the statement + +### Requirement: Non-query execution returns the affected row count +`execute_non_query(conn, query)` SHALL execute via `conn.execute(query)` and return +`int(stmt.rowcount())`. `rowcount` is a **method** on `ExaStatement`, not a property, so it +MUST be called. No explicit commit is issued, because the connection is opened with +`autocommit=True`. + +#### Scenario: Row count is returned +- **WHEN** an `INSERT` affecting 5 rows is executed +- **THEN** `execute_non_query` returns 5 + +#### Scenario: Rowcount is invoked as a method +- **WHEN** `execute_non_query` runs against a mocked statement +- **THEN** `rowcount()` is called +- **AND** the returned value is an integer + +### Requirement: Connection test bypasses the cursor-based default +The adapter SHALL override `execute_test_query(conn)`, because the inherited implementation +calls `conn.cursor()`, which pyexasol does not provide. The override SHALL run +`conn.execute(self.test_query)` and then `fetchval()` on the result. + +#### Scenario: Test query runs without a cursor +- **WHEN** `execute_test_query(conn)` is called +- **THEN** `conn.execute` is called with the inherited `SELECT 1` test query +- **AND** `fetchval()` is called on the result +- **AND** `conn.cursor` is never accessed + +### Requirement: Identifiers are quoted with doubled double quotes +`quote_identifier(name)` SHALL wrap the name in double quotes and escape any embedded double +quote by doubling it. + +#### Scenario: Plain identifier +- **WHEN** `quote_identifier` is called with `MY_TABLE` +- **THEN** it returns that name wrapped in double quotes + +#### Scenario: Embedded double quote is doubled +- **WHEN** `quote_identifier` is called with a name containing one double quote +- **THEN** that double quote appears doubled inside the surrounding quotes + +### Requirement: Select queries are schema-qualified and LIMIT-bounded +`build_select_query(table, limit, database=None, schema=None)` SHALL produce a +`SELECT * FROM` statement against the quoted schema-qualified table with a trailing `LIMIT` +clause, omitting the schema segment when the schema is empty. + +#### Scenario: Schema-qualified select +- **WHEN** `build_select_query` is called for table `T` in schema `S` with a limit of 100 +- **THEN** it selects from the quoted `S`-dot-`T` name and ends with `LIMIT 100` + +#### Scenario: Unqualified select +- **WHEN** `build_select_query` is called for table `T` with a limit of 50 and no schema +- **THEN** it selects from the quoted `T` name alone and ends with `LIMIT 50` + +#### Scenario: Identifiers in the select are quoted +- **WHEN** `build_select_query` is called with a lowercase table name +- **THEN** the name appears double-quoted, preserving its case diff --git a/openspec/changes/archive/2026-08-27-exasol-adapter/tasks.md b/openspec/changes/archive/2026-08-27-exasol-adapter/tasks.md new file mode 100644 index 00000000..22296784 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-adapter/tasks.md @@ -0,0 +1,121 @@ +## 1. Package skeleton and capability declaration + +Corresponds to plan.md step 1. + +- [x] 1.1 Create `sqlit/domains/connections/providers/exasol/__init__.py` containing exactly + the docstring `"""Provider package."""`. Do **not** create `provider.py` — see design D-context; + it would activate auto-discovery and break `tests/test_schema_capabilities.py`. +- [x] 1.2 Create `sqlit/domains/connections/providers/exasol/adapter.py` with the module + docstring, `from __future__ import annotations`, and imports of `ColumnInfo`, + `DatabaseAdapter`, `IndexInfo`, `SequenceInfo`, `TableInfo`, `TriggerInfo` from + `providers.adapters.base`, plus a `TYPE_CHECKING` import of `ConnectionConfig`. +- [x] 1.3 Declare `class ExasolAdapter(DatabaseAdapter)` (design D1 — not + `CursorBasedAdapter`, pyexasol has no `.cursor()`). +- [x] 1.4 Add the driver-metadata properties: `name` -> `"Exasol"`, `install_extra` -> + `"exasol"`, `install_package` -> `"pyexasol"`, `driver_import_names` -> `("pyexasol",)`. +- [x] 1.5 Add the capability properties per the spec table: `supports_multiple_databases` + `False`, `supports_cross_database_queries` `False`, `supports_stored_procedures` `True`, + `supports_indexes` `False`, `supports_triggers` `False`, `supports_sequences` `False`, + `default_schema` `""`. Do **not** define `supports_process_worker` (design D7). +- [x] 1.6 Stub every remaining abstract method (`connect`, `get_databases`, `get_tables`, + `get_views`, `get_columns`, `get_procedures`, `get_indexes`, `get_triggers`, + `get_sequences`, `quote_identifier`, `build_select_query`, `execute_query`, + `execute_non_query`) with correct signatures raising `NotImplementedError`, so the file + is importable while groups 2-4 fill it in. +- [x] 1.7 Verify: `uv run python -c "from sqlit.domains.connections.providers.exasol.adapter import ExasolAdapter; print(ExasolAdapter)"` + then `uv run ruff check sqlit && uv run mypy sqlit`. + +## 2. Connection and TLS + +Corresponds to plan.md step 2. Depends on group 1. + +- [x] 2.1 Add `import ssl` and import `TLS_MODE_DISABLE`, `TLS_MODE_REQUIRE`, `get_tls_files`, + `get_tls_mode`, `tls_mode_verifies_cert` from `providers.tls`; import `get_default_port` + from `providers.registry` (design D4 — matches all ten existing TCP adapters). +- [x] 2.2 Implement `_tls_args(self, config) -> dict[str, Any]` per the spec's mapping table: + `disable` -> `encryption=False`; `default` -> `encryption=True` with no + `websocket_sslopt`; `require` -> `cert_reqs=ssl.CERT_NONE`; verifying modes -> + `cert_reqs=ssl.CERT_REQUIRED`. +- [x] 2.3 In the verifying branch, add `ca_certs` / `certfile` / `keyfile` to + `websocket_sslopt` only for non-empty paths from `get_tls_files` (spec scenario + "Unconfigured certificate files are omitted"). +- [x] 2.4 Implement `connect()`: resolve `config.tcp_endpoint`, raise `ValueError` when it is + `None`, and lazily obtain the driver via `self._import_driver_module("pyexasol", + driver_name=self.name, extra_name=self.install_extra, package_name=self.install_package)`. + No module-scope `import pyexasol`. +- [x] 2.5 Build the base kwargs: `dsn` as host and port joined by a colon with port from + `int(endpoint.port or get_default_port("exasol"))`, `schema` from + `config.get_option("schema", "")`, and `autocommit=True`. +- [x] 2.6 Branch on `config.get_option("authenticator", "password")` and add **only** that + method's credentials — `user`+`password`, or `access_token`, or `refresh_token` (design + D5: the unused keys must be absent, not empty, or pyexasol rejects the combination). +- [x] 2.7 Apply `_tls_args(config)` and then `config.extra_options` last (design D6), and + return `pyexasol.connect(**connect_args)`. +- [x] 2.8 Verify: `uv run ruff check sqlit && uv run mypy sqlit`. Behaviour is covered later by + plan.md step 10. + +## 3. Introspection + +Corresponds to plan.md step 3. Depends on group 1. + +- [x] 3.1 Implement `get_databases` returning an empty list without touching `conn`. +- [x] 3.2 Implement `get_tables` from `conn.meta.list_tables()`, reading each row by the + `TABLE_SCHEMA` and `TABLE_NAME` **keys**. Design D2: these helpers return `list[dict]` + with UPPERCASE keys because pyexasol enforces `fetch_dict=True` — positional indexing + raises `KeyError` and `mypy` will not catch it. +- [x] 3.3 Implement `get_views` the same way from `conn.meta.list_views()`, keys `VIEW_SCHEMA` + and `VIEW_NAME`. +- [x] 3.4 Implement the primary-key lookup in `get_columns`: `conn.meta.execute_snapshot(...)` + against `SYS.EXA_ALL_CONSTRAINT_COLUMNS` filtered on `CONSTRAINT_TYPE = 'PRIMARY KEY'` + plus `CONSTRAINT_SCHEMA` and `CONSTRAINT_TABLE`, then an explicit `.fetchall()` — + `execute_snapshot` returns an `ExaStatement`, unlike the `list_*` helpers — collecting + `COLUMN_NAME` into a set. +- [x] 3.5 Complete `get_columns` from `conn.meta.list_columns(schema, table)`, mapping + `COLUMN_NAME` and `COLUMN_TYPE` into `ColumnInfo` with `is_primary_key` from the set + built in 3.4, preserving the order `list_columns` yields. +- [x] 3.6 Implement `get_procedures` via `conn.meta.execute_snapshot(...).fetchall()` on + `SYS.EXA_ALL_SCRIPTS` filtered to `SCRIPT_TYPE = 'SCRIPTING'` (the scripting-program + value, as opposed to `UDF` / `ADAPTER` / `PREPROCESSOR`), returning `SCRIPT_NAME` values. +- [x] 3.7 Implement `get_indexes`, `get_triggers` and `get_sequences` as empty-list returns + that never touch `conn` — abstract on the base class, so required despite the `False` + capability flags. +- [x] 3.8 Verify: `uv run ruff check sqlit && uv run mypy sqlit`. Behaviour is covered later by + plan.md step 11. + +## 4. Query execution and identifier quoting + +Corresponds to plan.md step 4. Depends on group 1. + +- [x] 4.1 Implement `execute_query`: call `conn.execute(query)`, then **first** test + `stmt.result_type != "resultSet"` and return empty columns/rows/`False`. Design D3: this + guard must precede any fetch, because `ExaStatement.__next__` raises `ExaRuntimeError` + ("Attempt to fetch from statement without result set") and `fetchmany()` iterates. +- [x] 4.2 Complete `execute_query`: with `max_rows` unset return `stmt.column_names()` and all + rows as tuples with `truncated=False`; with `max_rows` set fetch `max_rows + 1`, set + `truncated` from the overflow, and trim to `max_rows`. +- [x] 4.3 Implement `execute_non_query` as `int(conn.execute(query).rowcount())` — + `rowcount` is a **method** on `ExaStatement`, not a property. No explicit commit; + `autocommit=True` is set at connect time. +- [x] 4.4 Override `execute_test_query` to run `conn.execute(self.test_query).fetchval()`, + since the inherited implementation at `providers/adapters/base.py` calls `conn.cursor()`. +- [x] 4.5 Implement `quote_identifier`: wrap in double quotes, doubling any embedded double + quote. +- [x] 4.6 Implement `build_select_query` producing `SELECT * FROM` against the quoted + schema-qualified name with a trailing `LIMIT`, omitting the schema segment when the + schema is empty. +- [x] 4.7 Verify: `uv run ruff check sqlit && uv run mypy sqlit`. Behaviour is covered later by + plan.md step 11. + +## 5. Change gate and plan bookkeeping + +- [x] 5.1 Confirm no `NotImplementedError` stubs from 1.6 remain in `adapter.py`. +- [x] 5.2 Confirm the package still contains only `__init__.py` and `adapter.py`, and that + `uv run pytest tests/test_schema_capabilities.py -v` passes — proving Exasol is still + undiscovered (spec: "Provider package stays inert until registration"). +- [x] 5.3 Run the change gate: `uv run ruff check sqlit && uv run mypy sqlit`. +- [x] 5.4 In `plan.md`, set steps 1, 2, 3 and 4 to `done` in the Status table and update the + progress count to `4 / 15 done`. +- [x] 5.5 Append a `plan.md` Session log row recording the two resolutions from design D3 and + D2: `stmt.result_type` chosen over empty `column_names()` and checked before fetching; + and `conn.meta.*` returning dicts with UPPERCASE keys rather than tuples, which corrects + step 3's notation. diff --git a/openspec/changes/archive/2026-08-27-exasol-docs/.openspec.yaml b/openspec/changes/archive/2026-08-27-exasol-docs/.openspec.yaml new file mode 100644 index 00000000..f05b045c --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-docs/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-27 diff --git a/openspec/changes/archive/2026-08-27-exasol-docs/design.md b/openspec/changes/archive/2026-08-27-exasol-docs/design.md new file mode 100644 index 00000000..af2f0be2 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-docs/design.md @@ -0,0 +1,167 @@ +## Context + +Steps 1-14 of plan.md are `done`: Exasol has a provider package, a schema, a registered +`DatabaseType`, 55 unit tests, a compose service, an integration suite that passes 20/8-skipped +against a live server, and a CI job. Step 15 — the documentation — is the last `todo`, and it is +the first thing an upstream reviewer of `Maxteabag/sqlit` will read. + +The two target files have different audiences and different failure modes: + +- `README.md` is read by **users**. Its supported-database sentence (line 28) is a marketing + surface; its Driver Reference table (lines 277-297) is operational — a user reaches it because + sqlit just told them a driver is missing. +- `CONTRIBUTING.md` is read by **contributors** who are about to run the test suite. Line 49 lists + the enterprise-profile containers; lines 82-186 document per-engine environment variables. + +The binding constraint is that everything written here is a claim about code that already exists. +The env-var table in particular is a restatement of `tests/fixtures/exasol.py:15-24`, and the two +can silently diverge. This design treats each documented value as sourced from a specific line +rather than from memory of the implementation sessions. + +A second constraint is that both files are top-level documents in someone else's repository. Every +line of the diff is reviewed. Anything beyond the four insertions is noise the maintainer must read +and decide about. + +## Goals / Non-Goals + +**Goals:** + +- A user can discover Exasol support and get a working driver install command from `README.md` + alone. +- A contributor can bring up the Exasol container, know it will take minutes, and know which + variables repoint the tests, from `CONTRIBUTING.md` alone. +- Every documented default matches the code that reads it, verified by comparison rather than + recall. +- The diff is minimal and additive — a reviewer can approve it by reading four insertions. + +**Non-Goals:** + +- **No "adding a new provider" guide.** plan.md's Context section observes the repo has no such + documentation and that the convention had to be read off the code. Writing that guide is real + value, but it is a different change with a different reviewer conversation, and inventing repo + conventions inside a provider PR invites rejection of both. +- **No documentation of the `exasol` extra beyond the Driver Reference row.** The README documents + extras in prose for `ssh` only; no per-database extra has its own section. Adding one for Exasol + would make it the most-documented engine in the file. +- **No fixing of pre-existing gaps.** The Driver Reference table omits Db2, SAP HANA, Teradata, + Trino, Presto and RedShift. That is a real defect. It is not this change's defect. +- **No changes under `sqlit/`, `tests/`, `infra/`, `.github/` or `pyproject.toml`.** +- **Not the follow-up `get_columns` fix** (`O1` in plan.md's session log) and not plan.md's Final + verification block. Both were scoped out of this change deliberately. + +## Decisions + +### D1 — Exasol goes after Teradata in the supported-database sentence + +The sentence at `README.md:28` is neither alphabetical nor grouped by category; its order is +historical. That leaves two candidate rules: append at the end, or mirror the connection picker. + +Mirroring the picker wins. plan.md step 6 placed Exasol after Teradata in the `DatabaseType` display +order, and the step 5-7 session verified that position headlessly ("picker option after Teradata"). +A reader who scans the README list and then opens the picker sees the same neighbours. Appending +after `osquery` would put an enterprise analytical database at the end of a list that closes with +`SurrealDB and osquery`, and would require rewriting the `and osquery.` clause — a larger diff for a +worse result. + +*Alternative considered:* alphabetical insertion (after `DuckDB`). Rejected — the list is not +alphabetical, so this imposes a rule the file does not follow anywhere else. + +### D2 — The Driver Reference row goes between Spanner and Apache Arrow Flight SQL + +The table's row order roughly tracks the prose list, with omissions. Exasol's prose neighbours — +Teradata before it, Trino and Presto after it — are all absent from the table, so its position +cannot be read off directly. Walking outward from Exasol's prose position to the nearest engines +that *are* in the table gives `Spanner` before and `Apache Arrow Flight SQL` after. Inserting there +keeps the table's relative order consistent with the prose list it shadows. + +*Alternatives considered:* appending after `osquery` — simplest diff, but makes the table's ordering +rule strictly worse for the next person; grouping next to `Snowflake` as "the other analytical +warehouse" — rejected because every engine in this repo ships behind a named extra, so there is no +"extras-gated" grouping to join, and Snowflake's neighbours (`Cloudflare D1`, `Firebird`) show the +table is not grouped by engine category either. + +This is a low-stakes call. It is recorded because "why is the row here?" is otherwise unanswerable +in review, and an unanswerable question costs a round trip. + +### D3 — Every documented default is read out of the code, not recalled + +The six defaults come from `tests/fixtures/exasol.py`, which reads them via `os.environ.get` with +literal fallbacks. The implementation task extracts them by grepping that module and comparing +against the table, rather than transcribing from the proposal or from the session log. + +This matters because `EXASOL_PASSWORD` is `exasol` and `EXASOL_USER` is `sys` — the `docker-db` +defaults, values a reader has no independent way to check. A wrong value here is undetectable by +review and produces a failed login for the contributor who trusts it. + +### D4 — `EXASOL_READY_TIMEOUT` is documented despite having no analogue elsewhere + +No other engine's table has a timeout row, so including one is a visible asymmetry a reviewer may +question. It earns its place: the step 12-14 session measured port 8563 open at 21s and the first +successful login at 101s, and the fixture's readiness gate is a real connect retried to a deadline +precisely because of that gap. On a cold ~4 GB pull, or slower hardware, 300 seconds is reachable. +A contributor who hits the deadline needs to know the knob exists; nothing else in the repo tells +them. + +*Alternative considered:* documenting the five connection variables only and leaving the timeout to +be discovered from the source. Rejected — a contributor debugging a timeout is exactly the reader +who will not think to open a fixture module. + +### D5 — The boot-time warning is placed with the Exasol material, not by editing line 54 + +`CONTRIBUTING.md:54` says "Wait for the databases to be ready (about 30-45 seconds)". That sentence +is correct for the default profile, which is what most contributors run. Rewriting it to +accommodate Exasol would degrade accurate guidance for the common case in service of an opt-in +profile. + +The warning therefore attaches to the Exasol material, where the reader who started the enterprise +profile will be. The distinction it must draw is port-open versus login-accepted, because that is +the shape of the confusion — `is_port_open` returning true is exactly what makes the container look +ready when it is not. + +### D6 — Diff minimalism is a design constraint, not a style preference + +The intended diff is four insertions: one clause on `README.md:28`, one table row in the Driver +Reference, one clause on `CONTRIBUTING.md:49`, and one block (table plus warning) in the Environment +Variables section. No reflow of the line-28 paragraph, no realignment of the Driver Reference +table's column padding, no reordering of the env-var sections, no fixing of adjacent typos. + +The Driver Reference table's cells are space-padded to a common width. A new row whose content +exceeds the current column width would, under any auto-formatter, reflow every row in the table and +turn a 1-line diff into a 20-line one. `Exasol` / `pyexasol` / `pipx inject sqlit-tui pyexasol` / +`python -m pip install pyexasol` are all shorter than the widest existing cells +(`snowflake-connector-python` and its commands), so the new row pads into the existing widths and no +other row moves. This is a fact to verify in the diff, not to assume. + +### D7 — Verification is comparison against code plus a diff read + +There is no test to run: no Python changes, and pre-commit carries no markdown linter — only +`trailing-whitespace`, `end-of-file-fixer`, `check-yaml`, `check-toml`, `check-added-large-files` +and `check-merge-conflict`. Of those, `trailing-whitespace` is the one these edits can trip, since +markdown table rows are easy to leave padded. + +Verification is therefore mechanical where it can be: grep the fixture for `EXASOL_` and diff that +set against the table; grep `pyproject.toml` for the `exasol` extra and check the package name; +`git diff --stat` to confirm exactly two files; and read the rendered diff to confirm no +neighbouring row moved. + +## Risks / Trade-offs + +- **A documented default drifts from the fixture later** → The spec states the equality as a + testable scenario, and D3 makes the initial values code-derived. Nothing enforces it at CI time; + this is accepted, and matches how every other engine's table in the file already works. +- **The reviewer objects to `EXASOL_READY_TIMEOUT` as asymmetric** → D4 records the measured + 21s/101s figures as the justification. If pushed back on, dropping that one row costs nothing else + in the change. +- **The reviewer prefers a different row position in the Driver Reference** → D2 records the rule + used, so the discussion is about the rule rather than about taste. Moving the row is a one-line + change either way. +- **An editor auto-formats the markdown tables on save** → D6 identifies this as the main way a + 4-line diff becomes a 40-line one. The mitigation is checking `git diff` before committing, which + the tasks make an explicit step rather than an assumed habit. +- **Documentation claims something the code does not do** → The spec's final requirement makes + "every claim traces to existing code" a checkable condition. The nearest live example is the + `get_columns` casing defect the step 12-14 session found: the docs must not imply Exasol + introspection works in cases where it is known not to. +- **Trailing whitespace in new table rows trips pre-commit** → Caught by the hook itself if + pre-commit runs; the tasks include running it against the two files so it is caught before the + commit rather than during it. diff --git a/openspec/changes/archive/2026-08-27-exasol-docs/proposal.md b/openspec/changes/archive/2026-08-27-exasol-docs/proposal.md new file mode 100644 index 00000000..e74feb80 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-docs/proposal.md @@ -0,0 +1,68 @@ +## Why + +Exasol is now a registered, unit-tested, integration-tested provider — and it is invisible in +both documents a person reads before touching this repo. `README.md:28` enumerates every supported +engine and does not name Exasol; its Driver Reference table (`README.md:277-297`) tells users which +package to install when a driver is missing and has no `pyexasol` row. `CONTRIBUTING.md:49` names +the enterprise-profile containers as "(Db2, Trino, Presto, Oracle 11g)" and its Environment +Variables section (`CONTRIBUTING.md:82-186`) documents the env vars for eleven engines — neither +mentions the `exasol` compose service or the `EXASOL_*` variables that `tests/fixtures/exasol.py` +actually reads. + +Until this lands, a contributor who starts the enterprise profile has no way to know Exasol is in +it, and a user whose connection fails on a missing driver gets no install command. This is plan.md +step 15, the last `todo` in a 15-step plan. + +## What Changes + +- **Modified** `README.md` — Exasol added to the supported-database sentence at line 28, positioned + after Teradata to match the connection picker's display order established in plan.md step 6; and a + `pyexasol` row added to the Driver Reference table with its `pipx inject` and `pip install` + commands, in the position the table's existing ordering implies. +- **Modified** `CONTRIBUTING.md` — Exasol added to the enterprise-container list at line 49; a new + **Exasol:** environment-variable table in the Environment Variables section, documenting the six + variables `tests/fixtures/exasol.py` reads with their real defaults; and a note that the Exasol + container needs far longer than the "about 30-45 seconds" line 54 quotes for the standard profile. +- No changes to any file under `sqlit/`, `tests/`, `infra/`, `.github/` or `pyproject.toml`. This + change is documentation only. The behaviour being documented already exists and is verified. + +## Capabilities + +### New Capabilities + +- `exasol-documentation`: the user-facing and contributor-facing documentation surface for the + Exasol provider — that `README.md` lists Exasol among supported engines and gives the driver + install command, and that `CONTRIBUTING.md` tells a contributor how to bring up the Exasol test + container and which environment variables configure the tests against it. Each documented value + must match the code that reads it, which is the requirement that makes this capability testable + rather than decorative. + +### Modified Capabilities + +None. Nothing in `openspec/specs/exasol-adapter`, `exasol-driver-packaging`, +`exasol-integration-coverage`, `exasol-integration-harness`, `exasol-provider-registration` or +`exasol-unit-coverage` changes; this change describes their outcome to readers. + +## Impact + +- **Documented values are a contract with code, not prose.** Every default in the new env-var table + is read from a specific line of `tests/fixtures/exasol.py` (`EXASOL_HOST`=`localhost`, + `EXASOL_PORT`=`8563`, `EXASOL_USER`=`sys`, `EXASOL_PASSWORD`=`exasol`, + `EXASOL_SCHEMA`=`TEST_SQLIT`, `EXASOL_READY_TIMEOUT`=`300`). A wrong default here is worse than + no table — it sends a contributor debugging their environment instead of the code. +- **`EXASOL_READY_TIMEOUT` is undocumented elsewhere and has no analogue in any other engine's + table.** It exists because the step 12-14 session measured `exasol/docker-db` opening port 8563 at + 21s but refusing every login until 101s. Omitting it leaves the one knob a contributor on slow + hardware will need out of the docs entirely. +- **Boot time is a support question waiting to happen.** `CONTRIBUTING.md:54` says "about 30-45 + seconds"; Exasol needs minutes on a cold pull of a ~4 GB image. A contributor who waits 45 seconds + and sees connection refused will reasonably conclude the container is broken. +- **Driver Reference table placement**: the table lists 18 of ~30 engines and omits Db2, SAP HANA, + Teradata, Trino, Presto and RedShift, so Exasol's prose-order neighbours are all absent. See + design D2 for the ordering rule chosen and why. +- **Upstream review surface**: both files are top-level project documents in `Maxteabag/sqlit`, so + every line is read by the maintainer. Diff minimalism matters more here than anywhere else in the + Exasol work — the change must not reflow, reformat or "improve" adjacent rows. +- **Not affected**: no test runs, no CI job, no dependency. Verification is a read-through against + the code the tables claim to describe, plus the existing markdown lint in pre-commit if it covers + these files. diff --git a/openspec/changes/archive/2026-08-27-exasol-docs/specs/exasol-documentation/spec.md b/openspec/changes/archive/2026-08-27-exasol-docs/specs/exasol-documentation/spec.md new file mode 100644 index 00000000..8cf09f1f --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-docs/specs/exasol-documentation/spec.md @@ -0,0 +1,134 @@ +## ADDED Requirements + +### Requirement: Exasol is listed among supported databases + +`README.md` SHALL name Exasol in the sentence that enumerates supported engines, so that a reader +deciding whether sqlit talks to their database can answer the question without reading source. + +Placement SHALL follow the connection picker's display order rather than alphabetical order, so the +document and the running application agree on where Exasol sits among the engines. + +#### Scenario: Supported-database sentence names Exasol + +- **WHEN** the "Supports all major databases:" sentence in `README.md` is read +- **THEN** it contains `Exasol`, positioned immediately after `Teradata` +- **AND** every other engine named in that sentence is unchanged, in the same order, with the + trailing `and osquery.` still closing the list + +#### Scenario: Picker order and README order agree + +- **WHEN** the position of Exasol in that sentence is compared with the provider display order used + by the connection picker +- **THEN** Exasol follows Teradata in both + +### Requirement: Driver Reference gives the Exasol install command + +The Driver Reference table in `README.md` SHALL carry a row for Exasol naming `pyexasol` as the +driver package, with the `pipx inject` and `pip install` commands spelled out in the same form as +every other row. The table exists so that a user hitting a missing-driver error can copy one command +and continue; a row that omits either column fails that purpose for half its readers. + +The package name SHALL be the distribution actually declared by the `exasol` extra in +`pyproject.toml`, not a hand-written approximation of it. + +#### Scenario: Exasol row is present and complete + +- **WHEN** the Driver Reference table is read +- **THEN** it contains a row whose Database cell is `Exasol`, whose Driver package cell is + `pyexasol`, whose `pipx` cell is `pipx inject sqlit-tui pyexasol`, and whose `pip` / venv cell is + `python -m pip install pyexasol` + +#### Scenario: Documented package matches the declared extra + +- **WHEN** the driver package named in the Exasol row is compared with `exasol = [...]` in + `pyproject.toml` +- **THEN** both name the `pyexasol` distribution + +#### Scenario: Existing rows are untouched + +- **WHEN** the diff of `README.md` is inspected +- **THEN** the only change inside the table is the added Exasol row, with no other row's cells, + spacing or column alignment altered + +### Requirement: Enterprise profile documents its Exasol container + +`CONTRIBUTING.md` SHALL name Exasol in the list of containers started by the `enterprise` compose +profile. The Exasol service is declared under that profile precisely so it is not pulled by default, +which means a contributor learns it exists only from this list. + +#### Scenario: Enterprise container list names Exasol + +- **WHEN** the sentence introducing the `--profile enterprise` command in `CONTRIBUTING.md` is read +- **THEN** it names Exasol alongside Db2, Trino, Presto and Oracle 11g +- **AND** the `docker compose ... --profile enterprise up -d` command below it is unchanged, because + it already starts every service in the profile + +### Requirement: Exasol test environment variables are documented with their real defaults + +The Environment Variables section of `CONTRIBUTING.md` SHALL carry an Exasol table listing every +`EXASOL_*` variable that `tests/fixtures/exasol.py` reads, and each documented default SHALL equal +the default in that module. A table that drifts from the code is worse than no table: it sends a +contributor to debug their environment rather than the value. + +The table SHALL include `EXASOL_READY_TIMEOUT`, which has no analogue in any other engine's table +and is documented nowhere else, because it is the only knob a contributor on slow hardware or a cold +image pull can reach for. + +#### Scenario: Every fixture variable appears + +- **WHEN** the `EXASOL_`-prefixed names read by `tests/fixtures/exasol.py` are compared with the rows + of the Exasol table +- **THEN** the two sets are equal, covering `EXASOL_HOST`, `EXASOL_PORT`, `EXASOL_USER`, + `EXASOL_PASSWORD`, `EXASOL_SCHEMA` and `EXASOL_READY_TIMEOUT` + +#### Scenario: Documented defaults match the code + +- **WHEN** each default in the table is compared with the fallback passed to `os.environ.get` for the + same variable +- **THEN** they are identical: `localhost`, `8563`, `sys`, `exasol`, `TEST_SQLIT` and `300` + +#### Scenario: Table matches the shape of its neighbours + +- **WHEN** the Exasol table is compared with the SQL Server and Db2 tables above it +- **THEN** it uses the same `**Exasol:**` bold label, the same + `| Variable | Default | Description |` header, and the same separator row + +### Requirement: Contributors are told Exasol boots slowly + +`CONTRIBUTING.md` SHALL state that the Exasol container takes substantially longer to accept +connections than the "about 30-45 seconds" quoted for the standard profile, and SHALL distinguish +the port opening from the server accepting a login. + +Without this, a contributor who waits the documented 45 seconds, sees a refused login and concludes +the container is broken is behaving reasonably — the measured figures are 21 seconds to an open port +and 101 seconds to a first successful login on an already-pulled image. + +#### Scenario: Readiness expectation is stated + +- **WHEN** the Exasol documentation in `CONTRIBUTING.md` is read +- **THEN** it warns that Exasol needs minutes rather than seconds before it accepts connections, and + that an open port 8563 does not yet mean the database will authenticate + +#### Scenario: The standard-profile timing is not overwritten + +- **WHEN** the existing "about 30-45 seconds" guidance is inspected +- **THEN** it is unchanged, because it remains correct for the default profile that most + contributors run + +### Requirement: The change is documentation only + +This change SHALL modify no file outside `README.md` and `CONTRIBUTING.md`, and SHALL introduce no +claim about Exasol that the code does not already implement. Documentation lands after the behaviour +it describes; if a sentence cannot be written truthfully, that is a defect to file, not a sentence to +soften. + +#### Scenario: No source, test or configuration file changes + +- **WHEN** the diff for this change is listed +- **THEN** the only paths are `README.md` and `CONTRIBUTING.md` + +#### Scenario: Documented behaviour is already implemented + +- **WHEN** each factual claim added to either document is traced +- **THEN** each resolves to existing code or configuration — the provider registration, the `exasol` + extra, the compose service, or the fixture module — and none describes intended future behaviour diff --git a/openspec/changes/archive/2026-08-27-exasol-docs/tasks.md b/openspec/changes/archive/2026-08-27-exasol-docs/tasks.md new file mode 100644 index 00000000..a91e69ad --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-docs/tasks.md @@ -0,0 +1,61 @@ +## 1. Establish the facts before writing any prose + +Design D3: documented values are read out of the code, never recalled. Do this group first — its +output is what group 2 and 3 transcribe. + +- [x] 1.1 Extract every `EXASOL_`-prefixed name and its literal `os.environ.get` fallback from + `tests/fixtures/exasol.py` (`grep -n 'EXASOL_' tests/fixtures/exasol.py`). Expect six: + `EXASOL_HOST`, `EXASOL_PORT`, `EXASOL_USER`, `EXASOL_PASSWORD`, `EXASOL_SCHEMA`, + `EXASOL_READY_TIMEOUT`. If the set differs from six, the table follows the code, not this + plan. +- [x] 1.2 Confirm the driver distribution name from `pyproject.toml` (`grep -n 'exasol' pyproject.toml`) + — the `exasol = [...]` extra is the source of truth for the package the Driver Reference row + names. +- [x] 1.3 Confirm the compose service name and profile in `infra/docker/docker-compose.test.yml`, so + the `CONTRIBUTING.md` sentence names a profile that actually starts it. +- [x] 1.4 Record the widest cell in each column of the Driver Reference table (`README.md:277-297`) + and confirm each Exasol cell is no wider, per design D6 — this is what keeps the insertion + from reflowing every other row. + +## 2. `README.md` + +- [x] 2.1 Add `Exasol` to the supported-database sentence at `README.md:28`, immediately after + `Teradata` (design D1). Change nothing else in the sentence — same engines, same order, same + trailing `and osquery.` +- [x] 2.2 Insert one Driver Reference row between `Spanner` and `Apache Arrow Flight SQL` (design + D2): Database `Exasol`, Driver package `pyexasol`, `pipx` cell + `pipx inject sqlit-tui pyexasol`, `pip` / venv cell `python -m pip install pyexasol`. Pad the + cells to the existing column widths from task 1.4 so no neighbouring row shifts. +- [x] 2.3 Verify against the spec: `git diff README.md` shows exactly two insertions and zero + modified lines elsewhere; the package name matches task 1.2. + +## 3. `CONTRIBUTING.md` + +- [x] 3.1 Add Exasol to the enterprise-container list at `CONTRIBUTING.md:49`, alongside Db2, Trino, + Presto and Oracle 11g. Leave the `docker compose ... --profile enterprise up -d` command + below it untouched — it already starts every service in the profile. +- [x] 3.2 Add an `**Exasol:**` table to the Environment Variables section, matching the shape of the + neighbouring tables (same bold label, same `| Variable | Default | Description |` header, same + separator row), with one row per variable from task 1.1 and defaults copied from it verbatim. +- [x] 3.3 Add the readiness note with the Exasol material (design D5): Exasol needs minutes rather + than seconds, and an open port 8563 is not yet a database that will authenticate. Do **not** + edit the existing "about 30-45 seconds" line — it stays correct for the default profile. +- [x] 3.4 Verify against the spec: the set of variables in the new table equals the set from task + 1.1, and every default is character-identical to the fixture's fallback. + +## 4. Verify and close out + +- [x] 4.1 `git diff --stat` names exactly two files: `README.md` and `CONTRIBUTING.md`. Any third + path means something outside this change's scope was touched. +- [x] 4.2 Read the full `git diff` and confirm no table row, list item or paragraph other than the + four insertions moved or reflowed (design D6). +- [x] 4.3 Run the pre-commit hooks against the two files (`uv run pre-commit run --files README.md + CONTRIBUTING.md`) — `trailing-whitespace` and `end-of-file-fixer` are the two these edits can + trip. There is no markdown linter in this repo, so this is the whole automated gate (design + D7). +- [x] 4.4 Re-read each added claim and confirm it traces to code that exists today — the registered + provider, the `exasol` extra, the compose service, the fixture module. Nothing may describe + intended behaviour, and nothing may imply introspection works in the casing/schema cases the + step 12-14 session found broken. +- [x] 4.5 Mark step 15 `done` in plan.md's Status table, update the progress count to 15 / 15, and + append the session-log line the plan's protocol requires. diff --git a/openspec/changes/archive/2026-08-27-exasol-integration-tests/.openspec.yaml b/openspec/changes/archive/2026-08-27-exasol-integration-tests/.openspec.yaml new file mode 100644 index 00000000..f05b045c --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-integration-tests/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-27 diff --git a/openspec/changes/archive/2026-08-27-exasol-integration-tests/design.md b/openspec/changes/archive/2026-08-27-exasol-integration-tests/design.md new file mode 100644 index 00000000..8fba6645 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-integration-tests/design.md @@ -0,0 +1,202 @@ +## Context + +`ExasolAdapter` is registered, selectable and unit-tested — but only against `MagicMock`. The three +mocked test files pin the adapter's *intent*; nothing has checked that pyexasol agrees. Reading the +installed `pyexasol` 2.3.2 source while planning this change already surfaced one concrete +disagreement (design D10), which is the clearest possible argument for the change. + +Four constraints shape it: + +1. **The unit CI job installs no extras and imports `tests/conftest.py`.** `ci.yml:66` is + `uv sync --group test --no-dev`. `conftest.py` star-imports every fixture module, so + `tests/fixtures/exasol.py` must import with `pyexasol` absent. Driver imports go inside the + fixture bodies, guarded by `pytest.skip` on `ImportError` — the pattern + `tests/fixtures/clickhouse.py` uses for `clickhouse_connect`. +2. **The unit job's exclusion list is filename-based.** Adding `tests/test_exasol.py` without adding + `--ignore=tests/test_exasol.py` makes the driver-free job collect a container test. Steps 13 and + 14 are therefore one commit. +3. **Exasol folds unquoted identifiers to uppercase, and `pyexasol.meta.*` filters are + case-sensitive LIKE patterns.** This drives the seeding decision (D3) and the one adapter + disagreement (D10). +4. **`exasol/docker-db` is a privileged multi-gigabyte image with a multi-minute cold boot and no + shipped readiness probe.** It cannot be treated like the `postgres:16-alpine` of the default + profile — hence the `enterprise` profile and the connect-based readiness gate (D1, D2). + +## Goals / Non-Goals + +**Goals:** + +- One documented command brings up a local Exasol that `tests/test_exasol.py` runs against. +- The shared `BaseDatabaseTestsWithLimit` suite passes against a real Exasol 8 server, exercising + the adapter's introspection, query execution, streaming output and TLS paths end to end. +- Every Exasol fixture **skips** — never errors — when the container or the driver is absent, so + `uv run pytest tests/` stays green on a laptop with no Docker. +- A dedicated `test-exasol` CI job, shaped like every other integration job in `ci.yml`. +- Where the live server contradicts the adapter, the contradiction is **recorded**, not papered + over silently. + +**Non-Goals:** + +- No change to any file under `sqlit/`. Findings from this change become a follow-up (D10, O1). +- No new capability in the shared test base. Where Exasol needs different behaviour, it overrides in + its own subclass rather than adding an Exasol branch to `tests/test_database_base.py`. +- No SSH-tunnel, OpenID-token or multi-node coverage. `docker-db` authenticates with a password; + the token branches stay unit-tested only. +- No documentation. That is plan.md step 15. + +## Decisions + +### D1 — Compose service goes under the `enterprise` profile, with no healthcheck + +`exasol/docker-db:latest-8` needs `privileged: true` (it manages its own storage volumes and kernel +parameters) and a `stop_grace_period: 120s` (an abrupt kill leaves the data volume dirty and the +next boot slower). At several gigabytes it belongs with `db2` / `oracle11g` / `trino` in the opt-in +profile, not in the set that a plain `docker compose up` pulls. + +No `healthcheck:` block. The image ships no lightweight probe, and the two other enterprise +databases (`db2`, `oracle11g`) declare none either — readiness is the fixture's job (D2). +*Alternative considered:* a `nc -z localhost 8563` healthcheck, like `firebird`. Rejected: it is the +same false positive that D2 exists to avoid, and would put a misleading `healthy` in +`docker compose ps`. + +### D2 — Readiness is a real connect with a deadline, not `is_port_open` + +Exasol binds 8563 well before it will accept a login. `tests/fixtures/clickhouse.py` gets away with +`is_port_open` plus `time.sleep(2)`; for Exasol that turns a normal slow boot into a hard +authentication failure. So `exasol_server_ready` keeps `is_port_open` as the cheap "is anything +there at all" gate — an immediate `False` (skip) when nothing is listening — and then retries +`pyexasol.connect(...)` with a short sleep until a deadline, returning `False` only if the deadline +passes. Absent container → skip; present-but-still-booting container → wait; present-and-broken → +skip with the driver error in the message. `pyexasol` is imported inside this fixture, never at +module level (constraint 1). + +### D3 — Seed with unquoted identifiers + +The shared suite issues `SELECT * FROM test_users`. Unquoted, Exasol folds that to `TEST_USERS`, so +the DDL must also be unquoted (`CREATE TABLE test_users (...)` → `TEST_USERS`) and the two agree. +*Rejected alternative:* quoted lowercase DDL (`CREATE TABLE "test_users"`), which would create an +object the suite's own unquoted queries can never find. + +Column names come back uppercase. The base suite already tolerates that everywhere it matters +(`data[0].get("name") or data[0].get("NAME")`, `"id,name" in result.stdout.lower()`), so no +override is needed for the query tests. + +Seed exactly what the suite consumes: `test_users` (`id` PRIMARY KEY, `name`, `email`, three rows — +Alice / Bob / Charlie), `test_products`, and the view `test_user_emails`. **No** index, trigger or +sequence: `ExasolAdapter` reports `supports_indexes` / `supports_triggers` / `supports_sequences` +as `False`, so those six base tests self-skip, and seeding objects that exist only to be ignored +would be misleading. + +### D4 — The connection fixture passes `--tls-mode require` + +`docker-db` presents a self-signed certificate. `require` maps to `encryption=True` plus +`websocket_sslopt={"cert_reqs": ssl.CERT_NONE}`; the `default` mode also encrypts but leaves +websocket-client's verification on, which fails the handshake. This is not a workaround — +`--tls-mode require` makes the integration suite the first thing to exercise the `tls_mode` → +`websocket_sslopt` mapping against a real TLS negotiation. *Rejected alternative:* extract the +container's certificate and use `verify-ca`; more moving parts, no additional adapter coverage. + +### D5 — `exasol_db` is function-scoped and recreates the schema per test + +`test_query_insert` adds a fourth row to `test_users`; `test_query_select` asserts +`3 row(s) returned`. A session-scoped seed makes that pair order-dependent. So `exasol_db` does +`DROP SCHEMA TEST_SQLIT CASCADE` / `CREATE SCHEMA` / seed on every test, matching +`tests/fixtures/clickhouse.py`'s function scope. Only `exasol_server_ready` — the expensive part — +is session-scoped. + +### D6 — No `@pytest.mark.exasol` on the new test file + +`exasol-driver-packaging` registered the marker, but **no test file in this repo uses any of the +per-database markers** — `clickhouse`, `oracle`, `mssql` are all registered and unused, and both +`ci.yml` and the plan's verify commands select by filename. Marking Exasol alone would be an +inconsistency dressed as an improvement. The marker stays registered and inert. + +### D7 — The CI job mirrors `test-clickhouse`, with a longer boot poll + +Shape copied from `ci.yml:440`: `needs: build`, Python 3.12, `uv sync --group test --no-dev --extra +exasol`, a bare `docker run -d` (no CI job in this repo uses compose), a poll loop, then +`pytest tests/test_exasol.py`. Three deviations, all boot-time driven: + +- `docker run --privileged -p 8563:8563` — required by the image. +- Poll `60 × 10s` (10 minutes) instead of ClickHouse's `30 × 2s`. The fixture's own connect-retry + (D2) absorbs the remaining gap between "port open" and "accepts logins". +- `--timeout=300` rather than the ClickHouse job's `120`, because the first query against a cold + Exasol is slow. + +Gated exactly like every other integration job — on push and PR. Per the decision recorded with the +user: `workflow_dispatch`-gating would diverge from `ci.yml`'s convention on the PR being +upstreamed, and `continue-on-error` would hide regressions. + +### D8 — `BaseDatabaseTestsWithLimit`, not `BaseDatabaseTests` + +Exasol supports `LIMIT`. `BaseDatabaseTestsWithLimit` is a strict superset that adds +`test_query_limit`. plan.md step 13 says `BaseDatabaseTests`, written before the base-class split +was checked; taking the superset is free coverage. + +### D9 — `test_docker_container_connection` is overridden with a documented skip + +`DockerDiscoveryTests.test_docker_container_connection` builds a `ConnectionConfig` from the detected +container and connects with it. For Exasol that config cannot work, for two independent reasons: + +- `SPEC.docker_detector` declares `env_vars={}` because `exasol/docker-db` exposes its credentials + through no environment variable, so `container.password` is `None`. +- A discovery-built config carries no `tls_mode`, so the adapter negotiates verified TLS against a + self-signed certificate. + +Both are true properties of the image, not bugs in the adapter, and neither is fixable from +`tests/`. The subclass therefore overrides that one method with an unconditional `pytest.skip` whose +message states both reasons. `test_docker_container_detection` and +`test_docker_container_no_password_prompt_when_not_needed` are **not** overridden — the first passes +(the container is detected and its port mapped), and the second is a no-op for a `requires_auth` +provider. + +### D10 — `test_primary_key_detection` is overridden to call `get_columns` the way the app does + +The base version calls `get_columns(conn, "test_users", database=None)` — lowercase name, no schema. +Against a live server that returns zero columns, twice over: + +- `ExasolAdapter.get_columns` does `schema = schema or ""` and passes it as pyexasol's + `column_schema_pattern`. In `pyexasol/meta.py` that becomes `WHERE column_schema LIKE ''`, which + matches nothing. pyexasol's own pattern default is `'%'`, not `''`. +- pyexasol's meta patterns are case-sensitive, and `EXA_ALL_COLUMNS` stores `TEST_USERS`. The + primary-key snapshot query has the same problem with `CONSTRAINT_TABLE = 'test_users'`. + +The app never makes that call: `schema_service.py:86` and `process_worker.py:333` both pass the name +and schema straight through from `get_tables()`, which returns them uppercase. So the override passes +`schema=TEST_SQLIT` and the uppercase table name, and still asserts the whole contract — `id` is +flagged primary key, nothing else is. The `LIKE ''` weakness is recorded as a finding for a follow-up +change (O1); patching `sqlit/` is out of scope here. + +## Risks / Trade-offs + +- **[Cold boot exceeds the CI poll]** → 10-minute poll plus the fixture's connect-retry. If the + image regresses past that, the job fails loudly with the poll's own log lines rather than an + opaque authentication error. +- **[GitHub runner resources]** `docker-db` wants several GB of RAM and roughly 10 GB of disk; a + standard `ubuntu-latest` runner has both, with little headroom. → If the job proves flaky on + resources, the fallback is a pre-run `docker image prune -af`, recorded before reaching for + `continue-on-error`. +- **[Default credentials assumed]** The fixture defaults to `sys` / `exasol`. → Both the compose + service and the CI job read `EXASOL_USER` / `EXASOL_PASSWORD`, and a task verifies the defaults + against a running container before the suite is trusted. +- **[Two overridden base tests could mask a regression]** → Each is narrow, carries a comment naming + the reason, and D10's override still asserts the full primary-key contract. Neither weakens what + the other twenty-odd inherited tests check. +- **[`latest-8` is a floating tag]** A new Exasol 8 patch can change behaviour with no repo change. + → Accepted, matching the repo's convention (`clickhouse-server:latest`, `surrealdb:latest`; + `firebird` is the sole pinned exception). +- **[Per-test schema recreation]** D5 pays a `DROP` / `CREATE` / seed cycle roughly twenty times. + → Exasol schema DDL is fast next to the container boot this job already pays for; correctness wins. + +## Open Questions + +- **O1 — Should `get_columns`'s schema default be `"%"` instead of `""`?** For a schema-only + provider, "no schema given" plausibly means "search everywhere", which is what pyexasol's own + default expresses. **Recommendation:** yes, but in a separate change — this one adds no `sqlit/` + edits, and the fix wants its own unit test alongside the existing mocked assertion that currently + pins `""`. +- **O2 — `timezone_datetime_type` stays `None`,** so `test_timezone_aware_datetime` skips. Exasol has + `TIMESTAMP WITH LOCAL TIME ZONE`, but enabling the test needs an Exasol branch inside + `tests/test_database_base.py`, which Non-Goals excludes. Left as a known gap. +- **O3 — Confirm `exasol/docker-db:latest-8`'s default SYS credentials and whether the image accepts + a password override,** before treating the `EXASOL_PASSWORD` default as documented behaviour. diff --git a/openspec/changes/archive/2026-08-27-exasol-integration-tests/findings.md b/openspec/changes/archive/2026-08-27-exasol-integration-tests/findings.md new file mode 100644 index 00000000..315ab747 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-integration-tests/findings.md @@ -0,0 +1,176 @@ +# Pre-implementation findings + +Three things surfaced while reading the code to plan this change that `plan.md` steps 12-14 did not +anticipate. All three are *predictions* made by reading source, not observed test failures — no +Exasol container has been started yet. Each is handled by a decision in `design.md`; this file is the +evidence behind those decisions. + +Verified against the working tree at the time of writing: `pyexasol` 2.3.2 installed in `.venv`, +`ExasolAdapter` as committed by the `exasol-adapter` change. + +--- + +## 1. `test_primary_key_detection` will fail against a live server + +The inherited test from `BaseDatabaseTests` calls `get_columns` in a shape the application never +uses, and Exasol's metadata views will return nothing for it. + +**Evidence** + +- `tests/test_database_base.py:361-397` — the base test calls + `session.adapter.get_columns(session.connection, "test_users", database=...)`: lowercase table + name, **no schema argument**. It then asserts `len(columns) >= 3`. +- `sqlit/domains/connections/providers/exasol/adapter.py:161-183` — `get_columns` does + `schema = schema or ""` and passes that as the first positional argument to + `conn.meta.list_columns(schema, table)`. +- `.venv/Lib/site-packages/pyexasol/meta.py:276-306` — that first parameter is + `column_schema_pattern`, whose **default is `'%'`**, and it lands in the query as + `WHERE column_schema LIKE {column_schema_pattern}`. `LIKE ''` matches nothing. Passing `""` is + strictly narrower than passing nothing. +- `.venv/Lib/site-packages/pyexasol/meta.py:303-304` — the docstring states plainly: + *"Patterns are case-sensitive."* `EXA_ALL_COLUMNS` stores the folded, uppercase `TEST_USERS`, so + `column_table_pattern="test_users"` matches nothing either. +- The primary-key lookup in the same method has the same problem with exact equality: + `WHERE ... CONSTRAINT_SCHEMA = {schema!s} AND CONSTRAINT_TABLE = {table!s}`. + +So there are two independent reasons the call returns `[]`: the empty schema pattern and the +identifier casing. Fixing only one of them still yields zero columns. + +**Why the application is unaffected** + +Both real callers pass an explicit schema and a name that came from the server: + +- `sqlit/domains/explorer/app/schema_service.py:86` — `inspector.get_columns(conn, name, db_arg, schema)` +- `sqlit/domains/process_worker/app/process_worker.py:333` — same shape + +`name` and `schema` originate in `get_tables()`, which reads `TABLE_SCHEMA` / `TABLE_NAME` from +`EXA_ALL_TABLES` — already uppercase. The failure is confined to the base suite's call shape. + +**How this change handles it** + +`design.md` **D10**: `TestExasolIntegration` overrides `test_primary_key_detection` to call +`get_columns` with `schema=TEST_SQLIT` and the uppercase table name — the app's shape — while still +asserting the full contract (`ID` flagged primary key, nothing else flagged). Task 5.4. + +`design.md` **O1** records the underlying weakness as a follow-up: for a provider that declares +`supports_multiple_databases=False`, defaulting the schema pattern to `""` rather than `"%"` means +"no schema given" silently returns nothing instead of searching everywhere. That fix touches +`sqlit/` and wants its own unit test, which this change's Non-Goals exclude. + +**Note on the mocked suite.** `tests/connections/providers/exasol/test_adapter.py` asserts that +`schema=None` passes `""` to `list_columns` — it pins the current behaviour deliberately (recorded as +D8 of the `exasol-unit-tests` change). A `MagicMock` returns rows for any argument, so no mocked test +can distinguish `""` from `"%"`. This is the change's premise in miniature. + +--- + +## 2. `test_docker_container_connection` will fail while the container is running + +`DockerDiscoveryTests` builds a `ConnectionConfig` from the detected container and opens a real +connection with it. For Exasol that configuration cannot work — for two reasons, neither of which is +a bug in the adapter and neither of which is fixable from `tests/`. + +**Evidence** + +- `tests/test_database_docker.py:108-176` — the test skips only when Docker is unavailable, no + container matches, or the container is not connectable. With the container up it proceeds to + `adapter.connect(config)` and `adapter.execute_query(conn, "SELECT 1")`. +- `sqlit/domains/connections/providers/exasol/provider.py` — `SPEC.docker_detector` is + `DockerDetector(image_patterns=("exasol/docker-db",), env_vars={}, default_user="sys")`. The empty + `env_vars` is correct: `exasol/docker-db` publishes no credential through an environment variable. +- `sqlit/domains/connections/discovery/docker_detector.py:384-419` — + `container_to_connection_config` sets `password=container.password`, which is `None` when no env + var supplied one. +- `sqlit/domains/connections/providers/exasol/adapter.py:139` — the password branch passes + `endpoint.password` straight through to `pyexasol.connect`. +- The discovery-built config carries no `tls_mode` option, so `_tls_args` takes the default path: + `encryption=True` with **no** `websocket_sslopt`, leaving websocket-client's certificate + verification enabled — against the self-signed certificate `docker-db` presents. The handshake + fails before authentication is even reached. + +**How this change handles it** + +`design.md` **D9**: override that one method with an unconditional `pytest.skip` whose message states +both reasons. Task 5.3. + +The other two `DockerDiscoveryTests` methods are deliberately **not** overridden: + +- `test_docker_container_detection` (`tests/test_database_docker.py:11-53`) only asserts that a + matching, connectable container has a detected port — that passes. +- `test_docker_container_no_password_prompt_when_not_needed` asserts only for providers where + `requires_auth(db_type)` is false. `SPEC.requires_auth` is `True`, so it is a no-op. + +--- + +## 3. `is_port_open` is not a readiness check for Exasol + +The fixture pattern this change copies is not safe for this image. + +**Evidence** + +- `tests/fixtures/clickhouse.py:26-33` — `clickhouse_server_ready` returns `is_port_open(...)` + followed by `time.sleep(2)`. That is adequate for a container that is serving within seconds. +- `exasol/docker-db` binds 8563 during startup and refuses logins for minutes afterwards. A bare + port check therefore reports "ready" during a window where every connection attempt fails, turning + a normal slow boot into a hard authentication error instead of a wait. + +**How this change handles it** + +`design.md` **D2**: `exasol_server_ready` keeps `is_port_open` as the cheap "is anything there at +all" gate — an immediate `False`, and therefore a skip, when nothing is listening — then retries a +real `pyexasol.connect` with a sleep until a deadline. Absent container → skip; still booting → +wait; present but broken → skip with the driver's error in the message. Task 3.3. + +The same distinction drives the CI poll in **D7**: 60 attempts × 10 s on the port, with the +fixture's connect-retry absorbing the remaining gap between "port open" and "accepts logins" +(tasks 6.3, 2.4). + +--- + +## Smaller observations + +These did not change the shape of the work but are worth having written down. + +- **Quoted lowercase seed DDL would break the whole suite.** The shared tests issue + `SELECT * FROM test_users` unquoted, which Exasol folds to `TEST_USERS`. Seeding + `CREATE TABLE "test_users"` would create an object those queries can never find. The seed DDL + must stay unquoted so both sides fold identically — `design.md` **D3**, task 3.5. +- **The `exasol` pytest marker stays unused, and that is consistent.** `pyproject.toml:201` + registers it, but **no test file in this repo uses any per-database marker** — `clickhouse`, + `oracle` and `mssql` are all registered and unused; `ci.yml` and the plan's verify commands select + by filename. Marking Exasol alone would be an inconsistency, not an improvement — + `design.md` **D6**. +- **`BaseDatabaseTestsWithLimit` exists and is a free superset.** `tests/test_database_base.py:783` + adds `test_query_limit` on top of `BaseDatabaseTests`. Exasol supports `LIMIT`, so the subclass + costs nothing. `plan.md` step 13 names `BaseDatabaseTests`, written before the split was checked — + `design.md` **D8**. +- **Six inherited tests will skip on adapter capability flags.** `supports_indexes`, + `supports_triggers` and `supports_sequences` are all `False` + (`adapter.py:66-80`), which self-skips `test_get_indexes`, `test_get_triggers`, + `test_get_sequences` and the three `*_definition` tests. The fixture therefore seeds no index, + trigger or sequence — seeding objects that exist only to be ignored would mislead a reader. + Task 5.8 verifies that every skip in the run has a cause on this list. +- **`test_timezone_aware_datetime` will skip.** `DatabaseTestConfig.timezone_datetime_type` stays + `None`. Exasol has `TIMESTAMP WITH LOCAL TIME ZONE`, but the base test body branches per database + (`tests/test_database_base.py:687-729`), so enabling it means editing shared test code — excluded + by Non-Goals. Recorded as `design.md` **O2**. +- **The unit job's exclusion list is filename-based** (`ci.yml:70-82`), which is why tasks 5.1 and + 6.1 must land in the same commit. This one *was* in the plan, as the rationale for the `INT` atomic + group; repeated here because it is the single most likely way to leave the branch red. + +--- + +## Finding → decision → task + +| Finding | Decision | Task | Follow-up | +|---|---|---|---| +| `get_columns` empty schema pattern + case-sensitive matching | D10 | 5.4 | O1 — separate change, touches `sqlit/` | +| Docker discovery cannot build a working Exasol config | D9 | 5.3 | none — property of the image | +| Open port is not readiness | D2, D7 | 3.3, 6.3 | none | +| Uppercase folding in seed DDL | D3 | 3.5 | none | +| Per-database markers unused repo-wide | D6 | 5.1 | none | +| `WithLimit` base class available | D8 | 5.1 | none | +| Timezone test not applicable without shared-code edits | O2 | 5.2 | deferred | + +Anything the live container contradicts during implementation goes in `plan.md`'s Session log +(task 5.10) — not into a `sqlit/` edit inside this change. diff --git a/openspec/changes/archive/2026-08-27-exasol-integration-tests/proposal.md b/openspec/changes/archive/2026-08-27-exasol-integration-tests/proposal.md new file mode 100644 index 00000000..6dd6a1dc --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-integration-tests/proposal.md @@ -0,0 +1,79 @@ +## Why + +Exasol is a fully registered, unit-tested provider — but **no code path has ever opened a real +Exasol connection**. Every unit test added by `exasol-unit-coverage` runs against a `MagicMock`, +and a mock agrees with whatever the adapter asks of it by construction: if `conn.meta.list_tables()` +really returns lowercase keys, or `ExaStatement.rowcount` is really an attribute rather than a +method, or `websocket_sslopt` is really spelled differently, the mocked suite stays green. The +adapter's contract with pyexasol is currently unverified in both directions. + +Upstream `Maxteabag/sqlit` backs every non-file-based provider with a Docker-driven integration job +in `.github/workflows/ci.yml`. A provider PR without one is missing the repo's own convention. This +change adds that job and the harness it needs. It is plan.md steps 12, 13 and 14 — the `INT` atomic +group, taken together because step 13 alone leaves the default CI job collecting a test that needs a +container it does not have. + +## What Changes + +- **Modified** `infra/docker/docker-compose.test.yml` — a new `exasol` service under the existing + `enterprise` profile (alongside `db2` / `oracle11g` / `trino`), because `exasol/docker-db:latest-8` + is a multi-gigabyte image that needs `privileged: true`, a long boot and a + `stop_grace_period: 120s` to shut down cleanly. Ports `${EXASOL_PORT:-8563}:8563`. +- **New** `tests/fixtures/exasol.py` — modelled on `tests/fixtures/clickhouse.py`: env-var constants + (`EXASOL_HOST`/`PORT`/`USER`/`PASSWORD`/`SCHEMA`), an `exasol_available()` TCP guard, a session + `exasol_server_ready` fixture, an `exasol_db` fixture that drops and recreates the `TEST_SQLIT` + schema and seeds the objects `BaseDatabaseTests` expects (`test_users` with an `id` primary key, + `test_products`, the view `test_user_emails`), and an `exasol_connection` fixture that registers a + sqlit CLI connection. Defaults `sys` / `exasol` — the `docker-db` defaults. +- **Modified** `tests/conftest.py` — one `from tests.fixtures.exasol import *` line in the existing + alphabetical fixture block. +- **New** `tests/test_exasol.py` — `TestExasolIntegration(BaseDatabaseTestsWithLimit)` with a + `DatabaseTestConfig(db_type="exasol", display_name="Exasol", ...)`, plus CLI create/delete + connection tests mirroring `tests/test_clickhouse.py`. +- **Modified** `.github/workflows/ci.yml` — `--ignore=tests/test_exasol.py` added to the unit-test + job's exclude list, and a new `test-exasol` job modelled on `test-clickhouse` + (`uv sync --group test --no-dev --extra exasol`, start the container, poll port 8563, run + `pytest tests/test_exasol.py`). +- No changes to any file under `sqlit/`. The adapter is being exercised, not modified. If the + container disagrees with it, that is a finding for a follow-up change. + +## Capabilities + +### New Capabilities + +- `exasol-integration-harness`: a reproducible local and CI Exasol instance plus the pytest fixtures + that seed it — including the two hard constraints that the fixture module imports cleanly with no + `pyexasol` installed, and that every fixture skips rather than fails when the container is absent. +- `exasol-integration-coverage`: the shared database test suite runs against a real Exasol server, + and does so in a dedicated CI job without being collected by the driver-free unit job. + +### Modified Capabilities + +None. `openspec/specs/exasol-adapter` and `openspec/specs/exasol-provider-registration` describe the +behaviour this change verifies against a live server; no requirement in either changes. +`openspec/specs/exasol-driver-packaging` already registered the `exasol` pytest marker — this change +consumes nothing new from it (see design D6). + +## Impact + +- **CI cost**: one new job pulling `exasol/docker-db:latest-8` (~4 GB) and waiting several minutes + for the database to accept connections. Gated the same way as every other integration job — on + push and PR, `needs: build` — per the decision recorded in design D7. +- **CI safety**: the unit job's exclude list is *filename-based*, so `tests/test_exasol.py` and the + `--ignore` for it must land in the same commit. This is why plan.md marks 13 + 14 atomic. +- **`tests/conftest.py` import surface**: `conftest.py` is imported by the driver-free unit job, so + the new fixture module cannot import `pyexasol` at module level. It imports it inside `exasol_db` + and skips on `ImportError`, exactly as `tests/fixtures/clickhouse.py` does for + `clickhouse_connect`. +- **TLS**: `docker-db` presents a self-signed certificate, so the connection fixture must pass + `--tls-mode require`. This is not incidental — it makes the integration test the first thing to + exercise the `tls_mode` → `encryption` + `websocket_sslopt` mapping against a real TLS handshake. +- **Identifier case**: Exasol folds unquoted identifiers to uppercase. The shared suite's + assertions already tolerate uppercase column names (`data[0].get("name") or data[0].get("NAME")`, + `"id,name" in result.stdout.lower()`), and the seed DDL stays unquoted so `test_users` and + `TEST_USERS` resolve to the same object. See design D3. +- **Self-skipping base tests**: `test_get_indexes`, `test_get_triggers`, `test_get_sequences` and the + three `*_definition` tests skip on `session.adapter.supports_*`, all of which `ExasolAdapter` + reports `False`. The fixture therefore seeds no index, trigger or sequence. +- **Not affected**: no `sqlit/` source, no `pyproject.toml`, no `uv.lock`, no documentation. Docs are + plan.md step 15. diff --git a/openspec/changes/archive/2026-08-27-exasol-integration-tests/specs/exasol-integration-coverage/spec.md b/openspec/changes/archive/2026-08-27-exasol-integration-tests/specs/exasol-integration-coverage/spec.md new file mode 100644 index 00000000..dd944665 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-integration-tests/specs/exasol-integration-coverage/spec.md @@ -0,0 +1,127 @@ +## ADDED Requirements + +### Requirement: Exasol runs the shared database integration suite + +`tests/test_exasol.py` SHALL run the repository's shared database test suite against a live Exasol +server, including the suite's `LIMIT` coverage. + +#### Scenario: Suite is bound to the Exasol provider + +- **WHEN** `tests/test_exasol.py` is collected +- **THEN** it defines a single test class deriving from the shared base class that includes the + `LIMIT` test, configured with database type `exasol`, display name `Exasol`, and the Exasol + connection and database fixtures + +#### Scenario: Suite passes against a running server + +- **WHEN** `uv run pytest tests/test_exasol.py -v` is run with an Exasol container up and the + `exasol` extra installed +- **THEN** every inherited test either passes or skips for a reason the adapter declares, and none + fails + +#### Scenario: Suite skips without a server + +- **WHEN** the same command is run with no Exasol server reachable +- **THEN** every test in the file is skipped + +### Requirement: Exasol connection lifecycle is verified through the CLI + +The Exasol test file SHALL verify that a connection can be created and deleted through the sqlit +CLI, independently of the fixture that the shared suite uses. + +#### Scenario: Connection is created and listed + +- **WHEN** an Exasol connection is added with `connections add exasol` and the connection list is + printed +- **THEN** the command reports success and the listing shows the connection name alongside the + `Exasol` display name + +#### Scenario: Connection is deleted + +- **WHEN** that connection is deleted through the CLI +- **THEN** the command reports success and the connection no longer appears in the listing + +### Requirement: Capability-driven and image-driven skips are explicit + +Tests that cannot apply to Exasol SHALL skip for a stated reason rather than being deleted or +silently passing. + +#### Scenario: Unsupported object types self-skip + +- **WHEN** the inherited index, trigger and sequence tests run +- **THEN** they skip because `ExasolAdapter` reports those capabilities as unsupported, and the test + fixture seeds no such objects + +#### Scenario: Docker-discovery connection test is skipped with a reason + +- **WHEN** the inherited Docker-discovery connection test runs +- **THEN** it is skipped by an override whose message states that `exasol/docker-db` publishes no + credentials through environment variables and presents a self-signed certificate, so a + discovery-built configuration cannot connect + +#### Scenario: Docker container detection is not skipped + +- **WHEN** the inherited Docker container detection test runs with the Exasol container up +- **THEN** it is not overridden, and it passes by detecting the container and its published port + +### Requirement: Primary-key detection is verified through the app's call shape + +The primary-key test SHALL exercise `get_columns` with the schema and identifier casing that the +application itself supplies, and SHALL still assert the full primary-key contract. + +#### Scenario: Columns are requested with an explicit schema and server casing + +- **WHEN** the Exasol primary-key test calls the adapter's `get_columns` +- **THEN** it passes the seeded schema and the table name in the casing the server stores, matching + how the explorer and worker call it with values taken from `get_tables()` + +#### Scenario: Primary key flags are asserted both ways + +- **WHEN** the returned columns are inspected +- **THEN** the `id` column is flagged as a primary key and every other column is not + +### Requirement: The driver-free unit job does not collect the Exasol integration test + +The default CI unit job SHALL exclude `tests/test_exasol.py`, in the same commit that introduces the +file, because that job installs no database extras and starts no container. + +#### Scenario: Unit job excludes the file + +- **WHEN** the unit-test job's pytest invocation in `.github/workflows/ci.yml` is read +- **THEN** its exclude list contains `--ignore=tests/test_exasol.py` alongside the other integration + test files + +#### Scenario: Unit job still passes locally + +- **WHEN** that exact command is run locally +- **THEN** collection succeeds and no Exasol integration test is collected + +### Requirement: A dedicated CI job runs the Exasol integration suite + +`.github/workflows/ci.yml` SHALL contain a job that provisions an Exasol server and runs the Exasol +integration test file, following the same conventions as the repository's other per-database +integration jobs. + +#### Scenario: Job installs the driver + +- **WHEN** the job's dependency step runs +- **THEN** it installs the test group together with the `exasol` extra + +#### Scenario: Job starts the server and waits for it + +- **WHEN** the job's server step runs +- **THEN** it starts an `exasol/docker-db` container with the privileges the image needs and the + database port published, and polls that port until it accepts connections or a bounded number of + attempts is exhausted, logging each attempt + +#### Scenario: Job runs the suite with the harness environment + +- **WHEN** the job's test step runs +- **THEN** it invokes pytest on `tests/test_exasol.py` with the Exasol host, port, credential and + schema environment variables set, and with a per-test timeout large enough for a cold server + +#### Scenario: Job is gated like its peers + +- **WHEN** the workflow triggers are compared across integration jobs +- **THEN** the Exasol job runs on the same events as the other database jobs, depends on the same + upstream job, and is neither manually gated nor marked to continue on error diff --git a/openspec/changes/archive/2026-08-27-exasol-integration-tests/specs/exasol-integration-harness/spec.md b/openspec/changes/archive/2026-08-27-exasol-integration-tests/specs/exasol-integration-harness/spec.md new file mode 100644 index 00000000..d2c40113 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-integration-tests/specs/exasol-integration-harness/spec.md @@ -0,0 +1,146 @@ +## ADDED Requirements + +### Requirement: Local Exasol test server + +The test compose stack SHALL provide an Exasol service that a developer can start on demand, and +that service SHALL NOT be started by the default `docker compose up`. + +#### Scenario: Service is declared under the opt-in profile + +- **WHEN** `docker compose -f infra/docker/docker-compose.test.yml --profile enterprise config` is run +- **THEN** the rendered configuration contains an `exasol` service built from an `exasol/docker-db` + image, running `privileged`, publishing container port `8563` on `${EXASOL_PORT:-8563}`, and + declaring a `stop_grace_period` of at least 120 seconds + +#### Scenario: Default profile is unchanged + +- **WHEN** `docker compose -f infra/docker/docker-compose.test.yml config --services` is run with no + profile +- **THEN** `exasol` is absent from the listed services, so no developer pulls a multi-gigabyte image + without asking for it + +### Requirement: Fixture module imports without the driver + +`tests/fixtures/exasol.py` SHALL be importable in an environment where `pyexasol` is not installed, +because `tests/conftest.py` star-imports it and is loaded by the driver-free unit CI job. + +#### Scenario: Collection succeeds with no exasol extra installed + +- **WHEN** the unit-test command from `.github/workflows/ci.yml` is run in an environment installed + with `uv sync --group test --no-dev` (no `--extra exasol`) +- **THEN** collection completes with no `ImportError` and no collection error from `tests/conftest.py` + +#### Scenario: Driver is imported lazily + +- **WHEN** `tests/fixtures/exasol.py` is inspected +- **THEN** it contains no module-level `import pyexasol`, and every `pyexasol` import sits inside a + fixture body guarded so that an `ImportError` becomes `pytest.skip`, not a test failure + +### Requirement: Fixtures skip rather than fail when the server is absent + +Every Exasol fixture SHALL resolve to a skip when no reachable Exasol server is available, so that a +full `pytest tests/` run stays green on a machine with no Docker. + +#### Scenario: No container running + +- **WHEN** nothing is listening on the configured Exasol host and port +- **THEN** tests depending on the Exasol fixtures are reported as skipped, and no exception escapes + a fixture + +#### Scenario: Driver missing but container present + +- **WHEN** an Exasol server is reachable but `pyexasol` is not installed +- **THEN** the fixtures skip with a message naming the missing driver + +### Requirement: Readiness gate tolerates a slow boot + +The readiness fixture SHALL confirm readiness by opening a real database connection, retrying until +a deadline, rather than trusting an open port — Exasol accepts TCP connections on its port well +before it will accept a login. + +#### Scenario: Server is still booting + +- **WHEN** the port is open but the database refuses connections +- **THEN** the readiness fixture retries until its deadline, and only then reports the server as + unavailable + +#### Scenario: Server becomes ready during the wait + +- **WHEN** the database starts accepting connections before the deadline expires +- **THEN** the readiness fixture reports the server as ready and the dependent tests run + +#### Scenario: Readiness is computed once per session + +- **WHEN** more than one Exasol test runs in the same session +- **THEN** the readiness check is performed once, because it is session-scoped + +### Requirement: Seeded test schema matches the shared suite's expectations + +The `exasol_db` fixture SHALL create a dedicated test schema containing exactly the objects the +shared database test suite queries, and SHALL leave no trace of itself behind. + +#### Scenario: Schema is seeded + +- **WHEN** the `exasol_db` fixture runs +- **THEN** the test schema contains a `test_users` table whose `id` column is a primary key and + which holds the three rows Alice, Bob and Charlie; a `test_products` table; and a + `test_user_emails` view +- **AND** the fixture yields the schema name so a connection can be opened against it + +#### Scenario: Identifiers resolve unquoted + +- **WHEN** the seed DDL is executed +- **THEN** it uses unquoted identifiers, so that Exasol's uppercase folding makes + `SELECT * FROM test_users` — the form the shared suite issues — resolve to the seeded table + +#### Scenario: State does not leak between tests + +- **WHEN** one test inserts an extra row into `test_users` and a later test asserts a three-row + result +- **THEN** the later test still sees three rows, because the fixture recreates the schema per test + +#### Scenario: Teardown drops the schema + +- **WHEN** a test using `exasol_db` finishes, whether it passed or failed +- **THEN** the test schema is dropped, and a teardown failure does not fail the test + +### Requirement: CLI connection fixture negotiates TLS against a self-signed certificate + +The `exasol_connection` fixture SHALL register a sqlit connection through the CLI whose TLS settings +succeed against the self-signed certificate that `exasol/docker-db` presents, and SHALL remove that +connection afterwards. + +#### Scenario: Connection is created with an encrypting, non-verifying TLS mode + +- **WHEN** the `exasol_connection` fixture registers its connection +- **THEN** it passes `--tls-mode require`, so the adapter encrypts the connection without verifying + the certificate chain, and the connection is usable by the shared suite + +#### Scenario: Connection targets the seeded schema + +- **WHEN** the connection is created +- **THEN** it carries the seeded schema as its initial schema, so unqualified table names in the + shared suite's queries resolve without a schema prefix + +#### Scenario: Connection is cleaned up + +- **WHEN** a test using `exasol_connection` finishes +- **THEN** the connection is deleted from the sqlit connection store + +### Requirement: Fixtures are registered and environment-configurable + +The Exasol fixtures SHALL be discoverable by the test suite and SHALL take their host, port, +credentials and schema name from environment variables, so the same suite runs against a local +container and against a CI-provided server. + +#### Scenario: Fixtures are registered in conftest + +- **WHEN** `tests/conftest.py` is read +- **THEN** it star-imports `tests.fixtures.exasol` within its existing alphabetically ordered + fixture import block + +#### Scenario: Connection details are overridable + +- **WHEN** `EXASOL_HOST`, `EXASOL_PORT`, `EXASOL_USER`, `EXASOL_PASSWORD` or `EXASOL_SCHEMA` are set + in the environment +- **THEN** the fixtures use those values instead of their defaults diff --git a/openspec/changes/archive/2026-08-27-exasol-integration-tests/tasks.md b/openspec/changes/archive/2026-08-27-exasol-integration-tests/tasks.md new file mode 100644 index 00000000..32645209 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-integration-tests/tasks.md @@ -0,0 +1,152 @@ +## 1. Plan bookkeeping and preflight + +- [x] 1.1 In `plan.md`, set steps 12, 13 and 14 to `wip` in the Status table before touching any + file. The plan's own protocol requires the whole `INT` group to be taken in one session or not + at all. +- [x] 1.2 Confirm the environment has the driver: `uv sync --extra exasol` then + `uv run python -c "import pyexasol; print(pyexasol.__version__)"`. The integration suite is the + one place that genuinely needs it. +- [x] 1.3 Record the pre-change baseline of the unit-test command from `.github/workflows/ci.yml` + (lines 70-82) so a new failure can be told apart from the plan's already-logged + Windows-environment failures. + +## 2. Compose service (plan step 12a) + +- [x] 2.1 Add an `exasol` service to `infra/docker/docker-compose.test.yml`, placed with the other + `enterprise`-profile services (`db2`, `oracle11g`, `trino`, `presto`, `impala`): + `image: exasol/docker-db:latest-8`, `container_name: sqlit-test-exasol`, + `privileged: true`, `stop_grace_period: 120s`, `ports: ["${EXASOL_PORT:-8563}:8563"]`, + `profiles: [enterprise]`. Design D1: no `healthcheck` block. +- [x] 2.2 Verify the enterprise profile renders: + `docker compose -f infra/docker/docker-compose.test.yml --profile enterprise config`. +- [x] 2.3 Verify the default profile is untouched: + `docker compose -f infra/docker/docker-compose.test.yml config --services` does **not** list + `exasol`. +- [x] 2.4 Start the container — + `docker compose -f infra/docker/docker-compose.test.yml --profile enterprise up -d exasol` — + and note how long it takes before port 8563 accepts a login. That number is the input to task + 6.3's poll count. +- [x] 2.5 Resolve design O3 against the running container: confirm the default SYS credentials + (`sys` / `exasol`) and whether the image accepts a password override. Record the answer in the + plan's Session log; if the defaults differ, use the real ones in task 3.2 and 6.4. + +## 3. Fixture module (plan step 12b) + +- [x] 3.1 Create `tests/fixtures/exasol.py` modelled on `tests/fixtures/clickhouse.py`, importing + `cleanup_connection`, `is_port_open` and `run_cli` from `tests.fixtures.utils`. Design + constraint 1: **no module-level `import pyexasol`**. +- [x] 3.2 Add the env-var constants: `EXASOL_HOST` (default `localhost`), `EXASOL_PORT` (`8563`), + `EXASOL_USER` (`sys`), `EXASOL_PASSWORD` (`exasol`), `EXASOL_SCHEMA` (`TEST_SQLIT`), plus an + `exasol_available()` helper wrapping `is_port_open`. +- [x] 3.3 Add a session-scoped `exasol_server_ready` fixture: return `False` immediately when + `exasol_available()` is false; otherwise import `pyexasol` inside the fixture and retry + `pyexasol.connect(dsn=..., user=..., password=..., encryption=True, + websocket_sslopt={"cert_reqs": ssl.CERT_NONE}, autocommit=True)` with a sleep until a deadline, + returning `True` on the first success and `False` if the deadline passes. Design D2 — an open + port is not readiness. +- [x] 3.4 Add a function-scoped `exasol_db` fixture that skips when `exasol_server_ready` is false, + skips on `ImportError` for `pyexasol`, then `DROP SCHEMA IF EXISTS TEST_SQLIT CASCADE` / + `CREATE SCHEMA TEST_SQLIT` / `OPEN SCHEMA TEST_SQLIT`. Design D5. +- [x] 3.5 Seed inside `exasol_db`, all identifiers **unquoted** (design D3): `test_users` + (`id DECIMAL(18,0) PRIMARY KEY`, `name VARCHAR`, `email VARCHAR`), `test_products` + (`id`, `name`, `price`, `stock`), the view `test_user_emails` selecting `id, name, email` from + `test_users` where `email` is non-empty, three `test_users` rows (Alice, Bob, Charlie) and + three `test_products` rows. Seed **no** index, trigger or sequence. +- [x] 3.6 Wrap the setup in `try/except Exception` → `pytest.skip(f"Failed to setup Exasol schema: {e}")`, + matching `tests/fixtures/clickhouse.py`. `yield EXASOL_SCHEMA`, then drop the schema in a + teardown whose own failure is swallowed. +- [x] 3.7 Add a function-scoped `exasol_connection` fixture that `cleanup_connection`s a + pid-suffixed name, then `run_cli("connections", "add", "exasol", "--name", ..., "--server", ..., + "--port", ..., "--username", ..., "--password", ..., "--schema", exasol_db, + "--tls-mode", "require")`, yields the name, and cleans up after. Design D4 — `--tls-mode + require` is required by the self-signed certificate and is deliberate TLS coverage. +- [x] 3.8 Declare `__all__` listing the constants and fixtures, matching + `tests/fixtures/clickhouse.py`'s style — `test_exasol.py` reads the constants back through + `from .conftest import ...`, which relies on the star-import re-export. +- [x] 3.9 Verify the module imports with the driver absent — the constraint the unit CI job imposes. + Run the unit-test command from `ci.yml` in an environment installed without `--extra exasol` + (or temporarily rename the installed `pyexasol` package) and confirm no collection error. + +## 4. conftest registration (plan step 12c) + +- [x] 4.1 Add `from tests.fixtures.exasol import *` to `tests/conftest.py`, in the alphabetical block + that starts at line 5 — between the `duckdb` and `firebird` imports. +- [x] 4.2 Verify `uv run pytest tests/connections -v` still collects and passes: the conftest is + importable and nothing regressed. +- [x] 4.3 Verify `uv run pytest tests/ --collect-only -q` succeeds. This is the gate that catches a + fixture-module import error before CI does. + +## 5. Integration test file (plan step 13 — must land with group 6) + +- [x] 5.1 Create `tests/test_exasol.py` with + `class TestExasolIntegration(BaseDatabaseTestsWithLimit)` importing from + `.test_database_base`. Design D8 — the `WithLimit` superset, not `BaseDatabaseTests` as + plan.md step 13 says. No `@pytest.mark.exasol` (design D6). +- [x] 5.2 Implement the `config` property returning + `DatabaseTestConfig(db_type="exasol", display_name="Exasol", + connection_fixture="exasol_connection", db_fixture="exasol_db", + create_connection_args=lambda: [])`. Leave `uses_limit` at its `True` default and + `timezone_datetime_type` at `None` (design O2). +- [x] 5.3 Override `test_docker_container_connection` with an unconditional `pytest.skip` whose + message states both reasons from design D9: `exasol/docker-db` publishes no credentials through + environment variables (`SPEC.docker_detector` has `env_vars={}`), and a discovery-built config + carries no `tls_mode` so it verifies TLS against a self-signed certificate. Do **not** override + the other two `DockerDiscoveryTests` methods. +- [x] 5.4 Override `test_primary_key_detection` per design D10: load the connection config, open a + `ConnectionSession`, and call `session.adapter.get_columns(session.connection, "TEST_USERS", + database=None, schema=EXASOL_SCHEMA)` — the app's call shape. Assert at least three columns, + that `ID` is flagged primary key, and that no other column is. Comment the override with the + `LIKE ''` / case-sensitivity reason. +- [x] 5.5 Add `test_create_exasol_connection`, modelled on `tests/test_clickhouse.py:26`: create a + connection via `cli_runner`, assert `returncode == 0` and `"created successfully"` in stdout, + assert the name and `"Exasol"` appear in `connection list`, and delete it in a `finally`. +- [x] 5.6 Add `test_delete_exasol_connection`, mirroring the ClickHouse equivalent: create, delete, + assert `"deleted successfully"`, assert the name is gone from `connection list`. +- [x] 5.7 Run the suite against the live container: `uv run pytest tests/test_exasol.py -v`. + Every test must pass or skip; nothing may fail. +- [x] 5.8 Read the skip list from 5.7 and confirm each skip has a declared cause: indexes, triggers, + sequences and the three `*_definition` tests from adapter capability flags; the timezone test + from `timezone_datetime_type=None`; the Docker-discovery connection test from the 5.3 override. + Any **other** skip is unexplained — investigate before proceeding. +- [x] 5.9 Stop the container and re-run `uv run pytest tests/test_exasol.py -v`: every test must be + **skipped**, none errored. This is the guarantee that a laptop without Docker stays green. +- [x] 5.10 Record any place where the live server contradicted the adapter as a finding in the plan's + Session log. Do **not** edit anything under `sqlit/` — the proposal's Non-Goals exclude it, and + design O1 already names the follow-up. + +## 6. CI wiring (plan step 14 — must land with group 5) + +- [x] 6.1 Add `--ignore=tests/test_exasol.py` to the unit-test job's exclude list in + `.github/workflows/ci.yml`, after the `--ignore=tests/test_clickhouse.py` line (~line 82). + This must be in the same commit as task 5.1 — the exclude list is filename-based. +- [x] 6.2 Add a `test-exasol` job modelled on `test-clickhouse` (~line 440): `runs-on: + ubuntu-latest`, `needs: build`, checkout, Python 3.12, `astral-sh/setup-uv@v5`, then + `uv sync --group test --no-dev --extra exasol`. +- [x] 6.3 Add the server step: `docker run -d --name exasol --privileged -p 8563:8563 + exasol/docker-db:latest-8`, then a `for i in {1..60}` poll on port 8563 sleeping 10s, echoing + each attempt, and breaking on success. Design D7 — sized from the boot time measured in task + 2.4, with headroom. +- [x] 6.4 Add the test step with `EXASOL_HOST`, `EXASOL_PORT`, `EXASOL_USER`, `EXASOL_PASSWORD` and + `EXASOL_SCHEMA` in `env:`, running `uv run pytest tests/test_exasol.py -v --timeout=300`. +- [x] 6.5 Confirm the job's triggers, `needs:` and absence of `continue-on-error` match the other + per-database jobs. Design D7 — no manual gating, no soft failure. +- [x] 6.6 Verify the workflow file parses: `python -c "import yaml,sys; + yaml.safe_load(open('.github/workflows/ci.yml'))"`. +- [x] 6.7 Run the unit job's exact command (now with the exasol ignore) from `ci.yml:70-82` and + confirm it collects and passes, with no new failure against the task 1.3 baseline. + +## 7. Change gate and plan bookkeeping + +- [x] 7.1 Run `uv run ruff check tests` and confirm it is clean for the new and modified files. +- [x] 7.2 Confirm `git status` shows exactly the intended surface: `infra/docker/docker-compose.test.yml`, + `tests/conftest.py`, `.github/workflows/ci.yml`, new `tests/fixtures/exasol.py`, new + `tests/test_exasol.py`, plus the change's own openspec files and the pre-existing + modifications carried by earlier changes. **No file under `sqlit/`.** +- [x] 7.3 In `plan.md`, set steps 12, 13 and 14 to `done` and update the progress count to + `14 / 15 done`. +- [x] 7.4 Append one `plan.md` Session log row recording: the resolved O3 credentials, the measured + boot time and the poll count chosen from it, the two base-test overrides (D9, D10) and their + reasons, the `get_columns` `LIKE ''` finding earmarked for a follow-up change (O1), and the + `BaseDatabaseTestsWithLimit` deviation from step 13's wording (D8). +- [x] 7.5 Tear down the container: + `docker compose -f infra/docker/docker-compose.test.yml --profile enterprise down -v`. diff --git a/openspec/changes/archive/2026-08-27-exasol-provider-registration/.openspec.yaml b/openspec/changes/archive/2026-08-27-exasol-provider-registration/.openspec.yaml new file mode 100644 index 00000000..f05b045c --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-provider-registration/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-27 diff --git a/openspec/changes/archive/2026-08-27-exasol-provider-registration/design.md b/openspec/changes/archive/2026-08-27-exasol-provider-registration/design.md new file mode 100644 index 00000000..9d116c26 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-provider-registration/design.md @@ -0,0 +1,245 @@ +## Context + +sqlit auto-discovers providers: `providers/catalog.py::_discover_providers` walks every subpackage +of `sqlit/domains/connections/providers/` and imports `/provider.py` — skipping subpackages +that have no `provider` module, a tolerance added by the previous change. The moment `provider.py` +exists, the provider is live. + +The previous change (`exasol-adapter`, plan.md steps 1-4) landed a complete `ExasolAdapter` with no +`provider.py`, so Exasol is currently inert. This change is the registration step: plan.md's atomic +group `ACT`, steps 5, 6 and 7. + +**Why it is atomic.** `tests/test_schema_capabilities.py::TestCatalogConsistency` pins three +invariants at once: + +| Test | Assertion | Fails if | +|---|---|---| +| `test_database_type_enum_matches_schema` | `{t.value for t in DatabaseType} == set(get_supported_db_types())` | either the enum member or `provider.py` lands alone | +| `test_provider_schema_ids_match_keys` | `get_connection_schema(t).db_type == t` for every discovered type | `SCHEMA.db_type` is not exactly `"exasol"` | +| `test_display_names_match_schema` | `get_connection_schema(t).display_name == get_display_name(t)` | `SCHEMA.display_name` and `SPEC.display_name` disagree | + +Set *equality* is the key word — the enum is not a superset check. There is no ordering of these +three files that keeps the suite green in between, so they ship together. + +**Constraints.** + +- `pyexasol` is still not installed (the `exasol` extra is plan.md step 8). Registration must not + need it: everything here is metadata, and the adapter import inside `provider_factory` is lazy. +- No test file is added by this change; its gate is the existing capability suite plus the linters. +- `providers/exasol/adapter.py` is **not** modified. The schema's field names were fixed in the + previous change by what `connect()` already reads — `authenticator`, `access_token`, + `refresh_token`, `schema` — so this change adapts the schema to the adapter, not the reverse. + +**Reference implementations read for house style:** `providers/teradata/provider.py` and +`providers/hana/provider.py` (the `ProviderSpec` shape and the lazy factory), +`providers/snowflake/schema.py` (an `authenticator` dropdown with conditionally visible credential +fields), `providers/clickhouse/schema.py` (the `+ SSH_FIELDS + TLS_FIELDS` tail). + +## Goals / Non-Goals + +**Goals:** + +- Exasol appears in the connection picker, in `get_supported_db_types()`, and in the CLI. +- The auth dropdown shows exactly the credential fields the selected method needs, and validation + and the CLI agree with that visibility. +- `tests/test_schema_capabilities.py` stays green at 30 providers. +- No behaviour change for any of the 29 existing providers. +- Registration works with no driver installed. + +**Non-Goals:** + +- The `exasol` extra, the mypy override, the pytest marker — plan.md step 8. +- Any test file. Exasol's own unit tests are plan.md steps 9-11; this change is gated by the + existing suite. +- Docker compose service, integration test, CI job — plan.md steps 12-14. +- Docs — plan.md step 15. +- Any change to `adapter.py`. + +## Decisions + +### D1 — One change, three files, `provider.py` written last + +The three files are inseparable (see Context). Within the change the order still matters for the +working tree: `schema.py` and the `config.py` edit first, `provider.py` last. Until `provider.py` +exists, discovery skips the package and the suite is green; the instant it exists, all three +invariants are checked. Writing it last means at most one red interval, at the end, closed by the +gate. + +*Alternative considered:* land `schema.py` alone as a "dead" module first. Rejected as pointless — +nothing imports it, so it proves nothing and still leaves the same red interval later. + +### D2 — The dropdown field is named `authenticator`, not `auth_type` + +`ConnectionConfig.from_dict` special-cases two legacy top-level keys — `auth_type` and +`trusted_connection` — hoisting them into `options` (`domain/config.py:158-161`). A field literally +named `auth_type` would collide with that path and with whatever a legacy MSSQL-era config file +already carries under that key. `authenticator` avoids the collision entirely and mirrors +`providers/snowflake/schema.py`, which made the same choice. + +This is also not a free choice here: `adapter.connect()` already reads +`config.get_option("authenticator", "password")`. The name is fixed by the shipped adapter. + +### D3 — Declare `username` and `password` explicitly, with `visible_when` + +`_username_field()` and `_password_field()` return a **frozen** `SchemaField` with +`visible_when=None`, and `SchemaField` is `@dataclass(frozen=True)`. To hide them under token auth +they must either be rebuilt with `dataclasses.replace` or declared inline. Declared inline, because +that is what `providers/snowflake/schema.py` does for its conditional `password` field, and no +provider in the tree uses `replace` on these helpers. + +Hiding them is behaviour, not cosmetics — three separate consumers key off `visible_when`: + +| Consumer | Effect of hiding | +|---|---| +| `providers/validation.py:30` | skips `required` checks on hidden fields, so token auth is not blocked by a missing username | +| `cli/helpers.py:65` | `required=True` becomes an argparse-required flag only when `visible_when is None` — so `--username` stays optional | +| `ui/connection_form.py:219`, `ui/validation.py:76`, `ui/field_widgets.py:45` | the field is hidden and skipped in form validation | + +`username` keeps `required=True`: under password auth it genuinely is required, and the guard above +means that requirement simply does not apply when the field is hidden. `password` follows the house +pattern of *not* being required — `validation.py:33` explicitly allows an empty `PASSWORD` field so +it can be prompted at connect time. + +*Alternative considered:* always show `username`, Snowflake-style, since Snowflake's user is +meaningful under every method. Rejected for Exasol: `connect()` sends `user`/`password` only on the +password branch, so under token auth the field would be visible, CLI-required, and ignored. + +### D4 — Append `SSH_FIELDS + TLS_FIELDS`, and set `supports_ssh=True` + +`TLS_FIELDS` is what makes `adapter._tls_args()` reachable at all — without a `tls_mode` field, +`get_tls_mode(config)` always sees `default`, and the `require` mode that `exasol/docker-db`'s +self-signed certificate needs would be unselectable. plan.md step 13's integration test connects +with `--tls-mode require`, a CLI flag that exists only because `TLS_FIELDS` is in this tuple. + +`SSH_FIELDS` costs nothing — Exasol is a plain TCP endpoint, so sqlit's generic tunnel applies +unmodified, and `supports_ssh=True` matches `hana`, `teradata` and `clickhouse`. `SCHEMA` leaves +`supports_ssh` at its default `True`; `SPEC` states it explicitly, as the other providers do. + +### D5 — `display_name` is `"Exasol"` in both `SCHEMA` and `SPEC` + +`get_display_name()` resolves from the registered `ProviderSpec`, while +`test_display_names_match_schema` compares that against `SCHEMA.display_name`. The two literals +must match exactly, including case: `"Exasol"` — not `"EXASOL"`, not `"Exasol DB"`. Same for +`db_type` (`"exasol"`) and `default_port` (`"8563"`), each duplicated across the two objects by the +existing house pattern. + +*Alternative considered:* have `SPEC` read its `display_name` from `SCHEMA` to remove the +duplication. Rejected — it would deviate from all 29 providers to save one literal, and the +duplication is exactly what the test guards. + +### D6 — `DockerDetector` needs `env_vars={}` — plan.md step 7's snippet is invalid + +`DockerDetector` (`providers/docker.py:21-28`) declares `env_vars: dict[str, tuple[str, ...]]` +with **no default**, positioned before every defaulted field. The plan's +`DockerDetector(image_patterns=("exasol/docker-db",), default_user="sys")` therefore raises +`TypeError: missing 1 required positional argument` — at *import* time, inside `provider.py`, +inside discovery. The blast radius is all 30 providers, not just Exasol. + +`env_vars={}` is the right value, not a placeholder: `exasol/docker-db` takes no credential +environment variables — its `sys` / `exasol` defaults are baked into the image. `get_credentials` +handles the empty mapping natively (`get_first(())` returns `None`, then `default_user` applies), +so a detected container prefills user `sys` and leaves the password to be prompted. + +`default_user_requires_password` stays `False`, so `sys` is offered even when no password was +discovered from the environment — matching `postgresql` and `clickhouse`. + +*Alternative considered:* also set `default_database="SYS"`. Rejected — Exasol has no database +layer, `supports_multiple_databases` is `False`, and a database value would surface in the endpoint +and in display formatting for no reason. + +### D7 — Custom `display_info` rendering `host:port/SCHEMA` + +The default (`adapter_provider.py:61-73`) builds `host[:port][/database]` from the TCP endpoint. +Exasol never populates `endpoint.database` — the schema has no `database` field and +`supports_multiple_databases` is `False` — so the default degrades to a bare `host:port` and the +connection list cannot distinguish two connections into the same cluster. + +`_display_info` reads `config.get_option("schema", "")` (where `config.py:232-246` deposits it) and +appends `/` when non-empty, falling back to the default's `host:port` shape otherwise. Only +`motherduck` and `supabase` override this hook today, so the override is deliberate rather than +conventional — justified because `schema` is Exasol's only scoping dimension. + +### D8 — The `DATABASE_TYPE_DISPLAY_ORDER` entry is a silent requirement + +Two additions to `domain/config.py`, with very different failure modes: + +- `DatabaseType.EXASOL` — omitted, `test_database_type_enum_matches_schema` fails loudly. +- `DATABASE_TYPE_DISPLAY_ORDER` — `ui/screens/connection.py:331` assigns + `db_types = DATABASE_TYPE_DISPLAY_ORDER` and builds the picker's `Select` options from exactly + that list. Nothing else filters it, and `grep` finds **no test** referencing the constant. + Omitted, every test still passes and Exasol is simply absent from the picker — the change would + look complete and deliver nothing. + +Placement: `EXASOL = "exasol"` between `DB2` and `FIREBIRD` in the enum (which is alphabetical +apart from the `DB2` and `ORACLE`/`ORACLE_LEGACY` entries), and `DatabaseType.EXASOL` after +`DatabaseType.TERADATA` in the display order, whose comment declares it ordered "sqlite first, then +by popularity" — putting Exasol with the other enterprise engines (`DB2`, `HANA`, `TERADATA`) and +ahead of the cloud warehouses. + +### D9 — `default_port="8563"` in both objects + +8563 is Exasol's WebSocket port. It appears three times: `_port_field("8563")` (the form placeholder +and default), `SCHEMA.default_port` (consumed by `SchemaConfigValidator.normalize`, which backfills +an empty port), and `SPEC.default_port` (consumed by `get_default_port("exasol")`, which +`adapter.connect()` already calls as its port fallback). All three must agree, and the shipped +adapter fixes the value. + +Note the interaction: `SchemaConfigValidator.normalize` backfills the port only when a `port` field +exists in the schema — it does, so a config saved with an empty port is normalised to 8563 before +`connect()` ever needs its own fallback. + +### D10 — `provider_factory` imports `ExasolAdapter` lazily + +`provider.py` is imported for every provider at discovery, i.e. at startup. A module-scope +`from ...exasol.adapter import ExasolAdapter` would import `adapter.py` — and therefore `ssl` and +the whole adapter module — on every launch, for a provider the user may never select. Every existing +provider defers this into the factory body; matching that keeps startup cost flat. + +The factory returns `build_adapter_provider(spec, SCHEMA, ExasolAdapter())`, which instantiates the +adapter. `ExasolAdapter` is concrete as of the previous change, so this cannot raise +`TypeError: Can't instantiate abstract class` — but note that this change is the first code path +that ever instantiates it, so the previous change's static-only gate is effectively cashed here. +`SCHEMA` itself is imported at module scope, as in `teradata`/`hana`. + +## Risks / Trade-offs + +- **The change is red until the last file lands** (D1) → unavoidable; mitigated by writing + `provider.py` last and by the gate being a single fast test file. +- **`DockerDetector` misuse would break all 30 providers, not just Exasol** (D6) → caught by the + gate: `test_schema_capabilities.py` cannot even collect if discovery raises, so a `TypeError` + here fails loudly rather than silently. +- **A missing display-order entry passes every test** (D8) → mitigated by a spec scenario asserting + `DatabaseType.EXASOL in DATABASE_TYPE_DISPLAY_ORDER` and by an explicit `uv run sqlit` check in + the tasks. Adding a completeness test for the constant would be the durable fix, but that is a new + test for shared behaviour and belongs outside this change. +- **Exasol becomes selectable before `pyexasol` is installable via an extra** (step 8) → connecting + raises sqlit's normal driver-install prompt, naming an `exasol` extra that does not exist yet, so + the prompt is momentarily unactionable. Accepted: step 8 has no dependencies and can follow + immediately; the alternative is holding registration behind a `pyproject.toml` edit for no + technical reason. +- **Three duplicated literals** (`"exasol"`, `"Exasol"`, `"8563"`) across `SCHEMA` and `SPEC` (D5, + D9) → house pattern, and two of the three duplications are exactly what + `test_schema_capabilities.py` verifies. +- **`visible_when` predicates are untested by this change** (D3) → plan.md step 9 + (`tests/.../exasol/test_schema.py`) exists precisely for that and depends on this change. The + spec states the visibility matrix so step 9 has a contract to test against. + +## Migration Plan + +No migration. All three edits are additive, and no persisted connection config can carry +`db_type: "exasol"` yet — the type did not exist, and `from_dict` defaults an unknown or missing +`db_type` to `"mssql"`. + +Rollback is deleting `provider.py`: discovery then skips the package (the tolerance added by the +previous change), `get_supported_db_types()` returns to 29, and the two `config.py` lines become +inert — though `test_database_type_enum_matches_schema` would then fail on the orphaned enum member, +so a full rollback reverts all three files together, for the same reason the change ships as one. + +## Open Questions + +None blocking. Two deferred: + +- Whether `DATABASE_TYPE_DISPLAY_ORDER` deserves a completeness test asserting it covers + `DatabaseType`. Out of scope here (see Risks); worth raising in the upstream PR. +- Whether the `schema` field should offer a dropdown populated after connect, given the explorer + already lists schemas. Cosmetic, and no existing provider does this for a schema field. diff --git a/openspec/changes/archive/2026-08-27-exasol-provider-registration/proposal.md b/openspec/changes/archive/2026-08-27-exasol-provider-registration/proposal.md new file mode 100644 index 00000000..532d9a32 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-provider-registration/proposal.md @@ -0,0 +1,92 @@ +## Why + +The `exasol-adapter` change landed a complete `ExasolAdapter` but deliberately stopped short of +registration, so Exasol is still invisible: it is absent from the connection picker, from +`get_supported_db_types()`, from the CLI, and from every provider lookup. This change flips the +switch — it makes Exasol a selectable provider. + +Registration is **all-or-nothing** and cannot be split across sessions. +`tests/test_schema_capabilities.py::TestCatalogConsistency::test_database_type_enum_matches_schema` +asserts `{t.value for t in DatabaseType} == set(get_supported_db_types())` — an *exact set +equality*. Adding `provider.py` without the enum member fails it; adding the enum member without +`provider.py` fails it too. The same test class calls `get_connection_schema(db_type)` for every +discovered type and asserts `schema.db_type == db_type` and +`schema.display_name == get_display_name(db_type)`, so `schema.py` must exist and agree with +`ProviderSpec` in the same commit. This change is therefore plan.md's atomic group `ACT` +(steps 5, 6, 7) in full, and nothing less. + +## What Changes + +- **New** `sqlit/domains/connections/providers/exasol/schema.py` — a `ConnectionSchema` with + `db_type="exasol"`, `display_name="Exasol"`, `default_port="8563"`, `has_advanced_auth=True`, + and the field set: `server`, `port`, an `authenticator` dropdown (Username & Password / OpenID + Access Token / OpenID Refresh Token), `username`, `password`, `access_token`, `refresh_token`, + `schema`, followed by the shared `SSH_FIELDS + TLS_FIELDS` tails. +- Credential fields are **conditionally visible**: `username` and `password` only under + `authenticator == "password"`, `access_token` and `refresh_token` only under their own method. + This requires declaring `username`/`password` explicitly rather than reusing + `_username_field()` / `_password_field()`, whose returned `SchemaField` is frozen and carries no + `visible_when`. +- **Modified** `sqlit/domains/connections/domain/config.py` — `EXASOL = "exasol"` added to the + `DatabaseType` enum, and `DatabaseType.EXASOL` added to `DATABASE_TYPE_DISPLAY_ORDER` after + `TERADATA`. Both are required: the enum for catalog consistency, the display order because + `ui/screens/connection.py:331` builds the picker's `Select` options strictly from that list. +- **New** `sqlit/domains/connections/providers/exasol/provider.py` — a `ProviderSpec` plus + `register_provider(SPEC)`, with a `provider_factory` that imports `ExasolAdapter` lazily, a + `DockerDetector` for `exasol/docker-db`, and a `display_info` rendering `host:port/SCHEMA`. +- **Correction to plan.md step 7:** its snippet writes + `DockerDetector(image_patterns=("exasol/docker-db",), default_user="sys")`, but `env_vars` is a + **required** field on `DockerDetector` (`providers/docker.py:23`). As written the call raises + `TypeError` at import time — which, because `provider.py` is imported during discovery, would + break provider discovery for all 30 providers rather than just Exasol. This change passes + `env_vars={}`. + +Not breaking: no existing provider, schema, or test changes behaviour. The only shared file +touched is `domain/config.py`, and only by addition. + +## Capabilities + +### New Capabilities +- `exasol-provider-registration`: Exasol's presence in the provider catalog — its connection + schema and per-authenticator field visibility, its `DatabaseType` enum membership and position + in the connection picker, and the `ProviderSpec` that binds schema, adapter, Docker detection + and display formatting into a registered, selectable provider. + +### Modified Capabilities + + +## Impact + +- **New files:** `sqlit/domains/connections/providers/exasol/schema.py`, + `sqlit/domains/connections/providers/exasol/provider.py`. +- **Modified files:** `sqlit/domains/connections/domain/config.py` — two additive lines (one enum + member, one display-order entry). +- **Consumed unchanged:** `providers/schema_helpers.py` (`ConnectionSchema`, `SchemaField`, + `FieldType`, `SelectOption`, `SSH_FIELDS`, `TLS_FIELDS`, `_server_field`, `_port_field`), + `providers/model.py` (`ProviderSpec`, `DatabaseProvider`), `providers/catalog.py` + (`register_provider`), `providers/adapter_provider.py` (`build_adapter_provider`), + `providers/docker.py` (`DockerDetector`), and `providers/exasol/adapter.py` from the previous + change — untouched here. +- **Behaviour that turns on for free**, because it is all schema-driven: + - the connection picker gains an "Exasol" entry (`ui/screens/connection.py:331`); + - `cli/helpers.py:43` derives `--server/--host`, `--port`, `--authenticator`, + `--access-token`, `--refresh-token`, `--schema` flags from the schema fields; + - `providers/validation.py:30` enforces `required` only on *visible* fields, so `--username` is + not demanded when authenticating with a token; + - `config.py:232-246` routes the non-endpoint fields (`authenticator`, `access_token`, + `refresh_token`, `schema`) into `config.options`, which is exactly where + `adapter.connect()` reads them with `config.get_option(...)`. +- **Dependency:** none added. `pyexasol` is still not installed — the `exasol` extra is plan.md + step 8, a separate change. Registration does not need it: `provider_factory` imports the + adapter module lazily and `_import_driver_module` only runs on an actual connect, so + discovery, the picker, and the test suite all work driverless. +- **Test-suite impact:** `tests/test_schema_capabilities.py` moves from 29 to 30 discovered + providers and must stay 9/9 green — that is this change's gate. No test file is added here; + the Exasol-specific unit tests are plan.md steps 9-11. +- **User-visible risk:** selecting Exasol in the picker and connecting without `pyexasol` + installed now reaches sqlit's normal "install the driver" prompt rather than being impossible. + That is the intended end state, but it is the first time Exasol is reachable from the UI. +- **Gate:** `uv run pytest tests/test_schema_capabilities.py -v` plus + `uv run ruff check sqlit && uv run mypy sqlit`. As in the previous change, both linters are + dirty on `main` (and CI runs neither), so the lint half of the gate is applied as "zero findings + attributable to the changed files, repo totals unchanged". diff --git a/openspec/changes/archive/2026-08-27-exasol-provider-registration/specs/exasol-provider-registration/spec.md b/openspec/changes/archive/2026-08-27-exasol-provider-registration/specs/exasol-provider-registration/spec.md new file mode 100644 index 00000000..d92e4179 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-provider-registration/specs/exasol-provider-registration/spec.md @@ -0,0 +1,320 @@ +## ADDED Requirements + +### Requirement: Exasol declares a connection schema +The provider package SHALL contain `schema.py` exporting a module-level `SCHEMA` of type +`ConnectionSchema` with `db_type="exasol"`, `display_name="Exasol"`, `default_port="8563"` and +`has_advanced_auth=True`. `supports_ssh` MUST be `True` (the dataclass default) so the shared tunnel +fields apply. + +`db_type` and `display_name` are pinned by `tests/test_schema_capabilities.py`: +`test_provider_schema_ids_match_keys` requires `SCHEMA.db_type` to equal the registry key, and +`test_display_names_match_schema` requires `SCHEMA.display_name` to equal +`get_display_name("exasol")`, which resolves from `ProviderSpec`. + +#### Scenario: Schema identity +- **WHEN** `SCHEMA` is inspected +- **THEN** `SCHEMA.db_type == "exasol"` +- **AND** `SCHEMA.display_name == "Exasol"` +- **AND** `SCHEMA.default_port == "8563"` +- **AND** `SCHEMA.has_advanced_auth is True` +- **AND** `SCHEMA.supports_ssh is True` + +#### Scenario: Schema is reachable through the registry +- **WHEN** `get_connection_schema("exasol")` is called +- **THEN** it returns the same `ConnectionSchema` object exported by `schema.py` + +### Requirement: Schema declares the Exasol field set +`SCHEMA.fields` SHALL contain, in order: `server`, `port`, `authenticator`, `username`, `password`, +`access_token`, `refresh_token`, `schema` — followed by `SSH_FIELDS` and then `TLS_FIELDS` +appended as tuples. + +`server` and `port` SHALL use the shared `_server_field()` and `_port_field("8563")` helpers. +`schema` SHALL be optional with the placeholder `(empty = browse all)`, matching the adapter's +`default_schema` of `""`. + +The field names are fixed by the already-shipped `adapter.py`, which reads `authenticator`, +`access_token`, `refresh_token` and `schema` via `config.get_option(...)`, and takes `user` and +`password` from `config.tcp_endpoint`. + +#### Scenario: Endpoint and option fields are present +- **WHEN** `{f.name for f in SCHEMA.fields}` is computed +- **THEN** it is a superset of `{"server", "port", "authenticator", "username", "password", "access_token", "refresh_token", "schema"}` + +#### Scenario: Shared SSH and TLS tails are appended +- **WHEN** `SCHEMA.fields` is inspected +- **THEN** every field in `SSH_FIELDS` is present +- **AND** every field in `TLS_FIELDS` is present, including `tls_mode` +- **AND** they appear after the Exasol-specific fields + +#### Scenario: No database field +- **WHEN** `SCHEMA.fields` is inspected +- **THEN** no field is named `database`, because the adapter reports + `supports_multiple_databases is False` and `get_databases()` returns an empty list + +#### Scenario: Schema field is optional +- **WHEN** the `schema` field is inspected +- **THEN** `required is False` +- **AND** its placeholder communicates that leaving it empty browses all schemas + +### Requirement: Authentication method is chosen from a dropdown +The `authenticator` field SHALL be a `FieldType.DROPDOWN` with `default="password"` and exactly +three options, in this order: + +| value | label | +|---|---| +| `password` | `Username & Password` | +| `access_token` | `OpenID Access Token` | +| `refresh_token` | `OpenID Refresh Token` | + +The values MUST match the branch labels in `adapter.connect()`, which compares +`config.get_option("authenticator", "password")` against `"access_token"` and `"refresh_token"` and +treats anything else as password auth. + +The field MUST NOT be named `auth_type`: `ConnectionConfig.from_dict` special-cases that key as a +legacy top-level field and hoists it into `options`. + +#### Scenario: Dropdown options +- **WHEN** the `authenticator` field is inspected +- **THEN** `field_type is FieldType.DROPDOWN` +- **AND** `default == "password"` +- **AND** its option values are `("password", "access_token", "refresh_token")` + +#### Scenario: CLI exposes the choice +- **WHEN** the CLI parser is built from `SCHEMA` +- **THEN** an `--authenticator` flag exists +- **AND** its accepted `choices` are the three option values + +#### Scenario: Field is not named auth_type +- **WHEN** `{f.name for f in SCHEMA.fields}` is computed +- **THEN** `"auth_type"` is absent + +### Requirement: Credential fields are visible only for their own authentication method +Each credential field SHALL carry a `visible_when` predicate reading `authenticator` from the form +values, per this matrix: + +| `authenticator` | visible credential fields | hidden credential fields | +|---|---|---| +| `password` (default) | `username`, `password` | `access_token`, `refresh_token` | +| `access_token` | `access_token` | `username`, `password`, `refresh_token` | +| `refresh_token` | `refresh_token` | `username`, `password`, `access_token` | + +`username` and `password` MUST therefore be declared as explicit `SchemaField`s rather than reusing +`_username_field()` / `_password_field()`, whose returned frozen `SchemaField` carries +`visible_when=None`. + +`username` SHALL be `required=True` and `password` SHALL be `required=False`; both SHALL use +`group="credentials"`. `access_token` and `refresh_token` SHALL be `FieldType.PASSWORD` so their +values are masked and treated as promptable secrets. + +Visibility is behaviour, not presentation: `providers/validation.py:30` skips `required` checks on +hidden fields, and `cli/helpers.py:65` marks a flag argparse-required only when +`visible_when is None`. + +#### Scenario: Password auth shows only username and password +- **WHEN** `visible_when({"authenticator": "password"})` is evaluated for each credential field +- **THEN** `username` and `password` are visible +- **AND** `access_token` and `refresh_token` are hidden + +#### Scenario: Access token auth hides username and password +- **WHEN** `visible_when({"authenticator": "access_token"})` is evaluated for each credential field +- **THEN** `access_token` is visible +- **AND** `username`, `password` and `refresh_token` are hidden + +#### Scenario: Refresh token auth hides username and password +- **WHEN** `visible_when({"authenticator": "refresh_token"})` is evaluated for each credential field +- **THEN** `refresh_token` is visible +- **AND** `username`, `password` and `access_token` are hidden + +#### Scenario: Missing authenticator value falls back to password auth +- **WHEN** `visible_when({})` is evaluated — no `authenticator` key at all +- **THEN** `username` and `password` are visible +- **AND** both token fields are hidden +- **AND** this matches `adapter.connect()`, whose `get_option("authenticator", "password")` default + takes the password branch + +#### Scenario: Token auth does not demand a username +- **WHEN** a config with `authenticator == "access_token"` and an empty `username` is validated by + `SchemaConfigValidator.validate` +- **THEN** no `ValueError` is raised, because the hidden `username` field's `required` flag is + skipped + +#### Scenario: Password auth demands a username +- **WHEN** a config with `authenticator == "password"` and an empty `username` is validated +- **THEN** `ValueError` is raised naming the Username field + +#### Scenario: Username is not a globally required CLI flag +- **WHEN** the CLI parser is built from `SCHEMA` +- **THEN** `--username` is not argparse-required, because the field defines `visible_when` + +#### Scenario: Token fields are masked +- **WHEN** the `access_token` and `refresh_token` fields are inspected +- **THEN** both have `field_type is FieldType.PASSWORD` + +### Requirement: Exasol is a member of the DatabaseType enum +`DatabaseType` in `sqlit/domains/connections/domain/config.py` SHALL include +`EXASOL = "exasol"`, placed between `DB2` and `FIREBIRD`. + +This is required by `test_database_type_enum_matches_schema`, which asserts +`{t.value for t in DatabaseType} == set(get_supported_db_types())` as an exact set equality — so +the enum member and the provider registration must land together. + +#### Scenario: Enum member exists +- **WHEN** `DatabaseType.EXASOL` is accessed +- **THEN** its value is `"exasol"` + +#### Scenario: Enum and provider catalog agree +- **WHEN** `test_database_type_enum_matches_schema` runs +- **THEN** the enum value set exactly equals `set(get_supported_db_types())` +- **AND** both sets contain `"exasol"` + +### Requirement: Exasol appears in the connection picker +`DATABASE_TYPE_DISPLAY_ORDER` SHALL include `DatabaseType.EXASOL`, positioned after +`DatabaseType.TERADATA` among the enterprise engines. + +`ui/screens/connection.py:331` builds the database-type `Select` options from exactly this list with +no further filtering, so a type absent from it is unreachable in the UI. No test covers the +constant's completeness, so omitting the entry fails silently — every test passes and Exasol is +still unselectable. + +#### Scenario: Display order contains Exasol +- **WHEN** `DATABASE_TYPE_DISPLAY_ORDER` is inspected +- **THEN** `DatabaseType.EXASOL` is present +- **AND** it appears immediately after `DatabaseType.TERADATA` + +#### Scenario: Picker renders an Exasol option +- **WHEN** the connection screen builds its database-type `Select` +- **THEN** an option labelled `Exasol` with value `exasol` is present + +#### Scenario: Label resolves from the provider +- **WHEN** `get_database_type_labels()` is called +- **THEN** `labels[DatabaseType.EXASOL] == "Exasol"` + +### Requirement: Exasol is registered as a provider +The provider package SHALL contain `provider.py` that builds a `ProviderSpec` and calls +`register_provider(SPEC)` at module scope, following `providers/teradata/provider.py`. + +`SPEC` SHALL declare: + +| Field | Value | +|---|---| +| `db_type` | `"exasol"` | +| `display_name` | `"Exasol"` | +| `schema_path` | `("sqlit.domains.connections.providers.exasol.schema", "SCHEMA")` | +| `supports_ssh` | `True` | +| `is_file_based` | `False` | +| `has_advanced_auth` | `True` | +| `default_port` | `"8563"` | +| `requires_auth` | `True` | +| `badge_label` | `"Exasol"` | +| `url_schemes` | `("exasol", "exa")` | + +`display_name`, `db_type` and `default_port` MUST match `SCHEMA` exactly. + +#### Scenario: Provider is discovered +- **WHEN** `get_supported_db_types()` is called +- **THEN** `"exasol"` is present +- **AND** the total provider count is 30 + +#### Scenario: Registry metadata resolves +- **WHEN** the registry is queried for `"exasol"` +- **THEN** `get_display_name("exasol") == "Exasol"` +- **AND** `get_default_port("exasol") == "8563"` +- **AND** `has_advanced_auth("exasol") is True` +- **AND** `supports_ssh("exasol") is True` +- **AND** `is_file_based("exasol") is False` + +#### Scenario: URL schemes are claimed +- **WHEN** a connection URL with scheme `exasol://` or `exa://` is resolved +- **THEN** it maps to the Exasol provider + +#### Scenario: Existing capability suite stays green +- **WHEN** `uv run pytest tests/test_schema_capabilities.py` is run +- **THEN** all 9 tests pass + +### Requirement: Provider factory imports the adapter lazily +`SPEC.provider_factory` SHALL be a function that imports `ExasolAdapter` **inside its body** and +returns `build_adapter_provider(spec, SCHEMA, ExasolAdapter())`. `provider.py` MUST NOT import +`adapter.py` at module scope. + +`provider.py` is imported for every provider during discovery, i.e. at startup; deferring the +adapter import keeps `adapter.py` and its `ssl` import off the startup path for a provider the user +may never select. Every existing provider does this. + +#### Scenario: Adapter module is not imported at startup +- **WHEN** provider discovery completes without any Exasol connection being opened +- **THEN** `sqlit.domains.connections.providers.exasol.adapter` is absent from `sys.modules` + +#### Scenario: Factory produces a working provider +- **WHEN** `SPEC.provider_factory(SPEC)` is called +- **THEN** a `DatabaseProvider` is returned +- **AND** its `schema` is the Exasol `SCHEMA` +- **AND** no `TypeError` about abstract methods is raised, since `ExasolAdapter` is concrete + +#### Scenario: Registration needs no driver +- **WHEN** discovery, the picker and the capability suite run with `pyexasol` not installed +- **THEN** all succeed, because `_import_driver_module` runs only inside `connect()` + +### Requirement: Docker detection recognises exasol/docker-db +`SPEC.docker_detector` SHALL be a `DockerDetector` with `image_patterns=("exasol/docker-db",)`, +`env_vars={}` and `default_user="sys"`. + +`env_vars` is a **required** field on `DockerDetector` with no default. Omitting it raises +`TypeError` while `provider.py` is being imported during discovery, which breaks discovery for every +provider, not just Exasol. The empty mapping is also semantically correct: the `exasol/docker-db` +image takes no credential environment variables — its `sys` / `exasol` defaults are baked in — and +`get_credentials` resolves an empty mapping to `default_user`. + +`default_user_requires_password` SHALL remain `False`, so `sys` is offered even when no password is +discovered. + +#### Scenario: Image pattern matches +- **WHEN** `match_image("exasol/docker-db:latest-8")` is called +- **THEN** it returns `True` + +#### Scenario: Unrelated image does not match +- **WHEN** `match_image("postgres:16")` is called on the Exasol detector +- **THEN** it returns `False` + +#### Scenario: Default user is offered with no environment variables +- **WHEN** `get_credentials({})` is called +- **THEN** the returned `user` is `"sys"` +- **AND** no exception is raised for the empty `env_vars` mapping + +#### Scenario: Detector construction does not break discovery +- **WHEN** `provider.py` is imported +- **THEN** no `TypeError` is raised for a missing `env_vars` argument + +### Requirement: Connection display shows the target schema +`SPEC.display_info` SHALL render `host:port/SCHEMA` when a `schema` option is set, and fall back to +`host:port` when it is empty. + +The default implementation (`adapter_provider.py:61`) appends `endpoint.database`, which Exasol never +populates — there is no `database` field and `supports_multiple_databases` is `False` — so without +this override two connections into the same cluster are indistinguishable in the list. + +#### Scenario: Schema is shown when set +- **WHEN** `display_info` is called for a config with host `localhost`, port `8563` and the `schema` + option `TEST_SQLIT` +- **THEN** the result is `localhost:8563/TEST_SQLIT` + +#### Scenario: Falls back to host and port +- **WHEN** `display_info` is called for a config with no `schema` option +- **THEN** the result is `localhost:8563` with no trailing slash + +### Requirement: The change leaves existing providers untouched +Registering Exasol SHALL NOT change behaviour for any of the 29 existing providers. The only shared +file modified is `domain/config.py`, by addition only: one enum member and one display-order entry. + +`providers/exasol/adapter.py` MUST NOT be modified by this change. + +#### Scenario: Existing providers still resolve +- **WHEN** `get_supported_db_types()` is called +- **THEN** all 29 previously registered types are still present + +#### Scenario: Adapter is unchanged +- **WHEN** the diff for this change is inspected +- **THEN** `providers/exasol/adapter.py` does not appear in it + +#### Scenario: Full unit suite is unaffected +- **WHEN** the unit-test job's pytest command is run +- **THEN** no previously passing test fails diff --git a/openspec/changes/archive/2026-08-27-exasol-provider-registration/tasks.md b/openspec/changes/archive/2026-08-27-exasol-provider-registration/tasks.md new file mode 100644 index 00000000..62501107 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-provider-registration/tasks.md @@ -0,0 +1,129 @@ +## 1. Connection schema + +Corresponds to plan.md step 5. Write this group **before** group 3 — `provider.py` is the switch +that turns discovery on (design D1). + +- [x] 1.1 Create `sqlit/domains/connections/providers/exasol/schema.py` with the module docstring + `"""Connection schema for Exasol."""` and imports from `providers.schema_helpers`: + `SSH_FIELDS`, `TLS_FIELDS`, `ConnectionSchema`, `FieldType`, `SchemaField`, `SelectOption`, + `_port_field`, `_server_field`. Do **not** import `_username_field` / `_password_field` — + design D3 declares those two fields explicitly. +- [x] 1.2 Add a module-level `_get_exasol_auth_options()` returning the three `SelectOption`s in + order: `("password", "Username & Password")`, `("access_token", "OpenID Access Token")`, + `("refresh_token", "OpenID Refresh Token")`. Mirrors + `providers/snowflake/schema.py::_get_snowflake_auth_options`. +- [x] 1.3 Add the visibility predicates as module-level helpers taking a `dict` and reading + `v.get("authenticator", "password")` — the `"password"` default matters: the spec scenario + "Missing authenticator value falls back to password auth" requires `visible_when({})` to show + username/password, matching `adapter.connect()`'s own `get_option` default. +- [x] 1.4 Declare `SCHEMA = ConnectionSchema(...)` with `db_type="exasol"`, + `display_name="Exasol"` (exact case — design D5), `default_port="8563"`, + `has_advanced_auth=True`, and `supports_ssh` left at its `True` default. +- [x] 1.5 Fields, in order: `_server_field()`, `_port_field("8563")`, then the `authenticator` + `SchemaField` with `field_type=FieldType.DROPDOWN`, `options=_get_exasol_auth_options()` and + `default="password"`. +- [x] 1.6 Add the `username` field explicitly: `required=True`, `group="credentials"`, + `visible_when` = password-auth predicate. Required is safe because + `providers/validation.py:30` skips hidden fields (design D3). +- [x] 1.7 Add the `password` field explicitly: `field_type=FieldType.PASSWORD`, + `placeholder="(empty = ask every connect)"`, `group="credentials"`, `required=False`, + `visible_when` = password-auth predicate. +- [x] 1.8 Add `access_token` and `refresh_token`, both `FieldType.PASSWORD`, `required=False`, each + with a `visible_when` matching only its own `authenticator` value. +- [x] 1.9 Add the `schema` field: `required=False`, `placeholder="(empty = browse all)"`. Do + **not** add a `database` field — the adapter reports `supports_multiple_databases is False`. +- [x] 1.10 Close the tuple with `+ SSH_FIELDS + TLS_FIELDS` (design D4 — `TLS_FIELDS` is what makes + `adapter._tls_args()` reachable and step 13's `--tls-mode require` possible). +- [x] 1.11 Verify the module imports on its own and is still undiscovered: + `uv run python -c "from sqlit.domains.connections.providers.exasol.schema import SCHEMA; print(SCHEMA.db_type, SCHEMA.display_name, len(SCHEMA.fields))"` + then `uv run pytest tests/test_schema_capabilities.py -q` — must still pass at 29 providers, + since no `provider.py` exists yet. + +## 2. DatabaseType enum and picker order + +Corresponds to plan.md step 6. Independent of group 1; both must precede group 3. + +- [x] 2.1 In `sqlit/domains/connections/domain/config.py`, add `EXASOL = "exasol"` to + `DatabaseType`, between `DB2` and `FIREBIRD`. +- [x] 2.2 Add `DatabaseType.EXASOL` to `DATABASE_TYPE_DISPLAY_ORDER`, immediately after + `DatabaseType.TERADATA`. Design D8: this one is a **silent** requirement — + `ui/screens/connection.py:331` builds the picker from exactly this list and no test covers its + completeness, so omitting it passes every test and leaves Exasol unselectable. +- [x] 2.3 Confirm the enum edit alone now makes the capability suite fail (expected, and the reason + this change is atomic): `uv run pytest tests/test_schema_capabilities.py -q` reports + `test_database_type_enum_matches_schema` failing on exact set equality. Do not attempt to fix + it here — group 3 closes it. + +## 3. Provider registration + +Corresponds to plan.md step 7. Write this **last** (design D1). + +- [x] 3.1 Create `sqlit/domains/connections/providers/exasol/provider.py` with the docstring + `"""Provider registration."""` and imports of `build_adapter_provider`, `register_provider`, + `DockerDetector`, `DatabaseProvider`, `ProviderSpec`, and `SCHEMA` from the sibling + `schema` module — following `providers/teradata/provider.py`. +- [x] 3.2 Add `_provider_factory(spec: ProviderSpec) -> DatabaseProvider` that imports + `ExasolAdapter` **inside the function body** and returns + `build_adapter_provider(spec, SCHEMA, ExasolAdapter())`. No module-scope adapter import + (design D10). +- [x] 3.3 Add `_display_info(config: ConnectionConfig) -> str` returning `host:port/SCHEMA` from + `config.tcp_endpoint` plus `config.get_option("schema", "")`, and `host:port` with no trailing + slash when the schema is empty (design D7). Import `ConnectionConfig` under `TYPE_CHECKING`. +- [x] 3.4 Declare `SPEC = ProviderSpec(...)` with `db_type="exasol"`, `display_name="Exasol"`, + `schema_path=("sqlit.domains.connections.providers.exasol.schema", "SCHEMA")`, + `supports_ssh=True`, `is_file_based=False`, `has_advanced_auth=True`, `default_port="8563"`, + `requires_auth=True`, `badge_label="Exasol"`, `url_schemes=("exasol", "exa")`. +- [x] 3.5 Add `docker_detector=DockerDetector(image_patterns=("exasol/docker-db",), env_vars={}, + default_user="sys")`. Design D6: `env_vars` is a **required** field — plan.md step 7's snippet + omits it and would raise `TypeError` during discovery, breaking all 30 providers. +- [x] 3.6 Wire `display_info=_display_info` and `provider_factory=_provider_factory` into `SPEC`, + then call `register_provider(SPEC)` at module scope. +- [x] 3.7 Cross-check the duplicated literals now that both objects exist: `db_type`, + `display_name` and `default_port` must be character-identical between `SCHEMA` and `SPEC` + (design D5) — this is what `test_provider_schema_ids_match_keys` and + `test_display_names_match_schema` verify. + +## 4. Change gate + +The `ACT` group's gate. All of group 1, 2 and 3 must be complete before any of this runs. + +- [x] 4.1 Run the primary gate: `uv run pytest tests/test_schema_capabilities.py -v` — all 9 tests + pass, and `get_supported_db_types()` now returns 30 types including `"exasol"`. +- [x] 4.2 Confirm the registry metadata resolves: + `uv run python -c "from sqlit.domains.connections.providers.registry import get_default_port, get_display_name, has_advanced_auth, supports_ssh; print(get_display_name('exasol'), get_default_port('exasol'), has_advanced_auth('exasol'), supports_ssh('exasol'))"` + prints `Exasol 8563 True True`. +- [x] 4.3 Confirm the picker entry exists (the silent requirement from 2.2): + `uv run python -c "from sqlit.domains.connections.domain.config import DATABASE_TYPE_DISPLAY_ORDER, DatabaseType, get_database_type_labels; print(DatabaseType.EXASOL in DATABASE_TYPE_DISPLAY_ORDER, get_database_type_labels()[DatabaseType.EXASOL])"` + prints `True Exasol`. +- [x] 4.4 Confirm the adapter stays off the startup path: after importing the registry and listing + supported types, `sqlit.domains.connections.providers.exasol.adapter` is absent from + `sys.modules` (design D10 / spec scenario "Adapter module is not imported at startup"). +- [x] 4.5 Spot-check the visibility matrix by hand, ahead of the real tests in plan.md step 9: + evaluate each credential field's `visible_when` against `{"authenticator": "password"}`, + `{"authenticator": "access_token"}`, `{"authenticator": "refresh_token"}` and `{}`, and confirm + the results match the spec's matrix. +- [x] 4.6 Confirm no regressions in the wider suite: `uv run pytest tests/connections -q` and the + unit-test job command from `.github/workflows/ci.yml:70-82` — no previously passing test + fails. +- [x] 4.7 Run the lint half of the gate: `uv run ruff check sqlit && uv run mypy sqlit`. Both are + dirty on `main` (and CI runs neither), so the bar is "zero findings attributable to + `schema.py`, `provider.py` or the `config.py` edit, repo totals otherwise unchanged" — + compare against the pre-change totals. +- [x] 4.8 Confirm `providers/exasol/adapter.py` is absent from the diff (spec: "Adapter is + unchanged") and that the only shared file touched is `domain/config.py`, by addition only. +- [x] 4.9 Manual check: `uv run sqlit`, open the new-connection screen, select **Exasol**, and + confirm the port prefills to 8563, the auth dropdown shows the three methods, switching + methods swaps Password for Access Token / Refresh Token, and the TLS tab is present. + Connecting is expected to raise sqlit's driver-install prompt — `pyexasol` is not installed + until plan.md step 8. + +## 5. Plan bookkeeping + +- [x] 5.1 In `plan.md`, set steps 5, 6 and 7 to `done` in the Status table and update the progress + count to `7 / 15 done`. +- [x] 5.2 Append a `plan.md` Session log row recording the `DockerDetector` correction (design D6 — + `env_vars` is required, so step 7's snippet as written would break discovery for all 30 + providers) and the credential-visibility choice (design D3 — `username`/`password` declared + explicitly because the shared helpers return frozen fields with no `visible_when`). +- [x] 5.3 Note in the Session log that plan.md steps 9-11 (unit tests) are now unblocked, and that + step 8 remains the prerequisite for 10 and 11. diff --git a/openspec/changes/archive/2026-08-27-exasol-unit-tests/.openspec.yaml b/openspec/changes/archive/2026-08-27-exasol-unit-tests/.openspec.yaml new file mode 100644 index 00000000..f05b045c --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-unit-tests/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-27 diff --git a/openspec/changes/archive/2026-08-27-exasol-unit-tests/design.md b/openspec/changes/archive/2026-08-27-exasol-unit-tests/design.md new file mode 100644 index 00000000..a5827c7a --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-unit-tests/design.md @@ -0,0 +1,191 @@ +## Context + +`ExasolAdapter` (change `exasol-adapter`) and its registration (change +`exasol-provider-registration`) are both complete in the working tree. Exasol appears in the +connection picker, resolves through `get_supported_db_types()`, and passes +`tests/test_schema_capabilities.py`. What it does **not** have is an installable driver or a single +behavioural test. + +Three constraints shape this change: + +1. **The default CI unit job installs no extras.** `.github/workflows/ci.yml:66` is + `uv sync --group test --no-dev`. Every test added here is collected by that job — nothing under + `tests/connections/` is in its `--ignore` list — so **no test may require `pyexasol` to be + importable**. This is the hardest constraint and it drives D2. +2. **The adapter imports its driver lazily.** `connect()` calls + `self._import_driver_module("pyexasol", ...)`, which routes through + `providers/driver.py::import_driver_module` to `importlib.import_module`. The driver name only + ever exists as a *string* in `sqlit/`; there is no `import pyexasol` statement anywhere. This is + what makes the driver fakeable (D2) and what makes the mypy override inert (D4). +3. **The behaviour being pinned is already decided.** Both prior changes recorded their reasoning in + design documents and in plan.md's session log. This change adds no behaviour; it converts those + recorded decisions into assertions. Where a test and a recorded decision disagree, the recorded + decision wins and the test is wrong. + +## Goals / Non-Goals + +**Goals:** + +- `pip install sqlit-tui[exasol]` and `uv sync --extra exasol` install a working `pyexasol`. +- Every behavioural decision from the two prior changes has at least one assertion that fails if the + decision is reverted. +- All new tests pass with `pyexasol` **absent**, so the default unit job covers them on every push. +- The `exasol` pytest marker is registered, so plan step 13's integration test can use it without a + `PytestUnknownMarkWarning`. + +**Non-Goals:** + +- Testing `pyexasol` itself, or that its kwargs are spelled the way its 2.x API expects. Mocked + tests pin *sqlit's call shape*, not the driver's contract — that is the integration test's job + (plan step 13). See the first risk below. +- Any Docker service, integration test, CI workflow edit, or documentation (plan steps 12-15). +- Any change to `sqlit/` source. If a test reveals an adapter bug, that is a finding to record, not + a licence to broaden this change. +- Coverage thresholds or a coverage gate. + +## Decisions + +### D1 — Tests live in `tests/connections/providers/exasol/`, not `tests/unit/providers/` + +Both directories exist and both hold adapter tests. `tests/connections/providers/` is the +per-provider tree (`hana/`, `oracle/`), each a package with an empty `__init__.py`; +`tests/unit/providers/` holds standalone cross-cutting files such as `test_osquery_adapter.py`. +Exasol is a provider with three test files, so it takes the per-provider tree, matching plan.md and +giving `uv run pytest tests/connections/providers/exasol/ -v` as a single natural gate. + +*Alternative rejected:* one flat `tests/unit/test_exasol_adapter.py`. It would work, but it splits +Exasol away from the convention its two nearest templates already follow. + +### D2 — Fake the driver with `patch.dict("sys.modules", {"pyexasol": MagicMock()})` + +**This resolves plan.md step 10's open item.** `importlib.import_module` returns an existing +`sys.modules` entry without touching the filesystem, so seeding that dict makes +`_import_driver_module("pyexasol", ...)` hand back the mock. `adapter.connect(config)` then runs end +to end, and `mock.connect.call_args` holds the real kwargs. + +This is the established house pattern — `tests/unit/test_extra_options_passthrough.py:26` for +Snowflake, psycopg2 and PyMySQL, and `tests/unit/test_flight_adapter.py:15`. Using it keeps the +Exasol tests recognisable to a reviewer and satisfies constraint 1. + +*Alternatives rejected:* + +- **Patch `ExasolAdapter._import_driver_module`.** Shorter, but it stubs out the call site under + test: the `driver_name` / `extra_name` / `package_name` plumbing that produces sqlit's "install the + extra" prompt would stop being exercised, and a typo in the module name would pass. +- **Install the extra and monkeypatch `pyexasol.connect`.** This would make the tests depend on + `uv sync --extra exasol`, breaking them in the default unit job. Directly violates constraint 1. + +### D3 — Assert the *absence* of unused credential keys, not their emptiness + +For `authenticator == "password"`, the assertion is `"access_token" not in kwargs`, never +`kwargs.get("access_token") == ""`. This is the whole point of the branch: as recorded in plan.md's +session log, pyexasol's `_login()` branches on token *truthiness*, so a present-but-empty +`access_token` is falsy and silently falls back to password login — precisely the bug an +emptiness-tolerant assertion would let through. The same holds symmetrically: under token auth, +`"user"` and `"password"` must both be absent from the kwargs. + +### D4 — Add the `pyexasol` mypy override even though it is inert, and do not treat `mypy` as its verification + +`[tool.mypy]` sets `exclude = ["tests/"]`, and `sqlit/` never names `pyexasol` outside a string +literal, so mypy has no `pyexasol` import to resolve and the `ignore_missing_imports` entry changes +nothing today. It is added anyway because `hdbcli` and `teradatasql` — the two providers built the +same lazy way — are already in that list, so omitting `pyexasol` would make Exasol the inconsistent +entry, and because the override becomes load-bearing the moment anyone adds a `TYPE_CHECKING` import +of the driver. + +The consequence matters for the tasks: **`uv run mypy sqlit` passing does not verify this edit.** Its +verification is `uv sync --extra exasol` followed by `uv run python -c "import pyexasol"`. +(`clickhouse_connect` is the contrast: `clickhouse/adapter.py:98` has a real in-function +`import clickhouse_connect`, which mypy does analyse, so *that* override is genuinely required.) + +### D5 — Let `uv sync` fold in the pre-existing `uv.lock` drift rather than fighting it + +`uv.lock` is already modified on this branch, unrelated to Exasol: it drops the stale `mariadb` +package entry, because `pyproject.toml` in `HEAD` already points the `mariadb` extra at `PyMySQL` and +the lockfile had not caught up. `uv lock` regenerates the whole file, so that refresh cannot be +separated from the `pyexasol` addition — and reverting `uv.lock` first would simply re-drop +`mariadb` on the next resolve. + +Decision: accept both in one lockfile diff and call it out in the upstream PR description, so the +reviewer is not surprised by a `mariadb` deletion inside an Exasol PR. Hand-editing `uv.lock` to +isolate the change is not an option; a hand-edited lockfile is worse than an explained one. + +### D6 — Plain `pyexasol>=2.0.0` with no environment marker; let `uv` place the marker + +`pyexasol` declares `requires-python >=3.10,<3.15`; sqlit declares `>=3.10` with no ceiling. `uv` +resolves across the full declared range and attaches a `python_full_version < '3.15'` marker to the +locked entry itself, so `uv lock` succeeds and per-interpreter installs stay correct. No extra in +`pyproject.toml` carries an inline marker today, so adding one here would be the odd entry out. + +**Verification during implementation:** `uv lock` (or `uv sync --extra exasol`) must succeed without +a resolution error naming Python 3.15. If it fails, the escape hatch is +`"pyexasol>=2.0.0; python_version < '3.15'"` **in the `all` extra only**, leaving the dedicated +`exasol` extra unmarked so an explicit opt-in still fails loudly on an unsupported interpreter +rather than silently installing nothing. + +### D7 — Register the `exasol` marker now, apply it to nothing + +The three new test files are plain driver-free unit tests and carry no marker, matching +`tests/connections/providers/hana/test_get_columns.py`, which has no marker either (and there is no +`hana` marker). The registered `exasol` marker is therefore unused until plan step 13. That is +deliberate: registering it here keeps step 8 self-contained as "everything `pyproject.toml` needs for +Exasol", and an unused-but-registered marker is inert. Expect `-m exasol` to select zero tests after +this change. + +### D8 — Pin the empty-schema `get_columns` behaviour as-is; do not add a fallback + +plan.md's session log flags it: `default_schema` is `""`, so `get_columns(conn, table)` with no +schema passes an empty pattern to `conn.meta.list_columns`, which pyexasol turns into a pattern +matching nothing. It is unreachable through the UI — every Exasol table arrives from `get_tables()` +as a populated `(TABLE_SCHEMA, TABLE_NAME)` pair — and the prior change deliberately left it +spec-faithful rather than inventing a fallback. + +The test asserts the current behaviour (`list_columns` receives `""`), with a comment saying it +documents a deliberate choice rather than a desirable one. This is the honest option: a test that +pins the decision makes a future reversal visible, whereas no test at all leaves the next reader +guessing whether it was ever considered. + +## Risks / Trade-offs + +- **Mocked tests cannot catch a pyexasol API mismatch.** `MagicMock` accepts any kwarg, so if + `websocket_sslopt`, `access_token` or `refresh_token` were misspelled — or renamed in pyexasol 2.x + — every test here still passes while a live connect fails. → *Mitigation:* accept this explicitly. + These tests pin sqlit's intent; plan step 13's Docker integration test is the only thing that can + validate the driver contract. Keep the kwarg names as literal strings in the assertions so a + rename surfaces as a visible diff in the test file rather than passing silently. + +- **`MagicMock` makes the `result_type` guard pass for the wrong reason.** On a bare `MagicMock()` + statement, `stmt.result_type != "resultSet"` evaluates `True`, so `execute_query` returns + `([], [], False)` and a sloppy result-set test goes green while asserting nothing. → *Mitigation:* + set `result_type` explicitly on every statement mock, and add the inverse test — a `rowCount` + statement must leave `fetchall` / `fetchmany` **uncalled** + (`stmt.fetchall.assert_not_called()`), which is the actual invariant the guard protects. + +- **Unrelated `mariadb` deletion in the lockfile diff.** → *Mitigation:* D5; call it out in the PR + body. + +- **`pyexasol`'s `<3.15` ceiling constrains the `all` extra on future interpreters.** → + *Mitigation:* D6, with the marker escape hatch scoped to `all`. + +- **Bundling packaging with three test files makes the change wider than the plan's step + granularity.** → *Mitigation:* the tasks keep each plan step as its own numbered group with its own + verify command, and groups 2-4 are independently revertible. Only group 1 (packaging) unblocks the + others. + +## Migration Plan + +No migration. Additive `pyproject.toml` metadata plus new test files; no existing behaviour changes. +Rollback is a `git revert` of the commit — the `exasol` extra disappearing cannot break an existing +install, because no default dependency references `pyexasol`. + +## Open Questions + +- **Lower bound `>=2.0.0` or `>=2.3.2`?** plan.md specifies `>=2.0.0` and this change follows it; + every API detail was in fact verified against 2.3.2. The loose bound matches house style + (`hdbcli>=2.20.0`, `teradatasql>=20.0.0` are all loose lower bounds) and `uv.lock` pins the real + resolved version, so the exposure is limited to someone installing an old pyexasol outside the + lock. Raise it to `>=2.3.2` only if the upstream reviewer asks. +- **Does the maintainer want `pyexasol` in the `all` extra at all?** `all` is already 28 packages and + this makes 29. plan.md step 8 says add it and every other provider is in there, so this change + adds it. Flagged because it is the one line here a reviewer might ask to drop — and dropping it is + a one-line change with no test consequences. diff --git a/openspec/changes/archive/2026-08-27-exasol-unit-tests/proposal.md b/openspec/changes/archive/2026-08-27-exasol-unit-tests/proposal.md new file mode 100644 index 00000000..eeb31807 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-unit-tests/proposal.md @@ -0,0 +1,65 @@ +## Why + +The `exasol-adapter` and `exasol-provider-registration` changes made Exasol a fully selectable +provider, but two things are still missing before it can be upstreamed: **the driver cannot be +installed** (`pip install sqlit-tui[exasol]` fails — there is no `exasol` extra, and `pyexasol` is +absent from `uv.lock`), and **not one line of `ExasolAdapter` is covered by a test**. Every +behavioural decision recorded in the two previous changes — the token-vs-password credential +branch, the TLS mode mapping, `stmt.result_type` being checked before any fetch, `rowcount()` being +a method, `conn.meta.*` returning UPPERCASE-keyed dicts — is currently a claim in a design document +with nothing asserting it. + +This change closes both gaps: it makes the driver installable and locks the adapter's behaviour +into the default CI job. It is plan.md steps 8, 9, 10 and 11. + +## What Changes + +- **Modified** `pyproject.toml` — a new `exasol = ["pyexasol>=2.0.0"]` optional-dependency extra, + the same pin added to the aggregate `all` extra, `"pyexasol"` added to the mypy + `ignore_missing_imports` override list, and an `"exasol: Exasol database tests"` pytest marker. +- **Regenerated** `uv.lock` — `uv sync --extra exasol` resolves `pyexasol` (and its `websocket-client` + / `packaging` dependencies) into the lockfile. +- **New** `tests/connections/providers/exasol/test_schema.py` — asserts the `visible_when` + predicates on `SCHEMA` show exactly the credential fields belonging to the selected + `authenticator`, and hide the other two methods' fields. Needs no driver. +- **New** `tests/connections/providers/exasol/test_connect.py` — mocked `pyexasol` module, + asserting the kwargs `connect()` actually passes: the correct credentials per `authenticator` + **and the absence of the unused ones**, the `tls_mode` → `encryption` / `websocket_sslopt` + mapping for all five modes, and `extra_options` passthrough. +- **New** `tests/connections/providers/exasol/test_adapter.py` — mocked connection, asserting the + introspection row-shape contract (`conn.meta.*` read by UPPERCASE key), primary-key detection, + the `execute_query` truncation flag at the `max_rows` boundary, the `result_type` guard firing + before any fetch, `rowcount()` called as a method, and `quote_identifier` escaping. +- **New** `tests/connections/providers/exasol/__init__.py` — empty, matching the existing + `tests/connections/providers/hana/__init__.py`. +- No changes to any file under `sqlit/`. The adapter is not being modified; it is being pinned. + +## Capabilities + +### New Capabilities + +- `exasol-driver-packaging`: `pyexasol` is installable as a named extra, resolvable in the + lockfile, declared to mypy, and Exasol tests are addressable by marker. +- `exasol-unit-coverage`: driver-free unit tests that pin the schema's conditional field + visibility and every behavioural decision in `ExasolAdapter`, running in the default CI job. + +### Modified Capabilities + +None. `openspec/specs/exasol-provider-registration` describes behaviour that this change asserts +but does not alter — no requirement in it changes. + +## Impact + +- **Dependencies**: adds `pyexasol>=2.0.0` as an *optional* dependency. Nothing in the default + install changes; `sqlit` still imports the driver lazily, so a user without the extra sees the + normal install prompt rather than an `ImportError`. +- **Lockfile**: `uv.lock` is already carrying an unrelated pre-existing drift on this branch (the + stale `mariadb` package entry, dropped when the `mariadb` extra switched to `PyMySQL`). Running + `uv sync` folds that refresh in alongside the `pyexasol` addition — see design D5. +- **CI**: the three new test files are collected by the existing unit-test job with no workflow + change, because they live under `tests/connections/` and are not in its `--ignore` list. Test + count rises; runtime does not measurably. +- **Python support**: `pyexasol` declares `requires-python >=3.10,<3.15` against sqlit's unbounded + `>=3.10`. This constrains the aggregate `all` extra on future interpreters — see design D6. +- **Not affected**: no `sqlit/` source file, no Docker compose service, no integration test, no + workflow file, no documentation. Those are plan.md steps 12-15. diff --git a/openspec/changes/archive/2026-08-27-exasol-unit-tests/specs/exasol-driver-packaging/spec.md b/openspec/changes/archive/2026-08-27-exasol-unit-tests/specs/exasol-driver-packaging/spec.md new file mode 100644 index 00000000..957efa2a --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-unit-tests/specs/exasol-driver-packaging/spec.md @@ -0,0 +1,85 @@ +## ADDED Requirements + +### Requirement: Exasol driver is installable as a named optional extra + +`pyproject.toml` SHALL declare an `exasol` entry under `[project.optional-dependencies]` requiring +`pyexasol>=2.0.0`, and the same requirement SHALL appear in the aggregate `all` extra. The extra +MUST NOT be promoted into `[project].dependencies`: `ExasolAdapter` imports its driver lazily +through `_import_driver_module`, so a user without the extra MUST continue to receive sqlit's +install prompt rather than an `ImportError` at startup. + +The requirement string carries no inline environment marker even though `pyexasol` declares +`requires-python >=3.10,<3.15` against sqlit's unbounded `>=3.10`; `uv` attaches the interpreter +marker to the locked entry instead (design D6). + +#### Scenario: Dedicated extra exists +- **WHEN** `[project.optional-dependencies]` in `pyproject.toml` is read +- **THEN** it contains `exasol = ["pyexasol>=2.0.0"]` +- **AND** the entry sits with the other single-provider extras, not inside `all` + +#### Scenario: Aggregate extra includes the driver +- **WHEN** the `all` extra is read +- **THEN** it contains a `pyexasol>=2.0.0` requirement + +#### Scenario: Extra installs successfully +- **WHEN** `uv sync --extra exasol` is run +- **THEN** it completes without a resolution error +- **AND** `uv run python -c "import pyexasol"` succeeds + +#### Scenario: Driver is resolved in the lockfile +- **WHEN** `uv.lock` is inspected after the sync +- **THEN** it contains a `pyexasol` package entry +- **AND** that entry is reachable from the `exasol` and `all` extras of `sqlit-tui` + +#### Scenario: Default install is unaffected +- **WHEN** `[project].dependencies` is read +- **THEN** it does not mention `pyexasol` +- **AND** an environment without the extra can still import `sqlit` and open the connection picker + +#### Scenario: Resolution across the declared Python range succeeds +- **WHEN** `uv lock` resolves against sqlit's declared `requires-python >=3.10` +- **THEN** it completes without an error naming Python 3.15 +- **AND** the `pyexasol` lock entry carries the interpreter marker rather than the `pyproject.toml` + requirement string + +### Requirement: The driver module is declared to the type checker + +`"pyexasol"` SHALL be listed in the `[[tool.mypy.overrides]]` `module` array that sets +`ignore_missing_imports`, alongside the other driver modules. + +This entry is currently inert — `mypy` excludes `tests/` and `sqlit/` names `pyexasol` only inside a +string literal passed to `_import_driver_module`, so there is no import for mypy to resolve. It is +declared for consistency with `hdbcli` and `teradatasql`, which are lazily imported the same way and +are already listed, and so that the override is in place if a `TYPE_CHECKING` import of the driver is +ever added. Because the entry is inert, a clean `mypy` run does NOT constitute evidence that it was +added (design D4). + +#### Scenario: Override list includes the driver +- **WHEN** the mypy `ignore_missing_imports` override module list is read +- **THEN** it contains `"pyexasol"` + +#### Scenario: Type checking stays clean +- **WHEN** `uv run mypy sqlit` is run with the extra installed +- **THEN** it reports no new errors relative to the pre-change baseline + +### Requirement: Exasol tests are addressable by a registered marker + +`[tool.pytest.ini_options].markers` SHALL contain `"exasol: Exasol database tests"`, matching the +wording of the neighbouring per-database markers. + +No test introduced by this change carries the marker. The driver-free unit tests belong to the +default job and stay unmarked, matching `tests/connections/providers/hana/test_get_columns.py`. The +marker is registered ahead of the Docker integration test so that test can apply it without a +`PytestUnknownMarkWarning` (design D7). + +#### Scenario: Marker is registered +- **WHEN** `uv run pytest --markers` is run +- **THEN** `exasol` appears in the output with the description `Exasol database tests` + +#### Scenario: No unknown-mark warning is possible +- **WHEN** a test is decorated with `@pytest.mark.exasol` +- **THEN** pytest does not emit `PytestUnknownMarkWarning` for it + +#### Scenario: Marker selects nothing yet +- **WHEN** `uv run pytest -m exasol` is run against the tree produced by this change +- **THEN** zero tests are selected, because no test is marked yet diff --git a/openspec/changes/archive/2026-08-27-exasol-unit-tests/specs/exasol-unit-coverage/spec.md b/openspec/changes/archive/2026-08-27-exasol-unit-tests/specs/exasol-unit-coverage/spec.md new file mode 100644 index 00000000..a59a14e0 --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-unit-tests/specs/exasol-unit-coverage/spec.md @@ -0,0 +1,309 @@ +## ADDED Requirements + +### Requirement: Exasol unit tests run without the driver installed + +The test files introduced by this change SHALL pass in an environment where `pyexasol` is not +importable. Where a test needs the driver, it SHALL supply a fake by seeding +`sys.modules["pyexasol"]` (`unittest.mock.patch.dict`), which `importlib.import_module` — and +therefore `_import_driver_module` — returns without touching the filesystem. + +Tests MUST NOT patch `ExasolAdapter._import_driver_module` itself; the lazy-import call site, +including the `driver_name` / `extra_name` / `package_name` arguments that produce sqlit's install +prompt, is part of what is under test (design D2). + +#### Scenario: Tests pass with no extras installed +- **WHEN** `uv sync --group test --no-dev` is used, as the default CI unit job does +- **AND** `uv run pytest tests/connections/providers/exasol/ -v` is run +- **THEN** every test passes +- **AND** no test is skipped for a missing driver + +#### Scenario: Tests are collected by the default unit job +- **WHEN** the unit-test command from `.github/workflows/ci.yml` is run +- **THEN** the three new Exasol test files are collected +- **AND** no `--ignore` entry is required for them + +#### Scenario: The driver fake is installed through sys.modules +- **WHEN** a test that calls `adapter.connect(config)` is inspected +- **THEN** it seeds a mock under the `"pyexasol"` key of `sys.modules` +- **AND** it reads the recorded call from that mock's `connect` attribute + +### Requirement: Tests live in the per-provider test package + +The tests SHALL reside in `tests/connections/providers/exasol/` as a package with an empty +`__init__.py`, mirroring `tests/connections/providers/hana/`, split as `test_schema.py`, +`test_connect.py` and `test_adapter.py`. + +#### Scenario: Package layout +- **WHEN** the test tree is inspected +- **THEN** `tests/connections/providers/exasol/__init__.py` exists and is empty +- **AND** `test_schema.py`, `test_connect.py` and `test_adapter.py` sit beside it + +#### Scenario: The directory is a single runnable gate +- **WHEN** `uv run pytest tests/connections/providers/exasol/ -v` is run +- **THEN** it collects and runs every Exasol unit test and nothing else + +### Requirement: Conditional credential-field visibility is pinned + +`test_schema.py` SHALL evaluate the `visible_when` predicates on `SCHEMA.fields` against a form-values +dict for each `authenticator` value, asserting that exactly the selected method's credential fields +are visible and the other methods' fields are hidden. Fields with no `visible_when` (`server`, `port`, +`authenticator`, `schema`) are always visible. No driver is involved. + +#### Scenario: Password authentication +- **WHEN** the predicates are evaluated with `{"authenticator": "password"}` +- **THEN** `username` and `password` are visible +- **AND** `access_token` and `refresh_token` are hidden + +#### Scenario: Access-token authentication +- **WHEN** the predicates are evaluated with `{"authenticator": "access_token"}` +- **THEN** `access_token` is visible +- **AND** `username`, `password` and `refresh_token` are hidden + +#### Scenario: Refresh-token authentication +- **WHEN** the predicates are evaluated with `{"authenticator": "refresh_token"}` +- **THEN** `refresh_token` is visible +- **AND** `username`, `password` and `access_token` are hidden + +#### Scenario: Absent authenticator falls back to password +- **WHEN** the predicates are evaluated with an empty dict +- **THEN** the visibility matches the `"password"` case, because each predicate defaults the lookup to + `"password"` + +#### Scenario: Unconditional fields stay visible +- **WHEN** the predicates are evaluated with any `authenticator` value +- **THEN** `server`, `port`, `authenticator` and `schema` carry no `visible_when` and are visible + +### Requirement: Credentials passed to the driver match the selected authenticator, and only those + +`test_connect.py` SHALL assert, for each `authenticator` value, both which credential kwargs +`pyexasol.connect` receives **and that the other methods' kwargs are absent from the call +entirely**. Asserting that an unused key is empty is not sufficient: pyexasol's login branches on +token truthiness, so a present-but-empty `access_token` silently reverts to password login +(design D3). + +#### Scenario: Password authentication +- **WHEN** `connect()` runs with `authenticator` unset or `"password"` +- **THEN** the recorded kwargs contain `user` and `password` from the endpoint +- **AND** neither `access_token` nor `refresh_token` appears as a key + +#### Scenario: Access-token authentication +- **WHEN** `connect()` runs with `authenticator == "access_token"` +- **THEN** the recorded kwargs contain `access_token` +- **AND** none of `user`, `password` or `refresh_token` appears as a key + +#### Scenario: Refresh-token authentication +- **WHEN** `connect()` runs with `authenticator == "refresh_token"` +- **THEN** the recorded kwargs contain `refresh_token` +- **AND** none of `user`, `password` or `access_token` appears as a key + +### Requirement: Endpoint, schema and autocommit kwargs are pinned + +`test_connect.py` SHALL assert the non-credential kwargs `connect()` builds from the config: the +`dsn` string, the port fallback, the `schema` option and `autocommit`. It SHALL also cover the +rejection path for a config that carries no TCP endpoint. + +#### Scenario: DSN is host and port joined by a colon +- **WHEN** `connect()` runs against an endpoint with host `db.example.com` and port `1234` +- **THEN** the recorded kwargs contain `dsn == "db.example.com:1234"` + +#### Scenario: Absent port falls back to the registered default +- **WHEN** the endpoint carries no port +- **THEN** the `dsn` port segment is `8563`, resolved through `get_default_port("exasol")` + +#### Scenario: Schema option is forwarded +- **WHEN** the `schema` option is set to `TEST_SQLIT` +- **THEN** the recorded kwargs contain `schema == "TEST_SQLIT"` +- **AND** when the option is unset the value is the empty string, not omitted + +#### Scenario: Autocommit is enabled at connect time +- **WHEN** `connect()` runs with any authenticator +- **THEN** the recorded kwargs contain `autocommit is True` + +#### Scenario: A non-TCP configuration is rejected before importing the driver +- **WHEN** `connect()` is called with a config whose `tcp_endpoint` is `None` +- **THEN** `ValueError` is raised +- **AND** the fake driver's `connect` is never called + +### Requirement: The TLS mode mapping is pinned for every mode + +`test_connect.py` SHALL cover all five `tls_mode` values and assert the resulting `encryption` and +`websocket_sslopt` kwargs. + +#### Scenario: Encryption disabled +- **WHEN** `tls_mode` is `disable` +- **THEN** the recorded kwargs contain `encryption is False` +- **AND** no `websocket_sslopt` key is present + +#### Scenario: Driver default +- **WHEN** `tls_mode` is `default` or unset +- **THEN** the recorded kwargs contain `encryption is True` +- **AND** no `websocket_sslopt` key is present + +#### Scenario: Encrypted without verification +- **WHEN** `tls_mode` is `require` +- **THEN** the recorded kwargs contain `encryption is True` +- **AND** `websocket_sslopt == {"cert_reqs": ssl.CERT_NONE}` + +#### Scenario: Verifying modes request certificate validation +- **WHEN** `tls_mode` is `verify-ca` or `verify-full` +- **THEN** the recorded kwargs contain `encryption is True` +- **AND** `websocket_sslopt["cert_reqs"] == ssl.CERT_REQUIRED` + +#### Scenario: Configured certificate files are forwarded +- **WHEN** `tls_mode` is a verifying mode and `tls_ca`, `tls_cert` and `tls_key` are set +- **THEN** `websocket_sslopt` carries them as `ca_certs`, `certfile` and `keyfile` + +#### Scenario: Unconfigured certificate files are omitted +- **WHEN** `tls_mode` is a verifying mode and the certificate options are unset or whitespace +- **THEN** `websocket_sslopt` contains only `cert_reqs` +- **AND** no `ca_certs`, `certfile` or `keyfile` key is present + +### Requirement: extra_options pass through and win on conflict + +`test_connect.py` SHALL assert that `config.extra_options` reaches the driver verbatim and that it +overrides the kwargs `connect()` computes, since it is applied last. + +#### Scenario: Unknown options reach the driver +- **WHEN** `extra_options` contains a key sqlit does not know +- **THEN** the recorded kwargs contain that key and value verbatim + +#### Scenario: extra_options override computed kwargs +- **WHEN** `extra_options` sets a key that `connect()` also computes, such as `encryption` +- **THEN** the recorded kwargs carry the `extra_options` value + +### Requirement: The introspection row-shape contract is pinned + +`test_adapter.py` SHALL drive introspection off a mocked `conn.meta`, returning +already-fetched lists of dicts with UPPERCASE keys for the `list_*` helpers and an object requiring +`.fetchall()` for `execute_snapshot`. A test that fed tuples would pass against an index-based +implementation and so would not pin the contract at all. + +#### Scenario: Tables are read by key +- **WHEN** `conn.meta.list_tables()` yields dicts with `TABLE_SCHEMA` and `TABLE_NAME` +- **THEN** `get_tables` returns the corresponding `(schema, name)` tuples in order + +#### Scenario: Views are read by key +- **WHEN** `conn.meta.list_views()` yields dicts with `VIEW_SCHEMA` and `VIEW_NAME` +- **THEN** `get_views` returns the corresponding `(schema, name)` tuples in order + +#### Scenario: Columns combine type and primary-key information +- **WHEN** `execute_snapshot` yields a primary-key row for `ID` and `list_columns` yields `ID` then + `NAME` +- **THEN** `get_columns` returns `ColumnInfo("ID", , is_primary_key=True)` followed by + `ColumnInfo("NAME", , is_primary_key=False)` +- **AND** the order from `list_columns` is preserved + +#### Scenario: The primary-key query is snapshot-executed and parameterised +- **WHEN** `get_columns` runs for schema `S` and table `T` +- **THEN** the query goes through `conn.meta.execute_snapshot`, not `conn.execute` +- **AND** it filters on `CONSTRAINT_TYPE = 'PRIMARY KEY'` +- **AND** the schema and table are passed as query parameters rather than interpolated + +#### Scenario: A table with no primary key yields no flagged column +- **WHEN** the primary-key snapshot returns no rows +- **THEN** every returned `ColumnInfo` has `is_primary_key` false + +#### Scenario: An unspecified schema is passed through as empty +- **WHEN** `get_columns` is called without a schema +- **THEN** `list_columns` receives the empty string +- **AND** this pins a deliberate spec-faithful choice — the path is unreachable from the explorer, + and no fallback is substituted (design D8) + +#### Scenario: Procedures come from the scripting scripts +- **WHEN** `get_procedures` runs +- **THEN** it snapshot-executes against `SYS.EXA_ALL_SCRIPTS` filtered to `SCRIPT_TYPE = 'SCRIPTING'` +- **AND** returns the `SCRIPT_NAME` values + +#### Scenario: Unsupported object kinds return empty without querying +- **WHEN** `get_databases`, `get_indexes`, `get_triggers` or `get_sequences` is called +- **THEN** each returns an empty list +- **AND** the connection mock records no call at all + +### Requirement: Result-set detection precedes any fetch + +`test_adapter.py` SHALL assert that `execute_query` inspects `stmt.result_type` before any fetch, +because `ExaStatement.__next__` raises `ExaRuntimeError` for a `rowCount` statement and `fetchmany()` +iterates. + +Every statement mock MUST set `result_type` explicitly: on a bare `MagicMock` the inequality is +trivially true, which would let a result-set test pass while asserting nothing (see the second risk +in design.md). + +#### Scenario: A row-count statement returns empty without fetching +- **WHEN** `execute_query` runs against a statement whose `result_type` is `rowCount` +- **THEN** it returns empty columns, empty rows and `truncated` false +- **AND** neither `fetchall` nor `fetchmany` is called on the statement + +#### Scenario: A result-set statement is fetched +- **WHEN** `execute_query` runs against a statement whose `result_type` is `resultSet` +- **THEN** the column names come from `stmt.column_names()` +- **AND** the rows are returned as tuples + +### Requirement: The truncation flag is pinned at the max_rows boundary + +`test_adapter.py` SHALL cover the `max_rows` boundary in both directions: `execute_query` requests +`max_rows + 1` rows to detect truncation, MUST report it, and MUST trim the surplus before +returning. + +#### Scenario: Unlimited fetch is never truncated +- **WHEN** `execute_query` runs with `max_rows` unset +- **THEN** all rows from `fetchall` are returned +- **AND** `truncated` is false + +#### Scenario: Exactly max_rows rows available +- **WHEN** `max_rows` is 2 and the statement yields 2 rows +- **THEN** 2 rows are returned +- **AND** `truncated` is false + +#### Scenario: More than max_rows rows available +- **WHEN** `max_rows` is 2 and the statement yields 3 rows +- **THEN** exactly 2 rows are returned +- **AND** `truncated` is true + +#### Scenario: One extra row is requested +- **WHEN** `execute_query` runs with `max_rows` set to 2 +- **THEN** `fetchmany` is called with 3 + +### Requirement: Row counts and the test query use pyexasol's statement API + +`test_adapter.py` SHALL pin the two places the adapter departs from the DB-API shape of its base +class: `rowcount` is a **method** on `ExaStatement` rather than a property, and `execute_test_query` +is overridden because the inherited implementation calls `conn.cursor()`, which pyexasol does not +provide. + +#### Scenario: rowcount is invoked as a method +- **WHEN** `execute_non_query` runs +- **THEN** the statement's `rowcount` is **called**, not read as a property +- **AND** the returned value is an `int` + +#### Scenario: No explicit commit is issued +- **WHEN** `execute_non_query` runs +- **THEN** the connection mock records no `commit` call, because `autocommit=True` is set at connect + time + +#### Scenario: The connection test avoids cursor() +- **WHEN** `execute_test_query` runs +- **THEN** `conn.execute("SELECT 1")` is called and `fetchval()` is taken from the result +- **AND** `conn.cursor` is never accessed + +### Requirement: Identifier quoting and select building are pinned + +`test_adapter.py` SHALL assert that `quote_identifier` wraps in double quotes and doubles any +embedded double quote, and that `build_select_query` MUST omit the schema segment entirely when no +schema is given rather than emitting a leading dot. + +#### Scenario: Plain identifier +- **WHEN** `quote_identifier("MY_TABLE")` is called +- **THEN** it returns `"MY_TABLE"` wrapped in double quotes + +#### Scenario: Embedded double quote is doubled +- **WHEN** `quote_identifier` is called with an identifier containing a double quote +- **THEN** that character is doubled inside the quoted result + +#### Scenario: Schema-qualified select +- **WHEN** `build_select_query("T", 10, schema="S")` is called +- **THEN** it returns a `SELECT * FROM` against the quoted `"S"."T"` with `LIMIT 10` + +#### Scenario: Select without a schema omits the schema segment +- **WHEN** `build_select_query("T", 10)` is called with no schema +- **THEN** the result references only the quoted table, with no leading dot diff --git a/openspec/changes/archive/2026-08-27-exasol-unit-tests/tasks.md b/openspec/changes/archive/2026-08-27-exasol-unit-tests/tasks.md new file mode 100644 index 00000000..370e3bbc --- /dev/null +++ b/openspec/changes/archive/2026-08-27-exasol-unit-tests/tasks.md @@ -0,0 +1,158 @@ +## 1. Driver packaging + +Corresponds to plan.md step 8. Unblocks groups 3 and 4. + +- [x] 1.1 Add `exasol = ["pyexasol>=2.0.0"]` to `[project.optional-dependencies]` in + `pyproject.toml`, placed with the other single-provider extras (near `hana` / `teradata`). +- [x] 1.2 Add `"pyexasol>=2.0.0"` to the aggregate `all` extra. Do **not** add it to + `[project].dependencies` — the lazy import must keep producing sqlit's install prompt. +- [x] 1.3 Add `"pyexasol"` to the `[[tool.mypy.overrides]]` `module` list that sets + `ignore_missing_imports`. Design D4: this is inert today (mypy excludes `tests/` and `sqlit/` + names the driver only in a string literal) and is added for consistency with the equally lazy + `hdbcli` and `teradatasql`. +- [x] 1.4 Add `"exasol: Exasol database tests"` to `[tool.pytest.ini_options].markers`, matching the + wording of the neighbouring per-database markers. Design D7: nothing in this change uses it; + it is registered ahead of plan step 13. +- [x] 1.5 Run `uv sync --extra exasol` and confirm it resolves without an error naming Python 3.15 + (design D6 — `pyexasol` caps at `<3.15`, sqlit's `requires-python` has no ceiling). If it + fails, apply the D6 escape hatch: an inline `; python_version < '3.15'` marker on the **`all`** + entry only, and record it in the plan's Session log. +- [x] 1.6 Verify the driver is importable: `uv run python -c "import pyexasol; print(pyexasol.__version__)"`. + Record the resolved version in the plan's Session log — the spec's lower bound is `>=2.0.0` + but every API detail was verified against 2.3.2. +- [x] 1.7 Confirm `uv.lock` gained a `pyexasol` entry, and check whether `uv` attached a + `python_full_version < '3.15'` marker to it (design D6). +- [x] 1.8 Note the unrelated lockfile drift for the PR description: `uv.lock` on this branch also + drops the stale `mariadb` package, because `HEAD`'s `mariadb` extra already points at + `PyMySQL`. Design D5 — do not try to isolate it, and do not hand-edit `uv.lock`. +- [x] 1.9 Verify: `uv run mypy sqlit` is clean and `uv run ruff check sqlit` is clean. + **Remember mypy does not verify 1.3** (design D4); 1.6 is that task's real check. + +## 2. Test package skeleton + +- [x] 2.1 Create `tests/connections/providers/exasol/__init__.py`, empty, matching + `tests/connections/providers/hana/__init__.py`. +- [x] 2.2 Confirm `uv run pytest tests/connections/providers/exasol/ -v` collects cleanly (zero + tests, no collection error) before any test file is added. + +## 3. Schema visibility tests + +Corresponds to plan.md step 9. No driver involved. + +- [x] 3.1 Create `tests/connections/providers/exasol/test_schema.py` importing `SCHEMA` from + `providers.exasol.schema`, with a helper that maps a form-values dict to the set of visible + field names — a field is visible when `visible_when` is `None` or returns `True`. +- [x] 3.2 Test `{"authenticator": "password"}`: `username` and `password` visible, `access_token` + and `refresh_token` hidden. +- [x] 3.3 Test `{"authenticator": "access_token"}`: `access_token` visible; `username`, `password` + and `refresh_token` hidden. +- [x] 3.4 Test `{"authenticator": "refresh_token"}`: `refresh_token` visible; `username`, + `password` and `access_token` hidden. +- [x] 3.5 Test the empty dict: visibility matches the `password` case, since each predicate defaults + its lookup to `"password"`. +- [x] 3.6 Test that `server`, `port`, `authenticator` and `schema` carry no `visible_when` and stay + visible under every authenticator value. +- [x] 3.7 Verify: `uv run pytest tests/connections/providers/exasol/test_schema.py -v`. + +## 4. Connect and TLS tests + +Corresponds to plan.md step 10. Depends on group 1. + +- [x] 4.1 Create `tests/connections/providers/exasol/test_connect.py` with a helper that runs + `ExasolAdapter().connect(config)` under + `patch.dict("sys.modules", {"pyexasol": MagicMock()})` and returns the recorded + `connect` kwargs. Design D2 — seed `sys.modules`; do **not** patch + `_import_driver_module`, and do **not** rely on the real `pyexasol` being installed. +- [x] 4.2 Add a `ConnectionConfig` builder taking `options` and `extra_options`, building a + `TcpEndpoint` the way `tests/unit/test_extra_options_passthrough.py` does. +- [x] 4.3 Test password auth (both explicit `"password"` and the unset default): `user` and + `password` present from the endpoint, **and** `"access_token" not in kwargs` and + `"refresh_token" not in kwargs`. Design D3 — absence, not emptiness. +- [x] 4.4 Test `authenticator == "access_token"`: `access_token` present; `"user"`, `"password"` + and `"refresh_token"` all absent as keys. +- [x] 4.5 Test `authenticator == "refresh_token"`: `refresh_token` present; `"user"`, `"password"` + and `"access_token"` all absent as keys. +- [x] 4.6 Test `dsn == ":"` from the endpoint, and that an endpoint with no port falls + back to `8563` via `get_default_port("exasol")`. +- [x] 4.7 Test `schema` forwarding: a set `schema` option appears verbatim; an unset one appears as + the empty string rather than being omitted. Also assert `autocommit is True`. +- [x] 4.8 Test that a config whose `tcp_endpoint` is `None` raises `ValueError` and never calls the + fake driver's `connect`. +- [x] 4.9 Test `tls_mode="disable"`: `encryption is False` and no `websocket_sslopt` key. +- [x] 4.10 Test `tls_mode="default"` and unset: `encryption is True` and no `websocket_sslopt` key. +- [x] 4.11 Test `tls_mode="require"`: `encryption is True` and + `websocket_sslopt == {"cert_reqs": ssl.CERT_NONE}`. +- [x] 4.12 Test `verify-ca` and `verify-full`: `encryption is True` and + `websocket_sslopt["cert_reqs"] == ssl.CERT_REQUIRED`. +- [x] 4.13 Test that a verifying mode with `tls_ca` / `tls_cert` / `tls_key` set forwards them as + `ca_certs` / `certfile` / `keyfile`, and that with those options unset or whitespace-only + `websocket_sslopt` contains **only** `cert_reqs`. +- [x] 4.14 Test `extra_options` passthrough of an unknown key, and that `extra_options` overrides a + computed kwarg such as `encryption` (it is applied last). +- [x] 4.15 Verify: `uv run pytest tests/connections/providers/exasol/test_connect.py -v`. +- [x] 4.16 Confirm the file is driver-independent: the tests still pass in an environment without + the `exasol` extra (`uv run --no-sync` against a plain `--group test` environment, or a + temporary rename of the installed package). This is the constraint the default CI job imposes. + +## 5. Adapter behaviour tests + +Corresponds to plan.md step 11. Depends on group 1. + +- [x] 5.1 Create `tests/connections/providers/exasol/test_adapter.py` with a `mock_conn` fixture + whose `conn.meta.list_*` return **lists of dicts with UPPERCASE keys** and whose + `conn.meta.execute_snapshot` returns an object with a `fetchall()`. Feeding tuples here would + pass against an index-based implementation and pin nothing. +- [x] 5.2 Add a statement-mock helper that **always sets `result_type` explicitly**. On a bare + `MagicMock`, `result_type != "resultSet"` is trivially true, so an unset value makes + result-set tests pass while asserting nothing (design risk 2). +- [x] 5.3 Test `get_tables` maps `TABLE_SCHEMA` / `TABLE_NAME` to `(schema, name)` tuples in order. +- [x] 5.4 Test `get_views` maps `VIEW_SCHEMA` / `VIEW_NAME` the same way. +- [x] 5.5 Test `get_columns` combines the primary-key snapshot with `list_columns`: `ID` flagged + primary key, `NAME` not, `list_columns` order preserved. +- [x] 5.6 Test the primary-key lookup goes through `conn.meta.execute_snapshot` (not `conn.execute`), + filters on `CONSTRAINT_TYPE = 'PRIMARY KEY'`, and passes schema and table as query parameters + rather than interpolating them. +- [x] 5.7 Test a table with no primary key: every `ColumnInfo.is_primary_key` is false. +- [x] 5.8 Test that `get_columns` with no schema passes `""` to `list_columns`, with a comment + naming design D8 — this pins a deliberate, unreachable-from-the-UI choice, not a desirable one. +- [x] 5.9 Test `get_procedures` snapshot-executes `SYS.EXA_ALL_SCRIPTS` filtered to + `SCRIPT_TYPE = 'SCRIPTING'` and returns `SCRIPT_NAME` values. +- [x] 5.10 Test `get_databases`, `get_indexes`, `get_triggers` and `get_sequences` each return `[]` + **and** touch the connection mock not at all (`mock_conn.mock_calls == []`). +- [x] 5.11 Test the `rowCount` path of `execute_query`: returns `([], [], False)` and calls neither + `fetchall` nor `fetchmany` (`stmt.fetchall.assert_not_called()`). +- [x] 5.12 Test the `resultSet` path with `max_rows` unset: columns from `column_names()`, all rows + as tuples, `truncated` false. +- [x] 5.13 Test the truncation boundary: with `max_rows=2`, two available rows give 2 rows and + `truncated` false; three available rows give 2 rows and `truncated` true. +- [x] 5.14 Test that `fetchmany` is called with `max_rows + 1`. +- [x] 5.15 Test `execute_non_query` **calls** `stmt.rowcount()` as a method and returns an `int`, + and that no `commit` is issued on the connection. +- [x] 5.16 Test `execute_test_query` runs `conn.execute("SELECT 1")`, takes `fetchval()`, and never + accesses `conn.cursor`. +- [x] 5.17 Test `quote_identifier` on a plain identifier and on one containing a double quote + (doubled inside the quoted result). +- [x] 5.18 Test `build_select_query` with a schema (quoted `"S"."T"` plus `LIMIT`) and without one + (table only, no leading dot). +- [x] 5.19 Verify: `uv run pytest tests/connections/providers/exasol/ -v` — all three files green. + +## 6. Change gate and plan bookkeeping + +- [x] 6.1 Run `uv run ruff check sqlit tests` and `uv run mypy sqlit`; both clean relative to the + pre-change baseline. +- [x] 6.2 Confirm no file under `sqlit/` was modified by this change: `git status` shows only + `pyproject.toml`, `uv.lock` and the new test files (plus the pre-existing modifications to + `domain/config.py` and `providers/catalog.py` from earlier changes). +- [x] 6.3 Run the default unit job's command from `.github/workflows/ci.yml:70-82` verbatim and + confirm the new tests are collected and pass with no new failures. Pre-existing + Windows-environment failures recorded in the plan's Session log stay as they are. +- [x] 6.4 Confirm `uv run pytest --markers` lists `exasol`, and that `uv run pytest -m exasol` + selects zero tests (expected until plan step 13). +- [x] 6.5 If any test surfaced an adapter bug, record it in the plan's Session log as a finding — + do **not** fix `sqlit/` source in this change (design Non-Goals). +- [x] 6.6 In `plan.md`, set steps 8, 9, 10 and 11 to `done` in the Status table and update the + progress count to `11 / 15 done`. +- [x] 6.7 Append a `plan.md` Session log row recording: step 10's open item resolved via + `patch.dict("sys.modules", ...)` (design D2); the mypy override being inert (D4); the + `pyexasol` version actually resolved; whether the D6 Python-marker escape hatch was needed; + and the `mariadb` lockfile drift to disclose in the PR (D5). diff --git a/openspec/config.yaml b/openspec/config.yaml new file mode 100644 index 00000000..392946c6 --- /dev/null +++ b/openspec/config.yaml @@ -0,0 +1,20 @@ +schema: spec-driven + +# Project context (optional) +# This is shown to AI when creating artifacts. +# Add your tech stack, conventions, style guides, domain knowledge, etc. +# Example: +# context: | +# Tech stack: TypeScript, React, Node.js +# We use conventional commits +# Domain: e-commerce platform + +# Per-artifact rules (optional) +# Add custom rules for specific artifacts. +# Example: +# rules: +# proposal: +# - Keep proposals under 500 words +# - Always include a "Non-goals" section +# tasks: +# - Break tasks into chunks of max 2 hours diff --git a/openspec/specs/exasol-adapter/spec.md b/openspec/specs/exasol-adapter/spec.md new file mode 100644 index 00000000..7df21e0c --- /dev/null +++ b/openspec/specs/exasol-adapter/spec.md @@ -0,0 +1,382 @@ +# exasol-adapter Specification + +## Purpose +Defines the behaviour of `ExasolAdapter` — the `DatabaseAdapter` implementation backing sqlit's +Exasol provider, built on `pyexasol`. Covers the adapter's capability shape, lazy driver import, +connection and TLS parameter assembly, snapshot-based introspection, and query execution against +pyexasol's `ExaStatement` API. + +`pyexasol` is a native WebSocket client rather than a DB-API 2.0 driver, so this adapter extends +`DatabaseAdapter` directly instead of `CursorBasedAdapter`; most requirements here exist to pin the +consequences of that. + +Scope boundaries: the connection schema, `DatabaseType` membership and `ProviderSpec` registration +live in `exasol-provider-registration`. Driver packaging and unit-test coverage live in +`exasol-driver-packaging` and `exasol-unit-coverage`. + +## Requirements +### Requirement: Provider discovery tolerates a subpackage with no provider module +Provider discovery SHALL skip a subpackage that contains no `provider` module, resolved through +`importlib.util.find_spec`. `providers/catalog.py::_discover_providers` walks every subpackage of +`providers/` and imports its `provider.py`; without the skip that unconditional import raises +`ModuleNotFoundError` and no provider at all resolves, so one staged package breaks discovery +app-wide rather than merely being absent from it. + +This is what lets a provider package be built up incrementally: an adapter-only package is inert, +neither registered nor harmful, until its `provider.py` lands. + +> Superseded scenarios: this requirement originally also asserted that the Exasol package contained +> only `__init__.py` and `adapter.py`, that `get_supported_db_types()` returned 29 providers, and +> that `"exasol"` was absent from it. All three were true only while the adapter was staged below +> registration. The `exasol-provider-registration` change added `provider.py` and `schema.py`, so the +> package is now live and the catalog reports 30 providers including `exasol`. The discovery-skip +> behaviour below is the durable part. See `openspec/changes/archive/2026-08-27-exasol-adapter/` +> for the original wording. + +#### Scenario: A subpackage without provider.py does not break discovery +- **WHEN** `get_supported_db_types()` is called with an adapter-only provider package present +- **THEN** it returns every provider that does have a `provider` module +- **AND** no `ModuleNotFoundError` is raised +- **AND** the staged package's `db_type` is absent from the result + +#### Scenario: Package docstring matches house style +- **WHEN** `__init__.py` is read +- **THEN** its entire content is the single docstring `"""Provider package."""` + +### Requirement: Adapter subclasses DatabaseAdapter directly +`ExasolAdapter` SHALL extend `DatabaseAdapter`, not `CursorBasedAdapter`, because pyexasol is +a native WebSocket client that exposes no `.cursor()` method. Every abstract member of +`DatabaseAdapter` MUST be implemented. + +#### Scenario: Class is importable and concrete +- **WHEN** `ExasolAdapter()` is instantiated +- **THEN** instantiation succeeds without `TypeError` for unimplemented abstract methods + +#### Scenario: Base class choice +- **WHEN** the class declaration is inspected +- **THEN** `DatabaseAdapter` is its direct base +- **AND** `CursorBasedAdapter` does not appear in its MRO + +### Requirement: Adapter declares Exasol's capability shape +The adapter SHALL expose the following capability properties, which +`build_adapter_provider` reads via `getattr`. + +| Property | Value | +|---|---| +| `name` | `"Exasol"` | +| `install_extra` | `"exasol"` | +| `install_package` | `"pyexasol"` | +| `driver_import_names` | `("pyexasol",)` | +| `supports_multiple_databases` | `False` | +| `supports_cross_database_queries` | `False` | +| `supports_stored_procedures` | `True` | +| `supports_indexes` | `False` | +| `supports_triggers` | `False` | +| `supports_sequences` | `False` | +| `default_schema` | `""` | + +The adapter MUST NOT override `supports_process_worker`: `process_worker.py` calls +`provider.connection_factory.connect(...)` inside the child process, so it opens its own +WebSocket rather than pickling one, and pyexasol returns plain picklable tuples. + +#### Scenario: Capability properties report Exasol's shape +- **WHEN** each property in the table above is read from an `ExasolAdapter` instance +- **THEN** it returns the listed value + +#### Scenario: Process worker support is inherited +- **WHEN** the class body is inspected +- **THEN** `supports_process_worker` is not defined on `ExasolAdapter` +- **AND** the inherited value is `True` + +#### Scenario: Schema-only table display +- **WHEN** `format_table_name("MYSCHEMA", "T")` is called +- **THEN** it returns `"MYSCHEMA.T"`, because `default_schema` is empty so no schema is elided + +### Requirement: Driver import is lazy +`connect()` SHALL obtain the driver through the inherited +`self._import_driver_module("pyexasol", ...)`, passing `driver_name=self.name`, +`extra_name=self.install_extra` and `package_name=self.install_package`. The module MUST NOT +be imported at module scope, so that a missing driver produces sqlit's normal install prompt +instead of an `ImportError` at collection time. + +#### Scenario: Module imports without the driver installed +- **WHEN** `adapter.py` is imported in an environment with no `pyexasol` +- **THEN** the import succeeds + +#### Scenario: Missing driver surfaces the install prompt +- **WHEN** `connect()` is called with no `pyexasol` installed +- **THEN** the error raised by `import_driver_module` names the `exasol` extra and the + `pyexasol` package + +### Requirement: Connection parameters are assembled from the endpoint and options +`connect()` SHALL require a TCP endpoint and raise `ValueError` otherwise. It SHALL build +`dsn` as host and port joined by a colon, where port is +`int(endpoint.port or get_default_port("exasol"))`, pass `schema` from +`config.get_option("schema", "")`, pass `autocommit=True`, then apply the TLS kwargs, then +apply `config.extra_options` last so callers can override anything. + +#### Scenario: Endpoint is not TCP +- **WHEN** `connect()` is called with a config whose `tcp_endpoint` is `None` +- **THEN** `ValueError` is raised +- **AND** no connection attempt is made + +#### Scenario: DSN combines host and port +- **WHEN** `connect()` runs for host `db.example.com` and port `8563` +- **THEN** `pyexasol.connect` receives `dsn` equal to `db.example.com:8563` + +#### Scenario: Schema is forwarded +- **WHEN** the config option `schema` is `ANALYTICS` +- **THEN** `pyexasol.connect` receives `schema` equal to `ANALYTICS` + +#### Scenario: Empty schema browses everything +- **WHEN** the config option `schema` is unset +- **THEN** `pyexasol.connect` receives `schema` equal to the empty string + +#### Scenario: Autocommit is enabled +- **WHEN** `connect()` runs +- **THEN** `pyexasol.connect` receives `autocommit=True` + +#### Scenario: Extra options are applied last +- **WHEN** `config.extra_options` sets `autocommit` to `False` +- **THEN** `pyexasol.connect` receives `autocommit=False` + +### Requirement: Authentication method selects mutually exclusive credentials +`connect()` SHALL read `config.get_option("authenticator", "password")` and pass only that +method's credentials. Credentials for the other two methods MUST be absent from the kwargs, +not present-and-empty, because pyexasol rejects combinations of `password`, `access_token` +and `refresh_token`. + +#### Scenario: Username and password +- **WHEN** `authenticator` is `password` +- **THEN** `pyexasol.connect` receives `user` and `password` from the endpoint +- **AND** `access_token` and `refresh_token` are absent from the kwargs + +#### Scenario: OpenID access token +- **WHEN** `authenticator` is `access_token` +- **THEN** `pyexasol.connect` receives `access_token` from the `access_token` option +- **AND** `user`, `password` and `refresh_token` are absent from the kwargs + +#### Scenario: OpenID refresh token +- **WHEN** `authenticator` is `refresh_token` +- **THEN** `pyexasol.connect` receives `refresh_token` from the `refresh_token` option +- **AND** `user`, `password` and `access_token` are absent from the kwargs + +#### Scenario: Unset authenticator defaults to password +- **WHEN** the `authenticator` option is absent +- **THEN** the password credentials are used + +### Requirement: TLS mode maps onto pyexasol encryption settings +A private `_tls_args(config)` helper SHALL translate the shared `tls_mode` option into +pyexasol kwargs using `get_tls_mode`, `tls_mode_verifies_cert` and `get_tls_files` from +`providers/tls.py`. This mapping matters because pyexasol reads an omitted `websocket_sslopt` +as `cert_reqs` of `ssl.CERT_REQUIRED` while `exasol/docker-db` and most on-premise +installations present a self-signed certificate, so deferring to the driver default fails +certificate validation on every out-of-the-box connect. + +| `tls_mode` | kwargs | +|---|---| +| `default` | `encryption=True`, `websocket_sslopt` with `cert_reqs` of `ssl.CERT_NONE` | +| `disable` | `encryption=False` | +| `require` | same as `default` | +| `verify-ca` | `encryption=True`, `websocket_sslopt` with `cert_reqs` of `ssl.CERT_REQUIRED`, plus any configured files | +| `verify-full` | same as `verify-ca` | + +Under the verifying modes, `ca_certs`, `certfile` and `keyfile` SHALL be included in +`websocket_sslopt` only when the corresponding path from `get_tls_files` is non-empty. + +#### Scenario: Default mode encrypts without verifying +- **WHEN** `tls_mode` is absent or `default` +- **THEN** the kwargs contain `encryption=True` +- **AND** `cert_reqs` in `websocket_sslopt` is `ssl.CERT_NONE` + +#### Scenario: Disabled mode turns encryption off +- **WHEN** `tls_mode` is `disable` +- **THEN** the kwargs contain `encryption=False` +- **AND** no `websocket_sslopt` key is present + +#### Scenario: Require mode encrypts without verifying +- **WHEN** `tls_mode` is `require` +- **THEN** the kwargs contain `encryption=True` +- **AND** `cert_reqs` in `websocket_sslopt` is `ssl.CERT_NONE` + +#### Scenario: Verifying mode demands a valid certificate +- **WHEN** `tls_mode` is `verify-ca` or `verify-full` +- **THEN** the kwargs contain `encryption=True` +- **AND** `cert_reqs` in `websocket_sslopt` is `ssl.CERT_REQUIRED` + +#### Scenario: Certificate files are forwarded when configured +- **WHEN** `tls_mode` is `verify-full` and `tls_ca`, `tls_cert` and `tls_key` are all set +- **THEN** `websocket_sslopt` contains `ca_certs`, `certfile` and `keyfile` with those paths + +#### Scenario: Unconfigured certificate files are omitted +- **WHEN** `tls_mode` is `verify-ca` and only `tls_ca` is set +- **THEN** `websocket_sslopt` contains `ca_certs` +- **AND** `certfile` and `keyfile` are absent from `websocket_sslopt` + +### Requirement: Introspection uses snapshot metadata reads +All introspection SHALL go through `conn.meta.*`, which prefixes each query with Exasol's +snapshot-execution hint and therefore cannot be blocked by metadata locks. + +`conn.meta.list_tables()`, `list_views()` and `list_columns()` return **lists of dicts** with +UPPERCASE keys — pyexasol enforces `fetch_dict=True` on metadata reads specifically to stop +callers depending on column order — so results MUST be read by key, never by index. +`conn.meta.execute_snapshot()` returns an `ExaStatement`, so it requires an explicit +`.fetchall()`, which likewise yields dicts. + +#### Scenario: Databases are not a concept in Exasol +- **WHEN** `get_databases(conn)` is called +- **THEN** it returns an empty list without querying the connection + +#### Scenario: Tables are listed by schema and name +- **WHEN** `get_tables(conn)` is called +- **THEN** `conn.meta.list_tables()` is used +- **AND** each row yields a pair taken from the `TABLE_SCHEMA` and `TABLE_NAME` keys + +#### Scenario: Views are listed by schema and name +- **WHEN** `get_views(conn)` is called +- **THEN** `conn.meta.list_views()` is used +- **AND** each row yields a pair taken from the `VIEW_SCHEMA` and `VIEW_NAME` keys + +#### Scenario: Metadata rows are read by key +- **WHEN** a mocked `conn.meta.list_tables()` returns dicts whose keys are in a different + order than declared +- **THEN** `get_tables` still returns the correct schema and name pairs + +### Requirement: Column introspection reports primary keys +`get_columns(conn, table, database=None, schema=None)` SHALL read name and type from +`conn.meta.list_columns(schema, table)` using keys `COLUMN_NAME` and `COLUMN_TYPE`, and +determine the primary-key set with `conn.meta.execute_snapshot(...).fetchall()` against +`SYS.EXA_ALL_CONSTRAINT_COLUMNS` filtered on a `CONSTRAINT_TYPE` of `PRIMARY KEY` plus the +schema and table, reading `COLUMN_NAME` from each row. It returns `ColumnInfo` objects in the +order `list_columns` yields them. + +#### Scenario: Columns carry name and declared type +- **WHEN** `get_columns` runs for a table with a decimal `ID` and a varchar `NAME` +- **THEN** it returns `ColumnInfo` entries with those names and their `COLUMN_TYPE` strings + +#### Scenario: Primary key columns are flagged +- **WHEN** the constraint query reports `ID` as a primary-key column +- **THEN** the `ID` entry has `is_primary_key` set to `True` +- **AND** every other entry has `is_primary_key` set to `False` + +#### Scenario: Table without a primary key +- **WHEN** the constraint query returns no rows +- **THEN** every returned `ColumnInfo` has `is_primary_key` set to `False` + +#### Scenario: Composite primary key +- **WHEN** the constraint query reports both `A` and `B` as primary-key columns +- **THEN** both entries have `is_primary_key` set to `True` + +### Requirement: Stored procedures are read from EXA_ALL_SCRIPTS +`get_procedures(conn, database=None)` SHALL query `SYS.EXA_ALL_SCRIPTS` through +`conn.meta.execute_snapshot(...).fetchall()`, filtering on a `SCRIPT_TYPE` of `SCRIPTING` — +the value Exasol uses for scripting programs, as distinct from `UDF`, `ADAPTER` and +`PREPROCESSOR` — and return the script names. + +#### Scenario: Scripting programs are returned +- **WHEN** `EXA_ALL_SCRIPTS` contains a `SCRIPTING` entry named `MY_PROC` +- **THEN** `get_procedures` includes that name + +#### Scenario: UDFs are not stored procedures +- **WHEN** `EXA_ALL_SCRIPTS` contains a `UDF` entry +- **THEN** `get_procedures` excludes it + +### Requirement: Unsupported object types return empty lists +`get_indexes`, `get_triggers` and `get_sequences` SHALL each return an empty list without +querying the connection. All three MUST still be defined even though their capability flags +are `False`, because they are abstract on `DatabaseAdapter`. They are empty because Exasol's +indexes are auto-managed and unnamed, it has no triggers, and it uses IDENTITY columns rather +than sequences. + +#### Scenario: Indexes, triggers and sequences are empty +- **WHEN** `get_indexes(conn)`, `get_triggers(conn)` and `get_sequences(conn)` are called +- **THEN** each returns an empty list +- **AND** no method on `conn` is invoked + +### Requirement: Query execution returns columns, rows and a truncation flag +`execute_query(conn, query, max_rows=None)` SHALL execute via `conn.execute(query)` and +return a triple of columns, rows and a truncation flag. It MUST decide whether a result set +exists by testing `stmt.result_type` **before** fetching, because pyexasol raises +`ExaRuntimeError` with the message "Attempt to fetch from statement without result set" when +iterating a row-count statement. `result_type` is a public attribute whose values are exactly +`resultSet` and `rowCount`. + +When `max_rows` is set, the implementation SHALL fetch one row beyond the limit to detect +truncation, then trim to `max_rows`. Rows MUST be returned as tuples. + +#### Scenario: Unlimited fetch +- **WHEN** `execute_query` runs with `max_rows` of `None` on a statement returning 3 rows +- **THEN** it returns those 3 rows and the truncation flag is `False` + +#### Scenario: Fewer rows than the limit +- **WHEN** `max_rows` is 10 and the statement has 4 rows +- **THEN** all 4 rows are returned and the truncation flag is `False` + +#### Scenario: Exactly the limit is not truncation +- **WHEN** `max_rows` is 10 and the statement has exactly 10 rows +- **THEN** 10 rows are returned and the truncation flag is `False` + +#### Scenario: One row over the limit is truncation +- **WHEN** `max_rows` is 10 and the statement has 11 rows +- **THEN** 10 rows are returned and the truncation flag is `True` + +#### Scenario: Statement with no result set +- **WHEN** `execute_query` runs a statement whose `result_type` is `rowCount` +- **THEN** it returns empty columns, no rows, and a truncation flag of `False` +- **AND** no fetch method is called on the statement + +### Requirement: Non-query execution returns the affected row count +`execute_non_query(conn, query)` SHALL execute via `conn.execute(query)` and return +`int(stmt.rowcount())`. `rowcount` is a **method** on `ExaStatement`, not a property, so it +MUST be called. No explicit commit is issued, because the connection is opened with +`autocommit=True`. + +#### Scenario: Row count is returned +- **WHEN** an `INSERT` affecting 5 rows is executed +- **THEN** `execute_non_query` returns 5 + +#### Scenario: Rowcount is invoked as a method +- **WHEN** `execute_non_query` runs against a mocked statement +- **THEN** `rowcount()` is called +- **AND** the returned value is an integer + +### Requirement: Connection test bypasses the cursor-based default +The adapter SHALL override `execute_test_query(conn)`, because the inherited implementation +calls `conn.cursor()`, which pyexasol does not provide. The override SHALL run +`conn.execute(self.test_query)` and then `fetchval()` on the result. + +#### Scenario: Test query runs without a cursor +- **WHEN** `execute_test_query(conn)` is called +- **THEN** `conn.execute` is called with the inherited `SELECT 1` test query +- **AND** `fetchval()` is called on the result +- **AND** `conn.cursor` is never accessed + +### Requirement: Identifiers are quoted with doubled double quotes +`quote_identifier(name)` SHALL wrap the name in double quotes and escape any embedded double +quote by doubling it. + +#### Scenario: Plain identifier +- **WHEN** `quote_identifier` is called with `MY_TABLE` +- **THEN** it returns that name wrapped in double quotes + +#### Scenario: Embedded double quote is doubled +- **WHEN** `quote_identifier` is called with a name containing one double quote +- **THEN** that double quote appears doubled inside the surrounding quotes + +### Requirement: Select queries are schema-qualified and LIMIT-bounded +`build_select_query(table, limit, database=None, schema=None)` SHALL produce a +`SELECT * FROM` statement against the quoted schema-qualified table with a trailing `LIMIT` +clause, omitting the schema segment when the schema is empty. + +#### Scenario: Schema-qualified select +- **WHEN** `build_select_query` is called for table `T` in schema `S` with a limit of 100 +- **THEN** it selects from the quoted `S`-dot-`T` name and ends with `LIMIT 100` + +#### Scenario: Unqualified select +- **WHEN** `build_select_query` is called for table `T` with a limit of 50 and no schema +- **THEN** it selects from the quoted `T` name alone and ends with `LIMIT 50` + +#### Scenario: Identifiers in the select are quoted +- **WHEN** `build_select_query` is called with a lowercase table name +- **THEN** the name appears double-quoted, preserving its case + diff --git a/openspec/specs/exasol-documentation/spec.md b/openspec/specs/exasol-documentation/spec.md new file mode 100644 index 00000000..319871f0 --- /dev/null +++ b/openspec/specs/exasol-documentation/spec.md @@ -0,0 +1,155 @@ +# exasol-documentation Specification + +## Purpose +Defines the documentation surface for the Exasol provider — the two top-level documents a person +reads before touching this repo. `README.md` names Exasol among the supported engines and carries the +driver install command a user reaches for when sqlit reports a missing driver; `CONTRIBUTING.md` +tells a contributor which compose profile starts the Exasol container, which environment variables +repoint the tests at another server, and that the container needs minutes rather than seconds before +it accepts a login. + +Every value documented here is a claim about code that already exists, which is what makes this +capability testable rather than decorative: the driver package is the distribution declared by the +`exasol` extra, the container list names the profile that actually starts the service, and each +documented default equals the fallback the fixture module reads. A table that has drifted from the +code is worse than no table, because it sends a contributor to debug their environment rather than +the value. + +Scope boundaries: the extra itself lives in `exasol-driver-packaging`; `DatabaseType` membership and +the picker display order this documentation mirrors live in `exasol-provider-registration`; the +compose service, the fixtures and the `EXASOL_*` variables being described live in +`exasol-integration-harness`; adapter behaviour lives in `exasol-adapter`. + +## Requirements +### Requirement: Exasol is listed among supported databases + +`README.md` SHALL name Exasol in the sentence that enumerates supported engines, so that a reader +deciding whether sqlit talks to their database can answer the question without reading source. + +Placement SHALL follow the connection picker's display order rather than alphabetical order, so the +document and the running application agree on where Exasol sits among the engines. + +#### Scenario: Supported-database sentence names Exasol + +- **WHEN** the "Supports all major databases:" sentence in `README.md` is read +- **THEN** it contains `Exasol`, positioned immediately after `Teradata` +- **AND** every other engine named in that sentence is unchanged, in the same order, with the + trailing `and osquery.` still closing the list + +#### Scenario: Picker order and README order agree + +- **WHEN** the position of Exasol in that sentence is compared with the provider display order used + by the connection picker +- **THEN** Exasol follows Teradata in both + +### Requirement: Driver Reference gives the Exasol install command + +The Driver Reference table in `README.md` SHALL carry a row for Exasol naming `pyexasol` as the +driver package, with the `pipx inject` and `pip install` commands spelled out in the same form as +every other row. The table exists so that a user hitting a missing-driver error can copy one command +and continue; a row that omits either column fails that purpose for half its readers. + +The package name SHALL be the distribution actually declared by the `exasol` extra in +`pyproject.toml`, not a hand-written approximation of it. + +#### Scenario: Exasol row is present and complete + +- **WHEN** the Driver Reference table is read +- **THEN** it contains a row whose Database cell is `Exasol`, whose Driver package cell is + `pyexasol`, whose `pipx` cell is `pipx inject sqlit-tui pyexasol`, and whose `pip` / venv cell is + `python -m pip install pyexasol` + +#### Scenario: Documented package matches the declared extra + +- **WHEN** the driver package named in the Exasol row is compared with `exasol = [...]` in + `pyproject.toml` +- **THEN** both name the `pyexasol` distribution + +#### Scenario: Existing rows are untouched + +- **WHEN** the diff of `README.md` is inspected +- **THEN** the only change inside the table is the added Exasol row, with no other row's cells, + spacing or column alignment altered + +### Requirement: Enterprise profile documents its Exasol container + +`CONTRIBUTING.md` SHALL name Exasol in the list of containers started by the `enterprise` compose +profile. The Exasol service is declared under that profile precisely so it is not pulled by default, +which means a contributor learns it exists only from this list. + +#### Scenario: Enterprise container list names Exasol + +- **WHEN** the sentence introducing the `--profile enterprise` command in `CONTRIBUTING.md` is read +- **THEN** it names Exasol alongside Db2, Trino, Presto and Oracle 11g +- **AND** the `docker compose ... --profile enterprise up -d` command below it is unchanged, because + it already starts every service in the profile + +### Requirement: Exasol test environment variables are documented with their real defaults + +The Environment Variables section of `CONTRIBUTING.md` SHALL carry an Exasol table listing every +`EXASOL_*` variable that `tests/fixtures/exasol.py` reads, and each documented default SHALL equal +the default in that module. A table that drifts from the code is worse than no table: it sends a +contributor to debug their environment rather than the value. + +The table SHALL include `EXASOL_READY_TIMEOUT`, which has no analogue in any other engine's table +and is documented nowhere else, because it is the only knob a contributor on slow hardware or a cold +image pull can reach for. + +#### Scenario: Every fixture variable appears + +- **WHEN** the `EXASOL_`-prefixed names read by `tests/fixtures/exasol.py` are compared with the rows + of the Exasol table +- **THEN** the two sets are equal, covering `EXASOL_HOST`, `EXASOL_PORT`, `EXASOL_USER`, + `EXASOL_PASSWORD`, `EXASOL_SCHEMA` and `EXASOL_READY_TIMEOUT` + +#### Scenario: Documented defaults match the code + +- **WHEN** each default in the table is compared with the fallback passed to `os.environ.get` for the + same variable +- **THEN** they are identical: `localhost`, `8563`, `sys`, `exasol`, `TEST_SQLIT` and `300` + +#### Scenario: Table matches the shape of its neighbours + +- **WHEN** the Exasol table is compared with the SQL Server and Db2 tables above it +- **THEN** it uses the same `**Exasol:**` bold label, the same + `| Variable | Default | Description |` header, and the same separator row + +### Requirement: Contributors are told Exasol boots slowly + +`CONTRIBUTING.md` SHALL state that the Exasol container takes substantially longer to accept +connections than the "about 30-45 seconds" quoted for the standard profile, and SHALL distinguish +the port opening from the server accepting a login. + +Without this, a contributor who waits the documented 45 seconds, sees a refused login and concludes +the container is broken is behaving reasonably — the measured figures are 21 seconds to an open port +and 101 seconds to a first successful login on an already-pulled image. + +#### Scenario: Readiness expectation is stated + +- **WHEN** the Exasol documentation in `CONTRIBUTING.md` is read +- **THEN** it warns that Exasol needs minutes rather than seconds before it accepts connections, and + that an open port 8563 does not yet mean the database will authenticate + +#### Scenario: The standard-profile timing is not overwritten + +- **WHEN** the existing "about 30-45 seconds" guidance is inspected +- **THEN** it is unchanged, because it remains correct for the default profile that most + contributors run + +### Requirement: The change is documentation only + +This change SHALL modify no file outside `README.md` and `CONTRIBUTING.md`, and SHALL introduce no +claim about Exasol that the code does not already implement. Documentation lands after the behaviour +it describes; if a sentence cannot be written truthfully, that is a defect to file, not a sentence to +soften. + +#### Scenario: No source, test or configuration file changes + +- **WHEN** the diff for this change is listed +- **THEN** the only paths are `README.md` and `CONTRIBUTING.md` + +#### Scenario: Documented behaviour is already implemented + +- **WHEN** each factual claim added to either document is traced +- **THEN** each resolves to existing code or configuration — the provider registration, the `exasol` + extra, the compose service, or the fixture module — and none describes intended future behaviour diff --git a/openspec/specs/exasol-driver-packaging/spec.md b/openspec/specs/exasol-driver-packaging/spec.md new file mode 100644 index 00000000..5226a786 --- /dev/null +++ b/openspec/specs/exasol-driver-packaging/spec.md @@ -0,0 +1,100 @@ +# exasol-driver-packaging Specification + +## Purpose +Defines how the Exasol driver is packaged, resolved and addressed by tooling: the `exasol` +optional-dependency extra that installs `pyexasol`, the same requirement inside the aggregate `all` +extra, its resolution into `uv.lock`, its declaration to the type checker, and the registered +`exasol` pytest marker. + +The driver stays optional. `ExasolAdapter` imports it lazily through `_import_driver_module`, so an +environment without the extra receives sqlit's install prompt rather than an `ImportError`, and the +default install is unchanged. + +Scope boundaries: adapter behaviour lives in `exasol-adapter`; the connection schema, `DatabaseType` +membership and `ProviderSpec` registration live in `exasol-provider-registration`; the unit tests +themselves live in `exasol-unit-coverage`. + +## Requirements +### Requirement: Exasol driver is installable as a named optional extra + +`pyproject.toml` SHALL declare an `exasol` entry under `[project.optional-dependencies]` requiring +`pyexasol>=2.0.0`, and the same requirement SHALL appear in the aggregate `all` extra. The extra +MUST NOT be promoted into `[project].dependencies`: `ExasolAdapter` imports its driver lazily +through `_import_driver_module`, so a user without the extra MUST continue to receive sqlit's +install prompt rather than an `ImportError` at startup. + +The requirement string carries no inline environment marker even though `pyexasol` declares +`requires-python >=3.10,<3.15` against sqlit's unbounded `>=3.10`; `uv` attaches the interpreter +marker to the locked entry instead (design D6). + +#### Scenario: Dedicated extra exists +- **WHEN** `[project.optional-dependencies]` in `pyproject.toml` is read +- **THEN** it contains `exasol = ["pyexasol>=2.0.0"]` +- **AND** the entry sits with the other single-provider extras, not inside `all` + +#### Scenario: Aggregate extra includes the driver +- **WHEN** the `all` extra is read +- **THEN** it contains a `pyexasol>=2.0.0` requirement + +#### Scenario: Extra installs successfully +- **WHEN** `uv sync --extra exasol` is run +- **THEN** it completes without a resolution error +- **AND** `uv run python -c "import pyexasol"` succeeds + +#### Scenario: Driver is resolved in the lockfile +- **WHEN** `uv.lock` is inspected after the sync +- **THEN** it contains a `pyexasol` package entry +- **AND** that entry is reachable from the `exasol` and `all` extras of `sqlit-tui` + +#### Scenario: Default install is unaffected +- **WHEN** `[project].dependencies` is read +- **THEN** it does not mention `pyexasol` +- **AND** an environment without the extra can still import `sqlit` and open the connection picker + +#### Scenario: Resolution across the declared Python range succeeds +- **WHEN** `uv lock` resolves against sqlit's declared `requires-python >=3.10` +- **THEN** it completes without an error naming Python 3.15 +- **AND** the `pyexasol` lock entry carries the interpreter marker rather than the `pyproject.toml` + requirement string + +### Requirement: The driver module is declared to the type checker + +`"pyexasol"` SHALL be listed in the `[[tool.mypy.overrides]]` `module` array that sets +`ignore_missing_imports`, alongside the other driver modules. + +This entry is currently inert — `mypy` excludes `tests/` and `sqlit/` names `pyexasol` only inside a +string literal passed to `_import_driver_module`, so there is no import for mypy to resolve. It is +declared for consistency with `hdbcli` and `teradatasql`, which are lazily imported the same way and +are already listed, and so that the override is in place if a `TYPE_CHECKING` import of the driver is +ever added. Because the entry is inert, a clean `mypy` run does NOT constitute evidence that it was +added (design D4). + +#### Scenario: Override list includes the driver +- **WHEN** the mypy `ignore_missing_imports` override module list is read +- **THEN** it contains `"pyexasol"` + +#### Scenario: Type checking stays clean +- **WHEN** `uv run mypy sqlit` is run with the extra installed +- **THEN** it reports no new errors relative to the pre-change baseline + +### Requirement: Exasol tests are addressable by a registered marker + +`[tool.pytest.ini_options].markers` SHALL contain `"exasol: Exasol database tests"`, matching the +wording of the neighbouring per-database markers. + +No test introduced by this change carries the marker. The driver-free unit tests belong to the +default job and stay unmarked, matching `tests/connections/providers/hana/test_get_columns.py`. The +marker is registered ahead of the Docker integration test so that test can apply it without a +`PytestUnknownMarkWarning` (design D7). + +#### Scenario: Marker is registered +- **WHEN** `uv run pytest --markers` is run +- **THEN** `exasol` appears in the output with the description `Exasol database tests` + +#### Scenario: No unknown-mark warning is possible +- **WHEN** a test is decorated with `@pytest.mark.exasol` +- **THEN** pytest does not emit `PytestUnknownMarkWarning` for it + +#### Scenario: Marker selects nothing yet +- **WHEN** `uv run pytest -m exasol` is run against the tree produced by this change +- **THEN** zero tests are selected, because no test is marked yet diff --git a/openspec/specs/exasol-integration-coverage/spec.md b/openspec/specs/exasol-integration-coverage/spec.md new file mode 100644 index 00000000..b643b87b --- /dev/null +++ b/openspec/specs/exasol-integration-coverage/spec.md @@ -0,0 +1,151 @@ +# exasol-integration-coverage Specification + +## Purpose +Defines what running the repository's shared database suite against a live Exasol server proves, and +the dedicated CI job that runs it. Every other Exasol test in the repo runs against a `MagicMock`, +which agrees with whatever the adapter asks of it; these requirements are the only place the +adapter's contract with pyexasol is checked in both directions. + +Two inherited tests cannot apply as written and are overridden rather than deleted, each with its +reason stated in the skip or the comment. `test_docker_container_connection` cannot work because +`exasol/docker-db` publishes no credentials through any environment variable and a discovery-built +config carries no `tls_mode`, so it would verify TLS against the image's self-signed certificate - +both properties of the image, neither fixable from `tests/`. `test_primary_key_detection` is +re-issued through the call shape the application actually uses, because the base version passes a +lowercase name and no schema, and pyexasol's metadata patterns are case-sensitive `LIKE` patterns +that the adapter narrows further to `LIKE ''`. + +The exclusion of the integration file from the driver-free unit job is a requirement rather than an +incidental setting: that job's exclude list is matched by filename, so the file and its `--ignore` +have to land together or the job collects a test whose container it never starts. + +Scope boundaries: the server and fixtures this suite consumes live in +`exasol-integration-harness`; the adapter behaviour being asserted lives in `exasol-adapter` and +`exasol-provider-registration`; the mocked unit coverage lives in `exasol-unit-coverage`. + +## Requirements +### Requirement: Exasol runs the shared database integration suite + +`tests/test_exasol.py` SHALL run the repository's shared database test suite against a live Exasol +server, including the suite's `LIMIT` coverage. + +#### Scenario: Suite is bound to the Exasol provider + +- **WHEN** `tests/test_exasol.py` is collected +- **THEN** it defines a single test class deriving from the shared base class that includes the + `LIMIT` test, configured with database type `exasol`, display name `Exasol`, and the Exasol + connection and database fixtures + +#### Scenario: Suite passes against a running server + +- **WHEN** `uv run pytest tests/test_exasol.py -v` is run with an Exasol container up and the + `exasol` extra installed +- **THEN** every inherited test either passes or skips for a reason the adapter declares, and none + fails + +#### Scenario: Suite skips without a server + +- **WHEN** the same command is run with no Exasol server reachable +- **THEN** every test in the file is skipped + +### Requirement: Exasol connection lifecycle is verified through the CLI + +The Exasol test file SHALL verify that a connection can be created and deleted through the sqlit +CLI, independently of the fixture that the shared suite uses. + +#### Scenario: Connection is created and listed + +- **WHEN** an Exasol connection is added with `connections add exasol` and the connection list is + printed +- **THEN** the command reports success and the listing shows the connection name alongside the + `Exasol` display name + +#### Scenario: Connection is deleted + +- **WHEN** that connection is deleted through the CLI +- **THEN** the command reports success and the connection no longer appears in the listing + +### Requirement: Capability-driven and image-driven skips are explicit + +Tests that cannot apply to Exasol SHALL skip for a stated reason rather than being deleted or +silently passing. + +#### Scenario: Unsupported object types self-skip + +- **WHEN** the inherited index, trigger and sequence tests run +- **THEN** they skip because `ExasolAdapter` reports those capabilities as unsupported, and the test + fixture seeds no such objects + +#### Scenario: Docker-discovery connection test is skipped with a reason + +- **WHEN** the inherited Docker-discovery connection test runs +- **THEN** it is skipped by an override whose message states that `exasol/docker-db` publishes no + credentials through environment variables and presents a self-signed certificate, so a + discovery-built configuration cannot connect + +#### Scenario: Docker container detection is not skipped + +- **WHEN** the inherited Docker container detection test runs with the Exasol container up +- **THEN** it is not overridden, and it passes by detecting the container and its published port + +### Requirement: Primary-key detection is verified through the app's call shape + +The primary-key test SHALL exercise `get_columns` with the schema and identifier casing that the +application itself supplies, and SHALL still assert the full primary-key contract. + +#### Scenario: Columns are requested with an explicit schema and server casing + +- **WHEN** the Exasol primary-key test calls the adapter's `get_columns` +- **THEN** it passes the seeded schema and the table name in the casing the server stores, matching + how the explorer and worker call it with values taken from `get_tables()` + +#### Scenario: Primary key flags are asserted both ways + +- **WHEN** the returned columns are inspected +- **THEN** the `id` column is flagged as a primary key and every other column is not + +### Requirement: The driver-free unit job does not collect the Exasol integration test + +The default CI unit job SHALL exclude `tests/test_exasol.py`, in the same commit that introduces the +file, because that job installs no database extras and starts no container. + +#### Scenario: Unit job excludes the file + +- **WHEN** the unit-test job's pytest invocation in `.github/workflows/ci.yml` is read +- **THEN** its exclude list contains `--ignore=tests/test_exasol.py` alongside the other integration + test files + +#### Scenario: Unit job still passes locally + +- **WHEN** that exact command is run locally +- **THEN** collection succeeds and no Exasol integration test is collected + +### Requirement: A dedicated CI job runs the Exasol integration suite + +`.github/workflows/ci.yml` SHALL contain a job that provisions an Exasol server and runs the Exasol +integration test file, following the same conventions as the repository's other per-database +integration jobs. + +#### Scenario: Job installs the driver + +- **WHEN** the job's dependency step runs +- **THEN** it installs the test group together with the `exasol` extra + +#### Scenario: Job starts the server and waits for it + +- **WHEN** the job's server step runs +- **THEN** it starts an `exasol/docker-db` container with the privileges the image needs and the + database port published, and polls that port until it accepts connections or a bounded number of + attempts is exhausted, logging each attempt + +#### Scenario: Job runs the suite with the harness environment + +- **WHEN** the job's test step runs +- **THEN** it invokes pytest on `tests/test_exasol.py` with the Exasol host, port, credential and + schema environment variables set, and with a per-test timeout large enough for a cold server + +#### Scenario: Job is gated like its peers + +- **WHEN** the workflow triggers are compared across integration jobs +- **THEN** the Exasol job runs on the same events as the other database jobs, depends on the same + upstream job, and is neither manually gated nor marked to continue on error diff --git a/openspec/specs/exasol-integration-harness/spec.md b/openspec/specs/exasol-integration-harness/spec.md new file mode 100644 index 00000000..1294c662 --- /dev/null +++ b/openspec/specs/exasol-integration-harness/spec.md @@ -0,0 +1,168 @@ +# exasol-integration-harness Specification + +## Purpose +Defines the reproducible Exasol server that the integration suite runs against - the opt-in compose +service, the pytest fixtures that seed it, and the environment variables that repoint them at a +CI-provided server instead. + +Two hard constraints shape every requirement below. First, `tests/conftest.py` star-imports the +fixture module and is loaded by the driver-free unit CI job, so the module must import cleanly with +no `pyexasol` installed - every driver import sits inside a fixture body. Second, absence of a +server is a **skip**, never a failure, so a full `pytest tests/` run stays green on a machine with no +Docker. + +Readiness is a real connect rather than an open port because the two are measurably different: on a +warm image the container published 8563 after 21 seconds but refused every login until 101 seconds. +A port-only gate turns that 80-second window into a hard authentication failure. The seeded view +tests `email IS NOT NULL` rather than `email != ''` for a related reason - Exasol folds the empty +string to NULL, so the `!= ''` form matches no row at all. + +Scope boundaries: what the seeded server is then used to assert lives in +`exasol-integration-coverage`; the `exasol` extra and the pytest marker live in +`exasol-driver-packaging`; the adapter behaviour being exercised lives in `exasol-adapter`. + +## Requirements +### Requirement: Local Exasol test server + +The test compose stack SHALL provide an Exasol service that a developer can start on demand, and +that service SHALL NOT be started by the default `docker compose up`. + +#### Scenario: Service is declared under the opt-in profile + +- **WHEN** `docker compose -f infra/docker/docker-compose.test.yml --profile enterprise config` is run +- **THEN** the rendered configuration contains an `exasol` service built from an `exasol/docker-db` + image, running `privileged`, publishing container port `8563` on `${EXASOL_PORT:-8563}`, and + declaring a `stop_grace_period` of at least 120 seconds + +#### Scenario: Default profile is unchanged + +- **WHEN** `docker compose -f infra/docker/docker-compose.test.yml config --services` is run with no + profile +- **THEN** `exasol` is absent from the listed services, so no developer pulls a multi-gigabyte image + without asking for it + +### Requirement: Fixture module imports without the driver + +`tests/fixtures/exasol.py` SHALL be importable in an environment where `pyexasol` is not installed, +because `tests/conftest.py` star-imports it and is loaded by the driver-free unit CI job. + +#### Scenario: Collection succeeds with no exasol extra installed + +- **WHEN** the unit-test command from `.github/workflows/ci.yml` is run in an environment installed + with `uv sync --group test --no-dev` (no `--extra exasol`) +- **THEN** collection completes with no `ImportError` and no collection error from `tests/conftest.py` + +#### Scenario: Driver is imported lazily + +- **WHEN** `tests/fixtures/exasol.py` is inspected +- **THEN** it contains no module-level `import pyexasol`, and every `pyexasol` import sits inside a + fixture body guarded so that an `ImportError` becomes `pytest.skip`, not a test failure + +### Requirement: Fixtures skip rather than fail when the server is absent + +Every Exasol fixture SHALL resolve to a skip when no reachable Exasol server is available, so that a +full `pytest tests/` run stays green on a machine with no Docker. + +#### Scenario: No container running + +- **WHEN** nothing is listening on the configured Exasol host and port +- **THEN** tests depending on the Exasol fixtures are reported as skipped, and no exception escapes + a fixture + +#### Scenario: Driver missing but container present + +- **WHEN** an Exasol server is reachable but `pyexasol` is not installed +- **THEN** the fixtures skip with a message naming the missing driver + +### Requirement: Readiness gate tolerates a slow boot + +The readiness fixture SHALL confirm readiness by opening a real database connection, retrying until +a deadline, rather than trusting an open port — Exasol accepts TCP connections on its port well +before it will accept a login. + +#### Scenario: Server is still booting + +- **WHEN** the port is open but the database refuses connections +- **THEN** the readiness fixture retries until its deadline, and only then reports the server as + unavailable + +#### Scenario: Server becomes ready during the wait + +- **WHEN** the database starts accepting connections before the deadline expires +- **THEN** the readiness fixture reports the server as ready and the dependent tests run + +#### Scenario: Readiness is computed once per session + +- **WHEN** more than one Exasol test runs in the same session +- **THEN** the readiness check is performed once, because it is session-scoped + +### Requirement: Seeded test schema matches the shared suite's expectations + +The `exasol_db` fixture SHALL create a dedicated test schema containing exactly the objects the +shared database test suite queries, and SHALL leave no trace of itself behind. + +#### Scenario: Schema is seeded + +- **WHEN** the `exasol_db` fixture runs +- **THEN** the test schema contains a `test_users` table whose `id` column is a primary key and + which holds the three rows Alice, Bob and Charlie; a `test_products` table; and a + `test_user_emails` view +- **AND** the fixture yields the schema name so a connection can be opened against it + +#### Scenario: Identifiers resolve unquoted + +- **WHEN** the seed DDL is executed +- **THEN** it uses unquoted identifiers, so that Exasol's uppercase folding makes + `SELECT * FROM test_users` — the form the shared suite issues — resolve to the seeded table + +#### Scenario: State does not leak between tests + +- **WHEN** one test inserts an extra row into `test_users` and a later test asserts a three-row + result +- **THEN** the later test still sees three rows, because the fixture recreates the schema per test + +#### Scenario: Teardown drops the schema + +- **WHEN** a test using `exasol_db` finishes, whether it passed or failed +- **THEN** the test schema is dropped, and a teardown failure does not fail the test + +### Requirement: CLI connection fixture negotiates TLS against a self-signed certificate + +The `exasol_connection` fixture SHALL register a sqlit connection through the CLI whose TLS settings +succeed against the self-signed certificate that `exasol/docker-db` presents, and SHALL remove that +connection afterwards. + +#### Scenario: Connection is created with an encrypting, non-verifying TLS mode + +- **WHEN** the `exasol_connection` fixture registers its connection +- **THEN** it passes `--tls-mode require`, so the adapter encrypts the connection without verifying + the certificate chain, and the connection is usable by the shared suite + +#### Scenario: Connection targets the seeded schema + +- **WHEN** the connection is created +- **THEN** it carries the seeded schema as its initial schema, so unqualified table names in the + shared suite's queries resolve without a schema prefix + +#### Scenario: Connection is cleaned up + +- **WHEN** a test using `exasol_connection` finishes +- **THEN** the connection is deleted from the sqlit connection store + +### Requirement: Fixtures are registered and environment-configurable + +The Exasol fixtures SHALL be discoverable by the test suite and SHALL take their host, port, +credentials and schema name from environment variables, so the same suite runs against a local +container and against a CI-provided server. + +#### Scenario: Fixtures are registered in conftest + +- **WHEN** `tests/conftest.py` is read +- **THEN** it star-imports `tests.fixtures.exasol` within its existing alphabetically ordered + fixture import block + +#### Scenario: Connection details are overridable + +- **WHEN** `EXASOL_HOST`, `EXASOL_PORT`, `EXASOL_USER`, `EXASOL_PASSWORD` or `EXASOL_SCHEMA` are set + in the environment +- **THEN** the fixtures use those values instead of their defaults diff --git a/openspec/specs/exasol-provider-registration/spec.md b/openspec/specs/exasol-provider-registration/spec.md new file mode 100644 index 00000000..743ad549 --- /dev/null +++ b/openspec/specs/exasol-provider-registration/spec.md @@ -0,0 +1,324 @@ +# exasol-provider-registration Specification + +## Purpose +TBD - created by archiving change exasol-provider-registration. Update Purpose after archive. +## Requirements +### Requirement: Exasol declares a connection schema +The provider package SHALL contain `schema.py` exporting a module-level `SCHEMA` of type +`ConnectionSchema` with `db_type="exasol"`, `display_name="Exasol"`, `default_port="8563"` and +`has_advanced_auth=True`. `supports_ssh` MUST be `True` (the dataclass default) so the shared tunnel +fields apply. + +`db_type` and `display_name` are pinned by `tests/test_schema_capabilities.py`: +`test_provider_schema_ids_match_keys` requires `SCHEMA.db_type` to equal the registry key, and +`test_display_names_match_schema` requires `SCHEMA.display_name` to equal +`get_display_name("exasol")`, which resolves from `ProviderSpec`. + +#### Scenario: Schema identity +- **WHEN** `SCHEMA` is inspected +- **THEN** `SCHEMA.db_type == "exasol"` +- **AND** `SCHEMA.display_name == "Exasol"` +- **AND** `SCHEMA.default_port == "8563"` +- **AND** `SCHEMA.has_advanced_auth is True` +- **AND** `SCHEMA.supports_ssh is True` + +#### Scenario: Schema is reachable through the registry +- **WHEN** `get_connection_schema("exasol")` is called +- **THEN** it returns the same `ConnectionSchema` object exported by `schema.py` + +### Requirement: Schema declares the Exasol field set +`SCHEMA.fields` SHALL contain, in order: `server`, `port`, `authenticator`, `username`, `password`, +`access_token`, `refresh_token`, `schema` — followed by `SSH_FIELDS` and then `TLS_FIELDS` +appended as tuples. + +`server` and `port` SHALL use the shared `_server_field()` and `_port_field("8563")` helpers. +`schema` SHALL be optional with the placeholder `(empty = browse all)`, matching the adapter's +`default_schema` of `""`. + +The field names are fixed by the already-shipped `adapter.py`, which reads `authenticator`, +`access_token`, `refresh_token` and `schema` via `config.get_option(...)`, and takes `user` and +`password` from `config.tcp_endpoint`. + +#### Scenario: Endpoint and option fields are present +- **WHEN** `{f.name for f in SCHEMA.fields}` is computed +- **THEN** it is a superset of `{"server", "port", "authenticator", "username", "password", "access_token", "refresh_token", "schema"}` + +#### Scenario: Shared SSH and TLS tails are appended +- **WHEN** `SCHEMA.fields` is inspected +- **THEN** every field in `SSH_FIELDS` is present +- **AND** every field in `TLS_FIELDS` is present, including `tls_mode` +- **AND** they appear after the Exasol-specific fields + +#### Scenario: No database field +- **WHEN** `SCHEMA.fields` is inspected +- **THEN** no field is named `database`, because the adapter reports + `supports_multiple_databases is False` and `get_databases()` returns an empty list + +#### Scenario: Schema field is optional +- **WHEN** the `schema` field is inspected +- **THEN** `required is False` +- **AND** its placeholder communicates that leaving it empty browses all schemas + +### Requirement: Authentication method is chosen from a dropdown +The `authenticator` field SHALL be a `FieldType.DROPDOWN` with `default="password"` and exactly +three options, in this order: + +| value | label | +|---|---| +| `password` | `Username & Password` | +| `access_token` | `OpenID Access Token` | +| `refresh_token` | `OpenID Refresh Token` | + +The values MUST match the branch labels in `adapter.connect()`, which compares +`config.get_option("authenticator", "password")` against `"access_token"` and `"refresh_token"` and +treats anything else as password auth. + +The field MUST NOT be named `auth_type`: `ConnectionConfig.from_dict` special-cases that key as a +legacy top-level field and hoists it into `options`. + +#### Scenario: Dropdown options +- **WHEN** the `authenticator` field is inspected +- **THEN** `field_type is FieldType.DROPDOWN` +- **AND** `default == "password"` +- **AND** its option values are `("password", "access_token", "refresh_token")` + +#### Scenario: CLI exposes the choice +- **WHEN** the CLI parser is built from `SCHEMA` +- **THEN** an `--authenticator` flag exists +- **AND** its accepted `choices` are the three option values + +#### Scenario: Field is not named auth_type +- **WHEN** `{f.name for f in SCHEMA.fields}` is computed +- **THEN** `"auth_type"` is absent + +### Requirement: Credential fields are visible only for their own authentication method +Each credential field SHALL carry a `visible_when` predicate reading `authenticator` from the form +values, per this matrix: + +| `authenticator` | visible credential fields | hidden credential fields | +|---|---|---| +| `password` (default) | `username`, `password` | `access_token`, `refresh_token` | +| `access_token` | `access_token` | `username`, `password`, `refresh_token` | +| `refresh_token` | `refresh_token` | `username`, `password`, `access_token` | + +`username` and `password` MUST therefore be declared as explicit `SchemaField`s rather than reusing +`_username_field()` / `_password_field()`, whose returned frozen `SchemaField` carries +`visible_when=None`. + +`username` SHALL be `required=True` and `password` SHALL be `required=False`; both SHALL use +`group="credentials"`. `access_token` and `refresh_token` SHALL be `FieldType.PASSWORD` so their +values are masked and treated as promptable secrets. + +Visibility is behaviour, not presentation: `providers/validation.py:30` skips `required` checks on +hidden fields, and `cli/helpers.py:65` marks a flag argparse-required only when +`visible_when is None`. + +#### Scenario: Password auth shows only username and password +- **WHEN** `visible_when({"authenticator": "password"})` is evaluated for each credential field +- **THEN** `username` and `password` are visible +- **AND** `access_token` and `refresh_token` are hidden + +#### Scenario: Access token auth hides username and password +- **WHEN** `visible_when({"authenticator": "access_token"})` is evaluated for each credential field +- **THEN** `access_token` is visible +- **AND** `username`, `password` and `refresh_token` are hidden + +#### Scenario: Refresh token auth hides username and password +- **WHEN** `visible_when({"authenticator": "refresh_token"})` is evaluated for each credential field +- **THEN** `refresh_token` is visible +- **AND** `username`, `password` and `access_token` are hidden + +#### Scenario: Missing authenticator value falls back to password auth +- **WHEN** `visible_when({})` is evaluated — no `authenticator` key at all +- **THEN** `username` and `password` are visible +- **AND** both token fields are hidden +- **AND** this matches `adapter.connect()`, whose `get_option("authenticator", "password")` default + takes the password branch + +#### Scenario: Token auth does not demand a username +- **WHEN** a config with `authenticator == "access_token"` and an empty `username` is validated by + `SchemaConfigValidator.validate` +- **THEN** no `ValueError` is raised, because the hidden `username` field's `required` flag is + skipped + +#### Scenario: Password auth demands a username +- **WHEN** a config with `authenticator == "password"` and an empty `username` is validated +- **THEN** `ValueError` is raised naming the Username field + +#### Scenario: Username is not a globally required CLI flag +- **WHEN** the CLI parser is built from `SCHEMA` +- **THEN** `--username` is not argparse-required, because the field defines `visible_when` + +#### Scenario: Token fields are masked +- **WHEN** the `access_token` and `refresh_token` fields are inspected +- **THEN** both have `field_type is FieldType.PASSWORD` + +### Requirement: Exasol is a member of the DatabaseType enum +`DatabaseType` in `sqlit/domains/connections/domain/config.py` SHALL include +`EXASOL = "exasol"`, placed between `DB2` and `FIREBIRD`. + +This is required by `test_database_type_enum_matches_schema`, which asserts +`{t.value for t in DatabaseType} == set(get_supported_db_types())` as an exact set equality — so +the enum member and the provider registration must land together. + +#### Scenario: Enum member exists +- **WHEN** `DatabaseType.EXASOL` is accessed +- **THEN** its value is `"exasol"` + +#### Scenario: Enum and provider catalog agree +- **WHEN** `test_database_type_enum_matches_schema` runs +- **THEN** the enum value set exactly equals `set(get_supported_db_types())` +- **AND** both sets contain `"exasol"` + +### Requirement: Exasol appears in the connection picker +`DATABASE_TYPE_DISPLAY_ORDER` SHALL include `DatabaseType.EXASOL`, positioned after +`DatabaseType.TERADATA` among the enterprise engines. + +`ui/screens/connection.py:331` builds the database-type `Select` options from exactly this list with +no further filtering, so a type absent from it is unreachable in the UI. No test covers the +constant's completeness, so omitting the entry fails silently — every test passes and Exasol is +still unselectable. + +#### Scenario: Display order contains Exasol +- **WHEN** `DATABASE_TYPE_DISPLAY_ORDER` is inspected +- **THEN** `DatabaseType.EXASOL` is present +- **AND** it appears immediately after `DatabaseType.TERADATA` + +#### Scenario: Picker renders an Exasol option +- **WHEN** the connection screen builds its database-type `Select` +- **THEN** an option labelled `Exasol` with value `exasol` is present + +#### Scenario: Label resolves from the provider +- **WHEN** `get_database_type_labels()` is called +- **THEN** `labels[DatabaseType.EXASOL] == "Exasol"` + +### Requirement: Exasol is registered as a provider +The provider package SHALL contain `provider.py` that builds a `ProviderSpec` and calls +`register_provider(SPEC)` at module scope, following `providers/teradata/provider.py`. + +`SPEC` SHALL declare: + +| Field | Value | +|---|---| +| `db_type` | `"exasol"` | +| `display_name` | `"Exasol"` | +| `schema_path` | `("sqlit.domains.connections.providers.exasol.schema", "SCHEMA")` | +| `supports_ssh` | `True` | +| `is_file_based` | `False` | +| `has_advanced_auth` | `True` | +| `default_port` | `"8563"` | +| `requires_auth` | `True` | +| `badge_label` | `"Exasol"` | +| `url_schemes` | `("exasol", "exa")` | + +`display_name`, `db_type` and `default_port` MUST match `SCHEMA` exactly. + +#### Scenario: Provider is discovered +- **WHEN** `get_supported_db_types()` is called +- **THEN** `"exasol"` is present +- **AND** the total provider count is 30 + +#### Scenario: Registry metadata resolves +- **WHEN** the registry is queried for `"exasol"` +- **THEN** `get_display_name("exasol") == "Exasol"` +- **AND** `get_default_port("exasol") == "8563"` +- **AND** `has_advanced_auth("exasol") is True` +- **AND** `supports_ssh("exasol") is True` +- **AND** `is_file_based("exasol") is False` + +#### Scenario: URL schemes are claimed +- **WHEN** a connection URL with scheme `exasol://` or `exa://` is resolved +- **THEN** it maps to the Exasol provider + +#### Scenario: Existing capability suite stays green +- **WHEN** `uv run pytest tests/test_schema_capabilities.py` is run +- **THEN** all 9 tests pass + +### Requirement: Provider factory imports the adapter lazily +`SPEC.provider_factory` SHALL be a function that imports `ExasolAdapter` **inside its body** and +returns `build_adapter_provider(spec, SCHEMA, ExasolAdapter())`. `provider.py` MUST NOT import +`adapter.py` at module scope. + +`provider.py` is imported for every provider during discovery, i.e. at startup; deferring the +adapter import keeps `adapter.py` and its `ssl` import off the startup path for a provider the user +may never select. Every existing provider does this. + +#### Scenario: Adapter module is not imported at startup +- **WHEN** provider discovery completes without any Exasol connection being opened +- **THEN** `sqlit.domains.connections.providers.exasol.adapter` is absent from `sys.modules` + +#### Scenario: Factory produces a working provider +- **WHEN** `SPEC.provider_factory(SPEC)` is called +- **THEN** a `DatabaseProvider` is returned +- **AND** its `schema` is the Exasol `SCHEMA` +- **AND** no `TypeError` about abstract methods is raised, since `ExasolAdapter` is concrete + +#### Scenario: Registration needs no driver +- **WHEN** discovery, the picker and the capability suite run with `pyexasol` not installed +- **THEN** all succeed, because `_import_driver_module` runs only inside `connect()` + +### Requirement: Docker detection recognises exasol/docker-db +`SPEC.docker_detector` SHALL be a `DockerDetector` with `image_patterns=("exasol/docker-db",)`, +`env_vars={}` and `default_user="sys"`. + +`env_vars` is a **required** field on `DockerDetector` with no default. Omitting it raises +`TypeError` while `provider.py` is being imported during discovery, which breaks discovery for every +provider, not just Exasol. The empty mapping is also semantically correct: the `exasol/docker-db` +image takes no credential environment variables — its `sys` / `exasol` defaults are baked in — and +`get_credentials` resolves an empty mapping to `default_user`. + +`default_user_requires_password` SHALL remain `False`, so `sys` is offered even when no password is +discovered. + +#### Scenario: Image pattern matches +- **WHEN** `match_image("exasol/docker-db:latest-8")` is called +- **THEN** it returns `True` + +#### Scenario: Unrelated image does not match +- **WHEN** `match_image("postgres:16")` is called on the Exasol detector +- **THEN** it returns `False` + +#### Scenario: Default user is offered with no environment variables +- **WHEN** `get_credentials({})` is called +- **THEN** the returned `user` is `"sys"` +- **AND** no exception is raised for the empty `env_vars` mapping + +#### Scenario: Detector construction does not break discovery +- **WHEN** `provider.py` is imported +- **THEN** no `TypeError` is raised for a missing `env_vars` argument + +### Requirement: Connection display shows the target schema +`SPEC.display_info` SHALL render `host:port/SCHEMA` when a `schema` option is set, and fall back to +`host:port` when it is empty. + +The default implementation (`adapter_provider.py:61`) appends `endpoint.database`, which Exasol never +populates — there is no `database` field and `supports_multiple_databases` is `False` — so without +this override two connections into the same cluster are indistinguishable in the list. + +#### Scenario: Schema is shown when set +- **WHEN** `display_info` is called for a config with host `localhost`, port `8563` and the `schema` + option `TEST_SQLIT` +- **THEN** the result is `localhost:8563/TEST_SQLIT` + +#### Scenario: Falls back to host and port +- **WHEN** `display_info` is called for a config with no `schema` option +- **THEN** the result is `localhost:8563` with no trailing slash + +### Requirement: The change leaves existing providers untouched +Registering Exasol SHALL NOT change behaviour for any of the 29 existing providers. The only shared +file modified is `domain/config.py`, by addition only: one enum member and one display-order entry. + +`providers/exasol/adapter.py` MUST NOT be modified by this change. + +#### Scenario: Existing providers still resolve +- **WHEN** `get_supported_db_types()` is called +- **THEN** all 29 previously registered types are still present + +#### Scenario: Adapter is unchanged +- **WHEN** the diff for this change is inspected +- **THEN** `providers/exasol/adapter.py` does not appear in it + +#### Scenario: Full unit suite is unaffected +- **WHEN** the unit-test job's pytest command is run +- **THEN** no previously passing test fails + diff --git a/openspec/specs/exasol-unit-coverage/spec.md b/openspec/specs/exasol-unit-coverage/spec.md new file mode 100644 index 00000000..4c93fa9e --- /dev/null +++ b/openspec/specs/exasol-unit-coverage/spec.md @@ -0,0 +1,326 @@ +# exasol-unit-coverage Specification + +## Purpose +Defines the driver-free unit-test coverage that pins `ExasolAdapter` and the Exasol connection +schema: where the tests live, how they fake `pyexasol` by seeding `sys.modules` so they pass with no +extras installed, and which behavioural decision each one asserts. + +These tests pin *sqlit's call shape*, not pyexasol's contract. A mocked `connect` accepts any +keyword, so a driver-side rename cannot fail them — validating the real driver API is the Docker +integration test's job. Two consequences shape the requirements below: unused credential keys are +asserted **absent** rather than empty, because pyexasol's login branches on token truthiness; and +every statement mock sets `result_type` explicitly, because on a bare mock the guard's inequality is +trivially true and a result-set test would pass while asserting nothing. + +Scope boundaries: the behaviour being pinned is specified in `exasol-adapter` and +`exasol-provider-registration`; driver installability and the pytest marker live in +`exasol-driver-packaging`. + +## Requirements +### Requirement: Exasol unit tests run without the driver installed + +The test files introduced by this change SHALL pass in an environment where `pyexasol` is not +importable. Where a test needs the driver, it SHALL supply a fake by seeding +`sys.modules["pyexasol"]` (`unittest.mock.patch.dict`), which `importlib.import_module` — and +therefore `_import_driver_module` — returns without touching the filesystem. + +Tests MUST NOT patch `ExasolAdapter._import_driver_module` itself; the lazy-import call site, +including the `driver_name` / `extra_name` / `package_name` arguments that produce sqlit's install +prompt, is part of what is under test (design D2). + +#### Scenario: Tests pass with no extras installed +- **WHEN** `uv sync --group test --no-dev` is used, as the default CI unit job does +- **AND** `uv run pytest tests/connections/providers/exasol/ -v` is run +- **THEN** every test passes +- **AND** no test is skipped for a missing driver + +#### Scenario: Tests are collected by the default unit job +- **WHEN** the unit-test command from `.github/workflows/ci.yml` is run +- **THEN** the three new Exasol test files are collected +- **AND** no `--ignore` entry is required for them + +#### Scenario: The driver fake is installed through sys.modules +- **WHEN** a test that calls `adapter.connect(config)` is inspected +- **THEN** it seeds a mock under the `"pyexasol"` key of `sys.modules` +- **AND** it reads the recorded call from that mock's `connect` attribute + +### Requirement: Tests live in the per-provider test package + +The tests SHALL reside in `tests/connections/providers/exasol/` as a package with an empty +`__init__.py`, mirroring `tests/connections/providers/hana/`, split as `test_schema.py`, +`test_connect.py` and `test_adapter.py`. + +#### Scenario: Package layout +- **WHEN** the test tree is inspected +- **THEN** `tests/connections/providers/exasol/__init__.py` exists and is empty +- **AND** `test_schema.py`, `test_connect.py` and `test_adapter.py` sit beside it + +#### Scenario: The directory is a single runnable gate +- **WHEN** `uv run pytest tests/connections/providers/exasol/ -v` is run +- **THEN** it collects and runs every Exasol unit test and nothing else + +### Requirement: Conditional credential-field visibility is pinned + +`test_schema.py` SHALL evaluate the `visible_when` predicates on `SCHEMA.fields` against a form-values +dict for each `authenticator` value, asserting that exactly the selected method's credential fields +are visible and the other methods' fields are hidden. Fields with no `visible_when` (`server`, `port`, +`authenticator`, `schema`) are always visible. No driver is involved. + +#### Scenario: Password authentication +- **WHEN** the predicates are evaluated with `{"authenticator": "password"}` +- **THEN** `username` and `password` are visible +- **AND** `access_token` and `refresh_token` are hidden + +#### Scenario: Access-token authentication +- **WHEN** the predicates are evaluated with `{"authenticator": "access_token"}` +- **THEN** `access_token` is visible +- **AND** `username`, `password` and `refresh_token` are hidden + +#### Scenario: Refresh-token authentication +- **WHEN** the predicates are evaluated with `{"authenticator": "refresh_token"}` +- **THEN** `refresh_token` is visible +- **AND** `username`, `password` and `access_token` are hidden + +#### Scenario: Absent authenticator falls back to password +- **WHEN** the predicates are evaluated with an empty dict +- **THEN** the visibility matches the `"password"` case, because each predicate defaults the lookup to + `"password"` + +#### Scenario: Unconditional fields stay visible +- **WHEN** the predicates are evaluated with any `authenticator` value +- **THEN** `server`, `port`, `authenticator` and `schema` carry no `visible_when` and are visible + +### Requirement: Credentials passed to the driver match the selected authenticator, and only those + +`test_connect.py` SHALL assert, for each `authenticator` value, both which credential kwargs +`pyexasol.connect` receives **and that the other methods' kwargs are absent from the call +entirely**. Asserting that an unused key is empty is not sufficient: pyexasol's login branches on +token truthiness, so a present-but-empty `access_token` silently reverts to password login +(design D3). + +#### Scenario: Password authentication +- **WHEN** `connect()` runs with `authenticator` unset or `"password"` +- **THEN** the recorded kwargs contain `user` and `password` from the endpoint +- **AND** neither `access_token` nor `refresh_token` appears as a key + +#### Scenario: Access-token authentication +- **WHEN** `connect()` runs with `authenticator == "access_token"` +- **THEN** the recorded kwargs contain `access_token` +- **AND** none of `user`, `password` or `refresh_token` appears as a key + +#### Scenario: Refresh-token authentication +- **WHEN** `connect()` runs with `authenticator == "refresh_token"` +- **THEN** the recorded kwargs contain `refresh_token` +- **AND** none of `user`, `password` or `access_token` appears as a key + +### Requirement: Endpoint, schema and autocommit kwargs are pinned + +`test_connect.py` SHALL assert the non-credential kwargs `connect()` builds from the config: the +`dsn` string, the port fallback, the `schema` option and `autocommit`. It SHALL also cover the +rejection path for a config that carries no TCP endpoint. + +#### Scenario: DSN is host and port joined by a colon +- **WHEN** `connect()` runs against an endpoint with host `db.example.com` and port `1234` +- **THEN** the recorded kwargs contain `dsn == "db.example.com:1234"` + +#### Scenario: Absent port falls back to the registered default +- **WHEN** the endpoint carries no port +- **THEN** the `dsn` port segment is `8563`, resolved through `get_default_port("exasol")` + +#### Scenario: Schema option is forwarded +- **WHEN** the `schema` option is set to `TEST_SQLIT` +- **THEN** the recorded kwargs contain `schema == "TEST_SQLIT"` +- **AND** when the option is unset the value is the empty string, not omitted + +#### Scenario: Autocommit is enabled at connect time +- **WHEN** `connect()` runs with any authenticator +- **THEN** the recorded kwargs contain `autocommit is True` + +#### Scenario: A non-TCP configuration is rejected before importing the driver +- **WHEN** `connect()` is called with a config whose `tcp_endpoint` is `None` +- **THEN** `ValueError` is raised +- **AND** the fake driver's `connect` is never called + +### Requirement: The TLS mode mapping is pinned for every mode + +`test_connect.py` SHALL cover all five `tls_mode` values and assert the resulting `encryption` and +`websocket_sslopt` kwargs. + +#### Scenario: Encryption disabled +- **WHEN** `tls_mode` is `disable` +- **THEN** the recorded kwargs contain `encryption is False` +- **AND** no `websocket_sslopt` key is present + +#### Scenario: Driver default +- **WHEN** `tls_mode` is `default` or unset +- **THEN** the recorded kwargs contain `encryption is True` +- **AND** no `websocket_sslopt` key is present + +#### Scenario: Encrypted without verification +- **WHEN** `tls_mode` is `require` +- **THEN** the recorded kwargs contain `encryption is True` +- **AND** `websocket_sslopt == {"cert_reqs": ssl.CERT_NONE}` + +#### Scenario: Verifying modes request certificate validation +- **WHEN** `tls_mode` is `verify-ca` or `verify-full` +- **THEN** the recorded kwargs contain `encryption is True` +- **AND** `websocket_sslopt["cert_reqs"] == ssl.CERT_REQUIRED` + +#### Scenario: Configured certificate files are forwarded +- **WHEN** `tls_mode` is a verifying mode and `tls_ca`, `tls_cert` and `tls_key` are set +- **THEN** `websocket_sslopt` carries them as `ca_certs`, `certfile` and `keyfile` + +#### Scenario: Unconfigured certificate files are omitted +- **WHEN** `tls_mode` is a verifying mode and the certificate options are unset or whitespace +- **THEN** `websocket_sslopt` contains only `cert_reqs` +- **AND** no `ca_certs`, `certfile` or `keyfile` key is present + +### Requirement: extra_options pass through and win on conflict + +`test_connect.py` SHALL assert that `config.extra_options` reaches the driver verbatim and that it +overrides the kwargs `connect()` computes, since it is applied last. + +#### Scenario: Unknown options reach the driver +- **WHEN** `extra_options` contains a key sqlit does not know +- **THEN** the recorded kwargs contain that key and value verbatim + +#### Scenario: extra_options override computed kwargs +- **WHEN** `extra_options` sets a key that `connect()` also computes, such as `encryption` +- **THEN** the recorded kwargs carry the `extra_options` value + +### Requirement: The introspection row-shape contract is pinned + +`test_adapter.py` SHALL drive introspection off a mocked `conn.meta`, returning +already-fetched lists of dicts with UPPERCASE keys for the `list_*` helpers and an object requiring +`.fetchall()` for `execute_snapshot`. A test that fed tuples would pass against an index-based +implementation and so would not pin the contract at all. + +#### Scenario: Tables are read by key +- **WHEN** `conn.meta.list_tables()` yields dicts with `TABLE_SCHEMA` and `TABLE_NAME` +- **THEN** `get_tables` returns the corresponding `(schema, name)` tuples in order + +#### Scenario: Views are read by key +- **WHEN** `conn.meta.list_views()` yields dicts with `VIEW_SCHEMA` and `VIEW_NAME` +- **THEN** `get_views` returns the corresponding `(schema, name)` tuples in order + +#### Scenario: Columns combine type and primary-key information +- **WHEN** `execute_snapshot` yields a primary-key row for `ID` and `list_columns` yields `ID` then + `NAME` +- **THEN** `get_columns` returns `ColumnInfo("ID", , is_primary_key=True)` followed by + `ColumnInfo("NAME", , is_primary_key=False)` +- **AND** the order from `list_columns` is preserved + +#### Scenario: The primary-key query is snapshot-executed and parameterised +- **WHEN** `get_columns` runs for schema `S` and table `T` +- **THEN** the query goes through `conn.meta.execute_snapshot`, not `conn.execute` +- **AND** it filters on `CONSTRAINT_TYPE = 'PRIMARY KEY'` +- **AND** the schema and table are passed as query parameters rather than interpolated + +#### Scenario: A table with no primary key yields no flagged column +- **WHEN** the primary-key snapshot returns no rows +- **THEN** every returned `ColumnInfo` has `is_primary_key` false + +#### Scenario: An unspecified schema is passed through as empty +- **WHEN** `get_columns` is called without a schema +- **THEN** `list_columns` receives the empty string +- **AND** this pins a deliberate spec-faithful choice — the path is unreachable from the explorer, + and no fallback is substituted (design D8) + +#### Scenario: Procedures come from the scripting scripts +- **WHEN** `get_procedures` runs +- **THEN** it snapshot-executes against `SYS.EXA_ALL_SCRIPTS` filtered to `SCRIPT_TYPE = 'SCRIPTING'` +- **AND** returns the `SCRIPT_NAME` values + +#### Scenario: Unsupported object kinds return empty without querying +- **WHEN** `get_databases`, `get_indexes`, `get_triggers` or `get_sequences` is called +- **THEN** each returns an empty list +- **AND** the connection mock records no call at all + +### Requirement: Result-set detection precedes any fetch + +`test_adapter.py` SHALL assert that `execute_query` inspects `stmt.result_type` before any fetch, +because `ExaStatement.__next__` raises `ExaRuntimeError` for a `rowCount` statement and `fetchmany()` +iterates. + +Every statement mock MUST set `result_type` explicitly: on a bare `MagicMock` the inequality is +trivially true, which would let a result-set test pass while asserting nothing (see the second risk +in design.md). + +#### Scenario: A row-count statement returns empty without fetching +- **WHEN** `execute_query` runs against a statement whose `result_type` is `rowCount` +- **THEN** it returns empty columns, empty rows and `truncated` false +- **AND** neither `fetchall` nor `fetchmany` is called on the statement + +#### Scenario: A result-set statement is fetched +- **WHEN** `execute_query` runs against a statement whose `result_type` is `resultSet` +- **THEN** the column names come from `stmt.column_names()` +- **AND** the rows are returned as tuples + +### Requirement: The truncation flag is pinned at the max_rows boundary + +`test_adapter.py` SHALL cover the `max_rows` boundary in both directions: `execute_query` requests +`max_rows + 1` rows to detect truncation, MUST report it, and MUST trim the surplus before +returning. + +#### Scenario: Unlimited fetch is never truncated +- **WHEN** `execute_query` runs with `max_rows` unset +- **THEN** all rows from `fetchall` are returned +- **AND** `truncated` is false + +#### Scenario: Exactly max_rows rows available +- **WHEN** `max_rows` is 2 and the statement yields 2 rows +- **THEN** 2 rows are returned +- **AND** `truncated` is false + +#### Scenario: More than max_rows rows available +- **WHEN** `max_rows` is 2 and the statement yields 3 rows +- **THEN** exactly 2 rows are returned +- **AND** `truncated` is true + +#### Scenario: One extra row is requested +- **WHEN** `execute_query` runs with `max_rows` set to 2 +- **THEN** `fetchmany` is called with 3 + +### Requirement: Row counts and the test query use pyexasol's statement API + +`test_adapter.py` SHALL pin the two places the adapter departs from the DB-API shape of its base +class: `rowcount` is a **method** on `ExaStatement` rather than a property, and `execute_test_query` +is overridden because the inherited implementation calls `conn.cursor()`, which pyexasol does not +provide. + +#### Scenario: rowcount is invoked as a method +- **WHEN** `execute_non_query` runs +- **THEN** the statement's `rowcount` is **called**, not read as a property +- **AND** the returned value is an `int` + +#### Scenario: No explicit commit is issued +- **WHEN** `execute_non_query` runs +- **THEN** the connection mock records no `commit` call, because `autocommit=True` is set at connect + time + +#### Scenario: The connection test avoids cursor() +- **WHEN** `execute_test_query` runs +- **THEN** `conn.execute("SELECT 1")` is called and `fetchval()` is taken from the result +- **AND** `conn.cursor` is never accessed + +### Requirement: Identifier quoting and select building are pinned + +`test_adapter.py` SHALL assert that `quote_identifier` wraps in double quotes and doubles any +embedded double quote, and that `build_select_query` MUST omit the schema segment entirely when no +schema is given rather than emitting a leading dot. + +#### Scenario: Plain identifier +- **WHEN** `quote_identifier("MY_TABLE")` is called +- **THEN** it returns `"MY_TABLE"` wrapped in double quotes + +#### Scenario: Embedded double quote is doubled +- **WHEN** `quote_identifier` is called with an identifier containing a double quote +- **THEN** that character is doubled inside the quoted result + +#### Scenario: Schema-qualified select +- **WHEN** `build_select_query("T", 10, schema="S")` is called +- **THEN** it returns a `SELECT * FROM` against the quoted `"S"."T"` with `LIMIT 10` + +#### Scenario: Select without a schema omits the schema segment +- **WHEN** `build_select_query("T", 10)` is called with no schema +- **THEN** the result references only the quoted table, with no leading dot diff --git a/plan.md b/plan.md new file mode 100644 index 00000000..e75cf9cf --- /dev/null +++ b/plan.md @@ -0,0 +1,578 @@ +# Add Exasol provider to sqlit + +> **Multi-session plan.** Read `## How to use this plan` before doing anything. + +--- + +## How to use this plan + +This plan is designed to be implemented across several sessions. The status table below is the +single source of truth for what is done. + +**Protocol for every session:** + +1. Read the **Status** table. Ignore prose elsewhere until you know where you are. +2. Pick work: the lowest-numbered step whose `Depends on` steps are all `done`. +3. **If that step belongs to an atomic group, you must take the entire group in this session + or take nothing.** Atomic groups leave the repo red (failing tests / failing CI) if split + across sessions. See `## Atomic groups`. +4. Set the picked steps to `wip` in the Status table before starting. +5. Implement, then run the step's **Verify** command. Only when it passes, set the step to `done`. +6. Append one line to the **Session log**. +7. If you stop mid-step, leave it `wip` and write what is half-done in the Session log. A `wip` + step means "read the diff before continuing" — do not assume it is untouched. + +**States:** `todo` / `wip` / `done` / `skipped` (with a reason in the Session log). + +**Never** mark a step `done` without running its Verify command. **Never** start a step in an +atomic group unless the whole group fits in the session. + +--- + +## Status + +| # | Step | Group | Depends on | State | +|---|------|-------|-----------|-------| +| 1 | Package skeleton + `ExasolAdapter` shell & capabilities | — | — | `done` | +| 2 | `adapter.py`: `connect()` + `_tls_args()` | — | 1 | `done` | +| 3 | `adapter.py`: introspection methods | — | 1 | `done` | +| 4 | `adapter.py`: query execution + identifier quoting | — | 1 | `done` | +| 5 | `schema.py` | **ACT** | 1 | `done` | +| 6 | `DatabaseType.EXASOL` + display order | **ACT** | — | `done` | +| 7 | `provider.py` + `register_provider(SPEC)` | **ACT** | 2, 3, 4, 5, 6 | `done` | +| 8 | `pyproject.toml`: extra, mypy override, pytest marker | — | — | `done` | +| 9 | Unit tests: `test_schema.py` | — | 5, 7 | `done` | +| 10 | Unit tests: `test_connect.py` | — | 2, 7, 8 | `done` | +| 11 | Unit tests: `test_adapter.py` | — | 3, 4, 7, 8 | `done` | +| 12 | Docker compose service + `tests/fixtures/exasol.py` + conftest | **INT** | 7, 8 | `done` | +| 13 | `tests/test_exasol.py` | **INT** | 12 | `done` | +| 14 | `.github/workflows/ci.yml`: unit-job ignore + `test-exasol` job | **INT** | 13 | `done` | +| 15 | Docs: `CONTRIBUTING.md` + `README.md` | — | 7 | `done` | + +**Progress: 15 / 15 done.** Update this count when you change the table. + +--- + +## Atomic groups + +A group must be completed within one session. Splitting it leaves the branch failing. + +### Group `ACT` — steps 5, 6, 7 (activation) + +**Why atomic:** providers are auto-discovered. `providers/catalog.py:22` (`_discover_providers`) +walks every subpackage of `providers/` and imports `/provider.py`. The instant `provider.py` +exists, Exasol is a live registered provider, and: + +- `tests/test_schema_capabilities.py::TestCatalogConsistency::test_database_type_enum_matches_schema` + asserts `{t.value for t in DatabaseType} == set(get_supported_db_types())` — **exact set + equality**. `provider.py` without step 6 fails. Step 6 without `provider.py` also fails. +- The same test class calls `get_connection_schema(db_type)` for every discovered type and asserts + `schema.db_type == db_type` and `schema.display_name == get_display_name(db_type)`. So `schema.py` + (step 5) must exist and agree with `ProviderSpec` in the same commit. + +**Estimated size:** small — three short files/edits. Comfortably one session. + +**Ordering within the group:** write 5 and 6 first, `provider.py` last. `provider.py` is the switch +that turns discovery on. + +### Group `INT` — steps 13, 14 (integration test + CI) + +**Why atomic:** the unit-test CI job excludes integration tests *by filename* +(`.github/workflows/ci.yml:71-82`). Creating `tests/test_exasol.py` without adding +`--ignore=tests/test_exasol.py` to that job means CI collects and runs a test that needs a Docker +container it does not have. + +Step 12 (compose + fixtures) is safe on its own because the fixture guards with `pytest.skip`, but +13 and 14 must land together. + +**Recommended merge:** take 12 + 13 + 14 in one session. 12 alone delivers nothing testable. + +### Steps 2, 3, 4 — *not* atomic, but keep them before `ACT` + +`provider.py`'s `provider_factory` imports `ExasolAdapter` **lazily**, so an incomplete `adapter.py` +breaks no test (mypy flags abstract classes only at instantiation, and nothing instantiates it until +a connection is opened). This is why the adapter is built first, in pieces, and activation comes +after: the repo is never in a "provider is selectable but crashes on connect" state between sessions. + +If you have a large session available, 1-4 together is a clean unit of work. + +--- + +## Context + +sqlit ships ~29 database providers but not Exasol. We want to add one and upstream it as a PR +to `Maxteabag/sqlit`. + +**There is no "adding a new dialect" documentation in this repo.** `CONTRIBUTING.md` covers only +dev setup, test commands, per-database env vars, and the product vision. `docs/` is screenshots +and demo GIFs. `.github/` has only CI workflows. So the convention has to be read off the code — +which is what this plan encodes. + +The convention: providers are **auto-discovered** (see Group `ACT` above). Adding a provider is +therefore: drop in a 4-file package, then patch the handful of places that are *not* auto-discovered. + +Closest existing templates: `providers/hana/` (single database, many schemas, enterprise DB) and +`providers/snowflake/` (auth-method dropdown). + +## Decisions + +| Decision | Choice | +|---|---| +| Driver | `pyexasol` (2.3.2, requires-python `>=3.10,<3.15` — compatible with sqlit's `>=3.10`) | +| Base class | `DatabaseAdapter` directly, **not** `CursorBasedAdapter` — pyexasol is a native WebSocket client, not DB-API 2.0 | +| Schema model | Schema-only, HANA-style: `supports_multiple_databases=False`, `get_databases()` returns `[]`, tables grouped by schema in the explorer | +| Auth | Dropdown: Username & Password / OpenID Access Token / OpenID Refresh Token | +| TLS | Reuse shared `TLS_FIELDS` + `providers/tls.py` helpers, mapped to pyexasol `encryption` + `websocket_sslopt` | +| Tests | Mocked unit tests **and** Docker integration tests (`exasol/docker-db`, enterprise profile) | + +--- + +# Steps + +## Step 1 — Package skeleton + `ExasolAdapter` shell & capabilities + +**Group:** — | **Depends on:** — | **State:** `done` + +**Files:** `sqlit/domains/connections/providers/exasol/__init__.py`, +`sqlit/domains/connections/providers/exasol/adapter.py` + +`__init__.py` is `"""Provider package."""` — matches every other provider. + +Create `class ExasolAdapter(DatabaseAdapter)` with the capability properties below and every +abstract method present but raising `NotImplementedError` (filled in by steps 2-4). Capabilities are +declared as properties; `build_adapter_provider` reads them via `getattr`. + +| Property | Value | Why | +|---|---|---| +| `supports_multiple_databases` | `False` | Exasol has no database layer | +| `supports_cross_database_queries` | `False` | Mirrors `HanaAdapter`; only affects the unused `requires_database_selection()` and the database segment in `qualified_name()`, which is always empty here | +| `supports_stored_procedures` | `True` | Exposed from `EXA_ALL_SCRIPTS` | +| `supports_indexes` | `False` | Exasol indexes are auto-managed and unnamed | +| `supports_triggers` | `False` | Exasol has no triggers | +| `supports_sequences` | `False` | Exasol uses IDENTITY columns | +| `default_schema` | `""` | No universal default; every table stays schema-qualified | + +Do **not** override `supports_process_worker`. `process_worker.py:327` calls +`provider.connection_factory.connect(...)` *inside* the child process — it opens its own connection +rather than pickling one — so the WebSocket is fine, and pyexasol returns plain picklable tuples. +(`SurrealDBAdapter` disables it, but that reasoning does not apply here.) + +**Verify:** + +``` +uv run python -c "from sqlit.domains.connections.providers.exasol.adapter import ExasolAdapter; print(ExasolAdapter)" +uv run ruff check sqlit && uv run mypy sqlit +``` + +--- + +## Step 2 — `adapter.py`: `connect()` + `_tls_args()` + +**Group:** — | **Depends on:** 1 | **State:** `done` + +**Files:** `providers/exasol/adapter.py` + +Lazily import the driver through the inherited `self._import_driver_module("pyexasol", ...)`, so a +missing driver produces sqlit's normal install prompt: + +```python +connect_args = {"dsn": f"{host}:{port}", "schema": config.get_option("schema", ""), "autocommit": True} +# authenticator == "password" -> user=..., password=... +# authenticator == "access_token" -> access_token=... +# authenticator == "refresh_token" -> refresh_token=... +connect_args.update(self._tls_args(config)) +connect_args.update(config.extra_options) +return pyexasol.connect(**connect_args) +``` + +`autocommit=True` matches the house convention (`postgresql/adapter.py:124`, `mysql/adapter.py:87`, +`mssql/adapter.py:265`) and is also pyexasol's default. + +`_tls_args(config)` uses `get_tls_mode`, `tls_mode_verifies_cert`, `get_tls_files` from +`providers/tls.py`: + +| `tls_mode` | pyexasol kwargs | +|---|---| +| `default` | `encryption=True` (pyexasol default) | +| `disable` | `encryption=False` | +| `require` | `encryption=True, websocket_sslopt={"cert_reqs": ssl.CERT_NONE}` | +| `verify-ca` / `verify-full` | `encryption=True, websocket_sslopt={"cert_reqs": ssl.CERT_REQUIRED, "ca_certs": ..., "certfile": ..., "keyfile": ...}` | + +This matters: pyexasol defaults to `encryption=True`, and both `exasol/docker-db` and most on-prem +installs use a self-signed cert, so without `require` a naive connect fails cert validation. + +**Verify:** `uv run ruff check sqlit && uv run mypy sqlit` (behaviour is covered by step 10). + +--- + +## Step 3 — `adapter.py`: introspection methods + +**Group:** — | **Depends on:** 1 | **State:** `done` + +**Files:** `providers/exasol/adapter.py` + +Use `conn.meta.*`, which wraps every query in Exasol's `/*snapshot execution*/` hint and so cannot be +blocked by metadata locks. This is the pyexasol-recommended path and is strictly better than +hand-rolled SQL. + +| Method | Implementation | +|---|---| +| `get_databases` | `return []` | +| `get_tables` | `conn.meta.list_tables()` -> `(TABLE_SCHEMA, TABLE_NAME)` | +| `get_views` | `conn.meta.list_views()` -> `(VIEW_SCHEMA, VIEW_NAME)` | +| `get_columns` | `conn.meta.list_columns(schema, table)` for name/type + `conn.meta.execute_snapshot` on `SYS.EXA_ALL_CONSTRAINT_COLUMNS WHERE CONSTRAINT_TYPE = 'PRIMARY KEY'` for the PK set -> `ColumnInfo(name, data_type, is_primary_key)` | +| `get_procedures` | `conn.meta.execute_snapshot` on `SYS.EXA_ALL_SCRIPTS` (columns `SCRIPT_SCHEMA`, `SCRIPT_NAME`, `SCRIPT_TYPE`) | +| `get_indexes` / `get_triggers` / `get_sequences` | `return []` (abstract on the base class, must be defined even though the capability flags are `False`) | + +All system-table column names above are verified against the Exasol docs. + +**Verify:** `uv run ruff check sqlit && uv run mypy sqlit` (behaviour is covered by step 11). + +--- + +## Step 4 — `adapter.py`: query execution + identifier quoting + +**Group:** — | **Depends on:** 1 | **State:** `done` + +**Files:** `providers/exasol/adapter.py` + +This is the reason this cannot be `CursorBasedAdapter` — pyexasol has no `.cursor()`. + +```python +def execute_query(self, conn, query, max_rows=None): + stmt = conn.execute(query) + columns = stmt.column_names() + if not columns: + return [], [], False + if max_rows is None: + return columns, list(stmt.fetchall()), False + rows = stmt.fetchmany(max_rows + 1) # one extra to detect truncation + truncated = len(rows) > max_rows + return columns, [tuple(r) for r in rows[:max_rows]], truncated + +def execute_non_query(self, conn, query): + return int(conn.execute(query).rowcount()) # rowcount() is a method, not a property +``` + +Also override `execute_test_query` (the base implementation at `adapters/base.py:215` calls +`conn.cursor()`): `conn.execute("SELECT 1").fetchval()`. + +`quote_identifier` -> double quotes with `"` doubled. `build_select_query` -> +`SELECT * FROM "S"."T" LIMIT n`. + +**Open item to resolve in this step:** the code above uses `column_names()` being empty to mean "no +result set". `stmt.result_type` exists and may be cleaner — confirm against the installed pyexasol +and use whichever is public API. Record the choice in the Session log. + +**Verify:** `uv run ruff check sqlit && uv run mypy sqlit` (behaviour is covered by step 11). + +--- + +## Step 5 — `schema.py` + +**Group:** **ACT** (with 6, 7) | **Depends on:** 1 | **State:** `done` + +**Files:** `providers/exasol/schema.py` + +Follows `providers/snowflake/schema.py` for the conditional-visibility pattern and +`providers/clickhouse/schema.py` for the `+ SSH_FIELDS + TLS_FIELDS` tail. + +```python +SCHEMA = ConnectionSchema( + db_type="exasol", + display_name="Exasol", + fields=( + _server_field(), # from schema_helpers + _port_field("8563"), + SchemaField("authenticator", "Authentication", FieldType.DROPDOWN, + options=(SelectOption("password", "Username & Password"), + SelectOption("access_token", "OpenID Access Token"), + SelectOption("refresh_token", "OpenID Refresh Token")), + default="password"), + _username_field(), # visible_when authenticator == "password" + _password_field(), # visible_when authenticator == "password" + SchemaField("access_token", "Access Token", FieldType.PASSWORD, ...), # visible_when access_token + SchemaField("refresh_token", "Refresh Token", FieldType.PASSWORD, ...), # visible_when refresh_token + SchemaField("schema", "Schema", placeholder="(empty = browse all)"), + ) + SSH_FIELDS + TLS_FIELDS, + default_port="8563", + has_advanced_auth=True, +) +``` + +Field name `authenticator` (not `auth_type`) deliberately mirrors Snowflake and avoids the legacy +top-level `auth_type` key that `ConnectionConfig.from_dict` special-cases (`domain/config.py:158`). + +Non-endpoint fields (`authenticator`, `access_token`, `refresh_token`, `schema`) land in +`config.options` automatically (`config.py:240-246`) and are read with `config.get_option(...)`. +The CLI derives `--authenticator` / `--access-token` / `--schema` flags from these fields for free +(`cli/helpers.py:43`). + +`display_name` must be exactly `"Exasol"` — step 7's `ProviderSpec.display_name` has to match, or +`test_display_names_match_schema` fails. + +**Verify:** part of the Group `ACT` verify at step 7. + +--- + +## Step 6 — `DatabaseType.EXASOL` + display order + +**Group:** **ACT** (with 5, 7) | **Depends on:** — | **State:** `done` + +**Files:** `sqlit/domains/connections/domain/config.py` + +Add `EXASOL = "exasol"` to `DatabaseType` (enum starts line 11; place it between `DB2` and +`FIREBIRD`) **and** an entry in `DATABASE_TYPE_DISPLAY_ORDER` (line 44) near the other enterprise +engines, after `TERADATA`. + +Both are needed: `tests/test_schema_capabilities.py` asserts the enum set *exactly equals* the +discovered provider set, and the connection picker renders only from the display order +(`ui/screens/connection.py:331`). Nothing tests the display order for completeness, so a missing +entry here fails silently as "Exasol is not in the picker". + +**Verify:** part of the Group `ACT` verify at step 7. + +--- + +## Step 7 — `provider.py` + `register_provider(SPEC)` + +**Group:** **ACT** (with 5, 6) | **Depends on:** 2, 3, 4, 5, 6 | **State:** `done` + +**Files:** `providers/exasol/provider.py` + +Same shape as `providers/teradata/provider.py`: + +```python +SPEC = ProviderSpec( + db_type="exasol", display_name="Exasol", + schema_path=("sqlit.domains.connections.providers.exasol.schema", "SCHEMA"), + supports_ssh=True, has_advanced_auth=True, default_port="8563", + badge_label="Exasol", url_schemes=("exasol", "exa"), + display_info=_display_info, # "host:port/SCHEMA" + provider_factory=_provider_factory, # lazily imports ExasolAdapter + docker_detector=DockerDetector(image_patterns=("exasol/docker-db",), default_user="sys"), +) +register_provider(SPEC) +``` + +**Verify (this is the Group `ACT` gate — must pass before marking 5, 6, 7 `done`):** + +``` +uv run pytest tests/test_schema_capabilities.py -v +uv run ruff check sqlit && uv run mypy sqlit +``` + +--- + +## Step 8 — `pyproject.toml`: extra, mypy override, pytest marker + +**Group:** — | **Depends on:** — | **State:** `todo` + +**Files:** `pyproject.toml` + +- `exasol = ["pyexasol>=2.0.0"]` extra; add the same pin to `all`. +- Add `"pyexasol"` to the mypy `ignore_missing_imports` override list (~line 228). +- Add `"exasol: Exasol database tests"` to `[tool.pytest.ini_options] markers`. + +**Verify:** + +``` +uv sync --extra exasol +uv run python -c "import pyexasol; print(pyexasol.__version__)" +uv run mypy sqlit +``` + +--- + +## Step 9 — Unit tests: `test_schema.py` + +**Group:** — | **Depends on:** 5, 7 | **State:** `todo` + +**Files:** `tests/connections/providers/exasol/__init__.py`, +`tests/connections/providers/exasol/test_schema.py` + +Assert the `visible_when` predicates hide/show the right credential fields for each `authenticator` +value. No driver needed. + +**Verify:** `uv run pytest tests/connections/providers/exasol/test_schema.py -v` + +--- + +## Step 10 — Unit tests: `test_connect.py` + +**Group:** — | **Depends on:** 2, 7, 8 | **State:** `todo` + +**Files:** `tests/connections/providers/exasol/test_connect.py` + +Mocked, runs in the default CI job. Follow +`tests/connections/providers/hana/test_get_columns.py` (`MagicMock` connection, assert on the SQL and +kwargs actually passed): + +- each `authenticator` value produces the right `pyexasol.connect` kwargs (`password` vs + `access_token` vs `refresh_token`, **and that the unused ones are absent**); +- each `tls_mode` produces the right `encryption` / `websocket_sslopt`; +- `extra_options` passthrough. + +**Note:** patch the module that `_import_driver_module("pyexasol", ...)` returns rather than relying +on pyexasol being installed, so this test also passes in the no-extras unit job. If patching the lazy +import proves awkward, that is the one place this step may need a design decision — record it in the +Session log. + +**Verify:** `uv run pytest tests/connections/providers/exasol/test_connect.py -v` + +--- + +## Step 11 — Unit tests: `test_adapter.py` + +**Group:** — | **Depends on:** 3, 4, 7, 8 | **State:** `todo` + +**Files:** `tests/connections/providers/exasol/test_adapter.py` + +- `get_tables` / `get_views` / `get_columns` shapes off mocked `conn.meta.*`; PK detection. +- `execute_query` truncation flag at the `max_rows` boundary (exactly `max_rows` rows -> not + truncated; `max_rows + 1` -> truncated and trimmed). +- `execute_non_query` calls `rowcount()` (method, not property). +- `quote_identifier` escaping of an embedded double quote. + +**Verify:** `uv run pytest tests/connections/providers/exasol/ -v` + +--- + +## Step 12 — Docker compose service + fixtures + conftest + +**Group:** **INT** (recommended: merge with 13, 14) | **Depends on:** 7, 8 | **State:** `done` + +**Files:** `infra/docker/docker-compose.test.yml`, `tests/fixtures/exasol.py`, `tests/conftest.py` + +Compose: an `exasol` service under the existing `enterprise` profile (alongside `db2` / +`oracle11g`), since the image is large and slow: `image: exasol/docker-db:latest-8`, +`privileged: true`, `stop_grace_period: 120s`, `ports: ["${EXASOL_PORT:-8563}:8563"]`. + +`tests/fixtures/exasol.py` mirrors `tests/fixtures/clickhouse.py`: env-var constants, `is_port_open` +guard, `pytest.skip` when the container or driver is absent, and an `exasol_db` fixture that +creates/drops a `TEST_SQLIT` schema. Defaults `sys` / `exasol` (the `docker-db` defaults). + +Register with `from tests.fixtures.exasol import *` in `tests/conftest.py` (the fixture import block +is alphabetical, starting line 5). + +**The fixture module must be import-safe with no pyexasol installed** — no top-level driver import — +because `conftest.py` is imported by the unit-test job. + +**Verify:** + +``` +docker compose -f infra/docker/docker-compose.test.yml --profile enterprise config +uv run pytest tests/connections -v # conftest still imports cleanly +``` + +--- + +## Step 13 — `tests/test_exasol.py` + +**Group:** **INT** — must land with 14 | **Depends on:** 12 | **State:** `done` + +**Files:** `tests/test_exasol.py` + +`TestExasolIntegration(BaseDatabaseTests)` with a `DatabaseTestConfig(db_type="exasol", +display_name="Exasol", ...)`, plus a `test_create_exasol_connection` CLI test like +`tests/test_clickhouse.py:26`. + +The connection must be created with `--tls-mode require`, since `docker-db` presents a self-signed +cert — which conveniently makes the integration test exercise the TLS mapping from step 2. + +**Do not mark `done` without step 14.** This file breaks the unit CI job until 14 adds its ignore. + +**Verify:** with the container up (see step 12; this image takes several minutes to boot): +`uv run pytest tests/test_exasol.py -v` + +--- + +## Step 14 — `.github/workflows/ci.yml` + +**Group:** **INT** — must land with 13 | **Depends on:** 13 | **State:** `done` + +**Files:** `.github/workflows/ci.yml` + +- Add `--ignore=tests/test_exasol.py` to the unit-test job's exclude list (after line 82, + `--ignore=tests/test_clickhouse.py`). +- Add a `test-exasol` job modelled on `test-clickhouse` (line ~440): + `uv sync --group test --no-dev --extra exasol`, start the container, poll until port 8563 accepts, + then run `pytest tests/test_exasol.py`. + +**Verify:** the unit-test command from `ci.yml:70-82` (now including the exasol ignore) collects and +passes locally: + +``` +uv run pytest tests/ -v --ignore=tests/test_sqlite.py --ignore=tests/test_mssql.py \ + --ignore=tests/test_postgresql.py --ignore=tests/test_mysql.py --ignore=tests/test_oracle.py \ + --ignore=tests/test_mariadb.py --ignore=tests/test_duckdb.py --ignore=tests/test_cockroachdb.py \ + --ignore=tests/test_turso.py --ignore=tests/test_firebird.py --ignore=tests/test_ssh.py \ + --ignore=tests/test_clickhouse.py --ignore=tests/test_exasol.py +``` + +--- + +## Step 15 — Docs: `CONTRIBUTING.md` + `README.md` + +**Group:** — | **Depends on:** 7 | **State:** `todo` + +**Files:** `CONTRIBUTING.md`, `README.md` + +- `CONTRIBUTING.md` — add Exasol to the enterprise-profile list (line ~50) and an env-var table + (`EXASOL_HOST` / `EXASOL_PORT` / `EXASOL_USER` / `EXASOL_PASSWORD` / `EXASOL_SCHEMA`). +- `README.md` — add Exasol to the database list (line 28) and a `pyexasol` row in the Driver + Reference table (line ~286). + +**Verify:** manual read-through; both tables list Exasol consistently with the neighbouring engines. + +--- + +# Final verification (run once all steps are `done`) + +1. `uv run pytest tests/connections/providers/exasol/ -v` -> new unit tests pass. +2. `uv run pytest tests/test_schema_capabilities.py -v` -> catalog/enum consistency holds. +3. The full step-14 unit-job command -> no regressions across the existing suite. +4. `uv run ruff check sqlit tests && uv run mypy sqlit` -> clean. +5. `uv run sqlit` -> Exasol appears in the connection picker; the auth dropdown shows/hides + Password vs Access Token vs Refresh Token; the TLS tab is present. +6. Docker end-to-end: + `docker compose -f infra/docker/docker-compose.test.yml --profile enterprise up -d exasol`, + wait for readiness, then `uv run pytest tests/test_exasol.py -v`. Then connect interactively with + `uv run sqlit` against `localhost:8563`, `sys`/`exasol`, TLS mode `require` — confirm schemas and + tables list in the explorer, a `SELECT` returns rows, and an `INSERT` reports a row count. + +--- + +# Risks / open items + +- **`exasol/docker-db` in CI** (step 14): needs `--privileged` and at least 4 GB RAM, and boots in + minutes rather than seconds. The job is isolated (`needs: build`, its own runner) so it will not + slow the unit job, but expect the reviewer to question it. Fallback if pushed back on: keep the + compose service and `tests/test_exasol.py` for local use and drop the CI job — mark step 14's job + addition `skipped` and keep the `--ignore` line. +- **`ExaStatement` result detection** (step 4): `column_names()`-empty vs `stmt.result_type` — decide + during step 4 against the installed pyexasol. +- **Patching the lazy driver import in unit tests** (step 10) — see the note in that step. +- **Context7 MCP was not connected when this plan was written**, so the pyexasol and Exasol + system-table details above were verified against the published docs (`exasol.github.io/pyexasol`, + `docs.exasol.com`) rather than through Context7. Re-check with Context7 if available. + +--- + +# Session log + +Append one line per session: date, steps touched, outcome, anything half-done. + +| Date | Steps | Outcome / notes | +|---|---|---| +| 2026-08-27 | — | Plan restructured into 15 numbered steps with atomic groups `ACT` (5-7) and `INT` (13-14). No implementation yet. | +| 2026-08-27 | 1, 2, 3, 4 | All four `done`. **Step 4 open item resolved:** use `stmt.result_type` (plain attribute, values `resultSet` / `rowCount`) rather than empty `column_names()`, and test it **before** any fetch — `ExaStatement.__next__` raises `ExaRuntimeError` ("Attempt to fetch from statement without result set") and `fetchmany()` iterates, so reversing guard and fetch would turn every `INSERT` into an error. | +| 2026-08-27 | 3 | **Step 3 notation corrected:** `conn.meta.list_tables()` / `list_views()` / `list_columns()` return **already-fetched `list[dict]` with UPPERCASE keys**, not tuples — `execute_snapshot` hard-codes `fetch_dict=True`. Rows are read by key; positional indexing would raise `KeyError` at runtime and `mypy` would not catch it. `execute_snapshot` itself returns an `ExaStatement`, so it needs an explicit `.fetchall()`. Verified via Context7 + the pyexasol source (`meta.py`, `statement.py`, `formatter.py`). | +| 2026-08-27 | 1 | **Blocker found and fixed — the plan's isolation premise was wrong.** `catalog.py::_discover_providers` imports `/provider.py` for **every** subpackage unconditionally (no existence check, no `except`), so an adapter-only package did not stay inert: it broke discovery app-wide (`get_supported_db_types()` raised `ModuleNotFoundError`; `test_schema_capabilities.py` went 9-passed -> 4-failed). Fixed by skipping subpackages whose `provider` module is absent, via `importlib.util.find_spec`, in `catalog.py`. **This adds a modified shared file the proposal did not anticipate** — carry it into the upstream PR as a small robustness fix. | +| 2026-08-27 | 2 | **Note on design D5:** pyexasol does **not** client-side reject `password` + `access_token`/`refresh_token`; `_login()` simply branches on token truthiness. The spec's "unused credentials absent, not present-and-empty" rule is still correct — and matters more than D5 implies, since a present-but-empty `access_token` is falsy and would silently fall back to password login. | +| 2026-08-27 | 3 | **Edge case left for step 11:** `default_schema` is `""`, so an unset `schema` passes an empty pattern to `list_columns` (pyexasol defaults to `"%"`), which matches nothing. Unreachable via the explorer — every Exasol table arrives from `get_tables()` as a populated `(TABLE_SCHEMA, TABLE_NAME)` pair — so left spec-faithful rather than adding a fallback. Worth a unit test. | +| 2026-08-27 | 5, 6, 7 | Group `ACT` complete — Exasol is registered and selectable. **Step 7 snippet corrected (design D6):** `DockerDetector.env_vars` is a **required** field with no default, so the plan's `DockerDetector(image_patterns=("exasol/docker-db",), default_user="sys")` would raise `TypeError` at import time *inside discovery*, breaking all 30 providers rather than just Exasol. Passed `env_vars={}` — semantically right too, since `exasol/docker-db` takes no credential env vars and `get_credentials({})` resolves to `default_user="sys"`. **Credential visibility (design D3):** `username`/`password` declared as explicit `SchemaField`s instead of reusing `_username_field()` / `_password_field()`, whose returned `SchemaField` is frozen with `visible_when=None` and so cannot be hidden under token auth. | +| 2026-08-27 | 5, 6, 7 | Gate green: `test_schema_capabilities.py` 9/9 at 30 providers; ruff 117 and mypy 430-in-44 both exactly at the pre-change baseline (3 `no-any-return` findings on the new visibility predicates fixed by wrapping the lookup in `str(...)`, matching `schema_helpers._tls_mode_is_custom`). The 21 wider-suite failures are pre-existing Windows-environment ones — confirmed identical with the change backed out. **Steps 9-11 (unit tests) are now unblocked:** step 9 (`test_schema.py`) depends only on 5 and 7 and can start immediately; steps 10 and 11 still need step 8 (the `exasol` extra) first. Step 4.9's interactive `uv run sqlit` check is the one item outstanding — verified headlessly instead through the same code paths (picker option after Teradata, port 8563, three auth methods, visibility swap, TLS tab). | +| 2026-08-27 | 8, 9, 10, 11 | All four `done` — 55 unit tests green, and green with `pyexasol` genuinely uninstalled (`uv sync --group test --no-dev`, zero skips), so the default CI job covers them. **Step 10's open item resolved (design D2):** the driver is faked with `patch.dict("sys.modules", {"pyexasol": MagicMock()})` — `importlib.import_module` returns an existing `sys.modules` entry without touching the filesystem, so `_import_driver_module` stays under test rather than being patched out. **`pyexasol` resolved: 2.3.2** — the exact version every API detail was verified against, so the loose `>=2.0.0` bound cost nothing here. **D6 escape hatch NOT needed:** `uv sync --extra exasol` resolved cleanly despite `pyexasol`'s `requires-python >=3.10,<3.15` against sqlit's unbounded `>=3.10` — but note D6 predicted the wrong mechanism: `uv` attached **no** `python_full_version < '3.15'` marker to the lock entry, only `extra == 'all'` / `extra == 'exasol'`. Resolution succeeds because no fork is required; the `<3.15` ceiling will surface at install time on 3.15+, not at lock time. **D4 confirmed inert:** the `pyexasol` mypy override changes nothing (mypy excludes `tests/`, `sqlit/` names the driver only in a string literal); `uv run python -c "import pyexasol"` is its real check, not a clean mypy run. **D5 lockfile drift to disclose in the PR:** `uv.lock` also drops the stale `mariadb` package entry (0 occurrences left), because `HEAD`'s `mariadb` extra already points at `PyMySQL`; not separable from a `uv lock` regenerate, not hand-edited. **No adapter bug surfaced (6.5) — and the tests are not vacuous:** nine mutations of `adapter.py` (inverted `result_type` guard, `rowcount` read as a property, index-based row reads, `fetchmany(max_rows)` instead of `+ 1`, PK lookup via `conn.execute`, token auth also sending `password`, `cert_reqs` dropped, `quote_identifier` escaping removed, `execute_test_query` via `cursor()`) were each caught by 1-6 failing tests; `adapter.py` restored byte-identical (sha256 verified). Gate: ruff 268-on-`sqlit tests` and mypy 430-in-44 both exactly at the pre-change baseline with the new files themselves clean; CI's unit command verbatim gave **21 failed, 1703 passed, 395 skipped** — the same 21 pre-existing Windows-environment failures, none Exasol — and collected all 55 new tests (22 adapter + 24 connect + 9 schema) with no `--ignore` entry. `--markers` lists `exasol`; `-m exasol` selects zero, as D7 intended. **Steps 12-15 (Docker integration, CI job, docs) remain.** | +| 2026-08-27 | 12, 13, 14 | Group `INT` complete — Exasol now runs the shared suite against a live server: **20 passed, 8 skipped, 0 failed** (`uv run pytest tests/test_exasol.py`, container up), and **3 passed / 25 skipped / 0 errors** with the container stopped, so a Docker-less laptop stays green. **O3 resolved:** `sys` / `exasol` confirmed by a real login; the image *does* accept a password override, but only as a `docker run` argument (`exadt init-sc --sys-passwd`), **never an environment variable** — which independently confirms the `env_vars={}` behind D9. **Boot measured (2.4):** port 8563 opened at **21s**, first successful login at **101s** — an 80-second window in which `is_port_open` is true and every login fails, exactly the false positive design D2 exists to kill. CI poll kept at 60 × 10s (10 min) per D7: generous headroom over 101s for a cold runner that must also pull ~4 GB. **D8 deviation from step 13's wording:** subclassed `BaseDatabaseTestsWithLimit`, not `BaseDatabaseTests` — Exasol supports `LIMIT`, and `test_query_limit` passes, so it was free coverage. **Two base tests overridden:** `test_docker_container_connection` skips unconditionally (D9 — no credentials through env vars, and a discovery-built config carries no `tls_mode`, so it would verify TLS against the image's self-signed certificate); `test_primary_key_detection` is re-issued through the app's call shape (D10). `test_docker_container_detection` was **not** overridden and passes. **O1 confirmed against the live server, and it is worse than `findings.md` predicted — the two causes are independently fatal:** `get_columns(conn, 'test_users')` → 0 columns; `('TEST_USERS', no schema)` → **0**; `('test_users', schema=…)` → **0**; only the app's `('TEST_USERS', schema=…)` → 3 columns with `ID` correctly flagged and nothing else. `conn.meta.list_columns('%', 'TEST_USERS')` → 3, so O1's proposed `'%'` default *does* fix the no-schema case — but only when the table name is already in server casing. Earmarked for the follow-up change; **no file under `sqlit/` touched**. **New finding, not anticipated by the design — Exasol folds `''` to NULL:** the task's literal view body `WHERE email != ''` matches **0 rows** on a live server (verified: with an empty-string row inserted, `email IS NULL` → 1 and `email != ''` → 0), so `test_user_emails` would have returned nothing and failed `test_query_view`. The fixture uses `WHERE email IS NOT NULL` instead. **Constraint 2 demonstrated by accident:** while taking the pre-change baseline the fixture module was moved aside with `tests/test_exasol.py` still present and un-ignored, and the driver-free unit run went to **42 failed / 2 errors** — all 23 of them in `test_exasol.py`. That is precisely the breakage the `INT` atomicity rule exists to prevent. **Gate:** the unit command with the new `--ignore` gives **21 failed, 1703 passed, 395 skipped, 0 errors** — the failure set is identical to the pre-change baseline once `test_exasol.py` is netted out, and identical to the step 8-11 session's numbers; `ruff check tests` is 152 exactly at baseline with both new files clean; collection succeeds with `pyexasol` genuinely uninstalled (`uv sync --group test --no-dev`). **Local environment note:** an unrelated `exasoldb` container (`exasol/docker-db:latest`, port 9563) runs on this machine and is what satisfies the two Docker-discovery tests locally; on a Docker-less machine both skip. **Steps 8-11's per-step `**State:**` prose lines are still stale at `todo`** (the Status table is the source of truth); left as found. **Step 15 (docs) remains.** | +| 2026-08-27 | 15 | Step 15 `done` — the docs land as **four insertions across two files, zero lines reflowed**. `README.md`: `Exasol` after `Teradata` in the supported-database sentence (D1 — `config.py:55-56` independently confirms `EXASOL` follows `TERADATA` in the picker display order), plus one Driver Reference row between `Spanner` and `Apache Arrow Flight SQL` (D2). **D6's reflow risk did not materialise:** the new row is 176 characters, identical in width to the Spanner row, because all four Exasol cells are shorter than the `snowflake-connector-python` cells that set the column widths. `CONTRIBUTING.md`: `Exasol` appended to the enterprise-container list at line 49, and an `**Exasol:**` env-var table plus readiness note placed **after the Oracle 11g table, before Flight SQL** — grouped with the enterprise cluster so the table order matches the container list this change just edited. **D3 verified mechanically, not by eye:** a script re-derived the six `os.environ.get` fallbacks from `tests/fixtures/exasol.py` and compared them against the table it had just written — set equality and character-identical defaults (`localhost` / `8563` / `sys` / `exasol` / `TEST_SQLIT` / `300`). **One deviation from this change's own task 4.1:** `git diff --stat` names **nine** files, not two, because steps 1-14 are still uncommitted in this tree; the checkable condition — that this change added exactly `README.md` and `CONTRIBUTING.md` to the pre-existing modified set — holds. **Style fix caught in review:** the readiness note's em dash became a semicolon once `grep` showed the em-dash count in both files was 1, i.e. only the one just added; both files are pure ASCII, as found. **Claims traced to runtime rather than to this plan:** `get_provider(DatabaseType.EXASOL)` returns a live provider whose `DriverDescriptor(package_name='pyexasol', extra_name='exasol')` is exactly what the README row promises, and the note's port-open-versus-login claim restates the fixture's own comment (`exasol.py:21-22`) and its retry-to-deadline loop. Nothing added mentions introspection, so **O1 stays undocumented and unimplied**. **Gate:** pre-commit `trailing-whitespace` and `end-of-file-fixer` pass on both files and modified neither; there is no markdown linter in this repo, so that is the whole automated gate (D7). **Plan complete: 15 / 15.** Follow-up O1 (`get_columns` casing/schema) and the plan's Final verification block remain out of scope by design. | diff --git a/sqlit/domains/connections/providers/exasol/adapter.py b/sqlit/domains/connections/providers/exasol/adapter.py index b7b453b0..ba1d591c 100644 --- a/sqlit/domains/connections/providers/exasol/adapter.py +++ b/sqlit/domains/connections/providers/exasol/adapter.py @@ -16,7 +16,6 @@ from sqlit.domains.connections.providers.registry import get_default_port from sqlit.domains.connections.providers.tls import ( TLS_MODE_DISABLE, - TLS_MODE_REQUIRE, get_tls_files, get_tls_mode, tls_mode_verifies_cert, @@ -87,17 +86,17 @@ def default_schema(self) -> str: def _tls_args(self, config: ConnectionConfig) -> dict[str, Any]: """Map the shared tls_mode option onto pyexasol encryption kwargs. - pyexasol defaults to encryption=True, but exasol/docker-db and most - on-premise installations present a self-signed certificate, so an - unmapped connect fails certificate validation. + Since pyexasol 1.0.0 an omitted websocket_sslopt means CERT_REQUIRED, but + exasol/docker-db and most on-premise installations present a self-signed + certificate, so deferring to that driver default fails every out-of-the-box + connect. The default mode therefore encrypts without verifying, matching how + the other providers here treat it; verification starts at verify-ca. """ tls_mode = get_tls_mode(config) if tls_mode == TLS_MODE_DISABLE: return {"encryption": False} - if tls_mode == TLS_MODE_REQUIRE: - return {"encryption": True, "websocket_sslopt": {"cert_reqs": ssl.CERT_NONE}} if not tls_mode_verifies_cert(tls_mode): - return {"encryption": True} + return {"encryption": True, "websocket_sslopt": {"cert_reqs": ssl.CERT_NONE}} sslopt: dict[str, Any] = {"cert_reqs": ssl.CERT_REQUIRED} tls_ca, tls_cert, tls_key, _ = get_tls_files(config) diff --git a/tests/connections/providers/exasol/test_connect.py b/tests/connections/providers/exasol/test_connect.py index e6dba62e..6dd2b634 100644 --- a/tests/connections/providers/exasol/test_connect.py +++ b/tests/connections/providers/exasol/test_connect.py @@ -141,11 +141,13 @@ def test_tls_disable_turns_encryption_off() -> None: @pytest.mark.parametrize("options", [{}, {"tls_mode": "default"}], ids=["unset", "explicit"]) -def test_tls_default_encrypts_and_leaves_ssl_options_to_the_driver(options: dict[str, Any]) -> None: +def test_tls_default_encrypts_without_verifying_the_certificate(options: dict[str, Any]) -> None: + # Not left to the driver: pyexasol would demand CERT_REQUIRED, which no + # self-signed server - exasol/docker-db included - can satisfy. kwargs = _connect_kwargs(_config(options=options)) assert kwargs["encryption"] is True - assert "websocket_sslopt" not in kwargs + assert kwargs["websocket_sslopt"] == {"cert_reqs": ssl.CERT_NONE} def test_tls_require_encrypts_without_verifying_the_certificate() -> None: diff --git a/tests/test_exasol.py b/tests/test_exasol.py index 5ebe5443..cc109d79 100644 --- a/tests/test_exasol.py +++ b/tests/test_exasol.py @@ -27,17 +27,15 @@ def config(self) -> DatabaseTestConfig: def test_docker_container_connection(self, request): """Docker-discovered credentials cannot connect to exasol/docker-db. - Two independent properties of the image, neither fixable from tests/: - exasol/docker-db publishes no credentials through environment variables + A property of the image, not fixable from tests/: exasol/docker-db + publishes no credentials through environment variables (SPEC.docker_detector has env_vars={}), so the discovered config carries - no password; and a discovery-built config carries no tls_mode, so the - adapter verifies TLS against the image's self-signed certificate. + no password. """ pytest.skip( "exasol/docker-db publishes no credentials through environment " - "variables (docker_detector env_vars={}), and a discovery-built " - "config has no tls_mode, so it verifies TLS against the image's " - "self-signed certificate" + "variables (docker_detector env_vars={}), so the discovered config " + "carries no password" ) def test_primary_key_detection(self, request): From 66c61962ff3899265f9195a14fb26d747647600f Mon Sep 17 00:00:00 2001 From: Peter Adams <18162810+Maxteabag@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:05:58 +0200 Subject: [PATCH 3/5] fix(connections): protect provider token credentials and migration --- .../domains/connections/app/persist_utils.py | 3 + sqlit/domains/connections/domain/config.py | 29 ++++- .../connections/domain/credential_aliases.py | 67 ++++++++++ .../domains/connections/store/connections.py | 23 +++- .../unit/test_provider_credential_aliases.py | 119 ++++++++++++++++++ 5 files changed, 237 insertions(+), 4 deletions(-) create mode 100644 sqlit/domains/connections/domain/credential_aliases.py create mode 100644 tests/unit/test_provider_credential_aliases.py diff --git a/sqlit/domains/connections/app/persist_utils.py b/sqlit/domains/connections/app/persist_utils.py index c6c3babe..a9e9be17 100644 --- a/sqlit/domains/connections/app/persist_utils.py +++ b/sqlit/domains/connections/app/persist_utils.py @@ -21,6 +21,9 @@ def build_persist_connections( """ persist_connections = copy.deepcopy(connections) for conn in persist_connections: + from sqlit.domains.connections.domain.credential_aliases import normalize_credential_options + + normalize_credential_options(conn) endpoint = conn.tcp_endpoint if endpoint and endpoint.password is None and not endpoint.password_command: stored = credentials_service.get_password(conn.name) diff --git a/sqlit/domains/connections/domain/config.py b/sqlit/domains/connections/domain/config.py index ad05aa06..c87d059e 100644 --- a/sqlit/domains/connections/domain/config.py +++ b/sqlit/domains/connections/domain/config.py @@ -146,6 +146,11 @@ class ConnectionConfig: extra_options: dict[str, str] = field(default_factory=dict) options: dict[str, Any] = field(default_factory=dict) + def __post_init__(self) -> None: + from sqlit.domains.connections.domain.credential_aliases import normalize_credential_options + + normalize_credential_options(self) + @classmethod def from_dict(cls, data: Mapping[str, Any]) -> ConnectionConfig: payload = dict(data) @@ -264,10 +269,21 @@ def from_dict(cls, data: Mapping[str, Any]) -> ConnectionConfig: ) def get_option(self, name: str, default: Any | None = None) -> Any: + from sqlit.domains.connections.domain.credential_aliases import credential_option + + if name == credential_option(self) and self.tcp_endpoint is not None: + fallback = self.tcp_endpoint.password + return self.options.get(name, fallback if fallback is not None else default) return self.options.get(name, default) def set_option(self, name: str, value: Any) -> None: + from sqlit.domains.connections.domain.credential_aliases import credential_option, normalize_credential_options + + previous_credential = credential_option(self) self.options[name] = value + if previous_credential != credential_option(self) and self.tcp_endpoint is not None: + self.tcp_endpoint.password = None + normalize_credential_options(self) def get_field_value(self, name: str, default: Any = "") -> Any: values = self.to_form_values() @@ -319,17 +335,24 @@ def to_form_values(self) -> dict[str, Any]: values["ssh_enabled"] = "disabled" values.update(self.options) + from sqlit.domains.connections.domain.credential_aliases import credential_option + + alias = credential_option(self) + if alias: + values[alias] = self.get_option(alias) return values def to_dict(self, *, include_passwords: bool = True) -> dict[str, Any]: + from sqlit.domains.connections.domain.credential_aliases import public_connection_url, without_secret_options + data: dict[str, Any] = { "name": self.name, "db_type": self.db_type, "source": self.source, - "connection_url": self.connection_url, + "connection_url": self.connection_url if include_passwords else public_connection_url(self), "folder_path": self.folder_path, - "extra_options": dict(self.extra_options), - "options": dict(self.options), + "extra_options": dict(self.extra_options) if include_passwords else without_secret_options(self, self.extra_options), + "options": dict(self.options) if include_passwords else without_secret_options(self, self.options), } if isinstance(self.endpoint, FileEndpoint): diff --git a/sqlit/domains/connections/domain/credential_aliases.py b/sqlit/domains/connections/domain/credential_aliases.py new file mode 100644 index 00000000..b6342405 --- /dev/null +++ b/sqlit/domains/connections/domain/credential_aliases.py @@ -0,0 +1,67 @@ +"""Map mutually exclusive provider secrets onto the existing keyring credential. + +A Databricks/Exasol connection uses one authentication secret at a time. Keeping +it in endpoint.password reuses the credential store's save/load/rename semantics; +provider-specific field names remain available to forms and CLI configuration. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +if TYPE_CHECKING: + from typing import Any + + from sqlit.domains.connections.domain.config import ConnectionConfig + +_CREDENTIAL_MODES = { + 'databricks': ('auth_type', 'pat', {'pat': 'access_token', 'oauth-m2m': 'client_secret'}), + 'exasol': ('authenticator', 'password', {'access_token': 'access_token', 'refresh_token': 'refresh_token'}), +} + + +def credential_option(config: ConnectionConfig) -> str | None: + definition = _CREDENTIAL_MODES.get(config.db_type) + if definition is None: + return None + selector, default, aliases = definition + mode = config.options.get(selector, config.extra_options.get(selector, default)) + return aliases.get(mode) + + +def secret_option_names(config: ConnectionConfig) -> frozenset[str]: + definition = _CREDENTIAL_MODES.get(config.db_type) + return frozenset(definition[2].values()) if definition else frozenset() + + +def normalize_credential_options(config: ConnectionConfig) -> None: + definition = _CREDENTIAL_MODES.get(config.db_type) + if definition is None: + return + selector, _, _ = definition + config.options = dict(config.options) + config.extra_options = dict(config.extra_options) + if selector in config.extra_options: + config.options.setdefault(selector, config.extra_options.pop(selector)) + selected = credential_option(config) + for name in secret_option_names(config): + extra_value = config.extra_options.pop(name, None) + value = config.options.pop(name, extra_value) + if name == selected and value is not None and config.tcp_endpoint is not None: + config.tcp_endpoint.password = value + + +def without_secret_options(config: ConnectionConfig, options: dict[str, Any]) -> dict[str, Any]: + names = secret_option_names(config) + return {key: value for key, value in options.items() if key not in names} + + +def public_connection_url(config: ConnectionConfig) -> str | None: + url = config.connection_url + if not url or config.db_type not in _CREDENTIAL_MODES: + return url + parsed = urlsplit(url) + secret_names = secret_option_names(config) | {'password'} + query = [(key, value) for key, value in parse_qsl(parsed.query, keep_blank_values=True) + if key not in secret_names] + return urlunsplit(parsed._replace(netloc=parsed.netloc.rsplit('@', 1)[-1], query=urlencode(query))) diff --git a/sqlit/domains/connections/store/connections.py b/sqlit/domains/connections/store/connections.py index 20189d39..110b7568 100644 --- a/sqlit/domains/connections/store/connections.py +++ b/sqlit/domains/connections/store/connections.py @@ -79,19 +79,37 @@ def load_all(self, load_credentials: bool = True) -> list[ConnectionConfig]: return [] version, raw_connections, needs_migration = self._unpack_connections_payload(data) try: + from sqlit.domains.connections.domain.credential_aliases import credential_option, secret_option_names from sqlit.domains.connections.providers.config_service import normalize_connection_config configs = [] + migrated_provider_secret = False for conn in raw_connections: if not isinstance(conn, dict): continue config = ConnectionConfig.from_dict(conn) config = normalize_connection_config(config) + legacy_secrets = any( + source.get(key) + for source in (conn, conn.get("options") or {}, conn.get("extra_options") or {}) + if isinstance(source, dict) + for key in secret_option_names(config) + ) + if legacy_secrets: + # Move old plaintext token fields before rewriting any index. + # Otherwise saving an unrelated connection could redact the + # old token without ever storing it in the credential backend. + endpoint = config.tcp_endpoint + if credential_option(config) and endpoint and endpoint.password is not None: + self.credentials_service.set_password(config.name, endpoint.password) + migrated_provider_secret = True if load_credentials: # Retrieve passwords from credentials service self._load_credentials(config) configs.append(config) - if needs_migration: + if migrated_provider_secret: + self._write_index(configs) + elif needs_migration: self._migrate_connections_payload(raw_connections, version) return configs except (TypeError, KeyError): @@ -153,6 +171,9 @@ def _save_credentials(self, config: ConnectionConfig) -> list[CredentialsStoreEr Note: Empty string "" is a valid password (e.g., CockroachDB insecure mode). Only None means "delete/no password stored". """ + from sqlit.domains.connections.domain.credential_aliases import normalize_credential_options + + normalize_credential_options(config) errors: list[CredentialsStoreError] = [] endpoint = config.tcp_endpoint diff --git a/tests/unit/test_provider_credential_aliases.py b/tests/unit/test_provider_credential_aliases.py new file mode 100644 index 00000000..028c2690 --- /dev/null +++ b/tests/unit/test_provider_credential_aliases.py @@ -0,0 +1,119 @@ +"""Provider credentials must use the same protected slot as database passwords.""" +from __future__ import annotations + +import json +from dataclasses import replace + +import pytest + +from sqlit.domains.connections.app.credentials import PlaintextCredentialsService +from sqlit.domains.connections.domain.config import ConnectionConfig, TcpEndpoint +from sqlit.domains.connections.providers.registry import get_supported_db_types +from sqlit.domains.connections.store.connections import ConnectionStore + +CASES = [ + ('databricks', 'auth_type', 'pat', 'access_token'), + ('databricks', 'auth_type', 'oauth-m2m', 'client_secret'), + ('exasol', 'authenticator', 'access_token', 'access_token'), + ('exasol', 'authenticator', 'refresh_token', 'refresh_token'), +] + + +def config_for(provider, selector, mode, field): + return ConnectionConfig( + name='protected', db_type=provider, endpoint=TcpEndpoint(host='example.invalid'), + options={selector: mode, field: 'SYNTHETIC_SECRET', + 'http_path': '/sql/1.0/warehouses/example', 'client_id': 'example-client'}, + connection_url=f'{provider}://user:SYNTHETIC_URL_SECRET@example.invalid?{field}=SYNTHETIC_SECRET', + ) + + +@pytest.mark.parametrize(('provider', 'selector', 'mode', 'field'), CASES) +def test_secret_uses_protected_slot_and_redacts_serialization(provider, selector, mode, field): + config = config_for(provider, selector, mode, field) + assert config.tcp_endpoint.password == 'SYNTHETIC_SECRET' + assert field not in config.options + assert config.get_option(field) == 'SYNTHETIC_SECRET' + assert config.to_form_values()[field] == 'SYNTHETIC_SECRET' + public = json.dumps(config.to_dict(include_passwords=False)) + assert 'SYNTHETIC_SECRET' not in public + assert 'SYNTHETIC_URL_SECRET' not in public + transported = ConnectionConfig.from_dict(config.to_dict()) + assert transported.get_option(field) == 'SYNTHETIC_SECRET' + + +@pytest.mark.parametrize(('provider', 'selector', 'mode', 'field'), CASES) +def test_store_reload_and_rename_keep_secret_out_of_index(tmp_path, provider, selector, mode, field): + if provider not in get_supported_db_types(): + pytest.skip('Provider is not part of this branch') + credentials = PlaintextCredentialsService() + path = tmp_path / 'connections.json' + store = ConnectionStore(credentials, file_path=path) + config = config_for(provider, selector, mode, field) + store.save_one(config) + assert 'SYNTHETIC' not in path.read_text() + assert credentials.get_password('protected') == 'SYNTHETIC_SECRET' + loaded = store.load_all()[0] + assert loaded.get_option(field) == 'SYNTHETIC_SECRET' + renamed = replace(store.load_all(load_credentials=False)[0], name='renamed') + store.save_one(renamed, previous_name='protected') + assert credentials.get_password('protected') is None + assert store.load_all()[0].get_option(field) == 'SYNTHETIC_SECRET' + assert 'SYNTHETIC' not in path.read_text() + + +@pytest.mark.parametrize(('provider', 'selector', 'mode', 'field'), CASES) +def test_legacy_extra_options_secrets_are_protected(provider, selector, mode, field): + config = ConnectionConfig(name='legacy', db_type=provider, + options={selector: mode}, extra_options={field: 'SYNTHETIC_SECRET'}) + assert config.tcp_endpoint.password == 'SYNTHETIC_SECRET' + assert field not in config.extra_options + assert 'SYNTHETIC_SECRET' not in json.dumps(config.to_dict(include_passwords=False)) + + +def test_switching_authentication_does_not_reuse_a_different_secret(): + config = config_for('databricks', 'auth_type', 'pat', 'access_token') + config.set_option('auth_type', 'oauth-m2m') + assert config.get_option('client_secret') is None + config.set_option('client_secret', 'NEW_SECRET') + assert config.tcp_endpoint.password == 'NEW_SECRET' + + +@pytest.mark.parametrize(('provider', 'selector', 'mode', 'field'), CASES) +def test_saving_another_connection_preserves_legacy_token(tmp_path, provider, selector, mode, field): + if provider not in get_supported_db_types(): + pytest.skip('Provider is not part of this branch') + config = config_for(provider, selector, mode, field) + legacy = config.to_dict() + legacy['endpoint']['password'] = None + legacy['options'][field] = 'LEGACY_SECRET' + path = tmp_path / 'connections.json' + path.write_text(json.dumps({'version': 2, 'connections': [legacy]})) + credentials = PlaintextCredentialsService() + store = ConnectionStore(credentials, file_path=path) + other = ConnectionConfig(name='unrelated', db_type='postgresql', endpoint=TcpEndpoint(host='localhost')) + store.save_one(other) + assert credentials.get_password('protected') == 'LEGACY_SECRET' + assert 'LEGACY_SECRET' not in path.read_text() + assert store.get_by_name('protected').get_option(field) == 'LEGACY_SECRET' + + +def test_failed_legacy_migration_preserves_original_file(tmp_path): + from sqlit.domains.connections.app.credentials import CredentialsStoreError + + provider, selector, mode, field = next(case for case in CASES if case[0] in get_supported_db_types()) + legacy = config_for(provider, selector, mode, field).to_dict() + legacy['endpoint']['password'] = None + legacy['options'][field] = 'LEGACY_SECRET' + path = tmp_path / 'connections.json' + original = json.dumps({'version': 2, 'connections': [legacy]}) + path.write_text(original) + + class UnavailableCredentials(PlaintextCredentialsService): + def set_password(self, name, password): + raise CredentialsStoreError(connection_name=name, kind='db', action='store', reason=RuntimeError('locked')) + + store = ConnectionStore(UnavailableCredentials(), file_path=path) + with pytest.raises(CredentialsStoreError): + store.load_all() + assert path.read_text() == original From fdcebba225c12d46207f71f86e9f89f6a8b9981c Mon Sep 17 00:00:00 2001 From: Peter Adams <18162810+Maxteabag@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:48:59 +0200 Subject: [PATCH 4/5] fix(connections): cover token editing and cross-process persistence --- .../connections/domain/credential_aliases.py | 8 +- .../domains/connections/store/connections.py | 12 +- sqlit/domains/connections/ui/restart_cache.py | 12 ++ .../test_cloud_provider_credentials.py | 115 ++++++++++++++++++ tests/ui/test_provider_token_credentials.py | 32 +++++ .../unit/test_provider_credential_aliases.py | 36 ++++++ 6 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 tests/integration/test_cloud_provider_credentials.py create mode 100644 tests/ui/test_provider_token_credentials.py diff --git a/sqlit/domains/connections/domain/credential_aliases.py b/sqlit/domains/connections/domain/credential_aliases.py index b6342405..59f336ae 100644 --- a/sqlit/domains/connections/domain/credential_aliases.py +++ b/sqlit/domains/connections/domain/credential_aliases.py @@ -48,7 +48,13 @@ def normalize_credential_options(config: ConnectionConfig) -> None: extra_value = config.extra_options.pop(name, None) value = config.options.pop(name, extra_value) if name == selected and value is not None and config.tcp_endpoint is not None: - config.tcp_endpoint.password = value + config.tcp_endpoint.password = value or None + + +def credential_kind_changed(previous: ConnectionConfig | None, current: ConnectionConfig) -> bool: + if previous is None or not (secret_option_names(previous) or secret_option_names(current)): + return False + return (previous.db_type, credential_option(previous)) != (current.db_type, credential_option(current)) def without_secret_options(config: ConnectionConfig, options: dict[str, Any]) -> dict[str, Any]: diff --git a/sqlit/domains/connections/store/connections.py b/sqlit/domains/connections/store/connections.py index 110b7568..474c5864 100644 --- a/sqlit/domains/connections/store/connections.py +++ b/sqlit/domains/connections/store/connections.py @@ -273,10 +273,14 @@ def save_all(self, connections: list[ConnectionConfig]) -> None: connections: List of ConnectionConfig objects to save. """ from sqlit.domains.connections.app.persist_utils import build_persist_connections + from sqlit.domains.connections.domain.credential_aliases import credential_kind_changed errors: list[CredentialsStoreError] = [] + existing = {config.name: config for config in self.load_all(load_credentials=False)} persist_connections = build_persist_connections(connections, self.credentials_service) - for config in persist_connections: + for source, config in zip(connections, persist_connections, strict=True): + if credential_kind_changed(existing.get(source.name), source) and source.tcp_endpoint and source.tcp_endpoint.password is None and config.tcp_endpoint is not None: + config.tcp_endpoint.password = None errors.extend(self._save_credentials(config)) self._write_index(persist_connections) @@ -302,10 +306,13 @@ def save_one( previous_name: The connection's prior name when renaming. """ from sqlit.domains.connections.app.persist_utils import build_persist_connections + from sqlit.domains.connections.domain.credential_aliases import credential_kind_changed renamed = bool(previous_name and previous_name != connection.name) existing = self.load_all(load_credentials=False) + previous = next((config for config in existing if config.name == (previous_name or connection.name)), None) + credential_changed = credential_kind_changed(previous, connection) filtered = [ c for c in existing @@ -324,6 +331,7 @@ def save_one( endpoint and endpoint.password is None and not endpoint.password_command + and not credential_changed ): endpoint.password = self.credentials_service.get_password_for_migration(previous_name) # type: ignore[arg-type] if ( @@ -383,7 +391,7 @@ def save_one( errors.append(exc) else: self._write_index(filtered) - target = build_persist_connections([connection], self.credentials_service)[0] + target = copy.deepcopy(connection) if credential_changed else build_persist_connections([connection], self.credentials_service)[0] errors.extend(self._save_credentials(target)) if errors: raise CredentialsPersistError(errors) diff --git a/sqlit/domains/connections/ui/restart_cache.py b/sqlit/domains/connections/ui/restart_cache.py index e2daa69b..b316a39e 100644 --- a/sqlit/domains/connections/ui/restart_cache.py +++ b/sqlit/domains/connections/ui/restart_cache.py @@ -16,6 +16,18 @@ def get_restart_cache_path() -> Path: def write_restart_cache(payload: dict[str, Any]) -> None: """Persist restart cache payload to disk (best effort).""" try: + values = payload.get("values") + if isinstance(values, dict) and values.get("db_type") in {"databricks", "exasol"}: + from sqlit.domains.connections.domain.config import ConnectionConfig + from sqlit.domains.connections.domain.credential_aliases import secret_option_names + + config = ConnectionConfig.from_dict(values) + public = config.to_dict(include_passwords=False) + hidden = secret_option_names(config) | {"password", "ssh_password"} + safe_values = {key: value for key, value in values.items() if key not in hidden} + safe_values["extra_options"] = public["extra_options"] + safe_values["connection_url"] = public["connection_url"] + payload = {**payload, "values": safe_values} get_restart_cache_path().write_text(json.dumps(payload), encoding="utf-8") except Exception: # Best-effort; don't block installation due to caching failure. diff --git a/tests/integration/test_cloud_provider_credentials.py b/tests/integration/test_cloud_provider_credentials.py new file mode 100644 index 00000000..4e432c39 --- /dev/null +++ b/tests/integration/test_cloud_provider_credentials.py @@ -0,0 +1,115 @@ +"""Opt-in real-service proof of CLI, protected persistence and adapter behavior. + +Set SQLIT_LIVE_PROVIDER (databricks/exasol), SQLIT_LIVE_HOST and +SQLIT_LIVE_TOKEN. Databricks also needs SQLIT_LIVE_HTTP_PATH; Exasol needs +SQLIT_LIVE_USERNAME. The token is read only from the environment and passed to +CLI creation on stdin. The CLI must use a working OS keyring, never plaintext. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import uuid +from dataclasses import replace +from pathlib import Path +from urllib.parse import quote, urlencode + +import pytest + +from sqlit.domains.connections.app.credentials import KeyringCredentialsService, is_keyring_usable +from sqlit.domains.connections.providers.registry import get_adapter, get_supported_db_types +from sqlit.domains.connections.store.connections import ConnectionStore + + +def test_saved_cloud_connection_survives_cli_reload_and_rename(tmp_path): + provider = os.environ.get('SQLIT_LIVE_PROVIDER') + if not provider: + pytest.skip('Set SQLIT_LIVE_PROVIDER to run against an owned cloud test database') + if provider not in {'databricks', 'exasol'} or provider not in get_supported_db_types(): + pytest.fail('Requested cloud provider is not available in this branch') + required = ['SQLIT_LIVE_HOST', 'SQLIT_LIVE_TOKEN'] + required.append('SQLIT_LIVE_HTTP_PATH' if provider == 'databricks' else 'SQLIT_LIVE_USERNAME') + missing = [name for name in required if not os.environ.get(name)] + if missing: + pytest.fail(f'Missing live-test configuration: {missing}') + if not is_keyring_usable(): + pytest.fail('Live saved-connection proof requires a working OS keyring') + + secret = os.environ['SQLIT_LIVE_TOKEN'] + host = os.environ['SQLIT_LIVE_HOST'] + catalog = os.environ.get('SQLIT_LIVE_CATALOG', 'workspace') if provider == 'databricks' else '' + user = 'token' if provider == 'databricks' else os.environ['SQLIT_LIVE_USERNAME'] + port = '' if provider == 'databricks' else ':' + os.environ.get('SQLIT_LIVE_PORT', '8563') + query = urlencode({'http_path': os.environ['SQLIT_LIVE_HTTP_PATH']}) if provider == 'databricks' else '' + url = f'{provider}://{quote(user, safe="")}:{quote(secret, safe="")}@{host}{port}/{catalog}' + if query: + url += '?' + query + name = 'sqlit-live-' + uuid.uuid4().hex + renamed = name + '-renamed' + env = dict(os.environ, SQLIT_CONFIG_DIR=str(tmp_path)) + # Do not forward the credential environment variable to CLI processes. + env.pop('SQLIT_LIVE_TOKEN', None) + (tmp_path / 'settings.json').write_text('{"allow_plaintext_credentials": false}') + credentials = KeyringCredentialsService() + store = ConnectionStore(credentials, file_path=tmp_path / 'connections.json') + repo = Path(__file__).resolve().parents[2] + + def cli(*args, input_text=None): + process = subprocess.run([sys.executable, '-m', 'sqlit.cli', *args], + cwd=repo, env=env, input=input_text, text=True, capture_output=True, timeout=180) + output = (process.stdout + process.stderr).replace(secret, '[REDACTED]') + assert process.returncode == 0, output + return process.stdout + + def assert_index_is_protected(): + leaked = secret in (tmp_path / 'connections.json').read_text() + assert not leaked, 'Credential leaked into the saved connection index' + assert not (tmp_path / 'credentials.json').exists(), 'Unexpected plaintext credential file' + + conn = None + created = False + adapter = get_adapter(provider) + schema = ('SQLIT_TEST_' + uuid.uuid4().hex[:12]).lower() if provider == 'databricks' else 'SQLIT_TEST_' + uuid.uuid4().hex[:12].upper() + q = adapter.quote_identifier + namespace = f'{q(catalog)}.{q(schema)}' if catalog else q(schema) + table_name = 'probe' if provider == 'databricks' else 'PROBE' + view_name = 'probe_view' if provider == 'databricks' else 'PROBE_VIEW' + table = namespace + '.' + q(table_name) + try: + cli('connections', 'add', '--name', name, '--url-stdin', input_text=url + '\n') + assert_index_is_protected() + result = json.loads(cli('query', '-c', name, '-q', 'SELECT 1 AS probe', '--format', 'json')) + assert list(result[0].values()) == [1] + config = store.get_by_name(name) + assert config is not None + conn = adapter.connect(config) + adapter.execute_non_query(conn, f'CREATE SCHEMA {namespace}') + created = True + adapter.execute_non_query(conn, f'CREATE TABLE {table} (id INTEGER, label VARCHAR(40))') + adapter.execute_non_query(conn, f"INSERT INTO {table} VALUES (1, 'alpha'), (2, 'beta'), (3, NULL)") + _, rows, truncated = adapter.execute_query(conn, f'SELECT id, label FROM {table} ORDER BY id', max_rows=2) + assert rows == [(1, 'alpha'), (2, 'beta')] + assert truncated + adapter.execute_non_query(conn, f'CREATE VIEW {namespace}.{q(view_name)} AS SELECT * FROM {table}') + assert (schema, table_name) in adapter.get_tables(conn, catalog or None) + assert (schema, view_name) in adapter.get_views(conn, catalog or None) + columns = adapter.get_columns(conn, table_name, catalog or None, schema) + assert [column.name.lower() for column in columns] == ['id', 'label'] + store.save_one(replace(store.load_all(load_credentials=False)[0], name=renamed), previous_name=name) + assert_index_is_protected() + result = json.loads(cli('query', '-c', renamed, '-q', f'SELECT COUNT(*) AS n FROM {table}', '--format', 'json')) + assert list(result[0].values()) == [3] + finally: + try: + if conn is not None: + try: + if created: + adapter.execute_non_query(conn, f'DROP SCHEMA {namespace} CASCADE') + finally: + conn.close() + finally: + for connection_name in (name, renamed): + store.delete(connection_name) + credentials.delete_all_for_connection(connection_name) diff --git a/tests/ui/test_provider_token_credentials.py b/tests/ui/test_provider_token_credentials.py new file mode 100644 index 00000000..5ce95ed4 --- /dev/null +++ b/tests/ui/test_provider_token_credentials.py @@ -0,0 +1,32 @@ +"""Connection forms must preserve protected token credentials without extra prompts.""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from sqlit.domains.connections.providers.registry import get_supported_db_types +from tests.ui.conftest import ConnectionScreenTestApp +from tests.unit.test_provider_credential_aliases import CASES, config_for + + +@pytest.mark.parametrize(('provider', 'selector', 'mode', 'field'), CASES) +async def test_token_form_round_trip_and_test_connection(provider, selector, mode, field, monkeypatch): + if provider not in get_supported_db_types(): + pytest.skip('Provider is not part of this branch') + cfg = config_for(provider, selector, mode, field) + app = ConnectionScreenTestApp(config=cfg, editing=True) + async with app.run_test(size=(120, 45)) as pilot: + screen = app.screen + await pilot.pause() + assert screen.query_one(f'#field-{field}').value == 'SYNTHETIC_SECRET' + screen.query_one(f'#field-{field}').value = 'EDITED_SECRET' + await pilot.pause() + tested = [] + monkeypatch.setattr(screen, '_driver_status_controller', lambda: SimpleNamespace(missing_driver_error=None)) + monkeypatch.setattr(screen, '_run_test', tested.append) + screen.action_test_connection() + assert len(tested) == 1, 'Token auth opened an unrelated password prompt' + assert tested[0].get_option(field) == 'EDITED_SECRET' + assert field not in tested[0].options + assert tested[0].tcp_endpoint.password == 'EDITED_SECRET' diff --git a/tests/unit/test_provider_credential_aliases.py b/tests/unit/test_provider_credential_aliases.py index 028c2690..3199fc70 100644 --- a/tests/unit/test_provider_credential_aliases.py +++ b/tests/unit/test_provider_credential_aliases.py @@ -117,3 +117,39 @@ def set_password(self, name, password): with pytest.raises(CredentialsStoreError): store.load_all() assert path.read_text() == original + + +@pytest.mark.parametrize('save_all', [False, True]) +def test_auth_mode_change_without_a_new_secret_clears_old_credential(tmp_path, save_all): + provider = 'databricks' if 'databricks' in get_supported_db_types() else 'exasol' + selector = 'auth_type' if provider == 'databricks' else 'authenticator' + old_mode, new_mode, new_field = ('pat', 'oauth-m2m', 'client_secret') if provider == 'databricks' else ('access_token', 'refresh_token', 'refresh_token') + credentials = PlaintextCredentialsService() + store = ConnectionStore(credentials, file_path=tmp_path / 'connections.json') + store.save_one(config_for(provider, selector, old_mode, 'access_token')) + changed = store.load_all(load_credentials=False)[0] + changed.set_option(selector, new_mode) + if save_all: + store.save_all([changed]) + else: + store.save_one(changed) + assert credentials.get_password(changed.name) is None + assert store.load_all()[0].get_option(new_field) is None + + +def test_empty_token_field_means_no_new_credential(): + config = ConnectionConfig(name='blank', db_type='databricks', options={'access_token': ''}) + assert config.tcp_endpoint.password is None + + +@pytest.mark.parametrize(('provider', 'selector', 'mode', 'field'), CASES) +def test_driver_restart_cache_does_not_persist_provider_secrets(tmp_path, monkeypatch, provider, selector, mode, field): + from sqlit.domains.connections.ui import restart_cache + + path = tmp_path / 'restart.json' + monkeypatch.setattr(restart_cache, 'get_restart_cache_path', lambda: path) + values = config_for(provider, selector, mode, field).to_form_values() + restart_cache.write_restart_cache({'version': 1, 'values': values}) + assert 'SYNTHETIC_SECRET' not in path.read_text() + assert 'SYNTHETIC_URL_SECRET' not in path.read_text() + assert values[field] == 'SYNTHETIC_SECRET' From 3d9807cf24ace4035a02992f777b74fab836f15f Mon Sep 17 00:00:00 2001 From: Peter Adams <18162810+Maxteabag@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:57:51 +0200 Subject: [PATCH 5/5] fix(exasol): verify TLS and match metadata identifiers exactly --- .github/workflows/ci.yml | 5 +- CONTRIBUTING.md | 20 +++++++ README.md | 9 +++ .../connections/providers/exasol/adapter.py | 23 ++++---- .../providers/exasol/test_adapter.py | 27 ++++----- .../providers/exasol/test_connect.py | 9 ++- .../providers/exasol/test_regressions.py | 54 ++++++++++++++++++ tests/fixtures/exasol.py | 17 +++--- tests/test_exasol.py | 57 +++++++++++++++++++ 9 files changed, 184 insertions(+), 37 deletions(-) create mode 100644 tests/connections/providers/exasol/test_regressions.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f36ef32..2c382537 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -549,7 +549,8 @@ jobs: - name: Start Exasol run: | docker run -d --name exasol --privileged \ - -p 8563:8563 \ + --memory=4g --memory-swap=4g --cpus=2 --stop-timeout=120 \ + -p 127.0.0.1:8563:8563 \ exasol/docker-db:latest-8 for i in {1..60}; do if nc -z localhost 8563 > /dev/null 2>&1; then @@ -559,6 +560,7 @@ jobs: echo "Waiting for Exasol... ($i/60)" sleep 10 done + nc -z localhost 8563 - name: Run Exasol integration tests env: @@ -567,6 +569,7 @@ jobs: EXASOL_USER: sys EXASOL_PASSWORD: exasol EXASOL_SCHEMA: TEST_SQLIT + EXASOL_REQUIRE_LIVE: "1" run: uv run pytest tests/test_exasol.py -v --timeout=300 test-ssh: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 78a5a265..c874593e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -185,6 +185,7 @@ The database tests can be configured with these environment variables: | `EXASOL_PASSWORD` | `exasol` | Exasol password | | `EXASOL_SCHEMA` | `TEST_SQLIT` | Schema the Exasol fixtures create and drop | | `EXASOL_READY_TIMEOUT` | `300` | Seconds to wait for Exasol to accept a login | +| `EXASOL_REQUIRE_LIVE` | unset | Set to `1` to fail if the required driver/server is unavailable | **Note:** Exasol runs in the `enterprise` profile and needs minutes, not seconds, before it accepts connections. `exasol/docker-db` binds port 8563 long before it will authenticate, so an open port is not yet a database that accepts a login. The fixtures retry a real connect until `EXASOL_READY_TIMEOUT` elapses; raise that value on slower hardware or a cold image pull. @@ -274,3 +275,22 @@ sqlit should provide fun and a feeling of mastery and satisfaction for those who **Example:** = explorer pane, = query pane, = results pane. Rationale: E;Q;R satisfies both intuitiveness (each binding is the first letter of the pane), harmony (proximity: qwerty speaks for itself) + +### Cloud credential regression test + +`tests/integration/test_cloud_provider_credentials.py` tests an owned cloud database through the +real CLI and OS keyring. It creates a unique schema and connection, checks queries, metadata, +row limits and rename, and removes its test data afterward. It never opts into plaintext +credentials. Load the token from your secret manager into the process environment. + +Set `SQLIT_LIVE_PROVIDER=exasol`, `SQLIT_LIVE_HOST`, `SQLIT_LIVE_USERNAME`, and +`SQLIT_LIVE_TOKEN` (the SaaS PAT), with optional `SQLIT_LIVE_PORT` (default 8563), then run: + +```bash +uv run --no-sync pytest tests/integration/test_cloud_provider_credentials.py -v --timeout=240 +``` + +The ordinary CI lane has no cloud credentials. Configured live runs fail on missing settings or +a missing OS keyring. The dedicated Docker lane sets `EXASOL_REQUIRE_LIVE=1`; schema setup errors +fail the suite, rather than becoming skipped tests. The Docker TLS regression uses `openssl` to +retrieve the test server's public certificate chain. diff --git a/README.md b/README.md index 3b1d8f56..9c74d6fa 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,15 @@ Edit the `keymap.json` file in your sqlit config dir. See [`config/keymap.templa ## FAQ +### Exasol authentication and TLS + +For Exasol SaaS, select username/password authentication and use the database username from +connection details with your personal access token as the password. OpenID access/refresh token +modes are for those credential types, not SaaS PATs. The selected secret uses the OS credential +store. Default TLS follows pyexasol's certificate verification; use `--tls-mode require` only +when deliberately connecting to a self-signed development server. `verify-ca` validates the CA, +while `verify-full` also checks the hostname. + ### How are sensitive credentials stored? Connection details are stored in `connections.json` inside the config directory, but passwords are stored in your OS keyring when available (macOS Keychain, Windows Credential Locker, Linux Secret Service). diff --git a/sqlit/domains/connections/providers/exasol/adapter.py b/sqlit/domains/connections/providers/exasol/adapter.py index ba1d591c..a76149c9 100644 --- a/sqlit/domains/connections/providers/exasol/adapter.py +++ b/sqlit/domains/connections/providers/exasol/adapter.py @@ -15,10 +15,12 @@ ) from sqlit.domains.connections.providers.registry import get_default_port from sqlit.domains.connections.providers.tls import ( + TLS_MODE_DEFAULT, TLS_MODE_DISABLE, get_tls_files, get_tls_mode, tls_mode_verifies_cert, + tls_mode_verifies_hostname, ) if TYPE_CHECKING: @@ -84,21 +86,16 @@ def default_schema(self) -> str: return "" def _tls_args(self, config: ConnectionConfig) -> dict[str, Any]: - """Map the shared tls_mode option onto pyexasol encryption kwargs. - - Since pyexasol 1.0.0 an omitted websocket_sslopt means CERT_REQUIRED, but - exasol/docker-db and most on-premise installations present a self-signed - certificate, so deferring to that driver default fails every out-of-the-box - connect. The default mode therefore encrypts without verifying, matching how - the other providers here treat it; verification starts at verify-ca. - """ + """Preserve driver defaults and distinguish chain/hostname verification.""" tls_mode = get_tls_mode(config) + if tls_mode == TLS_MODE_DEFAULT: + return {"encryption": True} if tls_mode == TLS_MODE_DISABLE: return {"encryption": False} if not tls_mode_verifies_cert(tls_mode): return {"encryption": True, "websocket_sslopt": {"cert_reqs": ssl.CERT_NONE}} - sslopt: dict[str, Any] = {"cert_reqs": ssl.CERT_REQUIRED} + sslopt: dict[str, Any] = {"cert_reqs": ssl.CERT_REQUIRED, "check_hostname": tls_mode_verifies_hostname(tls_mode)} tls_ca, tls_cert, tls_key, _ = get_tls_files(config) if tls_ca: sslopt["ca_certs"] = tls_ca @@ -172,13 +169,19 @@ def get_columns( ).fetchall() pk_columns = {row["COLUMN_NAME"] for row in pk_rows} + columns = conn.meta.execute_snapshot( + "SELECT COLUMN_NAME, COLUMN_TYPE FROM SYS.EXA_ALL_COLUMNS " + "WHERE COLUMN_SCHEMA = {schema!s} AND COLUMN_TABLE = {table!s} " + "ORDER BY COLUMN_ORDINAL_POSITION", + {"schema": schema, "table": table}, + ).fetchall() return [ ColumnInfo( name=row["COLUMN_NAME"], data_type=row["COLUMN_TYPE"], is_primary_key=row["COLUMN_NAME"] in pk_columns, ) - for row in conn.meta.list_columns(schema, table) + for row in columns ] def get_procedures(self, conn: Any, database: str | None = None) -> list[str]: diff --git a/tests/connections/providers/exasol/test_adapter.py b/tests/connections/providers/exasol/test_adapter.py index 35c4dd02..8ab03b5c 100644 --- a/tests/connections/providers/exasol/test_adapter.py +++ b/tests/connections/providers/exasol/test_adapter.py @@ -72,11 +72,14 @@ def test_get_views_reads_schema_and_name_by_key(adapter: ExasolAdapter, mock_con def test_get_columns_combines_primary_key_and_column_information(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: - mock_conn.meta.execute_snapshot.return_value.fetchall.return_value = [{"COLUMN_NAME": "ID"}] - mock_conn.meta.list_columns.return_value = [ + pk = MagicMock() + pk.fetchall.return_value = [{"COLUMN_NAME": "ID"}] + columns = MagicMock() + columns.fetchall.return_value = [ {"COLUMN_NAME": "ID", "COLUMN_TYPE": "DECIMAL(18,0)"}, {"COLUMN_NAME": "NAME", "COLUMN_TYPE": "VARCHAR(200) UTF8"}, ] + mock_conn.meta.execute_snapshot.side_effect = [pk, columns] result = adapter.get_columns(mock_conn, "ORDERS", schema="SALES") @@ -92,9 +95,9 @@ def test_primary_key_lookup_is_snapshot_executed_and_parameterised(adapter: Exas # conn.meta.* wraps the query in Exasol's snapshot-execution hint, so it # cannot be blocked by a metadata lock; conn.execute would not be. mock_conn.execute.assert_not_called() - mock_conn.meta.execute_snapshot.assert_called_once() + assert mock_conn.meta.execute_snapshot.call_count == 2 - sql, params = mock_conn.meta.execute_snapshot.call_args.args + sql, params = mock_conn.meta.execute_snapshot.call_args_list[0].args assert "SYS.EXA_ALL_CONSTRAINT_COLUMNS" in sql assert "CONSTRAINT_TYPE = 'PRIMARY KEY'" in sql # Placeholders, not interpolated values. @@ -104,27 +107,25 @@ def test_primary_key_lookup_is_snapshot_executed_and_parameterised(adapter: Exas def test_table_without_a_primary_key_flags_no_column(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: - mock_conn.meta.execute_snapshot.return_value.fetchall.return_value = [] - mock_conn.meta.list_columns.return_value = [ + pk = MagicMock() + pk.fetchall.return_value = [] + columns = MagicMock() + columns.fetchall.return_value = [ {"COLUMN_NAME": "A", "COLUMN_TYPE": "BOOLEAN"}, {"COLUMN_NAME": "B", "COLUMN_TYPE": "DATE"}, ] + mock_conn.meta.execute_snapshot.side_effect = [pk, columns] result = adapter.get_columns(mock_conn, "LOG", schema="SALES") assert [column.is_primary_key for column in result] == [False, False] -def test_get_columns_without_a_schema_passes_an_empty_pattern(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: - # Design D8: default_schema is "", so an unset schema reaches list_columns as - # "" and matches nothing. The path is unreachable from the explorer - every - # Exasol table arrives from get_tables() as a populated (schema, name) pair - - # so it was left spec-faithful rather than given a fallback. This pins that - # deliberate choice; it does not endorse it. +def test_get_columns_without_a_schema_queries_the_empty_schema(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: adapter.get_columns(mock_conn, "ORDERS") assert adapter.default_schema == "" - assert mock_conn.meta.list_columns.call_args.args == ("", "ORDERS") + assert mock_conn.meta.execute_snapshot.call_args.args[1] == {"schema": "", "table": "ORDERS"} def test_get_procedures_returns_scripting_script_names(adapter: ExasolAdapter, mock_conn: MagicMock) -> None: diff --git a/tests/connections/providers/exasol/test_connect.py b/tests/connections/providers/exasol/test_connect.py index 6dd2b634..1965c47d 100644 --- a/tests/connections/providers/exasol/test_connect.py +++ b/tests/connections/providers/exasol/test_connect.py @@ -141,13 +141,11 @@ def test_tls_disable_turns_encryption_off() -> None: @pytest.mark.parametrize("options", [{}, {"tls_mode": "default"}], ids=["unset", "explicit"]) -def test_tls_default_encrypts_without_verifying_the_certificate(options: dict[str, Any]) -> None: - # Not left to the driver: pyexasol would demand CERT_REQUIRED, which no - # self-signed server - exasol/docker-db included - can satisfy. +def test_tls_default_preserves_driver_certificate_verification(options: dict[str, Any]) -> None: kwargs = _connect_kwargs(_config(options=options)) assert kwargs["encryption"] is True - assert kwargs["websocket_sslopt"] == {"cert_reqs": ssl.CERT_NONE} + assert "websocket_sslopt" not in kwargs def test_tls_require_encrypts_without_verifying_the_certificate() -> None: @@ -180,6 +178,7 @@ def test_verifying_modes_forward_configured_certificate_files(tls_mode: str) -> assert kwargs["websocket_sslopt"] == { "cert_reqs": ssl.CERT_REQUIRED, + "check_hostname": tls_mode == "verify-full", "ca_certs": "/certs/ca.pem", "certfile": "/certs/client.pem", "keyfile": "/certs/client.key", @@ -194,7 +193,7 @@ def test_verifying_modes_forward_configured_certificate_files(tls_mode: str) -> def test_unconfigured_certificate_files_are_omitted(certificate_options: dict[str, Any]) -> None: kwargs = _connect_kwargs(_config(options={"tls_mode": "verify-full", **certificate_options})) - assert kwargs["websocket_sslopt"] == {"cert_reqs": ssl.CERT_REQUIRED} + assert kwargs["websocket_sslopt"] == {"cert_reqs": ssl.CERT_REQUIRED, "check_hostname": True} # --- extra_options ---------------------------------------------------------- diff --git a/tests/connections/providers/exasol/test_regressions.py b/tests/connections/providers/exasol/test_regressions.py new file mode 100644 index 00000000..d493d8e4 --- /dev/null +++ b/tests/connections/providers/exasol/test_regressions.py @@ -0,0 +1,54 @@ +"""Behavioral regressions for TLS policy, auth prompts and fixture failures.""" +from __future__ import annotations + +import ssl +from unittest.mock import MagicMock + +import pytest + +from sqlit.domains.connections.cli.prompts import _needs_db_prompt +from sqlit.domains.connections.domain.config import ConnectionConfig +from sqlit.domains.connections.domain.passwords import needs_db_password +from sqlit.domains.connections.providers.exasol.adapter import ExasolAdapter + + +def test_default_tls_retains_driver_certificate_validation(): + kwargs = ExasolAdapter()._tls_args(ConnectionConfig(name='test', db_type='exasol')) + assert kwargs.get('encryption', True) + assert kwargs.get('websocket_sslopt', {}).get('cert_reqs', ssl.CERT_REQUIRED) == ssl.CERT_REQUIRED + + +@pytest.mark.parametrize(('mode', 'check_hostname'), [('verify-ca', False), ('verify-full', True)]) +def test_verifying_tls_modes_have_distinct_hostname_policy(mode, check_hostname): + cfg = ConnectionConfig(name='test', db_type='exasol', options={'tls_mode': mode}) + ssl_options = ExasolAdapter()._tls_args(cfg)['websocket_sslopt'] + assert ssl_options['cert_reqs'] == ssl.CERT_REQUIRED + assert ssl_options.get('check_hostname', True) is check_hostname + + +@pytest.mark.parametrize('auth', ['access_token', 'refresh_token']) +def test_provided_openid_secret_does_not_prompt_for_a_database_password(auth): + cfg = ConnectionConfig(name='test', db_type='exasol', options={'authenticator': auth, auth: 'SYNTHETIC_SECRET'}) + assert not needs_db_password(cfg) + assert not _needs_db_prompt(cfg) + + +def test_schema_setup_failure_fails_the_fixture_instead_of_skipping(monkeypatch): + from tests.fixtures import exasol + + conn = MagicMock() + conn.execute.side_effect = RuntimeError('seed statement failed') + monkeypatch.setattr(exasol, '_connect', lambda: conn) + setup = exasol.exasol_db.__wrapped__(True) + with pytest.raises(RuntimeError, match='seed statement failed'): + next(setup) + conn.close.assert_called_once() + + +def test_required_live_server_cannot_pass_by_skipping(monkeypatch): + from tests.fixtures import exasol + + monkeypatch.setenv('EXASOL_REQUIRE_LIVE', '1') + monkeypatch.setattr(exasol, 'exasol_available', lambda: False) + with pytest.raises(pytest.fail.Exception): + exasol.exasol_server_ready.__wrapped__() diff --git a/tests/fixtures/exasol.py b/tests/fixtures/exasol.py index 609eaf28..522561ca 100644 --- a/tests/fixtures/exasol.py +++ b/tests/fixtures/exasol.py @@ -5,6 +5,7 @@ import os import ssl import time +from contextlib import closing from typing import Any import pytest @@ -57,12 +58,17 @@ def exasol_server_ready() -> bool: """Check if Exasol is ready and return True/False.""" global _ready_error + required = os.environ.get("EXASOL_REQUIRE_LIVE") == "1" if not exasol_available(): + if required: + pytest.fail("Required Exasol server is not listening") return False try: import pyexasol # noqa: F401 except ImportError: + if required: + pytest.fail("Required pyexasol driver is not installed") pytest.skip("pyexasol is not installed") deadline = time.time() + EXASOL_READY_TIMEOUT @@ -73,6 +79,8 @@ def exasol_server_ready() -> bool: except Exception as e: _ready_error = str(e) if time.time() >= deadline: + if required: + pytest.fail(f"Required Exasol server did not become ready: {_ready_error}") return False time.sleep(_READY_INTERVAL) @@ -89,9 +97,7 @@ def exasol_db(exasol_server_ready: bool) -> str: except ImportError: pytest.skip("pyexasol is not installed") - try: - conn = _connect() - + with closing(_connect()) as conn: conn.execute(f"DROP SCHEMA IF EXISTS {EXASOL_SCHEMA} CASCADE") conn.execute(f"CREATE SCHEMA {EXASOL_SCHEMA}") conn.execute(f"OPEN SCHEMA {EXASOL_SCHEMA}") @@ -139,11 +145,6 @@ def exasol_db(exasol_server_ready: bool) -> str: (3, 'Gizmo', 29.99, 25) """) - conn.close() - - except Exception as e: - pytest.skip(f"Failed to setup Exasol schema: {e}") - yield EXASOL_SCHEMA try: diff --git a/tests/test_exasol.py b/tests/test_exasol.py index cc109d79..6eed2c98 100644 --- a/tests/test_exasol.py +++ b/tests/test_exasol.py @@ -166,3 +166,60 @@ def test_delete_exasol_connection(self, exasol_db, cli_runner): # Verify it's gone result = cli_runner("connection", "list") assert connection_name not in result.stdout + + +@pytest.mark.parametrize(('table', 'other'), [('A_B', 'AXB'), ('A%B', 'AXXB'), ("A'B", 'AXB')]) +def test_column_lookup_matches_literal_table_and_schema_names(exasol_db, table, other): + from sqlit.domains.connections.providers.exasol.adapter import ExasolAdapter + + from .fixtures.exasol import _connect + + adapter = ExasolAdapter() + quote = adapter.quote_identifier + conn = _connect() + other_schema = exasol_db.replace('_', 'X') + conn.execute(f'CREATE SCHEMA {quote(other_schema)}') + try: + conn.execute(f'CREATE TABLE {quote(exasol_db)}.{quote(table)} (EXPECTED_COL INTEGER PRIMARY KEY)') + conn.execute(f'CREATE TABLE {quote(exasol_db)}.{quote(other)} (WRONG_TABLE_COL INTEGER)') + conn.execute(f'CREATE TABLE {quote(other_schema)}.{quote(table)} (WRONG_SCHEMA_COL INTEGER)') + columns = adapter.get_columns(conn, table, schema=exasol_db) + assert [column.name for column in columns] == ['EXPECTED_COL'] + assert columns[0].is_primary_key + finally: + conn.execute(f'DROP SCHEMA {quote(other_schema)} CASCADE') + conn.close() + + +def test_tls_verification_modes_against_real_server(exasol_db, tmp_path): + import re + import subprocess + + from sqlit.domains.connections.domain.config import ConnectionConfig, TcpEndpoint + from sqlit.domains.connections.providers.exasol.adapter import ExasolAdapter + + from .fixtures.exasol import EXASOL_HOST, EXASOL_PASSWORD, EXASOL_PORT, EXASOL_USER + + peer = subprocess.run( + ['openssl', 's_client', '-connect', f'{EXASOL_HOST}:{EXASOL_PORT}', '-showcerts'], + input='', text=True, capture_output=True, timeout=15, + ) + chain = re.findall(r'-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----', peer.stdout, re.DOTALL) + assert chain, 'Server did not provide its test certificate chain' + ca_file = tmp_path / 'docker-server.pem' + ca_file.write_text('\n'.join(chain)) + adapter = ExasolAdapter() + cfg = ConnectionConfig(name='tls-proof', db_type='exasol', + endpoint=TcpEndpoint(host=EXASOL_HOST, port=str(EXASOL_PORT), username=EXASOL_USER, password=EXASOL_PASSWORD)) + # The Docker private CA issues the certificate for exacluster.local, not localhost. + with pytest.raises(Exception, match=r'(?i)certificate'): + adapter.connect(cfg) + cfg.options.update(tls_mode='verify-full', tls_ca=str(ca_file)) + with pytest.raises(Exception, match=r'(?i)certificate|hostname'): + adapter.connect(cfg) + cfg.options['tls_mode'] = 'verify-ca' + conn = adapter.connect(cfg) + try: + adapter.execute_test_query(conn) + finally: + conn.close()