feat(aicore): transparent TLS mode and reactive credential reload

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sap-cloud-sdk"
-version = "0.45.4"
+version = "0.46.0"
 description = "SAP Cloud SDK for Python"
 readme = "README.md"
 license = "Apache-2.0"

src/sap_cloud_sdk/aicore/__init__.py

@@ -7,6 +7,7 @@
 import json
 import logging
 import os
+import threading
 from typing import Optional
 
 from sap_cloud_sdk.core.secret_resolver import resolve_base_mount
@@ -175,8 +176,73 @@ def set_aicore_config(instance_name: str = "aicore-instance") -> None:
     set_filtering()
 
 
+def _get_secret_dir_mtime(instance_name: str = "aicore-instance") -> float:
+    """Return the mtime of the AI Core secret directory, or 0.0 if it does not exist."""
+    secret_dir = os.path.join(resolve_base_mount(), "aicore", instance_name)
+    try:
+        return os.stat(secret_dir).st_mtime
+    except OSError:
+        return 0.0
+
+
+def watch_aicore_config(
+    instance_name: str = "aicore-instance",
+    interval: float = 60.0,
+    stop_event: threading.Event | None = None,
+) -> threading.Thread:
+    """Start a daemon thread that proactively reloads AI Core credentials
+    when the mounted secret volume changes.
+
+    Polls the secret directory mtime every ``interval`` seconds. On change,
+    calls :func:`set_aicore_config` before LiteLLM's cached OAuth token
+    expires — avoiding 401 errors entirely rather than recovering from them.
+
+    Kubernetes projected volumes perform an atomic symlink swap on rotation,
+    which changes the directory mtime. Both ``secret`` and ``projected``
+    volume types are covered.
+
+    Returns the daemon thread. Stop it cleanly via ``stop_event.set()``.
+
+    Each call starts a new daemon thread — avoid calling more than once per process.
+
+    Typical usage::
+
+        import threading
+        from sap_cloud_sdk.aicore import set_aicore_config, watch_aicore_config
+
+        set_aicore_config()
+
+        _stop = threading.Event()
+        watch_aicore_config(stop_event=_stop)
+        # at shutdown: _stop.set()
+    """
+    if stop_event is None:
+        stop_event = threading.Event()
+
+    last_mtime = _get_secret_dir_mtime(instance_name)
+
+    def _watch() -> None:
+        nonlocal last_mtime
+        while not stop_event.wait(timeout=interval):
+            try:
+                current_mtime = _get_secret_dir_mtime(instance_name)
+                if current_mtime != last_mtime:
+                    logger.info(
+                        "AI Core secret volume changed — proactively reloading credentials"
+                    )
+                    set_aicore_config(instance_name=instance_name)
+                    last_mtime = current_mtime
+            except Exception:
+                logger.exception("Error during proactive AI Core credential reload")
+
+    thread = threading.Thread(target=_watch, daemon=True, name="aicore-secret-watcher")
+    thread.start()
+    return thread
+
+
 __all__ = [
     "set_aicore_config",
+    "watch_aicore_config",
     "set_filtering",
     "disable_filtering",
     "completion",

src/sap_cloud_sdk/aicore/completion.py

@@ -14,6 +14,16 @@
 re-raising as :class:`ContentFilteredError` so callers can rely on a
 single exception type for "filter blocked you."
 
+Credential rotation handling
+----------------------------
+When a credential (client_secret) is rotated while the pod is running,
+LiteLLM's cached token becomes invalid and the next token refresh attempt
+raises ``litellm.AuthenticationError``. The wrappers intercept this error,
+reload credentials from the mounted secret volume via
+:func:`sap_cloud_sdk.aicore.set_aicore_config`, and retry the call once.
+The caller is unaffected — rotation is transparent. If the retry also
+fails, the ``AuthenticationError`` propagates normally.
+
 Usage::
 
     from sap_cloud_sdk.aicore import completion, ContentFilteredError
@@ -39,12 +49,15 @@
 
 from __future__ import annotations
 
+import logging
 from typing import Any
 
 import litellm
 
 from .filtering.filters import _parse_input_filter_error
 
+logger = logging.getLogger(__name__)
+
 
 def _maybe_translate_filter_error(exc: BaseException) -> BaseException:
     """Return a :class:`ContentFilteredError` if ``exc`` is a wrapped
@@ -60,19 +73,22 @@ def _maybe_translate_filter_error(exc: BaseException) -> BaseException:
 
 
 def completion(*args: Any, **kwargs: Any) -> Any:
-    """Wrapper around :func:`litellm.completion` that normalises filter errors.
+    """Wrapper around :func:`litellm.completion` that normalises filter errors
+    and handles credential rotation transparently.
 
-    Forwards every argument unchanged. The only difference from calling
-    ``litellm.completion`` directly is that an input-filter rejection
-    (which litellm wraps in ``APIConnectionError``) is re-raised as
-    :class:`ContentFilteredError`. Output-filter rejections already
-    surface as :class:`ContentFilteredError` via the SDK's transport patch
-    and pass through unchanged.
-
-    All other exceptions surface verbatim.
+    On ``AuthenticationError`` (e.g. rotated client_secret), reloads
+    credentials from the mounted secret volume and retries once.
+    All other exceptions surface verbatim after the filter-error translation.
     """
     try:
         return litellm.completion(*args, **kwargs)
+    except litellm.AuthenticationError:
+        # Local import avoids circular dep: completion ← __init__ ← completion
+        from sap_cloud_sdk.aicore import set_aicore_config
+
+        logger.info("AI Core credentials reloading after authentication failure")
+        set_aicore_config()
+        return litellm.completion(*args, **kwargs)
     except Exception as exc:
         translated = _maybe_translate_filter_error(exc)
         if translated is exc:
@@ -83,10 +99,16 @@ def completion(*args: Any, **kwargs: Any) -> Any:
 async def acompletion(*args: Any, **kwargs: Any) -> Any:
     """Async wrapper around :func:`litellm.acompletion`.
 
-    Same translation semantics as :func:`completion`.
+    Same translation and credential-rotation semantics as :func:`completion`.
     """
     try:
         return await litellm.acompletion(*args, **kwargs)
+    except litellm.AuthenticationError:
+        from sap_cloud_sdk.aicore import set_aicore_config
+
+        logger.info("AI Core credentials reloading after authentication failure")
+        set_aicore_config()
+        return await litellm.acompletion(*args, **kwargs)
     except Exception as exc:
         translated = _maybe_translate_filter_error(exc)
         if translated is exc:

tests/aicore/unit/test_aicore_watcher.py

@@ -0,0 +1,184 @@
+"""Unit tests for watch_aicore_config() — proactive credential reload on secret mount change.
+
+The watcher polls the AI Core secret directory mtime every N seconds. When the mtime
+changes (Kubernetes projected volume atomic symlink swap on rotation), it calls
+set_aicore_config() proactively — before LiteLLM's cached OAuth token expires.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import time
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from sap_cloud_sdk.aicore import _get_secret_dir_mtime, watch_aicore_config
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_secret_dir(tmp_path: Path, instance_name: str = "aicore-instance") -> Path:
+    secret_dir = tmp_path / "aicore" / instance_name
+    secret_dir.mkdir(parents=True)
+    (secret_dir / "clientsecret").write_text("secret-v1")
+    return secret_dir
+
+
+# ---------------------------------------------------------------------------
+# _get_secret_dir_mtime
+# ---------------------------------------------------------------------------
+
+
+class TestGetSecretDirMtime:
+    def test_returns_float_for_existing_dir(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        _make_secret_dir(tmp_path)
+        mtime = _get_secret_dir_mtime()
+        assert isinstance(mtime, float)
+        assert mtime > 0.0
+
+    def test_returns_zero_for_missing_dir(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        # Do not create the secret dir
+        assert _get_secret_dir_mtime() == 0.0
+
+    def test_stable_without_modification(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        _make_secret_dir(tmp_path)
+        m1 = _get_secret_dir_mtime()
+        m2 = _get_secret_dir_mtime()
+        assert m1 == m2
+
+
+# ---------------------------------------------------------------------------
+# watch_aicore_config
+# ---------------------------------------------------------------------------
+
+
+class TestWatchAicoreConfig:
+    def test_thread_is_daemon(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        _make_secret_dir(tmp_path)
+        stop = threading.Event()
+        with patch("sap_cloud_sdk.aicore.set_aicore_config"):
+            t = watch_aicore_config(interval=60.0, stop_event=stop)
+        stop.set()
+        assert t.daemon is True
+
+    def test_no_reload_when_mtime_unchanged(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        _make_secret_dir(tmp_path)
+        stop = threading.Event()
+        with patch("sap_cloud_sdk.aicore.set_aicore_config") as mock_reload:
+            t = watch_aicore_config(interval=0.05, stop_event=stop)
+            time.sleep(0.2)
+            stop.set()
+            t.join(timeout=1.0)
+        mock_reload.assert_not_called()
+
+    def test_reloads_on_directory_mtime_change(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        secret_dir = _make_secret_dir(tmp_path)
+        stop = threading.Event()
+        reloaded = threading.Event()
+
+        def _fake_reload(**kwargs):
+            reloaded.set()
+
+        with patch("sap_cloud_sdk.aicore.set_aicore_config", side_effect=_fake_reload):
+            t = watch_aicore_config(interval=0.05, stop_event=stop)
+            # Advance directory mtime to simulate kubelet secret rotation
+            new_time = time.time() + 10
+            os.utime(secret_dir, (new_time, new_time))
+            assert reloaded.wait(timeout=1.0), "reload was not triggered after mtime change"
+            stop.set()
+            t.join(timeout=1.0)
+
+    def test_logs_info_on_reload(self, tmp_path, monkeypatch, caplog):
+        import logging
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        secret_dir = _make_secret_dir(tmp_path)
+        stop = threading.Event()
+        reloaded = threading.Event()
+
+        def _fake_reload(**kwargs):
+            reloaded.set()
+
+        with patch("sap_cloud_sdk.aicore.set_aicore_config", side_effect=_fake_reload):
+            with caplog.at_level(logging.INFO, logger="sap_cloud_sdk.aicore"):
+                t = watch_aicore_config(interval=0.05, stop_event=stop)
+                new_time = time.time() + 10
+                os.utime(secret_dir, (new_time, new_time))
+                reloaded.wait(timeout=1.0)
+                stop.set()
+                t.join(timeout=1.0)
+
+        assert any(
+            "proactively reloading credentials" in r.message for r in caplog.records
+        )
+
+    def test_exception_in_set_aicore_config_does_not_crash_thread(
+        self, tmp_path, monkeypatch
+    ):
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        secret_dir = _make_secret_dir(tmp_path)
+        stop = threading.Event()
+        errored = threading.Event()
+
+        def _boom(**kwargs):
+            errored.set()
+            raise RuntimeError("simulated reload failure")
+
+        with patch("sap_cloud_sdk.aicore.set_aicore_config", side_effect=_boom):
+            t = watch_aicore_config(interval=0.05, stop_event=stop)
+            new_time = time.time() + 10
+            os.utime(secret_dir, (new_time, new_time))
+            assert errored.wait(timeout=1.0)
+            # Thread must still be alive after the exception
+            assert t.is_alive()
+            stop.set()
+            t.join(timeout=1.0)
+
+    def test_stop_event_exits_loop(self, tmp_path, monkeypatch):
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        _make_secret_dir(tmp_path)
+        stop = threading.Event()
+        with patch("sap_cloud_sdk.aicore.set_aicore_config"):
+            t = watch_aicore_config(interval=0.05, stop_event=stop)
+            stop.set()
+            t.join(timeout=1.0)
+        assert not t.is_alive()
+
+    def test_custom_instance_name_forwarded_to_set_aicore_config(
+        self, tmp_path, monkeypatch
+    ):
+        custom = "my-aicore"
+        monkeypatch.setenv("SERVICE_BINDING_ROOT", str(tmp_path))
+        secret_dir = tmp_path / "aicore" / custom
+        secret_dir.mkdir(parents=True)
+        (secret_dir / "clientsecret").write_text("v1")
+        stop = threading.Event()
+        reloaded = threading.Event()
+        captured_kwargs: list = []
+
+        def _fake_reload(**kwargs):
+            captured_kwargs.append(kwargs)
+            reloaded.set()
+
+        with patch("sap_cloud_sdk.aicore.set_aicore_config", side_effect=_fake_reload):
+            t = watch_aicore_config(
+                instance_name=custom, interval=0.05, stop_event=stop
+            )
+            new_time = time.time() + 10
+            os.utime(secret_dir, (new_time, new_time))
+            assert reloaded.wait(timeout=1.0)
+            stop.set()
+            t.join(timeout=1.0)
+
+        assert captured_kwargs[0].get("instance_name") == custom