Index ingested result metadata in portal DB

docs/guides/portal-execution-profiles-handoff.md

@@ -34,7 +34,9 @@ Recommended order:
    `RESULT_SERVER_GITLAB_TRIGGER_TOKEN`; do not store it in SQLite, logs, or
    the OSS repository.
 6. Index received benchmark and estimation JSON metadata into SQLite while
-   keeping JSON/tgz artifacts as raw records.
+   keeping JSON/tgz artifacts as raw records. The first index should be an
+   auxiliary lookup table populated at ingest time; existing result and
+   estimate pages can remain file-backed until the indexed views are reviewed.
 7. Add environment snapshot storage after deciding which host/runtime metadata
    should define an environment identity.
 

result_server/routes/api.py

@@ -4,6 +4,7 @@
 import os
 import json
 import re
+import sqlite3
 import uuid
 import shutil
 import io
@@ -15,6 +16,7 @@
 from utils.auth import verify_ingest_key, verify_trusted_proxy_auth
 from utils.audit_logging import audit_event
 from utils.rate_limit import rate_limited
+from utils.result_metadata_index import index_result_metadata
 
 api_bp = Blueprint("api", __name__)
 _TIMESTAMP_RE = re.compile(r"^\d{8}_\d{6}$")
@@ -115,6 +117,41 @@ def save_json_file(data, prefix, out_dir, given_uuid=None):
         "id": unique_id,
         "timestamp": timestamp,
         "json_file": filename,
+        "payload": payload,
+    }
+
+
+def _index_saved_json(record_type, saved):
+    """Index saved JSON metadata when a Portal SQLite DB is configured."""
+    try:
+        indexed = index_result_metadata(
+            db_path=current_app.config.get("EXECUTION_PROFILE_DB_PATH"),
+            record_type=record_type,
+            payload=saved.get("payload", {}),
+            json_file=saved.get("json_file", ""),
+            fallback_uuid=saved.get("id", ""),
+            fallback_timestamp=saved.get("timestamp", ""),
+        )
+    except (sqlite3.Error, OSError, ValueError) as exc:
+        current_app.logger.exception("result metadata index update failed")
+        audit_event(
+            "result_metadata_index_failed",
+            target=saved.get("json_file", ""),
+            result="failure",
+            level=logging.ERROR,
+            details={"record_type": record_type, "error": str(exc)},
+        )
+        return False
+    return indexed
+
+
+def _saved_json_response(saved):
+    """Return the public API response fields for a saved JSON payload."""
+    return {
+        "status": saved["status"],
+        "id": saved["id"],
+        "timestamp": saved["timestamp"],
+        "json_file": saved["json_file"],
     }
 
 
@@ -280,14 +317,15 @@ def ingest_result():
         prefix="result",
         out_dir=current_app.config["RECEIVED_DIR"],
     )
+    _index_saved_json("result", saved)
     audit_event(
         "ingest_accepted",
         actor=runner_id,
         target=saved["json_file"],
         result="success",
         details={"ingest_type": "result", "id": saved["id"]},
     )
-    return saved, 200
+    return _saved_json_response(saved), 200
 
 
 @api_bp.route("/api/ingest/estimate", methods=["POST"])
@@ -306,14 +344,15 @@ def ingest_estimate():
         out_dir=current_app.config["ESTIMATED_DIR"],
         given_uuid=raw_uuid,
     )
+    _index_saved_json("estimate", saved)
     audit_event(
         "ingest_accepted",
         actor=runner_id,
         target=saved["json_file"],
         result="success",
         details={"ingest_type": "estimate", "id": saved["id"]},
     )
-    return saved, 200
+    return _saved_json_response(saved), 200
 
 
 @api_bp.route("/api/ingest/padata", methods=["POST"])

result_server/test_support.py

@@ -158,13 +158,16 @@ def build_api_route_app(
     received_padata_dir,
     received_estimation_artifacts_dir,
     estimated_dir,
+    execution_profile_db_path=None,
 ):
     """Build a Flask app with the API, results, and estimated blueprints for API tests."""
     app = Flask(__name__)
     app.config["RECEIVED_DIR"] = received_dir
     app.config["RECEIVED_PADATA_DIR"] = received_padata_dir
     app.config["RECEIVED_ESTIMATION_ARTIFACTS_DIR"] = received_estimation_artifacts_dir
     app.config["ESTIMATED_DIR"] = estimated_dir
+    if execution_profile_db_path is not None:
+        app.config["EXECUTION_PROFILE_DB_PATH"] = execution_profile_db_path
     app.config["TESTING"] = True
 
     from routes.api import api_bp

result_server/tests/test_api_routes.py

@@ -85,6 +85,60 @@ def test_post_valid_json(self, client, tmp_dirs):
         assert saved["code"] == "test"
         assert saved["_server_uuid"] == body["id"]
         assert saved["_server_timestamp"] == body["timestamp"]
+        assert "payload" not in body
+
+    def test_post_valid_json_indexes_metadata(self, tmp_dirs, tmp_path):
+        """Accepted result payloads should update the Portal SQLite metadata index."""
+        received, received_padata, received_estimation_artifacts, estimated = tmp_dirs
+        db_path = tmp_path / "cx_portal.sqlite3"
+        app = build_api_route_app(
+            received_dir=received,
+            received_padata_dir=received_padata,
+            received_estimation_artifacts_dir=received_estimation_artifacts,
+            estimated_dir=estimated,
+            execution_profile_db_path=str(db_path),
+        )
+        app.config["INGEST_KEYS"] = {API_KEY: "test-runner"}
+
+        with app.test_client() as client:
+            resp = client.post(
+                "/api/ingest/result",
+                data=json.dumps({
+                    "code": "qws",
+                    "system": "RIKYU",
+                    "Exp": "case0",
+                    "FOM": 42.0,
+                    "ci_trigger": "trigger",
+                    "pipeline_id": 3152,
+                }),
+                headers={
+                    "X-API-Key": API_KEY,
+                    "Content-Type": "application/json",
+                },
+            )
+
+        assert resp.status_code == 200
+        import sqlite3
+
+        with sqlite3.connect(db_path) as conn:
+            row = conn.execute(
+                """
+                SELECT record_type, result_uuid, json_file, code, system, exp,
+                       ci_trigger, pipeline_id
+                FROM result_metadata_index
+                """
+            ).fetchone()
+        body = resp.get_json()
+        assert row == (
+            "result",
+            body["id"],
+            body["json_file"],
+            "qws",
+            "RIKYU",
+            "case0",
+            "trigger",
+            "3152",
+        )
 
     def test_valid_key_logs_runner_id(self, client, caplog):
         """Accepted API requests should include the resolved runner id in logs."""
@@ -209,6 +263,60 @@ def test_post_valid_json(self, client, tmp_dirs):
         assert saved["code"] == "est-test"
         assert saved["estimate_metadata"]["estimation_result_uuid"] == body["id"]
         assert "estimation_result_timestamp" in saved["estimate_metadata"]
+        assert "payload" not in body
+
+    def test_post_valid_json_indexes_estimate_metadata(self, tmp_dirs, tmp_path):
+        """Accepted estimate payloads should update the Portal SQLite metadata index."""
+        received, received_padata, received_estimation_artifacts, estimated = tmp_dirs
+        db_path = tmp_path / "cx_portal.sqlite3"
+        app = build_api_route_app(
+            received_dir=received,
+            received_padata_dir=received_padata,
+            received_estimation_artifacts_dir=received_estimation_artifacts,
+            estimated_dir=estimated,
+            execution_profile_db_path=str(db_path),
+        )
+        app.config["INGEST_KEYS"] = {API_KEY: "test-runner"}
+
+        with app.test_client() as client:
+            resp = client.post(
+                "/api/ingest/estimate",
+                data=json.dumps({
+                    "code": "qws",
+                    "exp": "case0",
+                    "performance_ratio": 1.5,
+                    "current_system": {"system": "RIKYU"},
+                    "future_system": {"system": "FugakuNEXT"},
+                    "estimate_metadata": {
+                        "source_result_uuid": "11111111-2222-3333-4444-555555555555",
+                        "estimation_package": "weakscaling",
+                    },
+                }),
+                headers={
+                    "X-API-Key": API_KEY,
+                    "Content-Type": "application/json",
+                },
+            )
+
+        assert resp.status_code == 200
+        import sqlite3
+
+        with sqlite3.connect(db_path) as conn:
+            row = conn.execute(
+                """
+                SELECT record_type, result_uuid, json_file, code, system, exp
+                FROM result_metadata_index
+                """
+            ).fetchone()
+        body = resp.get_json()
+        assert row == (
+            "estimate",
+            body["id"],
+            body["json_file"],
+            "qws",
+            "RIKYU",
+            "case0",
+        )
 
     def test_post_valid_json_with_uuid_header(self, client, tmp_dirs):
         """A valid X-UUID header should be used as the persisted estimate id."""

result_server/tests/test_result_metadata_index.py

@@ -0,0 +1,114 @@
+"""Tests for the SQLite result metadata index."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from utils.result_metadata_index import (  # noqa: E402
+    extract_result_index_record,
+    index_result_metadata,
+    list_indexed_results,
+)
+
+
+def test_extract_result_index_record_from_benchmark_result():
+    payload = {
+        "code": "qws",
+        "system": "RIKYU",
+        "Exp": "case0",
+        "FOM": 42.5,
+        "_server_uuid": "11111111-2222-3333-4444-555555555555",
+        "_server_timestamp": "20260806_010203",
+        "ci_trigger": "pipeline",
+        "pipeline_id": 3152,
+        "source_info": {
+            "source_type": "git",
+            "repo_url": "https://example.org/repo.git",
+            "branch": "develop",
+            "commit_hash": "abcdef123456",
+        },
+    }
+
+    record = extract_result_index_record(
+        record_type="result",
+        payload=payload,
+        json_file="result_20260806_010203_11111111-2222-3333-4444-555555555555.json",
+    )
+
+    assert record["result_uuid"] == "11111111-2222-3333-4444-555555555555"
+    assert record["server_timestamp"] == "20260806_010203"
+    assert record["code"] == "qws"
+    assert record["system"] == "RIKYU"
+    assert record["exp"] == "case0"
+    assert record["pipeline_id"] == "3152"
+    assert record["source_type"] == "git"
+    assert record["source_ref"] == "abcdef123456"
+    metadata = json.loads(record["metadata_json"])
+    assert metadata["fom"] == 42.5
+    assert metadata["source_info"]["branch"] == "develop"
+
+
+def test_index_result_metadata_upserts_rows(tmp_path):
+    db_path = tmp_path / "cx_portal.sqlite3"
+    payload = {
+        "code": "qws",
+        "system": "RIKYU",
+        "_server_uuid": "11111111-2222-3333-4444-555555555555",
+        "_server_timestamp": "20260806_010203",
+    }
+
+    indexed = index_result_metadata(
+        db_path=str(db_path),
+        record_type="result",
+        payload=payload,
+        json_file="result.json",
+    )
+    payload["system"] = "FugakuNEXT"
+    index_result_metadata(
+        db_path=str(db_path),
+        record_type="result",
+        payload=payload,
+        json_file="result-renamed.json",
+    )
+
+    rows = list_indexed_results(str(db_path), record_type="result")
+    assert indexed is True
+    assert len(rows) == 1
+    assert rows[0]["json_file"] == "result-renamed.json"
+    assert rows[0]["system"] == "FugakuNEXT"
+
+
+def test_extract_result_index_record_from_estimate_result():
+    payload = {
+        "code": "qws",
+        "exp": "case0",
+        "performance_ratio": 2.5,
+        "current_system": {"system": "RIKYU"},
+        "future_system": {"system": "FugakuNEXT"},
+        "estimate_metadata": {
+            "estimation_result_uuid": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
+            "estimation_result_timestamp": "2026-08-06 01:02:03",
+            "source_result_uuid": "11111111-2222-3333-4444-555555555555",
+            "estimation_package": "weakscaling",
+        },
+        "applicability": {"status": "applicable"},
+    }
+
+    record = extract_result_index_record(
+        record_type="estimate",
+        payload=payload,
+        json_file="estimate_20260806_010203_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee.json",
+    )
+
+    assert record["result_uuid"] == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+    assert record["server_timestamp"] == "2026-08-06 01:02:03"
+    assert record["code"] == "qws"
+    assert record["system"] == "RIKYU"
+    assert record["exp"] == "case0"
+    metadata = json.loads(record["metadata_json"])
+    assert metadata["source_result_uuid"] == "11111111-2222-3333-4444-555555555555"
+    assert metadata["future_system"] == "FugakuNEXT"