diff --git a/infra/source_chips.py b/infra/source_chips.py new file mode 100644 index 00000000..f7267b0f --- /dev/null +++ b/infra/source_chips.py @@ -0,0 +1,208 @@ +"""Compact, generic display of knowledge-base retrieval sources. + +A retrieval step reports its sources as arbitrary strings: deep web URLs, +bare domains, URLs keyed by query parameters or opaque ids, object-store +paths (``s3://…``), DOIs, file names or plain titles. ``source_chip`` turns +any of them into a short readable label plus an optional safe link, and +``annotate_source_chips`` adds the chips to timeline rows, marking sources +an earlier step already returned. +""" + +from __future__ import annotations + +import re +from urllib.parse import parse_qsl, unquote, urlsplit + +REDACTION_MARK = "[REDACTED]" +LABEL_MAX = 60 + +_GENERIC_STEMS = {"index", "default", "home", "main", "view", "page", "article", "doc", "document", "content", "show"} +_DOI_RE = re.compile(r"^(?:doi:)?(10\.\d{4,9}/\S+)$", re.IGNORECASE) +# A scheme-less web address needs a path or "www." so "refunds.md" stays a file name. +_BARE_HOST_RE = re.compile(r"^(?:www\.[a-z0-9.-]+\.[a-z]{2,}(?:[/?#]\S*)?|[a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z]{2,}[/?#]\S*)$", re.IGNORECASE) +_UUID_RE = re.compile(r"^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", re.IGNORECASE) +_TITLE_KEYS = ("p", "page", "title", "q", "id", "doc", "docid", "article", "name", "slug") + + +def _shorten(text: str, limit: int = LABEL_MAX) -> str: + text = " ".join(text.split()) + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _humanize(token: str) -> str: + """URL-decode a slug, swap separators for spaces, capitalise all-lowercase slugs.""" + text = unquote(token or "").strip() + if not text: + return "" + slug = " " not in text + text = re.sub(r"[-_+]+", " ", text).strip() + if slug and text == text.lower(): + text = text[:1].upper() + text[1:] + return text + + +def _split_ext(segment: str) -> tuple[str, str]: + stem, dot, ext = segment.rpartition(".") + if dot and stem and 1 <= len(ext) <= 5 and ext.isalnum() and not ext.isdigit(): + return stem, ext.upper() + return segment, "" + + +def _is_opaque(token: str) -> bool: + """Machine ids (numbers, hashes, UUIDs, random tokens) that read badly as labels.""" + if token.isdigit() or _UUID_RE.match(token): + return True + has_digit = any(ch.isdigit() for ch in token) + if len(token) >= 8 and has_digit and re.fullmatch(r"[0-9a-fA-F]+", token): + return True + return len(token) >= 16 and has_digit and not re.search(r"[-_ ]", token) + + +def _path_label(segments: list[str]) -> tuple[str, str]: + """Best label from path segments: (label, extension badge).""" + ext = "" + for i in range(len(segments) - 1, -1, -1): + stem, seg_ext = _split_ext(unquote(segments[i])) + if i == len(segments) - 1: + ext = seg_ext + if stem.lower() in _GENERIC_STEMS: + continue + if _is_opaque(stem): + parent = next( + ( + _split_ext(unquote(s))[0] + for s in reversed(segments[:i]) + if not _is_opaque(_split_ext(unquote(s))[0]) and _split_ext(unquote(s))[0].lower() not in _GENERIC_STEMS + ), + "", + ) + short = stem if len(stem) <= 10 else stem[:8] + "…" + return (f"{_humanize(parent)} · {short}" if parent else short), ext + return _humanize(stem), ext + return "", ext + + +def _query_label(query: str) -> str: + pairs = [(k, v) for k, v in parse_qsl(query, keep_blank_values=False) if v] + for key in _TITLE_KEYS: + for k, v in pairs: + if k.lower() == key: + return _humanize(v) if not _is_opaque(v) else f"{k}={v[:12]}" + if pairs: + k, v = pairs[0] + return f"{k}={_shorten(v, 16)}" + return "" + + +def source_chip(source) -> dict | None: + """Normalise one retrieval source into a display chip. + + Returns ``{label, sublabel, href, host, full, key, kind, ext, redacted}`` + or ``None`` for an empty source. ``href`` is only set for safe http(s) + links whose text was not altered by redaction. + """ + title = url = None + if isinstance(source, dict): + title = source.get("title") or source.get("name") + url = source.get("url") or source.get("source") or "" + raw = str(url or title or "").strip() + else: + raw = str(source if source is not None else "").strip() + url = raw + if not raw: + return None + redacted = REDACTION_MARK in raw + chip = { + "label": "", "sublabel": "", "href": None, "host": "", "full": raw, + "key": raw.rstrip("/"), "kind": "text", "ext": "", "redacted": redacted, + } + text = str(url or "") + doi = _DOI_RE.match(text) + if doi: + chip.update(kind="doi", label=_shorten(doi.group(1)), host="doi.org") + if not redacted: + chip["href"] = "https://doi.org/" + doi.group(1) + else: + candidate = text + if "://" not in candidate and _BARE_HOST_RE.match(candidate) and " " not in candidate: + candidate = "https://" + candidate + parts = urlsplit(candidate) if "://" in candidate or candidate.startswith("file:") else None + if parts and parts.scheme: + scheme = parts.scheme.lower() + host = (parts.hostname or "").lower() + segments = [s for s in parts.path.split("/") if s] + label, ext = _path_label(segments) + sub = _humanize(parts.fragment) if parts.fragment else "" + q_label = _query_label(parts.query) + if not label: + label, sub = (q_label, sub) if q_label else (host or scheme, sub) + elif q_label and not sub: + sub = q_label + chip.update( + label=_shorten(label), sublabel=_shorten(sub, 40), ext=ext, + kind="web" if scheme in ("http", "https") else "uri", + host=host if scheme in ("http", "https") else scheme, + key=f"{scheme}://{host}{parts.path.rstrip('/')}" + (f"?{parts.query}" if parts.query else "") + (f"#{parts.fragment}" if parts.fragment else ""), + ) + if scheme in ("http", "https") and host and not redacted: + chip["href"] = candidate + else: + stem, ext = _split_ext(text.rsplit("/", 1)[-1]) if "/" in text or "." in text else (text, "") + label = _humanize(stem) if (ext or "/" in text) else text + chip.update(label=_shorten(label or text), ext=ext, kind="file" if ext else "text") + if title: + # A real title says more than the URL path; the URL stays in the tooltip. + chip.update(label=_shorten(str(title)), sublabel="") + chip["title"] = str(title) if title else "" + chip["doc_id"] = str(source.get("id") or "") if isinstance(source, dict) else "" + chip["score"] = None + if isinstance(source, dict) and source.get("score") is not None: + try: + chip["score"] = round(float(source["score"]), 2) + except (TypeError, ValueError): + pass + return chip + + +def _tooltip(chip: dict) -> str: + """Multi-line hover text: everything known about the source, nothing truncated.""" + lines = [chip["title"]] if chip["title"] else [] + if chip["score"] is not None: + lines.append(f"Score: {chip['score']}") + if chip["doc_id"]: + lines.append(f"ID: {chip['doc_id']}") + if chip["full"] != chip["title"]: + lines.append(chip["full"]) + if chip["seen_in"]: + lines.append(f"Also returned by {chip['seen_in']}") + if chip["redacted"]: + lines.append("Redacted: link disabled") + return "\n".join(lines) + + +def annotate_source_chips(rows: list[dict], *, visible: int = 3) -> None: + """Add ``kb_chips``, ``kb_hosts``, ``kb_hidden`` to each row with ``kb_sources``. + + A chip whose source an earlier row already returned gets ``seen_in`` set + to that row's label, so repeated retrievals read as repeats. + """ + first_seen: dict[str, str] = {} + for row in rows: + chips = [] + keys = set() + for src in row.get("kb_docs") or row.get("kb_sources") or []: + chip = source_chip(src) + if not chip or chip["key"] in keys: + continue + keys.add(chip["key"]) + chip["seen_in"] = first_seen.get(chip["key"], "") + chip["tooltip"] = _tooltip(chip) + chips.append(chip) + for chip in chips: + first_seen.setdefault(chip["key"], str(row.get("label") or row.get("kind") or "an earlier step")) + hosts = list(dict.fromkeys(c["host"] for c in chips if c["host"])) + row["kb_chips"] = chips + row["kb_visible"] = visible + row["kb_hidden"] = max(0, len(chips) - visible) + row["kb_hosts"] = hosts[:2] + row["kb_hosts_more"] = max(0, len(hosts) - 2) diff --git a/infra/tests/test_source_chips.py b/infra/tests/test_source_chips.py new file mode 100644 index 00000000..325ea079 --- /dev/null +++ b/infra/tests/test_source_chips.py @@ -0,0 +1,79 @@ +from django.test import SimpleTestCase + +from infra.source_chips import annotate_source_chips, source_chip + + +class SourceChipTests(SimpleTestCase): + def test_deep_url_uses_last_segment_and_anchor(self): + c = source_chip("https://legehandboka.no/handboken/psykiatri/selvmord-og-selvmordsforsok/#tiltak-og-rad") + self.assertEqual(c["label"], "Selvmord og selvmordsforsok") + self.assertEqual(c["sublabel"], "Tiltak og rad") + self.assertEqual(c["host"], "legehandboka.no") + self.assertTrue(c["href"].startswith("https://")) + + def test_domain_only(self): + c = source_chip("https://nhi.no/") + self.assertEqual(c["label"], "nhi.no") + self.assertEqual(c["href"], "https://nhi.no/") + + def test_query_id(self): + c = source_chip("https://example.org/view.aspx?p=Refund_Policy") + self.assertEqual(c["label"], "Refund Policy") + + def test_opaque_segment_keeps_parent(self): + c = source_chip("https://example.org/articles/3f9a2b7c1d4e5f60") + self.assertTrue(c["label"].startswith("Articles · 3f9a2b7c")) + + def test_generic_filename_uses_parent_and_ext_badge(self): + c = source_chip("https://example.org/guides/refunds/index.html") + self.assertEqual((c["label"], c["ext"]), ("Refunds", "HTML")) + + def test_non_http_scheme_is_not_linked(self): + c = source_chip("s3://bucket/policies/refund-policy.pdf") + self.assertEqual((c["label"], c["ext"], c["href"], c["host"]), ("Refund policy", "PDF", None, "s3")) + + def test_unsafe_scheme_never_linked(self): + self.assertIsNone(source_chip("javascript:alert(1)")["href"]) + self.assertIsNone(source_chip("data:text/html,x")["href"]) + + def test_doi_links_resolver(self): + c = source_chip("10.1000/xyz123") + self.assertEqual(c["href"], "https://doi.org/10.1000/xyz123") + + def test_bare_domain_path_is_linked(self): + self.assertEqual(source_chip("nhi.no/sykdommer/hodepine")["href"], "https://nhi.no/sykdommer/hodepine") + + def test_plain_title_and_file(self): + self.assertEqual(source_chip("Acme Retail Policy")["label"], "Acme Retail Policy") + self.assertIsNone(source_chip("Acme Retail Policy")["href"]) + c = source_chip("refunds.md") + self.assertEqual((c["label"], c["ext"], c["kind"]), ("Refunds", "MD", "file")) + + def test_redacted_url_not_linked(self): + c = source_chip("https://a.no/x/#klini[REDACTED]") + self.assertTrue(c["redacted"]) + self.assertIsNone(c["href"]) + + def test_document_object_title_and_score(self): + c = source_chip({"source": "https://a.no/selvmord/", "title": "Selvmord: Generelt", "score": 0.770743, "id": "d1"}) + self.assertEqual((c["label"], c["score"], c["doc_id"]), ("Selvmord: Generelt", 0.77, "d1")) + self.assertEqual(c["href"], "https://a.no/selvmord/") + + def test_empty(self): + self.assertIsNone(source_chip("")) + self.assertIsNone(source_chip(None)) + + def test_annotate_marks_repeats_and_hidden(self): + rows = [ + {"label": "RETRIEVAL-1", "kb_sources": ["https://a.no/x", "https://a.no/y", "https://a.no/y/", "b", "c"]}, + {"label": "RETRIEVAL-2", "kb_docs": [{"source": "https://a.no/x", "title": "X", "score": 0.5}]}, + {"label": "LLM-1"}, + ] + annotate_source_chips(rows) + self.assertEqual(len(rows[0]["kb_chips"]), 4) + self.assertEqual(rows[0]["kb_hidden"], 1) + chip = rows[1]["kb_chips"][0] + self.assertEqual(chip["seen_in"], "RETRIEVAL-1") + self.assertIn("Score: 0.5", chip["tooltip"]) + self.assertIn("Also returned by RETRIEVAL-1", chip["tooltip"]) + self.assertEqual(rows[2]["kb_chips"], []) diff --git a/infra/ui.py b/infra/ui.py index be38f8dc..4dd1ad94 100644 --- a/infra/ui.py +++ b/infra/ui.py @@ -52,6 +52,7 @@ from infra.exceptions import StableAPIError from infra.export_utils import csv_safe from infra.seo import SITE_NAME, _share_context +from infra.source_chips import annotate_source_chips from scenarios.models import ( Scenario, ScenarioSet, @@ -4265,6 +4266,23 @@ def _span_label(sid: str) -> str: return sid +def _retrieval_documents(value) -> list[dict]: + """Display metadata (source, title, score, id) of a ``gen_ai.retrieval.documents`` value.""" + if isinstance(value, str): + try: + value = json.loads(value) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(value, list): + return [] + keys = ("source", "url", "title", "name", "score", "id") + return [ + {k: d[k] for k in keys if d.get(k) not in (None, "")} + for d in value + if isinstance(d, dict) and (d.get("source") or d.get("url") or d.get("title")) + ] + + def _document_source_names(value) -> list[str]: """Source names of a ``gen_ai.retrieval.documents`` value (a list, or its JSON text).""" if isinstance(value, str): @@ -4334,6 +4352,7 @@ def _agentic_span_view(span: dict) -> dict | None: "decision_detail": decision_detail, "is_tool_decision": is_tool_decision, "kb_sources": list(dict.fromkeys(str(s) for s in kb_sources)), + "kb_docs": _retrieval_documents(attrs.get("gen_ai.retrieval.documents")), "started_at": start, "duration_ms": round(_trace_span_duration_ms(span), 1), "content_blocks": _trace_span_content_blocks(span), @@ -4693,6 +4712,7 @@ def _merge_retrieval_into_tool(tool_row: dict, retrieval_row: dict) -> None: tool_row["content_blocks"] = blocks + retrieved tool_row["raw_json"] = f"{tool_row['raw_json']}\n\n{retrieval_row['raw_json']}" tool_row["kb_sources"] = list(dict.fromkeys(tool_row["kb_sources"] + retrieval_row["kb_sources"])) + tool_row["kb_docs"] = tool_row.get("kb_docs", []) + retrieval_row.get("kb_docs", []) tool_row["arguments"] = tool_row["arguments"] or retrieval_row["arguments"] tool_row["is_retrieval"] = True tool_row["icon"], tool_row["tone"] = _AGENTIC_SPAN_KIND_ICONS["RETRIEVER"] @@ -4883,6 +4903,7 @@ def host(row: dict) -> dict: while parent["span_id"] not in by_id and parent["span_id"] in folded: parent = folded[parent["span_id"]] anchor_for.setdefault(span_id, parent["anchor"]) + annotate_source_chips(rows) all_spans = [ s for s in (rep.get("trace_evidence") or {}).get("all_spans") or [] if isinstance(s, dict) and not _is_workflow_wrapper(s) diff --git a/templates/partials/agentic_timeline.html b/templates/partials/agentic_timeline.html index daca710e..276e1b47 100644 --- a/templates/partials/agentic_timeline.html +++ b/templates/partials/agentic_timeline.html @@ -41,13 +41,26 @@

Trajectory timeline

{{ s.decision_detail }} {% elif s.arguments %}{{ s.arguments }} {% elif s.tool_name %}· {{ s.tool_name }}{% endif %} - {% if s.kb_sources %} - 📚 {{ s.kb_sources|join:", " }} - {% endif %} {% if s.duration_ms > 0 %}({{ s.duration_ms }} ms){% endif %} {% if s.offset_ms is not None %}{{ s.offset_ms }} ms{% endif %} + {% if s.kb_chips %} +
+ 📚 {{ s.kb_chips|length }} source{{ s.kb_chips|length|pluralize }} + {% if s.kb_hosts %}· {{ s.kb_hosts|join:", " }}{% if s.kb_hosts_more %} +{{ s.kb_hosts_more }}{% endif %}{% endif %} + {% for c in s.kb_chips %} + {% if c.href %} + {% if c.ext %}{{ c.ext }}{% endif %} + {{ c.label }}{% if c.sublabel %} › {{ c.sublabel }}{% endif %} + {% if c.seen_in %}↺{% endif %} + {% if c.href %}{% else %}{% endif %} + {% endfor %} + {% if s.kb_hidden %}{% endif %} +
+ {% endif %}
{% if s.content_blocks %} @@ -70,6 +83,26 @@

Trajectory timeline

{% endfor %}