Repository navigation
Expand file tree
/
Copy pathsourcegraph.py
More file actions
180 lines (163 loc) · 6.82 KB
/
Copy pathsourcegraph.py
File metadata and controls
180 lines (163 loc) · 6.82 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
"""Sourcegraph public-repo code search lane.
POST sourcegraph.com/.api/graphql, no auth, public repos only.
Verified live: resultCount + FileMatch (path/url/lineMatches) shape.
Complements PkgSeer (package-scoped) and searchcode (per-repo analysis):
Sourcegraph is the global public-code lane.
"""
from __future__ import annotations
import asyncio
from typing import Any
import httpx
from codequery import relax_queries
from config import get_http_client
_SG_GQL = "https://sourcegraph.com/.api/graphql"
_TIMEOUT = 30
# Server-side time budget: count:50 blows our 30s timeout and returns
# nothing (measured 2026-10-20); count:5..count:10 answers in ~2.4s.
SG_COUNT_MAX = 10
SG_RETRY_DELAY_S = 2.0
_SEARCH_QUERY = """query($q:String!){
search(query:$q,version:V3){
results{
resultCount
results{
__typename
... on FileMatch{
file{path url}
repository{name}
lineMatches{preview lineNumber}
}
... on CommitSearchResult{
commit{oid message url}
}
}
}
}
}"""
def _build_query(query: str, repo: str = "", lang: str = "",
count: int = 10) -> str:
# NOTE: no `type:code` filter — it suppresses results on the public
# endpoint (verified: same query 0 with / 3 without). FileMatch-only
# extraction keeps the lane code-typed regardless.
parts = ["context:global", f"count:{min(max(count, 1), SG_COUNT_MAX)}"]
if repo:
r = repo.strip().removesuffix(".git").rstrip("/")
if "github.com/" in r:
r = r.split("github.com/", 1)[1]
# Sourcegraph repo: filters need the github.com/ prefix to match.
# NOTE: do NOT re.escape — escaped dots/slashes match nothing.
if "/" in r:
if not r.startswith("github.com/"):
r = f"github.com/{r}"
parts.append(f"repo:^{r}$")
else:
parts.append(f"repo:{r}")
if lang:
parts.append(f"lang:{lang}")
parts.append(query)
return " ".join(parts)
def _extract(data: dict) -> list[dict]:
out = []
try:
res = data["data"]["search"]["results"]
except (KeyError, TypeError):
return out
total = res.get("resultCount", 0)
for item in (res.get("results", []) or []):
if item.get("__typename") == "FileMatch":
f = item.get("file", {}) or {}
repo = (item.get("repository", {}) or {}).get("name", "")
url = f.get("url", "")
if url and url.startswith("/"):
url = f"https://sourcegraph.com{url}"
matches = []
for lm in (item.get("lineMatches", []) or [])[:5]:
matches.append({"preview": lm.get("preview", "")[:300],
"line": lm.get("lineNumber", 0)})
out.append({"source": "sourcegraph",
"repo": repo.replace("github.com/", ""),
"file": f.get("path", ""),
"url": url,
"matches": matches,
"title": f.get("path", ""),
"text": " ".join(m.get("preview", "")
for m in matches)[:500]})
elif item.get("__typename") == "CommitSearchResult":
c = item.get("commit", {}) or {}
out.append({"source": "sourcegraph",
"repo": "",
"commit": c.get("oid", "")[:12],
"message": (c.get("message", "") or "")[:300],
"url": c.get("url", ""),
"title": (c.get("message", "") or "").split("\n")[0][:120],
"text": (c.get("message", "") or "")[:500]})
return out
async def search_sourcegraph(query: str, repo: str = "", lang: str = "",
count: int = 10,
relax: bool = True) -> dict[str, Any]:
"""Global code search over public repos. Keyless. Returns file matches.
`context:global` ANDs every term, so a natural-language query returns zero
even when all its words are common (measured 2026-10-31: "httpx vs aiohttp
benchmark" 0, "httpx aiohttp" 5). On an empty but successful result the
lane retries the shortened variants from `codequery.relax_queries` and
keeps `query_relaxed` / `query_requested` on the payload; an upstream
error stops the ladder instead of hammering a failing endpoint.
"""
if not query.strip():
return {"success": False, "error": "query required"}
r = await _sg_attempt(_build_query(query, repo, lang, count))
if relax and r.get("success") and not r.get("results"):
for cand in relax_queries(query):
r2 = await _sg_attempt(_build_query(cand, repo, lang, count))
if not r2.get("success"):
break
if r2.get("results"):
r2["query_relaxed"] = True
r2["query_requested"] = query.strip()
return r2
return r
async def _sg_attempt(q: str) -> dict[str, Any]:
"""One query string, with the existing single transient-error retry."""
last_err = "Sourcegraph request failed"
for attempt in range(2):
if attempt:
# One retry with a short backoff — upstream is flaky under load.
await asyncio.sleep(SG_RETRY_DELAY_S)
try:
c = get_http_client()
r = await c.post(_SG_GQL,
json={"query": _SEARCH_QUERY,
"variables": {"q": q}},
headers={"Content-Type": "application/json",
"User-Agent": "mcp-codesearch/1.0"},
timeout=_TIMEOUT)
if r.status_code != 200:
last_err = f"Sourcegraph HTTP {r.status_code} {r.text[:200]}"
continue
try:
data = r.json()
except ValueError as e:
last_err = f"Sourcegraph JSON parse error: {e}"
continue
if isinstance(data, dict) and data.get("errors"):
msgs = [e.get("message", str(e))
for e in data["errors"]][:3]
last_err = f"Sourcegraph: {'; '.join(msgs)}"
continue
results = _extract(data)
try:
total = data["data"]["search"]["results"].get("resultCount", 0)
except (KeyError, TypeError):
total = len(results)
if results or attempt:
return {"success": True, "results": results,
"total": total, "query": q}
last_err = "Sourcegraph returned no results"
continue
except httpx.TimeoutException:
last_err = "Sourcegraph timed out"
continue
except httpx.RequestError as e:
last_err = f"Sourcegraph request failed: {e}"
continue
return {"success": False, "error": last_err}