Repository navigation
Expand file tree
/
Copy pathdeepwiki.py
More file actions
174 lines (159 loc) · 8.27 KB
/
Copy pathdeepwiki.py
File metadata and controls
174 lines (159 loc) · 8.27 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
from __future__ import annotations
import asyncio
import json
import httpx
from config import DEEPWIKI_MCP, api_error
from config import _http_request_capped
MAX_DEEPWIKI_RETRIES = 3
BASE_DELAY = 1.0
# DeepWiki MCP replies (SSE frames with wiki text) are small; cap so a
# hostile endpoint cannot stream an unbounded body into RAM.
_MAX_BYTES = 2_000_000
_MCP_ERROR_MARKERS = ("Repository not found", "Error processing question:", "error while processing your question", "Invalid repoName format")
def _is_error_text(text: str) -> bool:
"""DeepWiki returns success:True even when the answer/structure text is an
error (e.g. 'Repository not found. Visit https://deepwiki.com to index it.').
Detect those so the caller treats them as failures instead of valid output."""
t = (text or "").strip()
if not t:
return False
first = t[:120]
return any(m in first for m in _MCP_ERROR_MARKERS)
def _parse_mcp_sse(text: str) -> dict | None:
# L10: SSE frames can split one JSON payload across multiple `data:` lines.
# Collect the current frame (data lines) and try each, not just the first.
buffered: list[str] = []
for line in text.split("\n"):
line = line.strip()
if line.startswith("data: "):
buffered.append(line[6:])
try:
return json.loads("\n".join(buffered))
except json.JSONDecodeError:
continue
elif line == "":
buffered = []
try:
return json.loads(text)
except json.JSONDecodeError:
return None
async def deepwiki_fetch(owner: str, repo: str, wiki_name: str = "") -> dict:
repo_label = f"{owner}/{repo}"
last_err = None
for attempt in range(MAX_DEEPWIKI_RETRIES):
try:
headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"}
init = await _http_request_capped("POST", DEEPWIKI_MCP, json={
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "codesearch", "version": "1.0"}}
}, headers=headers, max_bytes=_MAX_BYTES)
if init.status_code != 200:
last_err = api_error("init failed", init)
if attempt < MAX_DEEPWIKI_RETRIES - 1:
await asyncio.sleep(BASE_DELAY * (2 ** attempt))
continue
break
struct_args: dict[str, str] = {"repoName": repo_label}
struct = await _http_request_capped("POST", DEEPWIKI_MCP, json={
"jsonrpc": "2.0", "id": 2, "method": "tools/call",
"params": {"name": "read_wiki_structure", "arguments": struct_args}
}, headers=headers, max_bytes=_MAX_BYTES)
if struct.status_code != 200:
last_err = api_error("structure", struct)
if attempt < MAX_DEEPWIKI_RETRIES - 1:
await asyncio.sleep(BASE_DELAY * (2 ** attempt))
continue
break
sdata = _parse_mcp_sse(struct.text)
if not sdata or "result" not in sdata:
last_err = "no result from structure"
if attempt < MAX_DEEPWIKI_RETRIES - 1:
await asyncio.sleep(BASE_DELAY * (2 ** attempt))
continue
break
sections = []
for item in sdata["result"].get("content", []):
if item.get("type") == "text":
sections.append(item["text"][:3000])
joined = "".join(sections)
if _is_error_text(joined):
# surface a clean failure so the caller knows the repo is unindexed/wrong
return {"success": False, "error": sections[0][:300] if sections else "DeepWiki: repository not found/unindexed", "repo": repo_label}
detail_args: dict[str, str] = {"repoName": repo_label}
detail_data = await _http_request_capped("POST", DEEPWIKI_MCP, json={
"jsonrpc": "2.0", "id": 3, "method": "tools/call",
"params": {"name": "read_wiki_contents", "arguments": detail_args}
}, headers=headers, max_bytes=_MAX_BYTES)
detail = ""
if detail_data.status_code == 200:
ddata = _parse_mcp_sse(detail_data.text)
if ddata and "result" in ddata:
texts = []
for item in ddata["result"].get("content", []):
if item.get("type") == "text":
texts.append(item["text"])
detail = "\n\n".join(texts)[:10000]
return {"success": True, "source": "deepwiki_mcp", "repo": repo_label, "structure": sections[:5], "detail": detail}
except (httpx.HTTPError, json.JSONDecodeError, ValueError) as e:
last_err = str(e)
if attempt < MAX_DEEPWIKI_RETRIES - 1:
await asyncio.sleep(BASE_DELAY * (2 ** attempt))
continue
try:
r = await _http_request_capped("GET", f"https://deepwiki.com/{owner}/{repo}",
headers={"User-Agent": "mcp-codesearch/1.0"},
max_bytes=_MAX_BYTES)
if r.status_code != 200:
return {"success": False, "error": api_error("deepwiki", r)}
return {"success": True, "source": "deepwiki_html", "content": r.text[:10000]}
except (httpx.HTTPError, ValueError) as e:
return {"success": False, "error": str(e)}
async def deepwiki_ask(owner: str = "", repo: str = "", question: str = "",
wiki_name: str = "", repos: list[str] | None = None) -> dict:
repo_label = repos if repos else f"{owner}/{repo}"
last_err = None
headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"}
for attempt in range(MAX_DEEPWIKI_RETRIES):
try:
init = await _http_request_capped("POST", DEEPWIKI_MCP, json={
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "codesearch", "version": "1.0"}}
}, headers=headers, max_bytes=_MAX_BYTES)
if init.status_code != 200:
last_err = api_error("init failed", init)
if attempt < MAX_DEEPWIKI_RETRIES - 1:
await asyncio.sleep(BASE_DELAY * (2 ** attempt))
continue
return {"success": False, "error": last_err}
ask_args: dict = {"repoName": repo_label, "question": question}
r = await _http_request_capped("POST", DEEPWIKI_MCP, json={
"jsonrpc": "2.0", "id": 2, "method": "tools/call",
"params": {"name": "ask_question", "arguments": ask_args}
}, headers=headers, max_bytes=_MAX_BYTES)
if r.status_code != 200:
last_err = api_error("DeepWiki", r)
if attempt < MAX_DEEPWIKI_RETRIES - 1:
await asyncio.sleep(BASE_DELAY * (2 ** attempt))
continue
return {"success": False, "error": last_err}
data = _parse_mcp_sse(r.text)
if not data or "result" not in data:
last_err = "no result from DeepWiki"
if attempt < MAX_DEEPWIKI_RETRIES - 1:
await asyncio.sleep(BASE_DELAY * (2 ** attempt))
continue
return {"success": False, "error": last_err}
answer = ""
for item in data["result"].get("content", []):
if item.get("type") == "text":
answer += item["text"] + "\n"
answer = answer.strip()
if _is_error_text(answer):
return {"success": False, "error": answer[:300], "repo": repo_label if isinstance(repo_label, str) else "/".join(repo_label)}
return {"success": True, "answer": answer, "source": "deepwiki"}
except (httpx.HTTPError, json.JSONDecodeError, ValueError) as e:
last_err = str(e)
if attempt < MAX_DEEPWIKI_RETRIES - 1:
await asyncio.sleep(BASE_DELAY * (2 ** attempt))
continue
return {"success": False, "error": last_err}