Repository navigation
Expand file tree
/
Copy pathpkg_utils.py
More file actions
371 lines (343 loc) · 16.9 KB
/
Copy pathpkg_utils.py
File metadata and controls
371 lines (343 loc) · 16.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
"""Package utilities: resolution, extraction, changelog, upgrade review."""
from __future__ import annotations
import asyncio
import io
import logging
import os
import re
import shutil
import tarfile
import urllib.parse
from pathlib import Path
from typing import Any
import httpx
logger = logging.getLogger(__name__)
from config import get_http_client
PACKAGE_CACHE = os.path.expanduser("~/.local/share/cortexkit/aft/pkg-cache")
def _gh_repo_url(url: str) -> str | None:
"""Normalize various GitHub URL formats to 'owner/repo'."""
if not url:
return None
url = url.strip().rstrip("/").rstrip(".git")
m = re.match(r"(?:git\+)?https?://github\.com/([^/]+/[^/]+)", url)
if m:
return m.group(1)
m = re.match(r"git@github\.com:([^/]+/[^/]+)", url)
if m:
return m.group(1)
m = re.match(r"github:([^/]+/[^/]+)", url)
if m:
return m.group(1)
return None
async def resolve_package(registry: str, name: str, version: str = "") -> dict[str, Any]:
"""Resolve a package to metadata including repo URL.
Returns structured: success, name, version, registry, repo_owner, repo_name,
description, homepage, license, tarball_url, download_url.
"""
# Normalize scoped npm names
safe_name = urllib.parse.quote(name, safe="@")
try:
c = get_http_client()
if registry == "npm" or registry == "auto":
r = await c.get(f"https://registry.npmjs.org/{safe_name}",
headers={"User-Agent": "mcp-codesearch/1.0"})
if r.status_code == 200:
d = r.json()
latest = d.get("dist-tags", {}).get("latest", "")
ver = version or latest
repo_url = ""
repo_raw = d.get("repository", {})
if isinstance(repo_raw, dict):
repo_url = repo_raw.get("url", "")
elif isinstance(repo_raw, str):
repo_url = repo_raw
gh = _gh_repo_url(repo_url)
# Get version-specific data
ver_data = d.get("versions", {}).get(ver, {})
tarball_url = ver_data.get("dist", {}).get("tarball", "")
return {
"success": True, "name": name, "version": ver,
"registry": "npm", "description": d.get("description", ""),
"homepage": d.get("homepage", ""), "license": d.get("license", ""),
"repo_url": repo_url, "repo_owner": gh.split("/")[0] if gh else "",
"repo_name": gh.split("/")[1] if gh else "",
"tarball_url": tarball_url,
"download_url": f"https://www.npmjs.com/package/{name}",
}
if registry == "pypi" or registry == "auto":
r = await c.get(f"https://pypi.org/pypi/{urllib.parse.quote(name)}/json",
headers={"User-Agent": "mcp-codesearch/1.0"})
if r.status_code == 200:
d = r.json()
info = d.get("info", {})
latest = info.get("version", "")
ver = version or latest
repo_url = info.get("project_urls", {}).get("Source", "")
gh = _gh_repo_url(repo_url)
# Find tarball URL for the version
tarball_url = ""
for u in d.get("urls", []) or []:
if u.get("version") == ver and u.get("packagetype") == "sdist":
tarball_url = u.get("url", "")
break
if not tarball_url and d.get("urls"):
tarball_url = d["urls"][0].get("url", "")
return {
"success": True, "name": name, "version": ver,
"registry": "pypi", "description": info.get("summary", ""),
"homepage": info.get("home_page", ""), "license": info.get("license", ""),
"repo_url": repo_url, "repo_owner": gh.split("/")[0] if gh else "",
"repo_name": gh.split("/")[1] if gh else "",
"tarball_url": tarball_url,
"download_url": f"https://pypi.org/project/{name}/",
}
if registry == "crates" or registry == "auto":
r = await c.get(f"https://crates.io/api/v1/crates/{urllib.parse.quote(name)}",
headers={"User-Agent": "mcp-codesearch/1.0", "Accept": "application/json"})
if r.status_code == 200:
d = r.json()
crate = d.get("crate", {})
latest = crate.get("max_stable_version", crate.get("max_version", ""))
ver = version or latest
repo_url = crate.get("repository", "")
gh = _gh_repo_url(repo_url)
return {
"success": True, "name": name, "version": ver,
"registry": "crates", "description": crate.get("description", ""),
"homepage": crate.get("homepage", ""), "license": crate.get("license", ""),
"repo_url": repo_url, "repo_owner": gh.split("/")[0] if gh else "",
"repo_name": gh.split("/")[1] if gh else "",
"tarball_url": f"https://crates.io/api/v1/crates/{urllib.parse.quote(name)}/{urllib.parse.quote(ver)}/download",
"download_url": f"https://crates.io/crates/{name}",
}
except (httpx.HTTPError, ValueError, KeyError) as e:
logger.warning("resolve_package %s/%s failed: %s", registry, name, e)
return {"success": False, "error": f"registry lookup failed: {e}"}
return {"success": False, "error": f"package '{name}' not found in registry '{registry}'"}
async def _download_and_extract(registry: str, name: str, version: str) -> str | None:
"""Download package tarball and extract to cache dir. Returns path or None."""
cache_dir = Path(PACKAGE_CACHE) / registry / f"{name}@{version}"
if cache_dir.is_dir():
return str(cache_dir)
# Resolve tarball URL
info = await resolve_package(registry, name, version)
if not info.get("success") or not info.get("tarball_url"):
return None
tarball_url = info["tarball_url"]
try:
c = get_http_client()
# M1: stream + abort once over 50 MB (don't buffer the whole tarball
# into RAM before checking the guard).
async with c.stream("GET", tarball_url, headers={"User-Agent": "mcp-codesearch/1.0"}) as r:
if r.status_code != 200:
return None
chunks = []
size = 0
async for chunk in r.aiter_bytes():
size += len(chunk)
if size > 50 * 1024 * 1024:
return None # oversize; callers treat None as download-failed
chunks.append(chunk)
# mkdir AFTER the gate: an early return above must not leave
# an empty dir behind — is_dir() is the cache-hit check, so an
# empty dir reads as valid cache and lists 0 files forever.
cache_dir.mkdir(parents=True, exist_ok=True)
content = io.BytesIO(b"".join(chunks))
with tarfile.open(fileobj=content, mode="r:*") as tar:
first = tar.next()
if first is None:
return str(cache_dir)
strip = first.name.split("/")[0] if "/" in first.name else ""
tar.extractall(path=cache_dir, filter="data")
# If there's a strip prefix, move files up
if strip:
strip_dir = cache_dir / strip
if strip_dir.is_dir():
for f in strip_dir.iterdir():
f.rename(cache_dir / f.name)
shutil.rmtree(str(strip_dir))
return str(cache_dir)
except (httpx.HTTPError, OSError, tarfile.TarError, TypeError) as e:
logger.warning("download/extract failed for %s: %s", name, e)
return None
async def list_package_files(registry: str, name: str, version: str = "",
path_filter: str = "") -> dict[str, Any]:
"""List files in a package at the given registry/name/version."""
info = await resolve_package(registry, name, version)
if not info.get("success"):
return info
ver = info["version"]
extract_dir = await _download_and_extract(registry, name, ver)
if not extract_dir:
return {"success": False, "error": "could not download and extract package"}
base = Path(extract_dir)
files = []
for f in sorted(base.rglob("*")):
if f.is_dir():
continue
rel = str(f.relative_to(base))
if path_filter and not rel.startswith(path_filter):
continue
files.append({
"path": rel,
"size": f.stat().st_size,
"ext": f.suffix.lstrip("."),
})
return {"success": True, "files": files, "total": len(files),
"package": {"name": name, "version": ver, "registry": registry}}
async def read_package_file(registry: str, name: str, path: str,
version: str = "") -> dict[str, Any]:
"""Read a specific file from a downloaded package."""
info = await resolve_package(registry, name, version)
if not info.get("success"):
return info
ver = info["version"]
extract_dir = await _download_and_extract(registry, name, ver)
if not extract_dir:
return {"success": False, "error": "could not download and extract package"}
file_path = Path(extract_dir) / path
if not file_path.exists() or not file_path.is_file():
return {"success": False, "error": f"file '{path}' not found in package"}
try:
# H1: contain the path inside the extracted package. `path` is
# client-supplied — resolve both sides so `../`, absolute paths, and
# symlink escapes can't read arbitrary local files.
root = Path(extract_dir).resolve()
fp = (root / path).resolve()
if not fp.is_relative_to(root) or not fp.is_file():
return {"success": False, "error": f"invalid path: {path}"}
if fp.stat().st_size > 2_000_000: # M2: cap unbounded file reads
return {"success": False, "error": f"file too large to read (>2 MB): {path}"}
content = fp.read_text(encoding="utf-8", errors="replace")
lines = content.split("\n")
return {"success": True, "content": content, "path": path,
"total_lines": len(lines), "total_chars": len(content),
"package": {"name": name, "version": ver, "registry": registry}}
except (OSError, UnicodeDecodeError) as e:
return {"success": False, "error": f"could not read file: {e}"}
async def get_pkg_changelog(name: str, registry: str = "auto",
from_version: str = "", to_version: str = "",
count: int = 10) -> dict[str, Any]:
"""Get changelog/release notes for a package.
Resolves package → repo → GitHub releases API. Falls back to CHANGELOG.md.
"""
info = await resolve_package(registry, name)
if not info.get("success"):
return info
owner, repo_name = info.get("repo_owner", ""), info.get("repo_name", "")
if not owner or not repo_name:
return {"success": False, "error": f"no GitHub repo found for {registry}/{name}"}
# Fetch GitHub releases
try:
from github_api import gh_get_releases
r = await gh_get_releases(owner, repo_name, count=count * 2)
except ImportError:
return {"success": False, "error": "github_api module not available"}
if not r.get("success"):
return r
releases = r.get("releases", [])
filtered = []
def _vkey(s: str) -> tuple:
return tuple(int(p) if p.isdigit() else 0 for p in s.lstrip("v").split("."))
def _vcmp(a: tuple, b: tuple) -> int:
n = max(len(a), len(b))
a, b = a + (0,) * (n - len(a)), b + (0,) * (n - len(b))
return (a > b) - (a < b)
for rel in releases:
tag = rel.get("tag", "").lstrip("v")
if from_version and _vcmp(_vkey(tag), _vkey(from_version)) < 0:
continue
if to_version and _vcmp(_vkey(tag), _vkey(to_version)) > 0:
continue
filtered.append(rel)
if len(filtered) >= count:
break
if filtered:
return {"success": True, "package": {"name": name, "version": info["version"],
"registry": registry, "repo": f"{owner}/{repo_name}"},
"changelog": filtered}
# Fallback: try CHANGELOG.md from raw GitHub
try:
c = get_http_client()
for changelog_file in ["CHANGELOG.md", "CHANGELOG", "HISTORY.md", "RELEASE_NOTES.md"]:
r2 = await c.get(
f"https://raw.githubusercontent.com/{owner}/{repo_name}/master/{changelog_file}",
headers={"User-Agent": "mcp-codesearch/1.0"})
if r2.status_code != 200:
r2 = await c.get(
f"https://raw.githubusercontent.com/{owner}/{repo_name}/main/{changelog_file}",
headers={"User-Agent": "mcp-codesearch/1.0"})
if r2.status_code == 200:
return {"success": True, "package": {"name": name, "version": info["version"],
"registry": registry, "repo": f"{owner}/{repo_name}"},
"changelog": [{"file": changelog_file, "content": r2.text[:5000]}]}
except httpx.HTTPError:
pass
return {"success": False, "error": "no releases or changelog found",
"package": {"name": name, "version": info["version"],
"repo": f"{owner}/{repo_name}"}}
async def get_pkg_upgrade_review(name: str, registry: str = "auto",
current_version: str = "",
target_version: str = "") -> dict[str, Any]:
"""Compare two versions: vuln diff + changelog + deps changes."""
if not current_version or not target_version:
return {"success": False, "error": "current_version and target_version required"}
info = await resolve_package(registry, name, target_version)
if not info.get("success"):
return info
results = {"package": {"name": name, "registry": registry,
"current_version": current_version,
"target_version": target_version}}
# 1. Vuln scan both versions
try:
from oss_index import scan_vulnerabilities
cur_vulns, tgt_vulns = None, None
cur_task = asyncio.create_task(scan_vulnerabilities(registry, name, current_version))
tgt_task = asyncio.create_task(scan_vulnerabilities(registry, name, target_version))
cur_vulns = await cur_task
tgt_vulns = await tgt_task
cur_list = ((cur_vulns.get("reports") or [{}])[0].get("vulnerabilities", [])) if cur_vulns.get("success") else []
tgt_list = ((tgt_vulns.get("reports") or [{}])[0].get("vulnerabilities", [])) if tgt_vulns.get("success") else []
cur_ids = {v["id"] for v in cur_list}
tgt_ids = {v["id"] for v in tgt_list}
# KEV-exploited first: actively-exploited vulns sort above the rest.
def _kev_key(v: dict) -> tuple:
return (0 if v.get("exploited") else 1, v.get("id", ""))
results["vulnerabilities"] = {
"current_count": len(cur_ids), "target_count": len(tgt_ids),
"fixed": sorted(cur_ids - tgt_ids),
"added": sorted(tgt_ids - cur_ids),
"still_present": sorted(cur_ids & tgt_ids),
"still_present_exploited": sorted(
[v["id"] for v in tgt_list
if v["id"] in cur_ids and v.get("exploited")]),
"target_details": sorted(tgt_list, key=_kev_key),
}
except (ImportError, Exception) as e:
results["vulnerabilities"] = {"error": str(e)}
# 2. Changelog between versions
results["changelog"] = await get_pkg_changelog(name, registry,
from_version=current_version,
to_version=target_version, count=5)
# 3. Deps diff (npm only — registries expose no comparable dep lists)
if registry in ("npm", "auto"):
try:
from registries import npm_get_version
cur_deps, tgt_deps = set(), set()
cur = await npm_get_version(name, current_version)
tgt = await npm_get_version(name, target_version)
if cur.get("success"):
cur_deps = set(cur.get("dependencies", []))
if tgt.get("success"):
tgt_deps = set(tgt.get("dependencies", []))
results["dependencies"] = {
"current_deps": sorted(cur_deps),
"target_deps": sorted(tgt_deps),
"removed": sorted(cur_deps - tgt_deps),
"added": sorted(tgt_deps - cur_deps),
}
except Exception as e:
results["dependencies"] = {"error": str(e)}
else:
results["dependencies"] = {"skipped": f"dep diff not supported for registry '{registry}'"}
return {"success": True, **results}