From 53e911aab99327c2a18c0b85e4b0ddefcd206eb3 Mon Sep 17 00:00:00 2001 From: Darrien Rushing Date: Mon, 2 Nov 2026 13:30:06 -0600 Subject: [PATCH 1/2] feat(web): tabbed Repositories page for local, GitHub, GitLab and combined views The Repositories page now has Combined, Local, GitHub and GitLab tabs (kept in the URL as ?source=). Combined joins each local clone with the GitHub/GitLab repos its remotes point at, collapsing clones of the same remote into one row. Local keeps the existing full table. - lib/catalog: pure merge of clones with host listings (union-find over remote keys; only a clone's preferred remote per host is followed) - server: list the token user's GitHub repos and GitLab projects (paginated, cached for 5 minutes, failures reported per host without breaking the rest) - GitLab listing uses new optional GITLIB_GITLAB_TOKEN, or a username for public projects; contributions.ts now shares the GitLab env helpers - docs/env/compose updated; tests for the merge and the listing code Co-Authored-By: Claude Sonnet 5.5 --- .env.example | 3 + README.md | 12 + apps/web/.env.example | 4 + apps/web/src/components/catalog-table.tsx | 162 ++++++++ apps/web/src/lib/catalog.ts | 230 +++++++++++ apps/web/src/lib/library-state.ts | 5 +- apps/web/src/routes/index.tsx | 461 +++++++++++++++------- apps/web/src/server/catalog.ts | 131 ++++++ apps/web/src/server/contributions.ts | 4 +- apps/web/src/server/github.server.ts | 66 ++++ apps/web/src/server/gitlab.server.ts | 82 ++++ apps/web/src/server/provider-errors.ts | 4 + apps/web/src/server/repos.ts | 2 + apps/web/test/catalog.test.ts | 181 +++++++++ apps/web/test/remote-listing.test.ts | 132 +++++++ docker-compose.yml | 2 + docs/runbook.md | 2 + 17 files changed, 1347 insertions(+), 136 deletions(-) create mode 100644 apps/web/src/components/catalog-table.tsx create mode 100644 apps/web/src/lib/catalog.ts create mode 100644 apps/web/src/server/catalog.ts create mode 100644 apps/web/src/server/gitlab.server.ts create mode 100644 apps/web/src/server/provider-errors.ts create mode 100644 apps/web/test/catalog.test.ts create mode 100644 apps/web/test/remote-listing.test.ts diff --git a/.env.example b/.env.example index 0f3b098..d6f783e 100644 --- a/.env.example +++ b/.env.example @@ -34,5 +34,8 @@ GITLIB_GITHUB_TOKEN= # How often (in hours) GitHub community scores are fetched again. They are stored in the database, so this is # the only time they cost GitHub requests besides the Refresh buttons. 0 turns automatic refreshing off. #GITLIB_COMMUNITY_REFRESH_HOURS=24 +# Lists your GitLab projects on the Repositories page (GitLab and Combined tabs). A personal access token with +# the read_api scope also reaches private projects; with only a username, public projects are listed. +GITLIB_GITLAB_TOKEN= GITLIB_GITLAB_USERNAME= GITLIB_GITLAB_URL= diff --git a/README.md b/README.md index f944d80..f10295b 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,18 @@ npm run web # dev server at http://localhost:4000 npm run build -w @gitlib/web && npm run preview -w @gitlib/web # production build ``` +- The **Repositories** page has four tabs over one list: **Combined** (the default), **Local**, **GitHub** + and **GitLab**. Combined joins each local clone with the GitHub and GitLab repos its remotes point at, so + one row shows where a project lives (clone, GitHub, GitLab) and which of those it is missing; clones of the + same remote collapse into one row. GitHub lists every repo the token's user owns, collaborates on or reaches + through an organization (`GITLIB_GITHUB_TOKEN`; the token needs read access to repositories). GitLab lists + the projects the user is a member of with `GITLIB_GITLAB_TOKEN` (a personal access token with `read_api`), + or only a user's public projects with just `GITLIB_GITLAB_USERNAME`; set `GITLIB_GITLAB_URL` for a + self-hosted instance. Both hosts' lists are cached for a few minutes; **Refresh** asks again. A host that + isn't configured or fails is noted on the page and the rest still works, and clones link to GitHub or + GitLab from their remotes either way. The **Local** tab is the full table (favorites, descriptions, issue + counts, community scores, preview pane); the other tabs list repos with links out, and clones open their + detail page. The tab is kept in the URL (`?source=`). - Each repo has a detail page (click its name) with Overview, **Issues** and **Pull requests** tabs; the last two list open GitHub items and say so in a banner. Public repos work as-is; set `GITLIB_GITHUB_TOKEN` (or `GITHUB_TOKEN`) for private repos and a higher rate limit. The web server diff --git a/apps/web/.env.example b/apps/web/.env.example index 41caee9..d9c87e3 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -5,5 +5,9 @@ GITLIB_GITHUB_TOKEN= #GITLIB_COMMUNITY_REFRESH_HOURS=24 # Contribution graph "Local + GitHub & GitLab": GitHub uses the token above (any read-only token works). # GitLab reads your public contribution calendar; set GITLIB_GITLAB_URL too if you self-host. +# The GitLab and Combined tabs list your projects. A personal access token with the read_api scope also reaches +# private projects; with only a username, public projects are listed. The GitHub tabs use the token above, which +# needs to be able to read your repositories. +GITLIB_GITLAB_TOKEN= GITLIB_GITLAB_USERNAME= GITLIB_GITLAB_URL= diff --git a/apps/web/src/components/catalog-table.tsx b/apps/web/src/components/catalog-table.tsx new file mode 100644 index 0000000..f292a92 --- /dev/null +++ b/apps/web/src/components/catalog-table.tsx @@ -0,0 +1,162 @@ +import { Link } from "@tanstack/react-router"; +import { ExternalLink } from "lucide-react"; +import { TechBadge } from "#/components/tech-badge"; +import { formatAgo } from "#/lib/format"; +import type { CatalogEntry, RemoteLink } from "#/lib/catalog"; +import { cn } from "#/lib/utils"; +import type { ProviderStatus } from "#/server/catalog"; +import type { RepoView } from "#/server/repos"; + +const chip = "inline-flex h-5 items-center gap-1 border px-1.5 text-xs whitespace-nowrap"; + +function RemoteChip({ label, link, configured }: { label: string; link: RemoteLink | null; configured: boolean }) { + if (!link) { + // Only call a repo "missing" from a host gitlib could actually ask. + return configured ? ( + + {label} + — + + ) : null; + } + const flags = [link.isPrivate && "private", link.isFork && "fork", link.isArchived && "archived"].filter(Boolean); + return ( + 0 ? ` (${flags.join(", ")})` : ""}${link.listed ? "" : " — not in your account's list"}`} + className={cn(chip, "font-medium outline-none hover:bg-accent focus-visible:ring-2 focus-visible:ring-ring", !link.listed && "text-muted-foreground")} + > + {label} + + + ); +} + +function flagsOf(entry: CatalogEntry): string[] { + const links = [entry.github, entry.gitlab].filter((link): link is RemoteLink => link !== null); + return [ + links.some((link) => link.isPrivate) && "private", + links.some((link) => link.isFork) && "fork", + links.some((link) => link.isArchived) && "archived", + ].filter((flag): flag is string => Boolean(flag)); +} + +/** + * Repositories from every source in one list: where each lives (clone, GitHub, GitLab) is a column, so the + * combined view and the single-source views are the same table over different slices of the catalog. + */ +export function CatalogTable({ + entries, + reposByPath, + github, + gitlab, +}: { + entries: CatalogEntry[]; + /** Local repos by relativePath, for the stack detected in a clone. */ + reposByPath: ReadonlyMap; + github: ProviderStatus | undefined; + gitlab: ProviderStatus | undefined; +}) { + return ( +
+ + + + + + + + + + + + {entries.map((entry) => { + const local = entry.locals[0]; + const technologies = local ? (reposByPath.get(local)?.technologies ?? []) : []; + const external = entry.github?.url ?? entry.gitlab?.url; + const flags = flagsOf(entry); + return ( + + + + + + + + ); + })} + +
RepositoryDescriptionWhereStackUpdated
+ {local ? ( + + {entry.name} + + ) : ( + + {entry.name} + + )} + + {entry.fullName} + + {flags.length > 0 ? ( + {flags.join(" · ")} + ) : null} + + {entry.description ? ( + {entry.description} + ) : ( + — + )} + +
+ {local ? ( + + {entry.locals.length > 1 ? `Local ×${entry.locals.length}` : "Local"} + + ) : ( + + Local + — + + )} + + +
+ 		+ {technologies.length > 0 ? ( +
+ {technologies + .filter((tech) => tech.category === "language" || tech.category === "framework") + .slice(0, 3) + .map((tech) => ( + + ))} +
+ ) : entry.language ? ( + + ) : ( + — + )} + 		+ {entry.updatedAt ? formatAgo(entry.updatedAt) : "—"} +
+
+ ); +} diff --git a/apps/web/src/lib/catalog.ts b/apps/web/src/lib/catalog.ts new file mode 100644 index 0000000..19d0273 --- /dev/null +++ b/apps/web/src/lib/catalog.ts @@ -0,0 +1,230 @@ +/** + * The repository catalog: every repo gitlib can see, whether it is cloned locally, hosted on GitHub, hosted on + * GitLab, or any mix. Pure data shaping with no I/O, so the server and the tests share it. + */ + +export type RemoteProvider = "github" | "gitlab"; + +/** A repository as the hosting service lists it for the configured account. */ +export interface RemoteRepo { + provider: RemoteProvider; + /** `host/path`, lowercased; what a local clone's remote URL is matched against. */ + key: string; + /** `owner/repo` (GitHub) or `group/subgroup/project` (GitLab). */ + fullName: string; + name: string; + description: string; + url: string; + isPrivate: boolean; + isFork: boolean; + isArchived: boolean; + /** Last push (GitHub) or activity (GitLab), ISO 8601. */ + updatedAt: string | null; + language: string | null; +} + +/** A remote of a local clone that points at a known hosting service. */ +export interface LocalRemote { + provider: RemoteProvider; + key: string; + /** Path on the host, as written in the remote URL. */ + fullName: string; + url: string; +} + +/** What the catalog needs to know about a local clone. */ +export interface LocalClone { + relativePath: string; + name: string; + description: string; + /** Preferred remote first; only the first remote per provider is used to link the clone. */ + remotes: LocalRemote[]; +} + +/** One repo's presence on one hosting service. */ +export interface RemoteLink { + fullName: string; + url: string; + /** True when the provider's listing for the configured account contains it; false when only a clone's remote names it. */ + listed: boolean; + isPrivate: boolean; + isFork: boolean; + isArchived: boolean; +} + +export interface CatalogEntry { + /** Stable within one catalog; safe as a React key. */ + id: string; + name: string; + /** `owner/repo` when a host knows the repo, otherwise the clone's relative path. */ + fullName: string; + description: string; + /** Relative paths of the local clones (usually one); the first is the one links open. */ + locals: string[]; + github: RemoteLink | null; + gitlab: RemoteLink | null; + updatedAt: string | null; + language: string | null; +} + +/** `host/path`, lowercased and without a trailing `.git`: the identity two sources must share to be one repo. */ +export function remoteKey(host: string, path: string): string { + const trimmed = path.replace(/^\/+|\/+$/g, "").replace(/\.git$/i, ""); + return `${host.toLowerCase()}/${trimmed.toLowerCase()}`; +} + +/** Splits a remote landing page (`https://host/a/b`) into its `remoteKey` and path; null if it isn't one. */ +export function parseLandingPage(landingPageUrl: string | null): { key: string; path: string } | null { + if (!landingPageUrl) return null; + try { + const url = new URL(landingPageUrl); + const path = url.pathname.replace(/^\/+|\/+$/g, ""); + if (!url.hostname || !path) return null; + return { key: remoteKey(url.host, path), path }; + } catch { + return null; + } +} + +/** Disjoint sets over string ids; joins clones and remote repos that describe the same project. */ +class Groups { + private parent = new Map(); + + find(id: string): string { + let root = this.parent.get(id) ?? id; + if (root !== id) { + root = this.find(root); + this.parent.set(id, root); + } + return root; + } + + union(a: string, b: string): void { + this.parent.set(this.find(a), this.find(b)); + } +} + +const collator = new Intl.Collator(undefined, { sensitivity: "base", numeric: true }); + +function linkFromRemote(remote: RemoteRepo): RemoteLink { + return { + fullName: remote.fullName, + url: remote.url, + listed: true, + isPrivate: remote.isPrivate, + isFork: remote.isFork, + isArchived: remote.isArchived, + }; +} + +/** + * Joins local clones with the repos each hosting service lists. A clone belongs with the remote repos its + * preferred GitHub and GitLab remotes name, and clones that share a remote repo collapse into one entry. Remote + * repos nobody has cloned stand alone. Linked worktrees must be left out of `locals` by the caller. + */ +export function mergeCatalog( + locals: readonly LocalClone[], + github: readonly RemoteRepo[], + gitlab: readonly RemoteRepo[], +): CatalogEntry[] { + const remotes = new Map(); + for (const remote of [...github, ...gitlab]) remotes.set(`${remote.provider}:${remote.key}`, remote); + + const groups = new Groups(); + const nodeIds = new Set(); + const cloneByNode = new Map(); + /** The remote each clone uses per provider, in the clone's own preference order. */ + const cloneRemotes = new Map>>(); + + for (const clone of locals) { + const node = `local:${clone.relativePath}`; + cloneByNode.set(node, clone); + nodeIds.add(node); + const chosen: Partial> = {}; + for (const remote of clone.remotes) chosen[remote.provider] ??= remote; + cloneRemotes.set(node, chosen); + for (const remote of Object.values(chosen)) groups.union(node, `${remote.provider}:${remote.key}`); + } + for (const id of remotes.keys()) nodeIds.add(id); + + const members = new Map(); + for (const node of nodeIds) { + const root = groups.find(node); + members.set(root, [...(members.get(root) ?? []), node]); + } + + const entries: CatalogEntry[] = []; + for (const [root, nodes] of members) { + const clones = nodes.flatMap((node) => cloneByNode.get(node) ?? []); + const listed: Partial> = {}; + for (const node of nodes) { + const remote = remotes.get(node); + if (remote) listed[remote.provider] ??= remote; + } + + const links: Record = { github: null, gitlab: null }; + for (const provider of ["github", "gitlab"] as const) { + const hit = listed[provider]; + if (hit) { + links[provider] = linkFromRemote(hit); + continue; + } + // Not in the account's listing (other owner, no access, or the host isn't configured): the clone's own + // remote still says where it lives. + const named = clones.map((clone) => cloneRemotes.get(`local:${clone.relativePath}`)?.[provider]).find(Boolean); + if (named) { + links[provider] = { + fullName: named.fullName, + url: named.url, + listed: false, + isPrivate: false, + isFork: false, + isArchived: false, + }; + } + } + + const primary = clones[0]; + const remote = listed.github ?? listed.gitlab; + const hosted = links.github ?? links.gitlab; + entries.push({ + id: root, + name: primary?.name ?? remote?.name ?? root, + fullName: hosted?.fullName ?? primary?.relativePath ?? root, + description: primary?.description || remote?.description || "", + locals: clones.map((clone) => clone.relativePath).sort(collator.compare), + github: links.github, + gitlab: links.gitlab, + updatedAt: [listed.github?.updatedAt, listed.gitlab?.updatedAt].filter(Boolean).sort().at(-1) ?? null, + language: remote?.language ?? null, + }); + } + + return entries.sort((a, b) => collator.compare(a.fullName, b.fullName) || collator.compare(a.id, b.id)); +} + +export type CatalogSource = "combined" | "local" | "github" | "gitlab"; + +export const CATALOG_SOURCES: readonly CatalogSource[] = ["combined", "local", "github", "gitlab"]; + +/** The entries a catalog tab shows. "github"/"gitlab" mean repos that host lists for the account. */ +export function entriesForSource(entries: readonly CatalogEntry[], source: CatalogSource): CatalogEntry[] { + switch (source) { + case "combined": + return [...entries]; + case "local": + return entries.filter((entry) => entry.locals.length > 0); + case "github": + return entries.filter((entry) => entry.github?.listed); + case "gitlab": + return entries.filter((entry) => entry.gitlab?.listed); + } +} + +/** Case-insensitive match of every word in `query` against the name, full name and description. */ +export function matchesQuery(entry: CatalogEntry, query: string | undefined): boolean { + const words = query?.toLowerCase().split(/\s+/).filter(Boolean) ?? []; + if (words.length === 0) return true; + const haystack = `${entry.name} ${entry.fullName} ${entry.description}`.toLowerCase(); + return words.every((word) => haystack.includes(word)); +} diff --git a/apps/web/src/lib/library-state.ts b/apps/web/src/lib/library-state.ts index b0d4e28..76da991 100644 --- a/apps/web/src/lib/library-state.ts +++ b/apps/web/src/lib/library-state.ts @@ -1,5 +1,6 @@ import { useSyncExternalStore } from "react"; import type { SortDir, SortKey } from "#/components/repo-table"; +import type { CatalogSource } from "#/lib/catalog"; /** The Repositories table's filters and sort: the part of its URL worth restoring when you come back. */ export interface LibraryTableState { @@ -7,6 +8,8 @@ export interface LibraryTableState { tech?: string; /** Linked git worktrees are hidden unless this is set. */ worktrees?: true; + /** Which source tab is open; absent means the default (Combined). */ + source?: CatalogSource; sort?: SortKey; dir?: SortDir; } @@ -31,7 +34,7 @@ function read(): LibraryTableState { const isEmpty = (state: LibraryTableState) => Object.values(state).every((value) => value === undefined); const sameState = (a: LibraryTableState, b: LibraryTableState) => - a.q === b.q && a.tech === b.tech && a.worktrees === b.worktrees && a.sort === b.sort && a.dir === b.dir; + a.q === b.q && a.tech === b.tech && a.worktrees === b.worktrees && a.source === b.source && a.sort === b.sort && a.dir === b.dir; /** Remembers the table's state for this browser tab; session storage keeps it across reloads too. */ export function saveLibraryState(next: LibraryTableState): void { diff --git a/apps/web/src/routes/index.tsx b/apps/web/src/routes/index.tsx index 203db0d..2be803c 100644 --- a/apps/web/src/routes/index.tsx +++ b/apps/web/src/routes/index.tsx @@ -1,6 +1,7 @@ import { createFileRoute, useRouter, useRouterState } from "@tanstack/react-router"; import { RefreshCw, X } from "lucide-react"; import { useCallback, useEffect, useMemo, useState } from "react"; +import { CatalogTable } from "#/components/catalog-table"; import { ContributionGraph, type ContributionSource } from "#/components/contribution-graph"; import { RepoPreview } from "#/components/repo-preview"; import { @@ -15,9 +16,12 @@ import { } from "#/components/repo-table"; import { SearchBar } from "#/components/search-bar"; import { TechBadge } from "#/components/tech-badge"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "#/components/ui/tabs"; +import { CATALOG_SOURCES, entriesForSource, matchesQuery, type CatalogSource } from "#/lib/catalog"; import { formatAgo } from "#/lib/format"; import { saveLibraryState, type LibraryTableState } from "#/lib/library-state"; import { cn } from "#/lib/utils"; +import { getCatalog, refreshCatalog, type CatalogView, type ProviderStatus } from "#/server/catalog"; import { getCommunityScores, refreshCommunityScores, type CommunityScoresView } from "#/server/community"; import { getContributions, type ContributionsView } from "#/server/contributions"; import { getIssueCounts, type IssueCountsView } from "#/server/issues"; @@ -32,12 +36,23 @@ interface IndexSearch extends LibraryTableState { const SORT_KEYS: readonly SortKey[] = ["name", "description", "stack", "issues", "community", "scanned"]; const SORT_DIRS: readonly SortDir[] = ["asc", "desc"]; +/** The tab shown when the URL names none. */ +const DEFAULT_SOURCE: CatalogSource = "combined"; + +const SOURCE_LABELS: Record = { + combined: "Combined", + local: "Local", + github: "GitHub", + gitlab: "GitLab", +}; + export const Route = createFileRoute("/")({ validateSearch: (search: Record): IndexSearch => ({ q: typeof search.q === "string" && search.q ? search.q : undefined, tech: typeof search.tech === "string" && search.tech ? search.tech : undefined, repo: typeof search.repo === "string" && search.repo ? search.repo : undefined, worktrees: isTruthyParam(search.worktrees) ? true : undefined, + source: CATALOG_SOURCES.find((source) => source === search.source && source !== DEFAULT_SOURCE), sort: SORT_KEYS.find((key) => key === search.sort), dir: SORT_DIRS.find((dir) => dir === search.dir), }), @@ -145,6 +160,49 @@ function useCommunityScores(refreshKey: unknown) { return { view, refresh, busy: requesting || refreshing }; } +const catalogFailed = (error: string) => error; + +/** + * Every repo across clones, GitHub and GitLab, fetched after the page renders since it asks both hosts. The + * server caches the hosts' listings, so refetching is cheap; `refresh` makes it ask them again. + */ +function useCatalog(refreshKey: unknown) { + const [view, setView] = useState(); + const [error, setError] = useState(); + const [requesting, setRequesting] = useState(false); + + useEffect(() => { + let cancelled = false; + getCatalog() + .then((next) => { + if (cancelled) return; + setView(next); + setError(undefined); + }) + .catch((caught: unknown) => { + if (!cancelled) setError(catalogFailed(caught instanceof Error ? caught.message : "Something went wrong.")); + }); + return () => { + cancelled = true; + }; + // `refreshKey` is the only real input; the server function is stable. + }, [refreshKey]); + + const refresh = useCallback(async () => { + setRequesting(true); + try { + setView(await refreshCatalog()); + setError(undefined); + } catch (caught) { + setError(catalogFailed(caught instanceof Error ? caught.message : "Something went wrong.")); + } finally { + setRequesting(false); + } + }, []); + + return { view, error, refresh, busy: requesting }; +} + const CONTRIBUTION_SOURCE_KEY = "gitlib-contribution-source"; function readSource(): ContributionSource { @@ -186,7 +244,8 @@ function useContributions(refreshKey: unknown) { function RepoLibrary() { const { repos: allRepos, roots, found } = Route.useLoaderData(); - const { q, tech, repo: selectedPath, worktrees: showWorktrees, sort: sortKey, dir: sortDir } = Route.useSearch(); + const { q, tech, repo: selectedPath, worktrees: showWorktrees, sort: sortKey, dir: sortDir, source: sourceParam } = Route.useSearch(); + const source = sourceParam ?? DEFAULT_SOURCE; const navigate = Route.useNavigate(); const router = useRouter(); const [scanning, setScanning] = useState(false); @@ -195,6 +254,7 @@ function RepoLibrary() { const issueCountsResult = useBackgroundView(getIssueCounts, issueCountsFailed, allRepos); const issueCounts: IssueCounts = issueCountsResult?.counts; const community = useCommunityScores(allRepos); + const catalog = useCatalog(allRepos); const communityResult = community.view; const communityScores: CommunityScores = communityResult?.scores; @@ -222,8 +282,8 @@ function RepoLibrary() { // Filters and sort live in the URL (and are remembered for links back here), so they survive navigating away. useEffect( - () => saveLibraryState({ q, tech, worktrees: showWorktrees, sort: sortKey, dir: sortDir }), - [q, tech, showWorktrees, sortKey, sortDir], + () => saveLibraryState({ q, tech, worktrees: showWorktrees, source: sourceParam, sort: sortKey, dir: sortDir }), + [q, tech, showWorktrees, sourceParam, sortKey, sortDir], ); // Without a chosen column the rows keep search-relevance order (or name order when not searching). @@ -276,6 +336,24 @@ function RepoLibrary() { const described = repos.filter((repo) => repo.description).length; + const reposByPath = useMemo(() => new Map(allRepos.map((repo) => [repo.relativePath, repo])), [allRepos]); + const catalogEntries = catalog.view?.entries; + const sourceCounts = useMemo( + () => + Object.fromEntries( + CATALOG_SOURCES.map((key) => [key, catalogEntries ? entriesForSource(catalogEntries, key).length : undefined]), + ) as Record, + [catalogEntries], + ); + const catalogRows = useMemo( + () => (catalogEntries ? entriesForSource(catalogEntries, source).filter((entry) => matchesQuery(entry, q)) : undefined), + [catalogEntries, source, q], + ); + const setSource = (next: string) => { + const picked = CATALOG_SOURCES.find((candidate) => candidate === next); + if (picked) setSearch({ source: picked === DEFAULT_SOURCE ? undefined : picked, repo: undefined }); + }; + const selected = useMemo(() => repos.find((repo) => repo.relativePath === selectedPath), [repos, selectedPath]); const closePreview = useCallback(() => setSearch({ repo: undefined }), [setSearch]); @@ -317,146 +395,265 @@ function RepoLibrary() {

Repositories

-

- {repos.length} {repos.length === 1 ? "repository" : "repositories"} · {described} described - {!showWorktrees && worktreeCount > 0 ? ` · ${worktreeCount} worktrees hidden` : ""} - {roots.length > 0 ? ( - <> - {" · "} - {roots.join(", ")} - - ) : null} -

+ {source === "local" ? ( +

+ {repos.length} {repos.length === 1 ? "repository" : "repositories"} · {described} described + {!showWorktrees && worktreeCount > 0 ? ` · ${worktreeCount} worktrees hidden` : ""} + {roots.length > 0 ? ( + <> + {" · "} + {roots.join(", ")} + + ) : null} +

+ ) : ( +

+ {SOURCE_DESCRIPTIONS[source]} + {catalogRows ? ` · ${catalogRows.length} ${catalogRows.length === 1 ? "repository" : "repositories"}` : ""} +

+ )}
- - + {source === "local" ? ( + <> + + + + ) : ( + + )}
-
- {q && found ? ( -

- {found.mode === "meilisearch" ? ( - <> - {visible.length} {visible.length === 1 ? "result" : "results"} for{" "} - “{q}” - - ) : found.notice ? ( - Showing basic matches only — {found.notice} + + + {CATALOG_SOURCES.map((key) => { + const count = key === "local" ? repos.length : sourceCounts[key]; + return ( + + {SOURCE_LABELS[key]} + + {count ?? "…"} + + + ); + })} + + + {source === "local" ? ( + <> +

+ {q && found ? ( +

+ {found.mode === "meilisearch" ? ( + <> + {visible.length} {visible.length === 1 ? "result" : "results"} for{" "} + “{q}” + + ) : found.notice ? ( + Showing basic matches only — {found.notice} + ) : ( + <>Showing basic matches (full-text search is turned off). + )} +

+ ) : null} + + {issueCountsResult && issueCountsResult.status !== "ok" ? ( +

+ {issueCountsResult.status === "no-token" + ? "GitHub issue counts are off: set GITLIB_GITHUB_TOKEN (or GITHUB_TOKEN) to show and sort by them." + : `Some GitHub issue counts couldn't be loaded: ${issueCountsResult.error}`} +

+ ) : null} + + {communityResult && communityResult.status !== "ok" ? ( +

+ Some GitHub community scores couldn't be refreshed: {communityResult.error} +

+ ) : null} + {communityResult?.updatedAt ? ( +

+ Community scores last fetched from GitHub {formatAgo(communityResult.updatedAt)}. +

+ ) : null} + + {worktreeCount > 0 ? ( + + ) : null} + + {techCounts.length > 0 ? ( +
+ {techCounts.map(([name, { category, count }]) => { + const active = tech?.toLowerCase() === name.toLowerCase(); + return ( + + ); + })} + {tech || q ? ( + + ) : null} +
+ ) : null} +
+ + {repos.length === 0 ? ( + + No git repositories were found under {roots.join(", ") || "any root"}. + Add scan roots to ~/.git-library/config.toml and rescan. Running in + Docker? Set GITLIB_REPOS in .env to + the folder on your machine that holds your repos, then run{" "} + docker compose up -d. + + ) : visible.length === 0 ? ( + No repositories match your filters. ) : ( - <>Showing basic matches (full-text search is turned off). +
+
+ setSearch({ repo: repo.relativePath })} + /> +
+ +
)} -

- ) : null} + + ) : ( + + )} + + +
+ + ); +} - {issueCountsResult && issueCountsResult.status !== "ok" ? ( -

- {issueCountsResult.status === "no-token" - ? "GitHub issue counts are off: set GITLIB_GITHUB_TOKEN (or GITHUB_TOKEN) to show and sort by them." - : `Some GitHub issue counts couldn't be loaded: ${issueCountsResult.error}`} -

- ) : null} +const SOURCE_DESCRIPTIONS: Record, string> = { + combined: "Clones, GitHub and GitLab in one list", + github: "Repositories your GitHub account can reach", + gitlab: "Projects your GitLab account is a member of", +}; + +function statusNotice(label: string, status: ProviderStatus | undefined): string | null { + if (!status || status.status === "ok") return null; + return status.status === "not-configured" + ? `${label} isn't included. ${status.message ?? ""}`.trim() + : `${label} couldn't be listed: ${status.message ?? "unknown error"}`; +} - {communityResult && communityResult.status !== "ok" ? ( -

- Some GitHub community scores couldn't be refreshed: {communityResult.error} -

- ) : null} - {communityResult?.updatedAt ? ( -

- Community scores last fetched from GitHub {formatAgo(communityResult.updatedAt)}. +/** The Combined, GitHub and GitLab tabs: one catalog table, plus what is missing when a host can't be listed. */ +function SourceBody({ + source, + rows, + error, + github, + gitlab, + reposByPath, + query, +}: { + source: Exclude; + rows: ReturnType | undefined; + error: string | undefined; + github: ProviderStatus | undefined; + gitlab: ProviderStatus | undefined; + reposByPath: ReadonlyMap; + query: string | undefined; +}) { + if (error) return Couldn't load repositories: {error}; + if (!rows) return Loading repositories…; + + // The combined tab notes which host it left out; a tab for one host explains itself when it is empty. + const notices = + source === "combined" + ? [statusNotice("GitHub", github), statusNotice("GitLab", gitlab)].filter( + (notice): notice is string => notice !== null, + ) + : []; + const hostNotice = + source === "github" ? statusNotice("GitHub", github) : source === "gitlab" ? statusNotice("GitLab", gitlab) : null; + + return ( + <> + {notices.length > 0 ? ( +

+ {notices.map((notice) => ( +

+ {notice}

- ) : null} - - {worktreeCount > 0 ? ( - - ) : null} - - {techCounts.length > 0 ? ( -
- {techCounts.map(([name, { category, count }]) => { - const active = tech?.toLowerCase() === name.toLowerCase(); - return ( - - ); - })} - {tech || q ? ( - - ) : null} -
- ) : null} + ))}
- - {repos.length === 0 ? ( - - No git repositories were found under {roots.join(", ") || "any root"}. - Add scan roots to ~/.git-library/config.toml and rescan. Running in - Docker? Set GITLIB_REPOS in .env to - the folder on your machine that holds your repos, then run{" "} - docker compose up -d. - - ) : visible.length === 0 ? ( - No repositories match your filters. - ) : ( -
-
- setSearch({ repo: repo.relativePath })} - /> -
- -
- )} - + ) : null} + {rows.length === 0 ? ( + + {query ? "No repositories match your search." : (hostNotice ?? "No repositories found.")} + + ) : ( + + )} ); } diff --git a/apps/web/src/server/catalog.ts b/apps/web/src/server/catalog.ts new file mode 100644 index 0000000..deb2b97 --- /dev/null +++ b/apps/web/src/server/catalog.ts @@ -0,0 +1,131 @@ +import { createServerFn } from "@tanstack/react-start"; +import { classifyProvider, loadGitRemotes, selectPreferredRemote } from "@gitlib/core"; +import { + mergeCatalog, + parseLandingPage, + type CatalogEntry, + type LocalClone, + type LocalRemote, + type RemoteProvider, + type RemoteRepo, +} from "#/lib/catalog"; +import { listGitHubRepos } from "./github.server"; +import { gitlabHost, listGitLabProjects } from "./gitlab.server"; +import { getApp, loadAndIndexRepositories } from "./gitlib.server"; +import { ProviderSetupError } from "./provider-errors"; + +export interface ProviderStatus { + /** "not-configured" means gitlib has no credentials for the service; "error" means asking it failed. */ + status: "ok" | "not-configured" | "error"; + message: string | null; + /** Repositories the service listed for the account. */ + count: number; +} + +export interface CatalogView { + entries: CatalogEntry[]; + github: ProviderStatus; + gitlab: ProviderStatus; +} + +const REMOTES_TTL_MS = 5 * 60_000; + +/** Which hosting service a landing page belongs to; self-hosted GitLab is recognised by the configured URL. */ +function providerOf(host: string): RemoteProvider | null { + if (host.toLowerCase() === "github.com") return "github"; + if (host.toLowerCase() === gitlabHost().toLowerCase() || classifyProvider(host) === "GitLab") return "gitlab"; + return null; +} + +function localRemotes(absolutePath: string): LocalRemote[] { + const { remotes } = loadGitRemotes(absolutePath, getApp().system); + const preferred = selectPreferredRemote(remotes); + const ordered = preferred ? [preferred, ...remotes.filter((remote) => remote !== preferred)] : remotes; + return ordered.flatMap((remote) => { + const parsed = parseLandingPage(remote.landingPageUrl); + const provider = providerOf(remote.host); + if (!parsed || !provider || !remote.landingPageUrl) return []; + return [{ provider, key: parsed.key, fullName: parsed.path, url: remote.landingPageUrl }]; + }); +} + +/** Where each non-worktree clone points. One `git remote -v` per repo, so it is kept until the next rescan. */ +let cachedClones: Promise | undefined; + +async function loadClones(): Promise { + const repos = await loadAndIndexRepositories(); + return repos + .filter((repo) => !repo.isWorktree && !repo.isMissing) + .map((repo) => ({ + relativePath: repo.relativePath, + name: repo.name, + description: repo.description, + remotes: localRemotes(repo.absolutePath), + })); +} + +type Listing = { status: ProviderStatus; repos: RemoteRepo[] }; +const listings = new Map }>(); + +async function fetchListing(list: () => Promise): Promise { + try { + const repos = await list(); + return { status: { status: "ok", message: null, count: repos.length }, repos }; + } catch (caught) { + if (caught instanceof ProviderSetupError) { + return { status: { status: "not-configured", message: caught.message, count: 0 }, repos: [] }; + } + const message = caught instanceof Error ? caught.message : "Could not list repositories."; + return { status: { status: "error", message, count: 0 }, repos: [] }; + } +} + +function listingFor(provider: RemoteProvider): Promise { + const hit = listings.get(provider); + if (hit && Date.now() - hit.at <= REMOTES_TTL_MS) return hit.result; + const entry = { + at: Date.now(), + result: fetchListing(provider === "github" ? listGitHubRepos : listGitLabProjects), + }; + listings.set(provider, entry); + // Only successful lookups are worth keeping; a failed one should be retried next time. + void entry.result.then((result) => { + if (result.status.status !== "ok" && listings.get(provider) === entry) listings.delete(provider); + }); + return entry.result; +} + +/** Forgets the scanned clones' remotes (a rescan does this); the hosts' listings stay until they expire. */ +export function clearCatalogClones(): void { + cachedClones = undefined; +} + +function clearCatalog(): void { + cachedClones = undefined; + listings.clear(); +} + +async function buildCatalog(): Promise { + const clones = await (cachedClones ??= loadClones().catch((caught: unknown) => { + cachedClones = undefined; + throw caught; + })); + const [github, gitlab] = await Promise.all([listingFor("github"), listingFor("gitlab")]); + return { + entries: mergeCatalog(clones, github.repos, gitlab.repos), + github: github.status, + gitlab: gitlab.status, + }; +} + +/** + * Local clones joined with the repositories GitHub and GitLab list for the configured accounts. The hosts' + * listings are cached for a few minutes; the local half works whether or not either host is reachable. + */ +export const getCatalog = createServerFn({ method: "GET" }).handler(buildCatalog); + +/** Asks both hosts again and rereads every clone's remotes. */ +export const refreshCatalog = createServerFn({ method: "POST" }).handler(async () => { + clearCatalog(); + return buildCatalog(); +}); diff --git a/apps/web/src/server/contributions.ts b/apps/web/src/server/contributions.ts index ecd68fc..4b93d8f 100644 --- a/apps/web/src/server/contributions.ts +++ b/apps/web/src/server/contributions.ts @@ -2,6 +2,7 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { createServerFn } from "@tanstack/react-start"; import { githubToken } from "./github.server"; +import { gitlabBase, gitlabUsername } from "./gitlab.server"; import { loadAndIndexRepositories } from "./gitlib.server"; const run = promisify(execFile); @@ -68,9 +69,6 @@ async function computeLocal(): Promise { return { days, total: seen.size, repoCount, notices: [] }; } -const gitlabUsername = (): string | undefined => process.env["GITLIB_GITLAB_USERNAME"] || undefined; -const gitlabBase = (): string => (process.env["GITLIB_GITLAB_URL"] || "https://gitlab.com").replace(/\/+$/, ""); - interface Remote { days: Record; notice: string | null; diff --git a/apps/web/src/server/github.server.ts b/apps/web/src/server/github.server.ts index 809b652..80c56dc 100644 --- a/apps/web/src/server/github.server.ts +++ b/apps/web/src/server/github.server.ts @@ -1,6 +1,8 @@ import { loadGitRemotes, selectGitHubRepo, type GitHubRepoRef } from "@gitlib/core"; +import { remoteKey, type RemoteRepo } from "#/lib/catalog"; import { getApp, loadAndIndexRepositories } from "./gitlib.server"; import type { Provider } from "./provider"; +import { ProviderSetupError } from "./provider-errors"; export const githubToken = (): string | undefined => process.env["GITLIB_GITHUB_TOKEN"] || process.env["GITHUB_TOKEN"] || undefined; @@ -88,3 +90,67 @@ export async function githubGet( rateRemaining: remaining !== null && Number.isFinite(Number(remaining)) ? Number(remaining) : null, }; } + +interface GitHubAccountRepo { + full_name: string; + name: string; + description: string | null; + html_url: string; + private: boolean; + fork: boolean; + archived: boolean; + pushed_at: string | null; + updated_at: string | null; + language: string | null; +} + +/** Pages of 100; ten covers a thousand repos, which is plenty before a listing becomes unwieldy. */ +const ACCOUNT_REPO_PAGES = 10; + +/** Every repository the token's user owns, collaborates on, or can reach through an organization. */ +export async function listGitHubRepos(): Promise { + const token = githubToken(); + if (!token) throw new ProviderSetupError("Set GITLIB_GITHUB_TOKEN to list your GitHub repositories."); + + const repos: RemoteRepo[] = []; + for (let page = 1; page <= ACCOUNT_REPO_PAGES; page += 1) { + const url = new URL("https://api.github.com/user/repos"); + url.search = new URLSearchParams({ + per_page: "100", + page: String(page), + sort: "pushed", + affiliation: "owner,collaborator,organization_member", + }).toString(); + const response = await fetch(url, { + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "gitlib", + Authorization: `Bearer ${token}`, + }, + signal: AbortSignal.timeout(15_000), + }); + if (response.status === 401) throw new Error("GitHub rejected the token (401)."); + if (response.status === 403 || response.status === 429) throw new Error("GitHub rate limit reached. Try again later."); + if (!response.ok) throw new Error(`GitHub responded with ${response.status}.`); + + const batch = (await response.json()) as GitHubAccountRepo[]; + for (const item of batch) { + repos.push({ + provider: "github", + key: remoteKey("github.com", item.full_name), + fullName: item.full_name, + name: item.name, + description: item.description ?? "", + url: item.html_url, + isPrivate: item.private, + isFork: item.fork, + isArchived: item.archived, + updatedAt: item.pushed_at ?? item.updated_at, + language: item.language, + }); + } + if (batch.length < 100) break; + } + return repos; +} diff --git a/apps/web/src/server/gitlab.server.ts b/apps/web/src/server/gitlab.server.ts new file mode 100644 index 0000000..2089922 --- /dev/null +++ b/apps/web/src/server/gitlab.server.ts @@ -0,0 +1,82 @@ +import "./env.server"; +import { remoteKey, type RemoteRepo } from "#/lib/catalog"; +import { ProviderSetupError } from "./provider-errors"; + +export const gitlabUsername = (): string | undefined => process.env["GITLIB_GITLAB_USERNAME"] || undefined; + +/** A personal access token with `read_api`; without one only a user's public projects can be listed. */ +export const gitlabToken = (): string | undefined => process.env["GITLIB_GITLAB_TOKEN"] || undefined; + +export const gitlabBase = (): string => (process.env["GITLIB_GITLAB_URL"] || "https://gitlab.com").replace(/\/+$/, ""); + +export const gitlabHost = (): string => new URL(gitlabBase()).host; + +interface GitLabProject { + path_with_namespace: string; + path: string; + description: string | null; + web_url: string; + visibility?: string; + archived?: boolean; + last_activity_at: string | null; + forked_from_project?: unknown; +} + +/** Pages of 100, capped like the GitHub listing. */ +const PROJECT_PAGES = 10; + +/** + * The projects the token's user is a member of, or, with only a username configured, that user's public + * projects. Owned, group and shared projects all count as membership. + */ +export async function listGitLabProjects(): Promise { + const token = gitlabToken(); + const username = gitlabUsername(); + if (!token && !username) { + throw new ProviderSetupError("Set GITLIB_GITLAB_TOKEN (or GITLIB_GITLAB_USERNAME for public projects), and GITLIB_GITLAB_URL if self-hosted."); + } + + const endpoint = token + ? `${gitlabBase()}/api/v4/projects` + : `${gitlabBase()}/api/v4/users/${encodeURIComponent(username!)}/projects`; + const host = gitlabHost(); + + const repos: RemoteRepo[] = []; + for (let page = 1; page <= PROJECT_PAGES; page += 1) { + const url = new URL(endpoint); + url.search = new URLSearchParams({ + per_page: "100", + page: String(page), + order_by: "last_activity_at", + sort: "desc", + ...(token ? { membership: "true" } : {}), + }).toString(); + const response = await fetch(url, { + headers: { Accept: "application/json", "User-Agent": "gitlib", ...(token ? { "PRIVATE-TOKEN": token } : {}) }, + signal: AbortSignal.timeout(15_000), + }); + if (response.status === 401) throw new Error("GitLab rejected the token (401)."); + if (response.status === 404 && !token) throw new Error(`GitLab has no user named ${username}.`); + if (response.status === 429) throw new Error("GitLab rate limit reached. Try again later."); + if (!response.ok) throw new Error(`GitLab responded with ${response.status}.`); + + const batch = (await response.json()) as GitLabProject[]; + for (const item of batch) { + repos.push({ + provider: "gitlab", + key: remoteKey(host, item.path_with_namespace), + fullName: item.path_with_namespace, + name: item.path, + description: item.description ?? "", + url: item.web_url, + isPrivate: item.visibility !== undefined && item.visibility !== "public", + isFork: Boolean(item.forked_from_project), + isArchived: item.archived === true, + updatedAt: item.last_activity_at, + language: null, + }); + } + if (batch.length < 100) break; + } + return repos; +} diff --git a/apps/web/src/server/provider-errors.ts b/apps/web/src/server/provider-errors.ts new file mode 100644 index 0000000..47ab852 --- /dev/null +++ b/apps/web/src/server/provider-errors.ts @@ -0,0 +1,4 @@ +/** A hosting service that can't be listed because gitlib isn't set up for it, as opposed to a request that failed. */ +export class ProviderSetupError extends Error { + override name = "ProviderSetupError"; +} diff --git a/apps/web/src/server/repos.ts b/apps/web/src/server/repos.ts index 44e005b..d032eff 100644 --- a/apps/web/src/server/repos.ts +++ b/apps/web/src/server/repos.ts @@ -1,6 +1,7 @@ import { createServerFn } from "@tanstack/react-start"; import { loadRepoReadme, technologyCategory, type RepoRecord, type TechnologyCategory } from "@gitlib/core"; import { getApp, loadAndIndexRepositories } from "./gitlib.server"; +import { clearCatalogClones } from "./catalog"; import { clearContributions } from "./contributions"; import { clearIssueCounts } from "./issues"; @@ -67,6 +68,7 @@ export const getRepo = createServerFn({ method: "GET" }) export const rescanRepos = createServerFn({ method: "POST" }).handler(async () => { clearIssueCounts(); clearContributions(); + clearCatalogClones(); return listing(true); }); diff --git a/apps/web/test/catalog.test.ts b/apps/web/test/catalog.test.ts new file mode 100644 index 0000000..531aaaf --- /dev/null +++ b/apps/web/test/catalog.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from "vitest"; +import { + entriesForSource, + matchesQuery, + mergeCatalog, + parseLandingPage, + remoteKey, + type LocalClone, + type RemoteRepo, +} from "../src/lib/catalog"; + +const github = (fullName: string, extra: Partial = {}): RemoteRepo => ({ + provider: "github", + key: remoteKey("github.com", fullName), + fullName, + name: fullName.split("/").at(-1)!, + description: "", + url: `https://github.com/${fullName}`, + isPrivate: false, + isFork: false, + isArchived: false, + updatedAt: null, + language: null, + ...extra, +}); + +const gitlab = (fullName: string, extra: Partial = {}): RemoteRepo => ({ + ...github(fullName), + provider: "gitlab", + key: remoteKey("gitlab.com", fullName), + url: `https://gitlab.com/${fullName}`, + ...extra, +}); + +const clone = (relativePath: string, remotes: LocalClone["remotes"] = [], description = ""): LocalClone => ({ + relativePath, + name: relativePath.split("/").at(-1)!, + description, + remotes, +}); + +const remoteOf = (repo: RemoteRepo) => ({ + provider: repo.provider, + key: repo.key, + fullName: repo.fullName, + url: repo.url, +}); + +describe("remoteKey / parseLandingPage", () => { + it("ignores case, .git and slashes", () => { + expect(remoteKey("GitHub.com", "/Meddlin/Gitlib.git/")).toBe("github.com/meddlin/gitlib"); + }); + + it("reads the host and path of a landing page, keeping nested GitLab groups", () => { + expect(parseLandingPage("https://gitlab.com/group/sub/proj")).toEqual({ + key: "gitlab.com/group/sub/proj", + path: "group/sub/proj", + }); + expect(parseLandingPage("https://gitlab.com")).toBeNull(); + expect(parseLandingPage(null)).toBeNull(); + }); +}); + +describe("mergeCatalog", () => { + it("joins a clone with the GitHub repo its remote names", () => { + const repo = github("me/app", { description: "On GitHub", language: "TypeScript", updatedAt: "2026-01-02T00:00:00Z" }); + const [entry, ...rest] = mergeCatalog([clone("work/app", [remoteOf(repo)])], [repo], []); + expect(rest).toEqual([]); + expect(entry).toMatchObject({ + locals: ["work/app"], + fullName: "me/app", + description: "On GitHub", + language: "TypeScript", + updatedAt: "2026-01-02T00:00:00Z", + }); + expect(entry!.github).toMatchObject({ listed: true, url: "https://github.com/me/app" }); + expect(entry!.gitlab).toBeNull(); + }); + + it("prefers the clone's own description over the host's", () => { + const repo = github("me/app", { description: "On GitHub" }); + const [entry] = mergeCatalog([clone("app", [remoteOf(repo)], "My notes")], [repo], []); + expect(entry!.description).toBe("My notes"); + }); + + it("links one clone to both hosts when it has a remote on each", () => { + const gh = github("me/app"); + const gl = gitlab("me/app"); + const entries = mergeCatalog([clone("app", [remoteOf(gh), remoteOf(gl)])], [gh], [gl]); + expect(entries).toHaveLength(1); + expect(entries[0]!.github?.listed).toBe(true); + expect(entries[0]!.gitlab?.listed).toBe(true); + }); + + it("lists remote repos nobody has cloned on their own", () => { + const entries = mergeCatalog([], [github("me/a")], [gitlab("me/b")]); + expect(entries.map((e) => [e.fullName, e.locals.length, !!e.github, !!e.gitlab])).toEqual([ + ["me/a", 0, true, false], + ["me/b", 0, false, true], + ]); + }); + + it("keeps clones with no known remote as local-only", () => { + const [entry] = mergeCatalog([clone("scratch/thing")], [], []); + expect(entry).toMatchObject({ locals: ["scratch/thing"], fullName: "scratch/thing", github: null, gitlab: null }); + }); + + it("collapses clones of the same remote repo into one entry", () => { + const repo = github("me/app"); + const entries = mergeCatalog( + [clone("b/app", [remoteOf(repo)]), clone("a/app", [remoteOf(repo)])], + [repo], + [], + ); + expect(entries).toHaveLength(1); + expect(entries[0]!.locals).toEqual(["a/app", "b/app"]); + }); + + it("joins a GitHub-only clone and a GitLab-only clone of one project through a shared clone", () => { + const gh = github("me/app"); + const gl = gitlab("me/app"); + const entries = mergeCatalog( + [clone("one", [remoteOf(gh), remoteOf(gl)]), clone("two", [remoteOf(gl)])], + [gh], + [gl], + ); + expect(entries).toHaveLength(1); + expect(entries[0]!.locals).toEqual(["one", "two"]); + }); + + it("only follows the preferred remote per host, so a fork's upstream is not pulled in", () => { + const fork = github("me/app", { isFork: true }); + const upstream = github("them/app"); + const entries = mergeCatalog([clone("app", [remoteOf(fork), remoteOf(upstream)])], [fork, upstream], []); + expect(entries.map((e) => e.fullName)).toEqual(["me/app", "them/app"]); + expect(entries[0]!.locals).toEqual(["app"]); + expect(entries[1]!.locals).toEqual([]); + }); + + it("still names the host for a clone whose repo is not in the account's listing", () => { + const other = github("someone/else"); + const [entry] = mergeCatalog([clone("else", [remoteOf(other)])], [], []); + expect(entry!.github).toMatchObject({ listed: false, fullName: "someone/else", url: "https://github.com/someone/else" }); + }); + + it("matches hosts regardless of case in the remote URL", () => { + const repo = github("Me/App"); + const [entry] = mergeCatalog( + [clone("app", [{ ...remoteOf(repo), key: remoteKey("GITHUB.com", "me/app.git") }])], + [repo], + [], + ); + expect(entry!.github?.listed).toBe(true); + }); +}); + +describe("entriesForSource / matchesQuery", () => { + const gh = github("me/listed"); + const entries = mergeCatalog( + [clone("listed", [remoteOf(gh)]), clone("foreign", [remoteOf(github("them/foreign"))]), clone("scratch")], + [gh, github("me/remote-only")], + [gitlab("me/lab")], + ); + + it("slices the catalog by source", () => { + const names = (source: Parameters[1]) => + entriesForSource(entries, source).map((e) => e.fullName); + expect(names("combined")).toHaveLength(5); + expect(names("local")).toEqual(["me/listed", "scratch", "them/foreign"]); + expect(names("github")).toEqual(["me/listed", "me/remote-only"]); + expect(names("gitlab")).toEqual(["me/lab"]); + }); + + it("requires every word to appear in the name, full name or description", () => { + const [entry] = mergeCatalog([], [github("me/widget", { description: "Builds Things" })], []); + expect(matchesQuery(entry!, "widget things")).toBe(true); + expect(matchesQuery(entry!, "widget missing")).toBe(false); + expect(matchesQuery(entry!, "")).toBe(true); + expect(matchesQuery(entry!, undefined)).toBe(true); + }); +}); diff --git a/apps/web/test/remote-listing.test.ts b/apps/web/test/remote-listing.test.ts new file mode 100644 index 0000000..644cc66 --- /dev/null +++ b/apps/web/test/remote-listing.test.ts @@ -0,0 +1,132 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { listGitHubRepos } from "../src/server/github.server"; +import { gitlabHost, listGitLabProjects } from "../src/server/gitlab.server"; +import { ProviderSetupError } from "../src/server/provider-errors"; + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } }); + +const stubFetch = (...responses: Response[]) => { + const fetchMock = vi.fn(); + for (const response of responses) fetchMock.mockResolvedValueOnce(response); + vi.stubGlobal("fetch", fetchMock); + return fetchMock; +}; + +const githubRepo = (n: number) => ({ + full_name: `me/repo-${n}`, + name: `repo-${n}`, + description: n === 0 ? null : `repo ${n}`, + html_url: `https://github.com/me/repo-${n}`, + private: n === 1, + fork: false, + archived: false, + pushed_at: "2026-01-01T00:00:00Z", + updated_at: "2025-01-01T00:00:00Z", + language: "TypeScript", +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); +}); + +describe("listGitHubRepos", () => { + it("needs a token", async () => { + vi.stubEnv("GITLIB_GITHUB_TOKEN", ""); + vi.stubEnv("GITHUB_TOKEN", ""); + await expect(listGitHubRepos()).rejects.toBeInstanceOf(ProviderSetupError); + }); + + it("maps repos and keeps paging until a short page", async () => { + vi.stubEnv("GITLIB_GITHUB_TOKEN", "ghp_test"); + const fetchMock = stubFetch( + json(Array.from({ length: 100 }, (_, n) => githubRepo(n))), + json([githubRepo(100)]), + ); + const repos = await listGitHubRepos(); + + expect(repos).toHaveLength(101); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(String(fetchMock.mock.calls[1]![0])).toContain("page=2"); + expect((fetchMock.mock.calls[0]![1]!.headers as Record)["Authorization"]).toBe("Bearer ghp_test"); + expect(repos[1]).toMatchObject({ + provider: "github", + key: "github.com/me/repo-1", + fullName: "me/repo-1", + isPrivate: true, + updatedAt: "2026-01-01T00:00:00Z", + language: "TypeScript", + }); + expect(repos[0]!.description).toBe(""); + }); + + it("explains a rejected token", async () => { + vi.stubEnv("GITLIB_GITHUB_TOKEN", "bad"); + stubFetch(json({}, 401)); + await expect(listGitHubRepos()).rejects.toThrow("GitHub rejected the token"); + }); +}); + +describe("listGitLabProjects", () => { + const project = (path: string, extra: Record = {}) => ({ + path_with_namespace: path, + path: path.split("/").at(-1), + description: null, + web_url: `https://gitlab.com/${path}`, + visibility: "private", + archived: false, + last_activity_at: "2026-02-01T00:00:00Z", + ...extra, + }); + + it("needs a token or a username", async () => { + vi.stubEnv("GITLIB_GITLAB_TOKEN", ""); + vi.stubEnv("GITLIB_GITLAB_USERNAME", ""); + await expect(listGitLabProjects()).rejects.toBeInstanceOf(ProviderSetupError); + }); + + it("lists member projects with the token, including nested groups", async () => { + vi.stubEnv("GITLIB_GITLAB_TOKEN", "glpat_test"); + vi.stubEnv("GITLIB_GITLAB_URL", ""); + const fetchMock = stubFetch( + json([ + project("team/sub/app"), + project("me/fork", { visibility: "public", forked_from_project: { id: 1 }, archived: true }), + ]), + ); + const repos = await listGitLabProjects(); + + const [url, init] = fetchMock.mock.calls[0]!; + expect(String(url)).toContain("https://gitlab.com/api/v4/projects?"); + expect(String(url)).toContain("membership=true"); + expect((init!.headers as Record)["PRIVATE-TOKEN"]).toBe("glpat_test"); + expect(repos[0]).toMatchObject({ + provider: "gitlab", + key: "gitlab.com/team/sub/app", + fullName: "team/sub/app", + name: "app", + isPrivate: true, + }); + expect(repos[1]).toMatchObject({ isPrivate: false, isFork: true, isArchived: true }); + }); + + it("falls back to a user's public projects, on a self-hosted URL", async () => { + vi.stubEnv("GITLIB_GITLAB_TOKEN", ""); + vi.stubEnv("GITLIB_GITLAB_USERNAME", "me"); + vi.stubEnv("GITLIB_GITLAB_URL", "https://git.example.com/"); + const fetchMock = stubFetch(json([project("me/thing", { visibility: "public" })])); + const repos = await listGitLabProjects(); + + expect(String(fetchMock.mock.calls[0]![0])).toContain("https://git.example.com/api/v4/users/me/projects?"); + expect(gitlabHost()).toBe("git.example.com"); + expect(repos[0]!.key).toBe("git.example.com/me/thing"); + }); + + it("reports an unknown user", async () => { + vi.stubEnv("GITLIB_GITLAB_TOKEN", ""); + vi.stubEnv("GITLIB_GITLAB_USERNAME", "nobody"); + stubFetch(json({}, 404)); + await expect(listGitLabProjects()).rejects.toThrow("no user named nobody"); + }); +}); diff --git a/docker-compose.yml b/docker-compose.yml index f44b6c0..a5f0652 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ x-gitlib-env: &gitlib-env # Optional: hours between automatic refreshes of the GitHub community scores (blank = 24, 0 = off). GITLIB_COMMUNITY_REFRESH_HOURS: ${GITLIB_COMMUNITY_REFRESH_HOURS:-} # Optional: the contribution graph's "Local + GitHub & GitLab" mode. + # Optional: lists GitLab projects on the Repositories page (read_api scope; without it only public ones show). + GITLIB_GITLAB_TOKEN: ${GITLIB_GITLAB_TOKEN:-} GITLIB_GITLAB_USERNAME: ${GITLIB_GITLAB_USERNAME:-} GITLIB_GITLAB_URL: ${GITLIB_GITLAB_URL:-} GITLIB_MEILI_HOST: http://meilisearch:7700 diff --git a/docs/runbook.md b/docs/runbook.md index 3d517d4..ead883f 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -157,6 +157,7 @@ TZ=America/Chicago MEILI_ENV=production GITLIB_MEILI_API_KEY= GITLIB_GITHUB_TOKEN= +GITLIB_GITLAB_TOKEN= # plus GITLIB_GITLAB_URL if self-hosted ``` `GITLIB_BACKUPS` must exist and be writable by uid 1000 (`mkdir -p /srv/gitlib-backups` as your user normally @@ -399,6 +400,7 @@ docker compose run --rm gitlib list # one-off CLI command | `port is already allocated` / `address already in use` | Something else uses the port. Set `GITLIB_PORT` (or `GITLIB_MEILI_PORT`) in `.env` to a free one. `npm run web` also refuses to start on a busy 4000. | | No repositories listed | `GITLIB_REPOS` (default `~/git`) is wrong, not shared with Docker Desktop, or empty. Check `docker compose exec web ls /home/node/git`. Press **Rescan** after fixing. | | Repo pages show no GitHub data / community scores | Add `GITLIB_GITHUB_TOKEN` to `.env` (anonymous GitHub limit is 60 requests/hour), then `docker compose up -d`. Confirm the repo has a GitHub `origin` remote. Scores are stored and refreshed daily; press **Refresh scores** to fetch now. | +| GitHub / GitLab tab is empty or says "isn't included" | The Repositories page lists GitHub repos with `GITLIB_GITHUB_TOKEN` and GitLab projects with `GITLIB_GITLAB_TOKEN` (`read_api`; or `GITLIB_GITLAB_USERNAME` for public ones, plus `GITLIB_GITLAB_URL` if self-hosted). Add it to `.env`, run `docker compose up -d`, then press **Refresh**. A rejected token (401) is reported on the page. | | "Search falls back to substring matching" | Meilisearch is down or unreachable. `docker compose ps`, `docker compose logs meilisearch`. After changing `GITLIB_MEILI_API_KEY` run `docker compose up -d` so both containers get the new value. | | `meilisearch` will not start with `MEILI_ENV=production` | Production mode requires a key of at least 16 characters in `GITLIB_MEILI_API_KEY`. | | `permission denied` writing to `/backups` | The backups directory is not writable by uid 1000: `sudo chown 1000:1000 "$GITLIB_BACKUPS"`. | From 84a9e46200a52af5c5128b33220b732dd06d7e28 Mon Sep 17 00:00:00 2001 From: Darrien Rushing Date: Mon, 2 Nov 2026 13:38:35 -0600 Subject: [PATCH 2/2] feat(web): store GitHub/GitLab repo lists in the database and re-check in the background The Repositories page used to fetch the full GitHub and GitLab inventory into a 5-minute in-memory cache, so every restart or expiry cost a full listing. - Listings are stored per host in a new remote_repo_listings table and read from there on every view; no host call per page load, and they survive restarts - The server re-checks a host when its list is missing or older than GITLIB_REPO_REFRESH_HOURS (default 6, 0 = only the Refresh button), on page load and via a 15-minute timer; the UI polls while a first fetch runs - Checks send each page's ETag; a 304 reuses the stored page and costs no GitHub quota. Refresh fetches complete lists - A failed check keeps the stored list on screen with the error and pauses automatic retries for 30 minutes; a list for another account/host is ignored - Catalog logic moves to catalog.server.ts so it stays out of the client bundle - Tests for the store, ETag paging, stored-listing parsing and the end-to-end flow (first load, restart, recheck, failure, interval 0) Co-Authored-By: Claude Sonnet 5.5 --- .env.example | 3 + README.md | 11 +- apps/web/.env.example | 2 + apps/web/src/components/catalog-table.tsx | 14 +- apps/web/src/lib/catalog.ts | 25 ++ apps/web/src/lib/community.ts | 6 +- apps/web/src/routes/index.tsx | 61 ++++- apps/web/src/server/catalog.server.ts | 227 +++++++++++++++++++ apps/web/src/server/catalog.ts | 131 +---------- apps/web/src/server/github.server.ts | 67 +++--- apps/web/src/server/gitlab.server.ts | 72 +++--- apps/web/src/server/remote-listing.server.ts | 35 +++ apps/web/src/server/repos.ts | 2 +- apps/web/test/catalog-flow.test.ts | 204 +++++++++++++++++ apps/web/test/catalog.test.ts | 19 ++ apps/web/test/remote-listing.test.ts | 88 ++++++- docker-compose.yml | 2 + docs/runbook.md | 2 +- packages/core/src/store.ts | 33 +++ packages/core/test/core.test.ts | 18 ++ 20 files changed, 805 insertions(+), 217 deletions(-) create mode 100644 apps/web/src/server/catalog.server.ts create mode 100644 apps/web/src/server/remote-listing.server.ts create mode 100644 apps/web/test/catalog-flow.test.ts diff --git a/.env.example b/.env.example index d6f783e..7c85f6f 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,9 @@ GITLIB_GITHUB_TOKEN= # How often (in hours) GitHub community scores are fetched again. They are stored in the database, so this is # the only time they cost GitHub requests besides the Refresh buttons. 0 turns automatic refreshing off. #GITLIB_COMMUNITY_REFRESH_HOURS=24 +# How often (in hours) the GitHub and GitLab repository lists are checked again. They are stored in the database +# and checked with ETags, so a check that finds nothing new costs no GitHub quota. 0 = only the Refresh button. +#GITLIB_REPO_REFRESH_HOURS=6 # Lists your GitLab projects on the Repositories page (GitLab and Combined tabs). A personal access token with # the read_api scope also reaches private projects; with only a username, public projects are listed. GITLIB_GITLAB_TOKEN= diff --git a/README.md b/README.md index f10295b..81145d6 100644 --- a/README.md +++ b/README.md @@ -162,9 +162,14 @@ npm run build -w @gitlib/web && npm run preview -w @gitlib/web # production bu through an organization (`GITLIB_GITHUB_TOKEN`; the token needs read access to repositories). GitLab lists the projects the user is a member of with `GITLIB_GITLAB_TOKEN` (a personal access token with `read_api`), or only a user's public projects with just `GITLIB_GITLAB_USERNAME`; set `GITLIB_GITLAB_URL` for a - self-hosted instance. Both hosts' lists are cached for a few minutes; **Refresh** asks again. A host that - isn't configured or fails is noted on the page and the rest still works, and clones link to GitHub or - GitLab from their remotes either way. The **Local** tab is the full table (favorites, descriptions, issue + self-hosted instance. Each host's list is stored in the local database (`remote_repo_listings` table), so + the page reads it at once, survives restarts, and never calls GitHub or GitLab per view. The server checks + it in the background when it is missing or older than `GITLIB_REPO_REFRESH_HOURS` (default 6, `0` = only + on the button); a check sends each page's ETag, and a host that answers *Not Modified* costs no GitHub + quota and downloads nothing. **Refresh** fetches both complete lists now. If a check fails (a revoked + token, a rate limit) the stored list stays on screen with the error noted, and automatic retries pause for + half an hour. A host that isn't configured is noted on the page and the rest still works, and clones link + to GitHub or GitLab from their remotes either way. The **Local** tab is the full table (favorites, descriptions, issue counts, community scores, preview pane); the other tabs list repos with links out, and clones open their detail page. The tab is kept in the URL (`?source=`). - Each repo has a detail page (click its name) with Overview, **Issues** and **Pull requests** tabs; the diff --git a/apps/web/.env.example b/apps/web/.env.example index d9c87e3..3f9e39b 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -5,6 +5,8 @@ GITLIB_GITHUB_TOKEN= #GITLIB_COMMUNITY_REFRESH_HOURS=24 # Contribution graph "Local + GitHub & GitLab": GitHub uses the token above (any read-only token works). # GitLab reads your public contribution calendar; set GITLIB_GITLAB_URL too if you self-host. +# How often (in hours) the GitHub and GitLab repository lists are checked again; 0 = only the Refresh button. +#GITLIB_REPO_REFRESH_HOURS=6 # The GitLab and Combined tabs list your projects. A personal access token with the read_api scope also reaches # private projects; with only a username, public projects are listed. The GitHub tabs use the token above, which # needs to be able to read your repositories. diff --git a/apps/web/src/components/catalog-table.tsx b/apps/web/src/components/catalog-table.tsx index f292a92..3c78423 100644 --- a/apps/web/src/components/catalog-table.tsx +++ b/apps/web/src/components/catalog-table.tsx @@ -9,10 +9,14 @@ import type { RepoView } from "#/server/repos"; const chip = "inline-flex h-5 items-center gap-1 border px-1.5 text-xs whitespace-nowrap"; -function RemoteChip({ label, link, configured }: { label: string; link: RemoteLink | null; configured: boolean }) { +/** Whether gitlib holds a list from this host, so a repo absent from it really is missing there. */ +const wasListed = (status: ProviderStatus | undefined): boolean => + status !== undefined && status.status !== "not-configured" && status.fetchedAt !== null; + +function RemoteChip({ label, link, listed }: { label: string; link: RemoteLink | null; listed: boolean }) { if (!link) { - // Only call a repo "missing" from a host gitlib could actually ask. - return configured ? ( + // Only call a repo "missing" from a host gitlib has actually listed. + return listed ? ( {label} — @@ -129,8 +133,8 @@ export function CatalogTable({ — )} - - + + diff --git a/apps/web/src/lib/catalog.ts b/apps/web/src/lib/catalog.ts index 19d0273..9c414d0 100644 --- a/apps/web/src/lib/catalog.ts +++ b/apps/web/src/lib/catalog.ts @@ -23,6 +23,31 @@ export interface RemoteRepo { language: string | null; } +/** One page of a host's repository listing with the ETag it came with, so the next check can revalidate it. */ +export interface ListingPage { + etag: string | null; + repos: RemoteRepo[]; +} + +/** What is stored for one host: its pages, and the account they were listed for. */ +export interface StoredListing { + /** Which account and host the pages belong to; a listing for a different one is never reused. */ + scope: string; + pages: ListingPage[]; +} + +/** Reads a stored listing back; null when it is unreadable or was listed for a different `scope`. */ +export function parseStoredListing(payload: string, scope: string): StoredListing | null { + try { + const value = JSON.parse(payload) as Partial | null; + if (!value || value.scope !== scope || !Array.isArray(value.pages)) return null; + for (const page of value.pages) if (!page || !Array.isArray(page.repos)) return null; + return value as StoredListing; + } catch { + return null; + } +} + /** A remote of a local clone that points at a known hosting service. */ export interface LocalRemote { provider: RemoteProvider; diff --git a/apps/web/src/lib/community.ts b/apps/web/src/lib/community.ts index bea6df5..bb7a07c 100644 --- a/apps/web/src/lib/community.ts +++ b/apps/web/src/lib/community.ts @@ -89,9 +89,9 @@ export const DEFAULT_REFRESH_HOURS = 24; * (decimals are fine). Unset, blank or unusable means a day; `0` turns automatic refreshing off, leaving * only the buttons, in which case this returns null. */ -export function refreshIntervalMs(setting: string | undefined): number | null { - const hours = setting?.trim() ? Number(setting) : DEFAULT_REFRESH_HOURS; - if (!Number.isFinite(hours) || hours < 0) return DEFAULT_REFRESH_HOURS * 3_600_000; +export function refreshIntervalMs(setting: string | undefined, defaultHours = DEFAULT_REFRESH_HOURS): number | null { + const hours = setting?.trim() ? Number(setting) : defaultHours; + if (!Number.isFinite(hours) || hours < 0) return defaultHours * 3_600_000; return hours === 0 ? null : hours * 3_600_000; } diff --git a/apps/web/src/routes/index.tsx b/apps/web/src/routes/index.tsx index 2be803c..92f38e2 100644 --- a/apps/web/src/routes/index.tsx +++ b/apps/web/src/routes/index.tsx @@ -162,9 +162,12 @@ function useCommunityScores(refreshKey: unknown) { const catalogFailed = (error: string) => error; +/** How often to look again while the server lists a host in the background. */ +const CATALOG_POLL_MS = 3_000; + /** - * Every repo across clones, GitHub and GitLab, fetched after the page renders since it asks both hosts. The - * server caches the hosts' listings, so refetching is cheap; `refresh` makes it ask them again. + * Every repo across clones, GitHub and GitLab, fetched after the page renders. The hosts' listings come from + * gitlib's database and are re-checked in the background, so loading is cheap; `refresh` asks the hosts again. */ function useCatalog(refreshKey: unknown) { const [view, setView] = useState(); @@ -188,6 +191,22 @@ function useCatalog(refreshKey: unknown) { // `refreshKey` is the only real input; the server function is stable. }, [refreshKey]); + // While a host is being listed (the first time, or a scheduled check) keep looking so the table fills in. + const refreshing = view?.github.refreshing === true || view?.gitlab.refreshing === true; + useEffect(() => { + if (!refreshing) return; + let cancelled = false; + const timer = setTimeout(() => { + getCatalog() + .then((next) => !cancelled && setView(next)) + .catch(() => undefined); + }, CATALOG_POLL_MS); + return () => { + cancelled = true; + clearTimeout(timer); + }; + }, [view, refreshing]); + const refresh = useCallback(async () => { setRequesting(true); try { @@ -200,7 +219,7 @@ function useCatalog(refreshKey: unknown) { } }, []); - return { view, error, refresh, busy: requesting }; + return { view, error, refresh, busy: requesting || refreshing }; } const CONTRIBUTION_SOURCE_KEY = "gitlib-contribution-source"; @@ -441,7 +460,7 @@ function RepoLibrary() { type="button" onClick={catalog.refresh} disabled={catalog.busy} - title="Ask GitHub and GitLab for their repository lists again. They are otherwise cached for a few minutes." + title="Ask GitHub and GitLab for their complete repository lists now. They are otherwise re-checked in the background." className="inline-flex h-8 items-center gap-1.5 border px-3 text-sm outline-none hover:bg-accent focus-visible:ring-2 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50" > @@ -602,7 +621,29 @@ function statusNotice(label: string, status: ProviderStatus | undefined): string if (!status || status.status === "ok") return null; return status.status === "not-configured" ? `${label} isn't included. ${status.message ?? ""}`.trim() - : `${label} couldn't be listed: ${status.message ?? "unknown error"}`; + : `${label} couldn't be refreshed${status.fetchedAt ? ` (showing the list from ${formatAgo(status.fetchedAt)})` : ""}: ${ + status.message ?? "unknown error" + }`; +} + +/** When each host's list was last checked, for the hosts this tab shows. */ +function checkedNotice(source: Exclude, github: ProviderStatus | undefined, gitlab: ProviderStatus | undefined) { + const describe = (label: string, status: ProviderStatus | undefined): string | null => { + if (!status || status.status === "not-configured") return null; + if (status.refreshing) return `${label} is being checked…`; + return status.fetchedAt ? `${label} checked ${formatAgo(status.fetchedAt)}` : null; + }; + const parts = [ + source !== "gitlab" ? describe("GitHub", github) : null, + source !== "github" ? describe("GitLab", gitlab) : null, + ].filter((part): part is string => part !== null); + return parts.length > 0 ? `Lists stored in gitlib · ${parts.join(" · ")}` : null; +} + +/** Whether a host this tab depends on has nothing stored yet but is being listed. */ +function isChecking(source: Exclude, github: ProviderStatus | undefined, gitlab: ProviderStatus | undefined) { + const waiting = (status: ProviderStatus | undefined) => status?.refreshing === true && status.fetchedAt === null; + return source === "github" ? waiting(github) : source === "gitlab" ? waiting(gitlab) : waiting(github) || waiting(gitlab); } /** The Combined, GitHub and GitLab tabs: one catalog table, plus what is missing when a host can't be listed. */ @@ -633,23 +674,29 @@ function SourceBody({ (notice): notice is string => notice !== null, ) : []; + const checked = checkedNotice(source, github, gitlab); const hostNotice = source === "github" ? statusNotice("GitHub", github) : source === "gitlab" ? statusNotice("GitLab", gitlab) : null; return ( <> - {notices.length > 0 ? ( + {notices.length > 0 || checked ? (
{notices.map((notice) => (

{notice}

))} + {checked ?

{checked}

: null}
) : null} {rows.length === 0 ? ( - {query ? "No repositories match your search." : (hostNotice ?? "No repositories found.")} + {query + ? "No repositories match your search." + : isChecking(source, github, gitlab) + ? "Listing repositories for the first time…" + : (hostNotice ?? "No repositories found.")} ) : ( diff --git a/apps/web/src/server/catalog.server.ts b/apps/web/src/server/catalog.server.ts new file mode 100644 index 0000000..d309f59 --- /dev/null +++ b/apps/web/src/server/catalog.server.ts @@ -0,0 +1,227 @@ +import { classifyProvider, loadGitRemotes, selectPreferredRemote, utcNow } from "@gitlib/core"; +import { + mergeCatalog, + parseLandingPage, + parseStoredListing, + type CatalogEntry, + type ListingPage, + type LocalClone, + type LocalRemote, + type RemoteProvider, + type RemoteRepo, + type StoredListing, +} from "#/lib/catalog"; +import { isStale, refreshIntervalMs } from "#/lib/community"; +import { githubToken, listGitHubRepos } from "./github.server"; +import { getApp, loadAndIndexRepositories } from "./gitlib.server"; +import { gitlabHost, gitlabScope, gitlabToken, gitlabUsername, listGitLabProjects } from "./gitlab.server"; +import { ProviderSetupError } from "./provider-errors"; + +export interface ProviderStatus { + /** "not-configured" means gitlib has no credentials for the service; "error" means the last check failed. */ + status: "ok" | "not-configured" | "error"; + message: string | null; + /** Repositories the service listed for the account, as last stored. */ + count: number; + /** When the service was last asked (whether or not anything changed); null if never. */ + fetchedAt: string | null; + /** A check is running now. */ + refreshing: boolean; +} + +export interface CatalogView { + entries: CatalogEntry[]; + github: ProviderStatus; + gitlab: ProviderStatus; +} + +const PROVIDERS: readonly RemoteProvider[] = ["github", "gitlab"]; + +/** Hours between automatic checks. A check that finds nothing new costs no GitHub quota (ETag revalidation). */ +const DEFAULT_REFRESH_HOURS = 6; +const refreshInterval = (): number | null => refreshIntervalMs(process.env["GITLIB_REPO_REFRESH_HOURS"], DEFAULT_REFRESH_HOURS); + +/** How often the running server checks whether a stored listing has aged out. */ +const CHECK_EVERY_MS = 15 * 60_000; +/** After a check failed, page loads and the timer leave that host alone for this long. */ +const RETRY_AFTER_MS = 30 * 60_000; + +const isConfigured = (provider: RemoteProvider): boolean => + provider === "github" ? Boolean(githubToken()) : Boolean(gitlabToken() || gitlabUsername()); + +/** Identifies the account a listing is for; a stored listing from another account or host is ignored. */ +const scopeOf = (provider: RemoteProvider): string => (provider === "github" ? "github.com" : gitlabScope()); + +interface HostState { + run?: Promise; + lastAttemptAt: number | null; + lastError: string | null; +} + +interface CatalogState { + hosts: Record; + timer?: ReturnType; +} + +// Kept on globalThis so dev-server module reloads reuse one timer and one check per host. +const state = ((globalThis as { __gitlibCatalog?: CatalogState }).__gitlibCatalog ??= { + hosts: { github: { lastAttemptAt: null, lastError: null }, gitlab: { lastAttemptAt: null, lastError: null } }, +}); + +/** Which hosting service a landing page belongs to; self-hosted GitLab is recognised by the configured URL. */ +function providerOf(host: string): RemoteProvider | null { + if (host.toLowerCase() === "github.com") return "github"; + if (host.toLowerCase() === gitlabHost().toLowerCase() || classifyProvider(host) === "GitLab") return "gitlab"; + return null; +} + +function localRemotes(absolutePath: string): LocalRemote[] { + const { remotes } = loadGitRemotes(absolutePath, getApp().system); + const preferred = selectPreferredRemote(remotes); + const ordered = preferred ? [preferred, ...remotes.filter((remote) => remote !== preferred)] : remotes; + return ordered.flatMap((remote) => { + const parsed = parseLandingPage(remote.landingPageUrl); + const provider = providerOf(remote.host); + if (!parsed || !provider || !remote.landingPageUrl) return []; + return [{ provider, key: parsed.key, fullName: parsed.path, url: remote.landingPageUrl }]; + }); +} + +/** Where each non-worktree clone points. One `git remote -v` per repo, so it is kept until the next rescan. */ +let cachedClones: Promise | undefined; + +async function loadClones(): Promise { + const repos = await loadAndIndexRepositories(); + return repos + .filter((repo) => !repo.isWorktree && !repo.isMissing) + .map((repo) => ({ + relativePath: repo.relativePath, + name: repo.name, + description: repo.description, + remotes: localRemotes(repo.absolutePath), + })); +} + +/** Forgets the scanned clones' remotes (a rescan does this); stored host listings are untouched. */ +export function clearCatalogClones(): void { + cachedClones = undefined; +} + +interface StoredHost { + listing: StoredListing; + fetchedAt: string; +} + +/** The host's listing as last stored, if it is readable and for the account that is configured now. */ +function storedHost(provider: RemoteProvider): StoredHost | null { + const row = getApp().store.getRemoteListing(provider); + const listing = row ? parseStoredListing(row.payload, scopeOf(provider)) : null; + return row && listing ? { listing, fetchedAt: row.fetchedAt } : null; +} + +/** + * Asks the host for its listing and stores the answer. Unless `full`, pages the host says haven't changed are + * reused (free for GitHub). The stored time moves to now even when nothing changed, since it was just checked. + * Never rejects. + */ +async function runCheck(provider: RemoteProvider, full: boolean): Promise { + const host = state.hosts[provider]; + host.lastAttemptAt = Date.now(); + try { + const held = full ? [] : (storedHost(provider)?.listing.pages ?? []); + const pages = await (provider === "github" ? listGitHubRepos(held) : listGitLabProjects(held)); + const listing: StoredListing = { scope: scopeOf(provider), pages }; + getApp().store.saveRemoteListing(provider, JSON.stringify(listing), utcNow()); + host.lastError = null; + } catch (caught) { + host.lastError = caught instanceof ProviderSetupError ? null : caught instanceof Error ? caught.message : "Could not list repositories."; + } +} + +/** Starts a check, or joins the one running; a full one waits for a running conditional check first. */ +function startCheck(provider: RemoteProvider, full: boolean): Promise { + const host = state.hosts[provider]; + if (host.run && !full) return host.run; + const run = (host.run ?? Promise.resolve()).then(() => runCheck(provider, full)); + host.run = run; + void run.finally(() => { + if (host.run === run) host.run = undefined; + }); + return run; +} + +/** Starts a background check for each configured host whose stored listing is missing or older than the interval. */ +function checkIfDue(): void { + const interval = refreshInterval(); + for (const provider of PROVIDERS) { + const host = state.hosts[provider]; + if (!isConfigured(provider) || host.run) continue; + const stored = storedHost(provider); + // Nothing stored must be fetched whatever the interval says; otherwise 0 means "only on the button". + if (stored && (interval === null || !isStale(stored.fetchedAt, Date.now(), interval))) continue; + // A failure (an expired token, the rate limit) would repeat on every page load, so give the host a rest first. + if (host.lastError && host.lastAttemptAt !== null && Date.now() - host.lastAttemptAt < RETRY_AFTER_MS) continue; + void startCheck(provider, false); + } +} + +/** Keeps listings current while the server stays up even if nobody opens the page. */ +function ensureScheduler(): void { + if (state.timer) return; + state.timer = setInterval(() => { + try { + checkIfDue(); + } catch (caught) { + console.warn("Repository listing check failed:", caught instanceof Error ? caught.message : caught); + } + }, CHECK_EVERY_MS); + state.timer.unref(); +} + +function statusOf(provider: RemoteProvider, stored: StoredHost | null): ProviderStatus { + const host = state.hosts[provider]; + const refreshing = host.run !== undefined; + if (!isConfigured(provider)) { + const message = + provider === "github" + ? "Set GITLIB_GITHUB_TOKEN to list your GitHub repositories." + : "Set GITLIB_GITLAB_TOKEN (or GITLIB_GITLAB_USERNAME for public projects), and GITLIB_GITLAB_URL if self-hosted."; + return { status: "not-configured", message, count: 0, fetchedAt: null, refreshing: false }; + } + return { + status: host.lastError ? "error" : "ok", + message: host.lastError, + count: stored?.listing.pages.reduce((sum, page) => sum + page.repos.length, 0) ?? 0, + fetchedAt: stored?.fetchedAt ?? null, + refreshing, + }; +} + +export async function buildCatalog(): Promise { + ensureScheduler(); + checkIfDue(); + const clones = await (cachedClones ??= loadClones().catch((caught: unknown) => { + cachedClones = undefined; + throw caught; + })); + + const repos: Record = { github: [], gitlab: [] }; + const stored: Record = { github: null, gitlab: null }; + for (const provider of PROVIDERS) { + // Stored repos of a host that is no longer configured are not shown as if they were current. + stored[provider] = isConfigured(provider) ? storedHost(provider) : null; + repos[provider] = (stored[provider]?.listing.pages ?? []).flatMap((page: ListingPage) => page.repos); + } + return { + entries: mergeCatalog(clones, repos.github, repos.gitlab), + github: statusOf("github", stored.github), + gitlab: statusOf("gitlab", stored.gitlab), + }; +} + +/** Asks both configured hosts for their complete listings now (ignoring ETags), and rereads every clone's remotes. */ +export async function refreshAllListings(): Promise { + cachedClones = undefined; + await Promise.all(PROVIDERS.filter(isConfigured).map((provider) => startCheck(provider, true))); + return buildCatalog(); +} diff --git a/apps/web/src/server/catalog.ts b/apps/web/src/server/catalog.ts index deb2b97..35e75d2 100644 --- a/apps/web/src/server/catalog.ts +++ b/apps/web/src/server/catalog.ts @@ -1,131 +1,14 @@ import { createServerFn } from "@tanstack/react-start"; -import { classifyProvider, loadGitRemotes, selectPreferredRemote } from "@gitlib/core"; -import { - mergeCatalog, - parseLandingPage, - type CatalogEntry, - type LocalClone, - type LocalRemote, - type RemoteProvider, - type RemoteRepo, -} from "#/lib/catalog"; -import { listGitHubRepos } from "./github.server"; -import { gitlabHost, listGitLabProjects } from "./gitlab.server"; -import { getApp, loadAndIndexRepositories } from "./gitlib.server"; -import { ProviderSetupError } from "./provider-errors"; +import { buildCatalog, refreshAllListings } from "./catalog.server"; -export interface ProviderStatus { - /** "not-configured" means gitlib has no credentials for the service; "error" means asking it failed. */ - status: "ok" | "not-configured" | "error"; - message: string | null; - /** Repositories the service listed for the account. */ - count: number; -} - -export interface CatalogView { - entries: CatalogEntry[]; - github: ProviderStatus; - gitlab: ProviderStatus; -} - -const REMOTES_TTL_MS = 5 * 60_000; - -/** Which hosting service a landing page belongs to; self-hosted GitLab is recognised by the configured URL. */ -function providerOf(host: string): RemoteProvider | null { - if (host.toLowerCase() === "github.com") return "github"; - if (host.toLowerCase() === gitlabHost().toLowerCase() || classifyProvider(host) === "GitLab") return "gitlab"; - return null; -} - -function localRemotes(absolutePath: string): LocalRemote[] { - const { remotes } = loadGitRemotes(absolutePath, getApp().system); - const preferred = selectPreferredRemote(remotes); - const ordered = preferred ? [preferred, ...remotes.filter((remote) => remote !== preferred)] : remotes; - return ordered.flatMap((remote) => { - const parsed = parseLandingPage(remote.landingPageUrl); - const provider = providerOf(remote.host); - if (!parsed || !provider || !remote.landingPageUrl) return []; - return [{ provider, key: parsed.key, fullName: parsed.path, url: remote.landingPageUrl }]; - }); -} - -/** Where each non-worktree clone points. One `git remote -v` per repo, so it is kept until the next rescan. */ -let cachedClones: Promise | undefined; - -async function loadClones(): Promise { - const repos = await loadAndIndexRepositories(); - return repos - .filter((repo) => !repo.isWorktree && !repo.isMissing) - .map((repo) => ({ - relativePath: repo.relativePath, - name: repo.name, - description: repo.description, - remotes: localRemotes(repo.absolutePath), - })); -} - -type Listing = { status: ProviderStatus; repos: RemoteRepo[] }; -const listings = new Map }>(); - -async function fetchListing(list: () => Promise): Promise { - try { - const repos = await list(); - return { status: { status: "ok", message: null, count: repos.length }, repos }; - } catch (caught) { - if (caught instanceof ProviderSetupError) { - return { status: { status: "not-configured", message: caught.message, count: 0 }, repos: [] }; - } - const message = caught instanceof Error ? caught.message : "Could not list repositories."; - return { status: { status: "error", message, count: 0 }, repos: [] }; - } -} - -function listingFor(provider: RemoteProvider): Promise { - const hit = listings.get(provider); - if (hit && Date.now() - hit.at <= REMOTES_TTL_MS) return hit.result; - const entry = { - at: Date.now(), - result: fetchListing(provider === "github" ? listGitHubRepos : listGitLabProjects), - }; - listings.set(provider, entry); - // Only successful lookups are worth keeping; a failed one should be retried next time. - void entry.result.then((result) => { - if (result.status.status !== "ok" && listings.get(provider) === entry) listings.delete(provider); - }); - return entry.result; -} - -/** Forgets the scanned clones' remotes (a rescan does this); the hosts' listings stay until they expire. */ -export function clearCatalogClones(): void { - cachedClones = undefined; -} - -function clearCatalog(): void { - cachedClones = undefined; - listings.clear(); -} - -async function buildCatalog(): Promise { - const clones = await (cachedClones ??= loadClones().catch((caught: unknown) => { - cachedClones = undefined; - throw caught; - })); - const [github, gitlab] = await Promise.all([listingFor("github"), listingFor("gitlab")]); - return { - entries: mergeCatalog(clones, github.repos, gitlab.repos), - github: github.status, - gitlab: gitlab.status, - }; -} +export type { CatalogView, ProviderStatus } from "./catalog.server"; /** * Local clones joined with the repositories GitHub and GitLab list for the configured accounts. The hosts' - * listings are cached for a few minutes; the local half works whether or not either host is reachable. + * listings are read from gitlib's database, never fetched per view; one that is missing or older than + * `GITLIB_REPO_REFRESH_HOURS` is refreshed in the background and the host's status says `refreshing`. */ -export const getCatalog = createServerFn({ method: "GET" }).handler(buildCatalog); +export const getCatalog = createServerFn({ method: "GET" }).handler(() => buildCatalog()); -/** Asks both hosts again and rereads every clone's remotes. */ -export const refreshCatalog = createServerFn({ method: "POST" }).handler(async () => { - clearCatalog(); - return buildCatalog(); -}); +/** Asks both hosts for their complete listings now, ignoring ETags, and rereads every clone's remotes. */ +export const refreshCatalog = createServerFn({ method: "POST" }).handler(() => refreshAllListings()); diff --git a/apps/web/src/server/github.server.ts b/apps/web/src/server/github.server.ts index 80c56dc..65ee46b 100644 --- a/apps/web/src/server/github.server.ts +++ b/apps/web/src/server/github.server.ts @@ -1,8 +1,9 @@ import { loadGitRemotes, selectGitHubRepo, type GitHubRepoRef } from "@gitlib/core"; -import { remoteKey, type RemoteRepo } from "#/lib/catalog"; +import { remoteKey, type ListingPage } from "#/lib/catalog"; import { getApp, loadAndIndexRepositories } from "./gitlib.server"; import type { Provider } from "./provider"; import { ProviderSetupError } from "./provider-errors"; +import { PAGE_SIZE, fetchListingPages } from "./remote-listing.server"; export const githubToken = (): string | undefined => process.env["GITLIB_GITHUB_TOKEN"] || process.env["GITHUB_TOKEN"] || undefined; @@ -104,39 +105,37 @@ interface GitHubAccountRepo { language: string | null; } -/** Pages of 100; ten covers a thousand repos, which is plenty before a listing becomes unwieldy. */ -const ACCOUNT_REPO_PAGES = 10; - /** Every repository the token's user owns, collaborates on, or can reach through an organization. */ -export async function listGitHubRepos(): Promise { +export async function listGitHubRepos(previous: readonly ListingPage[] = []) { const token = githubToken(); if (!token) throw new ProviderSetupError("Set GITLIB_GITHUB_TOKEN to list your GitHub repositories."); - const repos: RemoteRepo[] = []; - for (let page = 1; page <= ACCOUNT_REPO_PAGES; page += 1) { - const url = new URL("https://api.github.com/user/repos"); - url.search = new URLSearchParams({ - per_page: "100", - page: String(page), - sort: "pushed", - affiliation: "owner,collaborator,organization_member", - }).toString(); - const response = await fetch(url, { - headers: { - Accept: "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "gitlib", - Authorization: `Bearer ${token}`, - }, - signal: AbortSignal.timeout(15_000), - }); - if (response.status === 401) throw new Error("GitHub rejected the token (401)."); - if (response.status === 403 || response.status === 429) throw new Error("GitHub rate limit reached. Try again later."); - if (!response.ok) throw new Error(`GitHub responded with ${response.status}.`); - - const batch = (await response.json()) as GitHubAccountRepo[]; - for (const item of batch) { - repos.push({ + return fetchListingPages( + async (page, etag) => { + const url = new URL("https://api.github.com/user/repos"); + url.search = new URLSearchParams({ + per_page: String(PAGE_SIZE), + page: String(page), + sort: "pushed", + affiliation: "owner,collaborator,organization_member", + }).toString(); + const response = await fetch(url, { + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "gitlib", + Authorization: `Bearer ${token}`, + ...(etag ? { "If-None-Match": etag } : {}), + }, + signal: AbortSignal.timeout(15_000), + }); + if (response.status === 401) throw new Error("GitHub rejected the token (401)."); + if (response.status === 403 || response.status === 429) throw new Error("GitHub rate limit reached. Try again later."); + if (!response.ok && response.status !== 304) throw new Error(`GitHub responded with ${response.status}.`); + return response; + }, + (body) => + (body as GitHubAccountRepo[]).map((item) => ({ provider: "github", key: remoteKey("github.com", item.full_name), fullName: item.full_name, @@ -148,9 +147,7 @@ export async function listGitHubRepos(): Promise { isArchived: item.archived, updatedAt: item.pushed_at ?? item.updated_at, language: item.language, - }); - } - if (batch.length < 100) break; - } - return repos; + })), + previous, + ); } diff --git a/apps/web/src/server/gitlab.server.ts b/apps/web/src/server/gitlab.server.ts index 2089922..4706a6a 100644 --- a/apps/web/src/server/gitlab.server.ts +++ b/apps/web/src/server/gitlab.server.ts @@ -1,6 +1,7 @@ import "./env.server"; -import { remoteKey, type RemoteRepo } from "#/lib/catalog"; +import { remoteKey, type ListingPage } from "#/lib/catalog"; import { ProviderSetupError } from "./provider-errors"; +import { PAGE_SIZE, fetchListingPages } from "./remote-listing.server"; export const gitlabUsername = (): string | undefined => process.env["GITLIB_GITLAB_USERNAME"] || undefined; @@ -22,18 +23,17 @@ interface GitLabProject { forked_from_project?: unknown; } -/** Pages of 100, capped like the GitHub listing. */ -const PROJECT_PAGES = 10; - /** * The projects the token's user is a member of, or, with only a username configured, that user's public * projects. Owned, group and shared projects all count as membership. */ -export async function listGitLabProjects(): Promise { +export async function listGitLabProjects(previous: readonly ListingPage[] = []) { const token = gitlabToken(); const username = gitlabUsername(); if (!token && !username) { - throw new ProviderSetupError("Set GITLIB_GITLAB_TOKEN (or GITLIB_GITLAB_USERNAME for public projects), and GITLIB_GITLAB_URL if self-hosted."); + throw new ProviderSetupError( + "Set GITLIB_GITLAB_TOKEN (or GITLIB_GITLAB_USERNAME for public projects), and GITLIB_GITLAB_URL if self-hosted.", + ); } const endpoint = token @@ -41,28 +41,33 @@ export async function listGitLabProjects(): Promise { : `${gitlabBase()}/api/v4/users/${encodeURIComponent(username!)}/projects`; const host = gitlabHost(); - const repos: RemoteRepo[] = []; - for (let page = 1; page <= PROJECT_PAGES; page += 1) { - const url = new URL(endpoint); - url.search = new URLSearchParams({ - per_page: "100", - page: String(page), - order_by: "last_activity_at", - sort: "desc", - ...(token ? { membership: "true" } : {}), - }).toString(); - const response = await fetch(url, { - headers: { Accept: "application/json", "User-Agent": "gitlib", ...(token ? { "PRIVATE-TOKEN": token } : {}) }, - signal: AbortSignal.timeout(15_000), - }); - if (response.status === 401) throw new Error("GitLab rejected the token (401)."); - if (response.status === 404 && !token) throw new Error(`GitLab has no user named ${username}.`); - if (response.status === 429) throw new Error("GitLab rate limit reached. Try again later."); - if (!response.ok) throw new Error(`GitLab responded with ${response.status}.`); - - const batch = (await response.json()) as GitLabProject[]; - for (const item of batch) { - repos.push({ + return fetchListingPages( + async (page, etag) => { + const url = new URL(endpoint); + url.search = new URLSearchParams({ + per_page: String(PAGE_SIZE), + page: String(page), + order_by: "last_activity_at", + sort: "desc", + ...(token ? { membership: "true" } : {}), + }).toString(); + const response = await fetch(url, { + headers: { + Accept: "application/json", + "User-Agent": "gitlib", + ...(token ? { "PRIVATE-TOKEN": token } : {}), + ...(etag ? { "If-None-Match": etag } : {}), + }, + signal: AbortSignal.timeout(15_000), + }); + if (response.status === 401) throw new Error("GitLab rejected the token (401)."); + if (response.status === 404 && !token) throw new Error(`GitLab has no user named ${username}.`); + if (response.status === 429) throw new Error("GitLab rate limit reached. Try again later."); + if (!response.ok && response.status !== 304) throw new Error(`GitLab responded with ${response.status}.`); + return response; + }, + (body) => + (body as GitLabProject[]).map((item) => ({ provider: "gitlab", key: remoteKey(host, item.path_with_namespace), fullName: item.path_with_namespace, @@ -74,9 +79,10 @@ export async function listGitLabProjects(): Promise { isArchived: item.archived === true, updatedAt: item.last_activity_at, language: null, - }); - } - if (batch.length < 100) break; - } - return repos; + })), + previous, + ); } + +/** Which host and account a stored listing was made for, so changing either never shows the old one's repos. */ +export const gitlabScope = (): string => `${gitlabBase()}|${gitlabToken() ? "member" : `user:${gitlabUsername() ?? ""}`}`; diff --git a/apps/web/src/server/remote-listing.server.ts b/apps/web/src/server/remote-listing.server.ts new file mode 100644 index 0000000..02329fc --- /dev/null +++ b/apps/web/src/server/remote-listing.server.ts @@ -0,0 +1,35 @@ +import type { ListingPage, RemoteRepo } from "#/lib/catalog"; + +/** Both hosts are asked for this many repos a page; a shorter page is the last one. */ +export const PAGE_SIZE = 100; + +/** Pages of 100; ten covers a thousand repos, which is plenty before a listing becomes unwieldy. */ +export const MAX_PAGES = 10; + +/** + * Fetches a paginated listing, revalidating pages we already hold. `request` returns the raw response (and + * throws for anything but 200 and 304). A page whose ETag still matches comes back as `304 Not Modified`, which + * GitHub doesn't count against the rate limit, so a check that finds nothing new costs no quota and no download. + * Every page is asked about, since a repo that changes can move to another page. + */ +export async function fetchListingPages( + request: (page: number, etag: string | null) => Promise, + parse: (body: unknown) => RemoteRepo[], + previous: readonly ListingPage[], +): Promise { + const pages: ListingPage[] = []; + for (let page = 1; page <= MAX_PAGES; page += 1) { + const held = previous[page - 1]; + const response = await request(page, held?.etag ?? null); + let next: ListingPage; + if (response.status === 304) { + if (!held) throw new Error("The host answered 304 Not Modified to a request with no ETag."); + next = held; + } else { + next = { etag: response.headers.get("etag"), repos: parse(await response.json()) }; + } + pages.push(next); + if (next.repos.length < PAGE_SIZE) break; + } + return pages; +} diff --git a/apps/web/src/server/repos.ts b/apps/web/src/server/repos.ts index d032eff..849cc46 100644 --- a/apps/web/src/server/repos.ts +++ b/apps/web/src/server/repos.ts @@ -1,7 +1,7 @@ import { createServerFn } from "@tanstack/react-start"; import { loadRepoReadme, technologyCategory, type RepoRecord, type TechnologyCategory } from "@gitlib/core"; import { getApp, loadAndIndexRepositories } from "./gitlib.server"; -import { clearCatalogClones } from "./catalog"; +import { clearCatalogClones } from "./catalog.server"; import { clearContributions } from "./contributions"; import { clearIssueCounts } from "./issues"; diff --git a/apps/web/test/catalog-flow.test.ts b/apps/web/test/catalog-flow.test.ts new file mode 100644 index 0000000..fcaf952 --- /dev/null +++ b/apps/web/test/catalog-flow.test.ts @@ -0,0 +1,204 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +/** A GitHub that lists one repo and honours If-None-Match, counting how it is asked. */ +function fakeGitHub() { + const calls: { ifNoneMatch: string | null; status: number }[] = []; + const listing = [ + { + full_name: "me/app", + name: "app", + description: "The app", + html_url: "https://github.com/me/app", + private: false, + fork: false, + archived: false, + pushed_at: "2026-01-01T00:00:00Z", + updated_at: null, + language: "TypeScript", + }, + { ...{ full_name: "me/remote-only", name: "remote-only" }, description: null, html_url: "https://github.com/me/remote-only", private: true, fork: false, archived: false, pushed_at: null, updated_at: "2025-01-01T00:00:00Z", language: null }, + ]; + const control = { etag: '"v1"', failWith: 0 }; + const fetchMock = vi.fn(async (_url, init) => { + const headers = init?.headers as Record; + const ifNoneMatch = headers["If-None-Match"] ?? null; + if (control.failWith) { + calls.push({ ifNoneMatch, status: control.failWith }); + return new Response("{}", { status: control.failWith }); + } + if (ifNoneMatch === control.etag) { + calls.push({ ifNoneMatch, status: 304 }); + return new Response(null, { status: 304 }); + } + calls.push({ ifNoneMatch, status: 200 }); + return new Response(JSON.stringify(listing), { status: 200, headers: { ETag: control.etag } }); + }); + return { calls, control, fetchMock }; +} + +let base: string; + +beforeEach(() => { + base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "gitlib-catalog-"))); + const gitRoot = path.join(base, "git"); + const clone = path.join(gitRoot, "app"); + fs.mkdirSync(clone, { recursive: true }); + execFileSync("git", ["init", "-q", clone]); + execFileSync("git", ["-C", clone, "remote", "add", "origin", "https://github.com/me/app.git"]); + fs.mkdirSync(path.join(base, "home")); + fs.writeFileSync( + path.join(base, "home", "config.toml"), + `roots = ["${gitRoot}"]\n\n[search]\nenabled = false\n`, + ); + + vi.stubEnv("GITLIB_HOME", path.join(base, "home")); + vi.stubEnv("GITLIB_GITHUB_TOKEN", "ghp_test"); + vi.stubEnv("GITHUB_TOKEN", ""); + vi.stubEnv("GITLIB_GITLAB_TOKEN", ""); + vi.stubEnv("GITLIB_GITLAB_USERNAME", ""); + vi.stubEnv("GITLIB_GITLAB_URL", ""); + vi.stubEnv("GITLIB_REPO_REFRESH_HOURS", ""); +}); + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + fs.rmSync(base, { recursive: true, force: true }); +}); + +/** A fresh copy of the server modules and their shared state, like a restarted server. */ +async function startServer() { + const shared = globalThis as Record; + delete shared["__gitlib"]; + delete shared["__gitlibCatalog"]; + vi.resetModules(); + return (await import("../src/server/catalog.server")).buildCatalog; +} + +/** Moves the clock forward without touching timers, so background checks still run on real time. */ +const hoursLater = (hours: number) => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + hours * 3_600_000); +}; + +const settled = (load: () => Promise<{ github: { refreshing: boolean } }>) => + vi.waitFor(async () => expect((await load()).github.refreshing).toBe(false)); + +describe("catalog listings", () => { + it("lists in the background the first time, then serves the stored list without asking GitHub", async () => { + const github = fakeGitHub(); + vi.stubGlobal("fetch", github.fetchMock); + const load = await startServer(); + + const first = await load(); + expect(first.github).toMatchObject({ status: "ok", fetchedAt: null, refreshing: true }); + // Until GitHub answers, the clone still shows, linked from its own remote. + expect(first.entries.map((entry) => entry.fullName)).toEqual(["me/app"]); + expect(first.entries[0]!.github?.listed).toBe(false); + + await settled(load); + const second = await load(); + expect(second.github).toMatchObject({ status: "ok", count: 2, refreshing: false }); + expect(second.entries.map((entry) => [entry.fullName, entry.locals.length, entry.github?.listed])).toEqual([ + ["me/app", 1, true], + ["me/remote-only", 0, true], + ]); + expect(second.entries[0]!.description).toBe("The app"); + + await load(); + await load(); + expect(github.calls).toHaveLength(1); + }); + + it("serves the stored list after a restart without asking GitHub", async () => { + const github = fakeGitHub(); + vi.stubGlobal("fetch", github.fetchMock); + const first = await startServer(); + await first(); + await settled(first); + expect(github.calls).toHaveLength(1); + + const restarted = await startServer(); + const view = await restarted(); + expect(view.github).toMatchObject({ count: 2, refreshing: false }); + expect(view.entries).toHaveLength(2); + expect(github.calls).toHaveLength(1); + }); + + it("rechecks once the list is old, revalidating with the stored ETag", async () => { + const github = fakeGitHub(); + vi.stubGlobal("fetch", github.fetchMock); + const load = await startServer(); + await load(); + await settled(load); + const before = (await load()).github.fetchedAt!; + + hoursLater(7); // past the 6 hour default + await load(); // notices the list is due and starts a check + await settled(load); + + expect(github.calls.map((call) => call.status)).toEqual([200, 304]); + expect(github.calls[1]!.ifNoneMatch).toBe('"v1"'); + const after = await load(); + expect(after.github.count).toBe(2); + expect(after.github.fetchedAt! > before).toBe(true); + }); + + it("checks nothing automatically when the interval is 0, but still fills an empty store", async () => { + vi.stubEnv("GITLIB_REPO_REFRESH_HOURS", "0"); + const github = fakeGitHub(); + vi.stubGlobal("fetch", github.fetchMock); + const load = await startServer(); + await load(); + await settled(load); + expect(github.calls).toHaveLength(1); + + hoursLater(1000); + await load(); + await load(); + expect(github.calls).toHaveLength(1); + }); + + it("keeps the stored list when a check fails, and leaves GitHub alone for a while", async () => { + const github = fakeGitHub(); + vi.stubGlobal("fetch", github.fetchMock); + const load = await startServer(); + await load(); + await settled(load); + + github.control.failWith = 401; + hoursLater(7); + await load(); + await settled(load); + + const failed = await load(); + expect(failed.github.status).toBe("error"); + expect(failed.github.message).toContain("rejected the token"); + expect(failed.github.count).toBe(2); + expect(failed.entries).toHaveLength(2); + + const attempts = github.calls.length; + await load(); + await load(); + expect(github.calls).toHaveLength(attempts); + }); + + it("does not show a stored list once GitHub is no longer configured", async () => { + const github = fakeGitHub(); + vi.stubGlobal("fetch", github.fetchMock); + const load = await startServer(); + await load(); + await settled(load); + + vi.stubEnv("GITLIB_GITLAB_TOKEN", ""); + vi.stubEnv("GITLIB_GITHUB_TOKEN", ""); + const view = await load(); + expect(view.github).toMatchObject({ status: "not-configured", count: 0 }); + expect(view.entries.map((entry) => entry.fullName)).toEqual(["me/app"]); + }); +}); diff --git a/apps/web/test/catalog.test.ts b/apps/web/test/catalog.test.ts index 531aaaf..b86ff3e 100644 --- a/apps/web/test/catalog.test.ts +++ b/apps/web/test/catalog.test.ts @@ -4,6 +4,7 @@ import { matchesQuery, mergeCatalog, parseLandingPage, + parseStoredListing, remoteKey, type LocalClone, type RemoteRepo, @@ -179,3 +180,21 @@ describe("entriesForSource / matchesQuery", () => { expect(matchesQuery(entry!, undefined)).toBe(true); }); }); + +describe("parseStoredListing", () => { + const listing = { scope: "github.com", pages: [{ etag: '"a"', repos: [github("me/app")] }] }; + + it("reads back what was stored", () => { + expect(parseStoredListing(JSON.stringify(listing), "github.com")).toEqual(listing); + }); + + it("ignores a listing made for another host or account", () => { + expect(parseStoredListing(JSON.stringify(listing), "https://gitlab.com|member")).toBeNull(); + }); + + it("ignores unreadable payloads", () => { + expect(parseStoredListing("not json", "github.com")).toBeNull(); + expect(parseStoredListing("null", "github.com")).toBeNull(); + expect(parseStoredListing(JSON.stringify({ scope: "github.com", pages: [null] }), "github.com")).toBeNull(); + }); +}); diff --git a/apps/web/test/remote-listing.test.ts b/apps/web/test/remote-listing.test.ts index 644cc66..4734343 100644 --- a/apps/web/test/remote-listing.test.ts +++ b/apps/web/test/remote-listing.test.ts @@ -3,8 +3,15 @@ import { listGitHubRepos } from "../src/server/github.server"; import { gitlabHost, listGitLabProjects } from "../src/server/gitlab.server"; import { ProviderSetupError } from "../src/server/provider-errors"; -const json = (body: unknown, status = 200) => - new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } }); +const json = (body: unknown, status = 200, etag?: string) => + new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json", ...(etag ? { ETag: etag } : {}) }, + }); + +const notModified = () => new Response(null, { status: 304 }); + +const flat = (pages: { repos: unknown[] }[]) => pages.flatMap((page) => page.repos); const stubFetch = (...responses: Response[]) => { const fetchMock = vi.fn(); @@ -26,6 +33,20 @@ const githubRepo = (n: number) => ({ language: "TypeScript", }); +const githubRepoView = (n: number) => ({ + provider: "github" as const, + key: `github.com/me/repo-${n}`, + fullName: `me/repo-${n}`, + name: `repo-${n}`, + description: "", + url: `https://github.com/me/repo-${n}`, + isPrivate: false, + isFork: false, + isArchived: false, + updatedAt: null, + language: null, +}); + afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); @@ -44,7 +65,8 @@ describe("listGitHubRepos", () => { json(Array.from({ length: 100 }, (_, n) => githubRepo(n))), json([githubRepo(100)]), ); - const repos = await listGitHubRepos(); + const pages = await listGitHubRepos(); + const repos = pages.flatMap((page) => page.repos); expect(repos).toHaveLength(101); expect(fetchMock).toHaveBeenCalledTimes(2); @@ -66,6 +88,60 @@ describe("listGitHubRepos", () => { stubFetch(json({}, 401)); await expect(listGitHubRepos()).rejects.toThrow("GitHub rejected the token"); }); + + it("keeps each page's ETag and revalidates with it, reusing pages GitHub says haven't changed", async () => { + vi.stubEnv("GITLIB_GITHUB_TOKEN", "ghp_test"); + stubFetch( + json(Array.from({ length: 100 }, (_, n) => githubRepo(n)), 200, '"page-1"'), + json([githubRepo(100)], 200, '"page-2"'), + ); + const first = await listGitHubRepos(); + expect(first.map((page) => page.etag)).toEqual(['"page-1"', '"page-2"']); + + const fetchMock = stubFetch(notModified(), notModified()); + const second = await listGitHubRepos(first); + + const sent = fetchMock.mock.calls.map(([, init]) => (init!.headers as Record)["If-None-Match"]); + expect(sent).toEqual(['"page-1"', '"page-2"']); + expect(second).toEqual(first); + expect(flat(second)).toHaveLength(101); + }); + + it("takes a changed page from the response and still reuses the rest", async () => { + vi.stubEnv("GITLIB_GITHUB_TOKEN", "ghp_test"); + const first = [ + { etag: '"a"', repos: Array.from({ length: 100 }, (_, n) => ({ ...githubRepoView(n) })) }, + { etag: '"b"', repos: [githubRepoView(100)] }, + ]; + stubFetch(json(Array.from({ length: 100 }, (_, n) => githubRepo(n + 1)), 200, '"a2"'), notModified()); + const second = await listGitHubRepos(first); + + expect(second[0]!.etag).toBe('"a2"'); + expect(second[0]!.repos[0]!.fullName).toBe("me/repo-1"); + expect(second[1]).toBe(first[1]); + }); + + it("drops pages a shorter listing no longer has", async () => { + vi.stubEnv("GITLIB_GITHUB_TOKEN", "ghp_test"); + const held = [ + { etag: '"a"', repos: Array.from({ length: 100 }, (_, n) => githubRepoView(n)) }, + { etag: '"b"', repos: [githubRepoView(100)] }, + ]; + // Repos were deleted, so page 1 is now short and page 2 is never asked for. + const fetchMock = stubFetch(json([githubRepo(0)], 200, '"a3"')); + const pages = await listGitHubRepos(held); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(pages).toHaveLength(1); + expect(pages[0]!.repos).toHaveLength(1); + }); + + it("ignores held pages when asked to start over", async () => { + vi.stubEnv("GITLIB_GITHUB_TOKEN", "ghp_test"); + const fetchMock = stubFetch(json([githubRepo(0)], 200, '"fresh"')); + await listGitHubRepos([]); + expect((fetchMock.mock.calls[0]![1]!.headers as Record)["If-None-Match"]).toBeUndefined(); + }); }); describe("listGitLabProjects", () => { @@ -95,7 +171,8 @@ describe("listGitLabProjects", () => { project("me/fork", { visibility: "public", forked_from_project: { id: 1 }, archived: true }), ]), ); - const repos = await listGitLabProjects(); + const pages = await listGitLabProjects(); + const repos = pages.flatMap((page) => page.repos); const [url, init] = fetchMock.mock.calls[0]!; expect(String(url)).toContain("https://gitlab.com/api/v4/projects?"); @@ -116,7 +193,8 @@ describe("listGitLabProjects", () => { vi.stubEnv("GITLIB_GITLAB_USERNAME", "me"); vi.stubEnv("GITLIB_GITLAB_URL", "https://git.example.com/"); const fetchMock = stubFetch(json([project("me/thing", { visibility: "public" })])); - const repos = await listGitLabProjects(); + const pages = await listGitLabProjects(); + const repos = pages.flatMap((page) => page.repos); expect(String(fetchMock.mock.calls[0]![0])).toContain("https://git.example.com/api/v4/users/me/projects?"); expect(gitlabHost()).toBe("git.example.com"); diff --git a/docker-compose.yml b/docker-compose.yml index a5f0652..92e4dbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ x-gitlib-env: &gitlib-env # Optional: hours between automatic refreshes of the GitHub community scores (blank = 24, 0 = off). GITLIB_COMMUNITY_REFRESH_HOURS: ${GITLIB_COMMUNITY_REFRESH_HOURS:-} # Optional: the contribution graph's "Local + GitHub & GitLab" mode. + # Optional: hours between checks of the GitHub/GitLab repository lists (blank = 6, 0 = only the Refresh button). + GITLIB_REPO_REFRESH_HOURS: ${GITLIB_REPO_REFRESH_HOURS:-} # Optional: lists GitLab projects on the Repositories page (read_api scope; without it only public ones show). GITLIB_GITLAB_TOKEN: ${GITLIB_GITLAB_TOKEN:-} GITLIB_GITLAB_USERNAME: ${GITLIB_GITLAB_USERNAME:-} diff --git a/docs/runbook.md b/docs/runbook.md index ead883f..6a764bc 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -400,7 +400,7 @@ docker compose run --rm gitlib list # one-off CLI command | `port is already allocated` / `address already in use` | Something else uses the port. Set `GITLIB_PORT` (or `GITLIB_MEILI_PORT`) in `.env` to a free one. `npm run web` also refuses to start on a busy 4000. | | No repositories listed | `GITLIB_REPOS` (default `~/git`) is wrong, not shared with Docker Desktop, or empty. Check `docker compose exec web ls /home/node/git`. Press **Rescan** after fixing. | | Repo pages show no GitHub data / community scores | Add `GITLIB_GITHUB_TOKEN` to `.env` (anonymous GitHub limit is 60 requests/hour), then `docker compose up -d`. Confirm the repo has a GitHub `origin` remote. Scores are stored and refreshed daily; press **Refresh scores** to fetch now. | -| GitHub / GitLab tab is empty or says "isn't included" | The Repositories page lists GitHub repos with `GITLIB_GITHUB_TOKEN` and GitLab projects with `GITLIB_GITLAB_TOKEN` (`read_api`; or `GITLIB_GITLAB_USERNAME` for public ones, plus `GITLIB_GITLAB_URL` if self-hosted). Add it to `.env`, run `docker compose up -d`, then press **Refresh**. A rejected token (401) is reported on the page. | +| GitHub / GitLab tab is empty or says "isn't included" | The Repositories page lists GitHub repos with `GITLIB_GITHUB_TOKEN` and GitLab projects with `GITLIB_GITLAB_TOKEN` (`read_api`; or `GITLIB_GITLAB_USERNAME` for public ones, plus `GITLIB_GITLAB_URL` if self-hosted). Add it to `.env`, run `docker compose up -d`, then press **Refresh**. A rejected token (401) is reported on the page, with the last stored list still shown. The lists are stored in the database and re-checked every `GITLIB_REPO_REFRESH_HOURS` (default 6). | | "Search falls back to substring matching" | Meilisearch is down or unreachable. `docker compose ps`, `docker compose logs meilisearch`. After changing `GITLIB_MEILI_API_KEY` run `docker compose up -d` so both containers get the new value. | | `meilisearch` will not start with `MEILI_ENV=production` | Production mode requires a key of at least 16 characters in `GITLIB_MEILI_API_KEY`. | | `permission denied` writing to `/backups` | The backups directory is not writable by uid 1000: `sudo chown 1000:1000 "$GITLIB_BACKUPS"`. | diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 33dc650..bc27637 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -83,6 +83,13 @@ const SCHEMA = ` payload TEXT NOT NULL, fetched_at TEXT NOT NULL ); + + -- What a hosting service (GitHub, GitLab) last listed for the configured account, one row per provider. + CREATE TABLE IF NOT EXISTS remote_repo_listings ( + provider TEXT PRIMARY KEY, + payload TEXT NOT NULL, + fetched_at TEXT NOT NULL + ); `; /** A GitHub community profile as last fetched; `payload` is opaque to the store (the caller serializes it). */ @@ -93,6 +100,14 @@ export interface StoredCommunityProfile { fetchedAt: string; } +/** A hosting service's repository listing as last checked; `payload` is opaque to the store. */ +export interface StoredRemoteListing { + provider: string; + payload: string; + /** When the service was last asked, whether or not the answer changed. */ + fetchedAt: string; +} + export class Store { constructor(readonly dbPath: string) { fs.mkdirSync(path.dirname(dbPath), { recursive: true }); @@ -323,6 +338,24 @@ export class Store { ); } + saveRemoteListing(provider: string, payload: string, fetchedAt: string): void { + this.transaction((db) => { + db.prepare( + `INSERT INTO remote_repo_listings(provider, payload, fetched_at) VALUES (?, ?, ?) + ON CONFLICT(provider) DO UPDATE SET payload = excluded.payload, fetched_at = excluded.fetched_at`, + ).run(provider, payload, fetchedAt); + }); + } + + getRemoteListing(provider: string): StoredRemoteListing | null { + return this.withConnection((db) => { + const row = db + .prepare("SELECT provider, payload, fetched_at FROM remote_repo_listings WHERE provider = ?") + .get(provider) as { provider: string; payload: string; fetched_at: string } | undefined; + return row ? { provider: row.provider, payload: row.payload, fetchedAt: row.fetched_at } : null; + }); + } + updateNotes(repoId: number, notes: string): void { this.transaction((db) => { db.prepare( diff --git a/packages/core/test/core.test.ts b/packages/core/test/core.test.ts index 4501f4b..33c0b3e 100644 --- a/packages/core/test/core.test.ts +++ b/packages/core/test/core.test.ts @@ -297,6 +297,24 @@ describe("application", () => { expect(reopened.store.listCommunityProfiles().map((profile) => profile.slug)).toEqual(["meddlin/gitlib", "other/repo"]); }); + it("persists each hosting service's repository listing and replaces it on the next save", () => { + fx.writeConfig(); + const app = createApplication(fx.env()); + expect(app.store.getRemoteListing("github")).toBeNull(); + + app.store.saveRemoteListing("github", '{"pages":[]}', "2026-11-01T00:00:00+00:00"); + app.store.saveRemoteListing("gitlab", '{"pages":[1]}', "2026-11-01T00:00:00+00:00"); + app.store.saveRemoteListing("github", '{"pages":[2]}', "2026-11-02T00:00:00+00:00"); + + const reopened = createApplication(fx.env()); + expect(reopened.store.getRemoteListing("github")).toEqual({ + provider: "github", + payload: '{"pages":[2]}', + fetchedAt: "2026-11-02T00:00:00+00:00", + }); + expect(reopened.store.getRemoteListing("gitlab")?.payload).toBe('{"pages":[1]}'); + }); + it("adds the community table to a database created before it existed", () => { fx.writeConfig(); const first = createApplication(fx.env());