Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
216 changes: 159 additions & 57 deletions .github/scripts/site-check.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,32 @@

Exit 1 on any of these, each printed as one ERROR line:
- a link or image whose target is not in the built site: a nav entry whose page does not
exist, a missing image, any other dangling href or src (Zensical validates Markdown page
links and anchors; it does not validate images, nav entries or raw HTML links). A redirect
stub's own links (canonical and body) are checked the same way;
exist, a missing image, any other dangling href or src, in any quoting and with HTML
entities decoded (Zensical validates Markdown page links; it does not validate images,
nav entries or raw HTML links). A full URL into the site itself
(https://geiserx.github.io/<repo>/...) is checked the same way. A redirect stub's own
links (canonical and body) are checked too;
- a link whose #fragment is not an id on the target page, including links that come in
through a snippet include (Zensical 0.1.0 does not check those; MkDocs strict did);
- an href ending in .md: a correct page link is rewritten to a directory URL, so a .md left
in the HTML is a raw-HTML or unprocessed link, and it opens the llmstxt Markdown copy;
- a root-absolute link (/getting-started/) outside the site's own path prefix: it breaks
under https://geiserx.github.io/<repo>/ (MkDocs' `absolute_links: warn`, which Zensical
ignores);
ignores). The prefix comes from site_url, else the home page's canonical link; with
neither, that is an error rather than a check that cannot fire;
- a page that no navigation entry links to: a stray file under docs/ published at a public
URL that nothing reaches (MkDocs' `nav.omitted_files: warn`, which Zensical ignores);
- a .py, .pyc or __pycache__ entry in the site: the awesome lists keep their Markdown
extension under docs/hooks/, and Zensical publishes every file not in exclude_docs;
- an awesome list's count token left in a page (`<!-- n:proyectos -->`, `n:categorias`,
`n:retirados`): the Markdown extension that fills them in did not run. Only those three
names: CashPilot writes other `<!-- n:... -->` comments of its own in its pages;
- a raw <title>, <textarea>, <xmp>, <plaintext>, <noembed> or <noframes> tag inside a page's
article: a browser's HTML parser treats everything after it as text, so the rest of the
page disappears (Markdown passes `<title>` written in prose through as a real tag;
Telegram-Archive shipped that way);
- a "Copy as Markdown" button (data-md-copy-url) whose Markdown copy does not exist: the
llmstxt plugin writes the copies, and only for pages its `sections` globs match.
- a raw <title>, or an unclosed <textarea>, <xmp>, <noembed>, <noframes>, or any
<plaintext>, in a page's article outside inline SVG and MathML: a browser's HTML parser
treats everything after it as text, so the rest of the page disappears (Markdown passes
`<title>` written in prose through as a real tag; Telegram-Archive shipped that way);
- a "Copy as Markdown" button (data-md-copy-url) whose Markdown copy does not exist in the
site: the llmstxt plugin writes the copies, and only for pages its `sections` globs match.
And, from zensical.toml (Zensical drops what it does not know without a word; under MkDocs
strict an unknown plugin aborted the build):
- a plugin name Zensical 0.1.0 does not know (`redirect` for `redirects`, `llmstxtt`): the
Expand All @@ -40,23 +45,25 @@
- `theme.variant` other than "modern" or "classic" (anything else is classic, silently),
and a "modern" config whose index.html does not load the modern stylesheet;
- a `redirect_maps` entry whose stub was not written;
- a `[project.plugins.llmstxt]` table with no site/llms.txt, or `content.action.copy` in
the features with a page that carries no copy button.
- an enabled llmstxt plugin with no site/llms.txt, or `content.action.copy` in the
features with a page that carries no copy button.
Plugin keys are read the way Zensical reads them (`material/redirects` is `redirects`), and
a plugin with `enabled = false` is treated as off. With directory URLs off (the offline
plugin turns them off) every .html file is a page.
Stdlib only (tomllib needs Python 3.11+), so it runs under `python -I` on any runner.
"""

import re
import sys
import tomllib
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import unquote, urlparse

SWALLOW = re.compile(r"<(title|textarea|xmp|plaintext|noembed|noframes)\b", re.IGNORECASE)
FOREIGN = re.compile(r"<(svg|math)\b.*?</\1\s*>", re.IGNORECASE | re.DOTALL)
TOKEN = re.compile(r"<!--\s*n:(?:proyectos|categorias|retirados)\s*-->")
ARTICLE = re.compile(r"<article\b.*?</article>", re.DOTALL)
ATTR = re.compile(r'\s(?:href|src)="([^"]*)"')
COPY = re.compile(r'\sdata-md-copy-url="([^"]*)"')
NAV = re.compile(r'<a\s+href="([^"]*)"\s+class="md-(?:nav|tabs)__link')
STUB = "<title>Redirecting...</title>"

# Keys of _PLUGIN_UNSUPPORTED_OPTIONS in zensical/config.py (0.1.0, same set as 0.0.74): every other name is
Expand Down Expand Up @@ -85,16 +92,86 @@
}


def body_urls(body: str) -> set[str]:
return set(ATTR.findall(body))


class Page(HTMLParser):
"""The parts of one built HTML file the checks read, from a real HTML parser: any
quoting, entities decoded, nothing taken from <script>/<style> text or escaped code."""

def __init__(self, text: str):
"""Parse one file's text completely."""
super().__init__(convert_charrefs=True)
self.urls: list[tuple[str, str, bool]] = [] # (tag, url, in <head>)
self.ids: set[str] = set()
self.copies: list[str] = []
self.nav: list[str] = []
self.canonical: str | None = None
self.in_head = True
self.feed(text)
self.close()

def handle_starttag(self, tag, attrs):
"""Record ids, URLs, copy targets, nav links and the canonical link of one tag."""
a = {k: v or "" for k, v in attrs}
if tag == "body":
self.in_head = False
for key in ("id", "name") if tag == "a" else ("id",):
if a.get(key):
self.ids.add(a[key])
for key in ("href", "src"):
if key in a:
self.urls.append((tag, a[key], self.in_head))
if "data-md-copy-url" in a:
self.copies.append(a["data-md-copy-url"])
if tag == "link" and "canonical" in a.get("rel", "").split():
self.canonical = a.get("href")
classes = a.get("class", "").split()
# The toc's #heading entries carry md-nav__link too: they point at the page itself,
# so only links with a path count toward reachability.
if tag == "a" and ("md-nav__link" in classes or "md-tabs__link" in classes) and urlparse(a.get("href", "")).path:
self.nav.append(a["href"])

handle_startendtag = handle_starttag


def canonical_plugin(name: str) -> str:
"""The plugin name Zensical resolves a config key to (`material/search` is `search`)."""
name = name.removeprefix("material/")
return name.split("/", 1)[0] if name.startswith(("social/", "privacy/", "optimize/")) else name


def plugin_on(plugins: dict, name: str) -> bool:
"""True when the plugin is configured under any of its key spellings and not `enabled = false`."""
for key, opts in plugins.items():
if canonical_plugin(key) == name:
return not (isinstance(opts, dict) and opts.get("enabled", True) is False)
return False


def plugin_opts(plugins: dict, name: str) -> dict:
"""The plugin's option table, under whichever key spelling the config uses."""
for key, opts in plugins.items():
if canonical_plugin(key) == name and isinstance(opts, dict):
return opts
return {}


def swallowing_tags(article: str) -> list[str]:
"""Raw tags in an article that make a browser treat the rest of the page as text."""
# <title> inside inline SVG is the SVG's accessible name, and MathML has its own
# content model: neither swallows the page, so both are cut out first.
art = FOREIGN.sub("", article)
found = []
for m in SWALLOW.finditer(art):
tag = m.group(1).lower()
closed = re.search(rf"</{tag}\s*>", art[m.end():], re.IGNORECASE)
if tag in ("title", "plaintext") or not closed:
found.append(tag)
return found


def read_config(path: Path, errors: set[str]) -> dict:
"""The [project] table of zensical.toml; an unreadable or empty config is an error."""
if not path.is_file():
errors.add(f"{path}: config file not found (run from the repository root, or pass its path)")
return {}
Expand All @@ -111,28 +188,49 @@ def read_config(path: Path, errors: set[str]) -> dict:


def main(site_dir: str, config_path: str) -> int:
"""Run every check over the built site and config; print one ERROR line each, return 1 if any."""
site = Path(site_dir).resolve()
index = site / "index.html"
if not index.is_file():
print(f"ERROR {site_dir}: no index.html (did the build run?)")
return 1
m = re.search(r'<link rel="canonical" href="([^"]+)"', index.read_text(errors="replace"))
# Directory URLs assumed (every repo's default): with use_directory_urls off or the
# offline plugin on, the prefix and page checks below do not apply as written.
prefix = urlparse(m.group(1)).path if m else "/"
prefix = prefix if prefix.endswith("/") else prefix + "/"
errors: set[str] = set()
nav_targets: set[Path] = set()
pages: list[Path] = []
cfg = read_config(Path(config_path), errors)
plugins = cfg.get("plugins") or {}
# The offline plugin turns directory URLs off (zensical/config.py, _apply_offline_plugin).
directory_urls = cfg.get("use_directory_urls", True) is not False and not plugin_on(plugins, "offline")

site_url = cfg.get("site_url") or Page(index.read_text(encoding="utf-8", errors="replace")).canonical or ""
origin = urlparse(site_url)
prefix = origin.path or ""
if not site_url:
errors.add("no site_url in zensical.toml and no canonical link in index.html: the "
"root-absolute link check has no prefix to check against")
prefix = "/"
prefix = prefix if prefix.endswith("/") else prefix + "/"

parsed: dict[Path, Page] = {}

def parse(path: Path) -> Page:
"""Each file is parsed once: fragment checks read target pages again."""
if path not in parsed:
parsed[path] = Page(path.read_text(encoding="utf-8", errors="replace"))
return parsed[path]

def resolve(page: Path, url: str):
"""None for a URL outside this site, "ABS" for a rooted path outside the prefix,
else the file the URL lands on (not necessarily existing)."""
u = urlparse(url)
if u.scheme or u.netloc or url == "" or url.startswith("#"):
if u.scheme in ("http", "https") and u.netloc and u.netloc == origin.netloc:
if not u.path.startswith(prefix) and u.path + "/" != prefix:
return None # another site on the same host (geiserx.github.io/<other repo>)
elif u.scheme or u.netloc:
return None
if url == "" or url.startswith("#"):
return page
path = unquote(u.path)
if path.startswith("/"):
if not path.startswith(prefix):
if not path.startswith(prefix) and path + "/" != prefix:
return "ABS"
target = site / path[len(prefix):]
else:
Expand All @@ -142,44 +240,53 @@ def resolve(page: Path, url: str):
return target.resolve()

for p in site.rglob("*"):
if p.suffix in (".py", ".pyc") or "__pycache__" in p.parts:
if p.suffix in (".py", ".pyc") or "__pycache__" in p.relative_to(site).parts:
errors.add(f"{p.relative_to(site)}: source file published (add its directory to exclude_docs)")

nav_targets: set[Path] = set()
pages: list[Path] = []
copy_on = "content.action.copy" in ((cfg.get("theme") or {}).get("features") or [])
for page in sorted(site.rglob("*.html")):
text = page.read_text(errors="replace")
text = page.read_text(encoding="utf-8", errors="replace")
rel = page.relative_to(site)
stub = STUB in text # a redirect stub: its links are checked, it is not a page
if rel.name == "index.html" and not stub:
is_page = not stub and (rel.name == "index.html" if directory_urls else rel.as_posix() != "404.html")
if is_page:
pages.append(page)
for art in ARTICLE.findall(text):
for tag in SWALLOW.findall(art):
for tag in swallowing_tags(art):
errors.add(f"{rel}: raw <{tag}> in the page body hides everything after it in a browser")
for tok in TOKEN.findall(art):
errors.add(f"{rel}: build token {tok} was not filled in (extension not loaded?)")
# <head> holds <link rel="alternate" type="text/markdown" href="./index.md"> from the
# llmstxt plugin, a legitimate .md href; the .md rule looks at the body only.
body = text.split("</head>", 1)[-1]
for url in ATTR.findall(text):
doc = parse(page)
for tag, url, in_head in doc.urls:
t = resolve(page, url)
if t is None:
continue
if t == "ABS":
errors.add(f"{rel}: absolute link {url} (outside {prefix})")
elif t is not None and not t.is_relative_to(site):
elif not t.is_relative_to(site):
# ../LICENSE from the top page: the file exists in the checkout, not on Pages.
errors.add(f"{rel}: {url} leaves the built site (resolves to {t})")
elif t is not None and not t.exists():
shown = t.relative_to(site) if t.is_relative_to(site) else t
errors.add(f"{rel}: {url} -> missing {shown}")
elif t is not None and unquote(urlparse(url).path).endswith(".md") and url in body_urls(body):
errors.add(f"{rel}: {url} links a Markdown source, not a page URL (raw HTML link?)")
copies = COPY.findall(text)
for url in copies:
elif not t.exists():
errors.add(f"{rel}: {url} -> missing {t.relative_to(site)}")
else:
# <head> holds <link rel="alternate" type="text/markdown" href="./index.md">
# from the llmstxt plugin, a legitimate .md href; this rule reads the body only.
if not in_head and unquote(urlparse(url).path).endswith(".md"):
errors.add(f"{rel}: {url} links a Markdown source, not a page URL (raw HTML link?)")
frag = unquote(urlparse(url).fragment)
# #top and text fragments (#:~:text=) are valid in a browser with no matching id.
special = frag.lower() == "top" or frag.startswith(":~:")
if tag == "a" and frag and not special and t.suffix == ".html" and frag not in parse(t).ids:
errors.add(f"{rel}: {url} -> no id {frag!r} on {t.relative_to(site)}")
for url in doc.copies:
t = resolve(page, url)
if isinstance(t, Path) and not t.is_file():
errors.add(f"{rel}: Copy as Markdown points at {url}, which was not written (llmstxt sections?)")
if rel.name == "index.html" and not stub and not copies \
and "content.action.copy" in ((cfg.get("theme") or {}).get("features") or []):
if t is None or t == "ABS" or not t.is_relative_to(site) or not t.is_file():
errors.add(f"{rel}: Copy as Markdown points at {url}, which is not in the site (llmstxt sections?)")
if is_page and copy_on and not doc.copies:
errors.add(f"{rel}: content.action.copy is on but the page has no Copy as Markdown button")
for url in NAV.findall(text):
for url in doc.nav:
t = resolve(page, url)
if isinstance(t, Path):
nav_targets.add(t)
Expand All @@ -189,7 +296,6 @@ def resolve(page: Path, url: str):
errors.add(f"{page.relative_to(site)}: page is in no navigation entry (stray page)")

if cfg:
plugins = cfg.get("plugins") or {}
for name in plugins:
if canonical_plugin(name) not in KNOWN_PLUGINS:
errors.add(f"zensical.toml: plugin {name!r} is unknown to Zensical and silently ignored")
Expand All @@ -200,14 +306,10 @@ def resolve(page: Path, url: str):
variant = theme.get("variant", "classic")
if variant not in ("modern", "classic"):
errors.add(f"zensical.toml: theme.variant {variant!r} is not modern or classic (Zensical falls back to classic silently)")
elif variant == "modern" and "assets/stylesheets/modern/" not in index.read_text(errors="replace"):
elif variant == "modern" and "assets/stylesheets/modern/" not in index.read_text(encoding="utf-8", errors="replace"):
errors.add("index.html does not load the modern stylesheet although theme.variant is modern")
# The offline plugin turns directory URLs off (zensical/config.py,
# _apply_offline_plugin), so a redirect stub becomes old.html there.
offline = plugins.get("offline", False)
offline_on = "offline" in plugins and (offline or {}).get("enabled", True) is not False
directory_urls = cfg.get("use_directory_urls", True) and not offline_on
for src in (plugins.get("redirects") or {}).get("redirect_maps") or {}:
redirects = plugin_opts(plugins, "redirects") if plugin_on(plugins, "redirects") else {}
for src in redirects.get("redirect_maps") or {}:
s = src.removesuffix(".md")
# A README page is its folder's index: sub/README.md -> sub/index.html in every mode.
if s == "README" or s.endswith("/README"):
Expand All @@ -216,9 +318,9 @@ def resolve(page: Path, url: str):
stub = site / (s + ".html")
else:
stub = site / s / "index.html"
if not stub.is_file() or STUB not in stub.read_text(errors="replace"):
if not stub.is_file() or STUB not in stub.read_text(encoding="utf-8", errors="replace"):
errors.add(f"redirect_maps {src!r}: no redirect stub at {stub.relative_to(site)}")
if "llmstxt" in plugins and not (site / "llms.txt").is_file():
if plugin_on(plugins, "llmstxt") and not (site / "llms.txt").is_file():
errors.add("llmstxt plugin is configured but site/llms.txt was not written")

for e in sorted(errors):
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ on:
workflow_dispatch:

# One run at a time per ref, never cancelled. GitHub keeps one running and one
# pending run per group and a newer push replaces the pending one, so the last
# deploy is the newest push; it does not promise strict push order beyond that.
# pending run per group, and a run that joins later replaces the pending one.
# Runs join in the order GitHub queues them, which is not promised to be push
# order, so a newer push is likely, not certain, to deploy last.
# Locking only the deploy job is not enough: an older push whose build finishes
# later could deploy last and publish a stale site. Pull requests get their own group.
concurrency:
Expand Down
Loading