diff --git a/.github/workflows/build-kotlin-docs-local.yaml b/.github/workflows/build-kotlin-docs-local.yaml new file mode 100644 index 00000000..f58113b1 --- /dev/null +++ b/.github/workflows/build-kotlin-docs-local.yaml @@ -0,0 +1,422 @@ +name: Build Kotlin Docs (Local) + +# Local-filesystem counterpart of build-kotlin-docs.yaml: same five steps +# (find_missing_assets -> populate_db -> insert_optimized_media -> +# build-stdlib-json-docs -> sync_kdoc_json_to_db), same ADFA-4737 blacklist, +# but reads its documentation.db/webHelpImages.zip inputs from paths on the +# runner's own disk (db_path / images_zip_path) instead of Google Drive, and +# writes its outputs (the updated database, the missing-assets report) back +# to disk (output_dir / db_path) instead of uploading them to Drive. No GCP +# Workload Identity Federation, Drive API, or associated secrets are used +# anywhere in this file. +# +# Since a GitHub-hosted runner is a fresh, disposable VM with no access to +# anyone's actual local disk, db_path/images_zip_path/output_dir only make +# sense here against a self-hosted runner, or when this workflow is run +# locally (e.g. via https://github.com/nektos/act) with those host paths +# bind-mounted into the job's container at the paths you pass as inputs. +# run-build-kotlin-docs-with-act.sh at the repo root drives exactly that and +# is the supported way to run this file locally. +# +# CAUTION when invoking act by hand rather than through that script: act does +# NOT apply workflow_dispatch input defaults. Every "default:" below applies +# on real GitHub and is simply absent under act, so an input you don't pass +# arrives empty. That matters most for dry_run, whose default is true: with +# it unset, "${{ !inputs.dry_run }}" evaluates to true and the final step +# writes the rebuilt database back over db_path. Always pass +# --input dry_run=true/false explicitly (the script always does). +# +# KNOWN LIMITATION: populate_db.py requires Writerside's own image export +# ("webHelpImages.zip"), which JetBrains only produces via IntelliJ IDEA's +# Writerside plugin build/export action - there is no headless/CLI way to +# generate it (see ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md, +# "Inputs you need before starting"). So this workflow expects that export +# to already exist on disk at images_zip_path rather than generating it +# itself. Use skip_website_docs to bypass this entirely and only refresh the +# kotlin-stdlib/-reflect/-test JSON content. +# +# Optional secret (Slack notifications are skipped with a warning if unset) - +# same as build-kotlin-docs.yaml: +# SLACK_WEBHOOK_URL - Incoming Webhook URL for the "Notify Slack" steps +# below ("Grabbing baton" on start, "...Dropping +# baton" on finish - org shorthand for lock +# acquire/release, since this workflow mutates a +# single shared local file, db_path). + +permissions: + contents: read + +# This workflow overwrites a single shared local file (db_path) - never let +# two runs race to write it at the same time. +concurrency: + group: build-kotlin-docs-local + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + kotlin_web_site_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin-web-site to check out for the + "docs" tree (topics/, images/, kr.tree, v.list). Leave empty to use + the repo's default branch. + required: false + default: '' + kotlin_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin to check out for the + kotlin-stdlib-docs build. Leave empty to use the repo's default + branch. Pin this to a real release tag for a reproducible build. + required: false + default: '' + db_path: + description: >- + Path on this runner's disk to the input documentation.db. Read + directly (no download/unzip) and, unless dry_run is true, written + back to this same path when the run finishes. + required: true + images_zip_path: + description: >- + Path on this runner's disk to Writerside's webHelpImages.zip + export matching kotlin_web_site_ref (see KNOWN LIMITATION above). + Required unless skip_website_docs is true. + required: false + default: '' + output_dir: + description: >- + Directory on this runner's disk to write outputs into: the + missing-assets QA report and a run-numbered copy of the built + database (documentation-db-.db). Created if it + doesn't already exist. + required: false + default: 'build-kotlin-docs-output' + kotlin_libs_version: + description: >- + Version of the published kotlin-stdlib/-reflect/-test artifacts to + document (Gradle -PdeployVersion). kotlin_big extracts the real + binaries at this version rather than requiring a local build of the + whole kotlin repo, which is what the checkout's own + defaultSnapshotVersion would otherwise demand. Keep this in step with + kotlin_ref. Set empty to fall back to that snapshot default, which + only resolves if you have built the kotlin repo yourself. + required: false + default: '2.4.10' + kotlin_libs_repo: + description: >- + Maven repository to resolve those artifacts from (Gradle + -PkotlinLibsRepo). kotlin_big already declares mavenCentral(), so a + released kotlin_libs_version needs nothing here; set it to point at a + private or snapshot repository instead. + required: false + default: '' + skip_website_docs: + description: 'Skip the kotlin-web-site steps and only refresh kotlin-stdlib/-reflect/-test JSON content.' + required: false + default: false + type: boolean + skip_stdlib_docs: + description: >- + Skip the kotlin-stdlib/-reflect/-test steps (cloning JetBrains/kotlin, + the Dokka JSON build, and the sync into the database) and only refresh + the kotlin-web-site content. The mirror image of skip_website_docs - + setting both leaves nothing for the run to do and is rejected. + required: false + default: false + type: boolean + dry_run: + description: >- + If true, build and verify everything but do NOT write the result + back to db_path - the input file on disk is left untouched. Set to + false only once you trust a given ref/path combination (see this + workflow's testing notes). + required: false + default: true + type: boolean + +jobs: + build-kotlin-docs: + runs-on: ubuntu-latest + timeout-minutes: 180 + env: + KOTLIN_WEB_SITE_REF: ${{ inputs.kotlin_web_site_ref }} + KOTLIN_REF: ${{ inputs.kotlin_ref }} + DB_PATH: ${{ inputs.db_path }} + IMAGES_ZIP_PATH: ${{ inputs.images_zip_path }} + OUTPUT_DIR: ${{ inputs.output_dir }} + SKIP_WEBSITE_DOCS: ${{ inputs.skip_website_docs }} + KOTLIN_LIBS_VERSION: ${{ inputs.kotlin_libs_version }} + KOTLIN_LIBS_REPO: ${{ inputs.kotlin_libs_repo }} + SKIP_STDLIB_DOCS: ${{ inputs.skip_stdlib_docs }} + # The ADFA-4737 blacklist, defined once and consumed by both the + # populate_db.py step and the verification step that checks its effect. + # Previously spelled out separately in each, which let the verification + # drift onto a different list than the one actually applied and still + # report PASS. run_e2e_pipeline_test.sh already had it right (a single + # BLACKLIST array expanded at both call sites); this matches that. + # One entry per line, read back with `mapfile -t`. Re-derive these from + # kotlin-web-site/docs/kr.tree if its nav structure has changed. + # "|-" (not "|") so there's no trailing blank line to become a 4th, + # empty array element. + BLACKLISTED_ELEMENT_TITLES: |- + Development\/Web development + Interoperability\/Swift/Objective-C and C interop + Interoperability\/JavaScript interop + steps: + - name: Checkout OfflineDocumentationTools + uses: actions/checkout@v4 + + - name: Resolve local file paths + run: | + if [ "$SKIP_WEBSITE_DOCS" = "true" ] && [ "$SKIP_STDLIB_DOCS" = "true" ]; then + echo "Error: skip_website_docs and skip_stdlib_docs are both true - that skips every step that changes the database, leaving nothing for this run to do" >&2 + exit 1 + fi + if [ ! -f "$DB_PATH" ]; then + echo "Error: db_path '$DB_PATH' does not exist on this runner - for a self-hosted runner this must be a path on that machine; for act, bind-mount it into the container so it's visible at this exact path" >&2 + exit 1 + fi + if [ "$SKIP_WEBSITE_DOCS" != "true" ]; then + if [ -z "$IMAGES_ZIP_PATH" ]; then + echo "Error: images_zip_path is required unless skip_website_docs is true" >&2 + exit 1 + fi + if [ ! -f "$IMAGES_ZIP_PATH" ]; then + echo "Error: images_zip_path '$IMAGES_ZIP_PATH' does not exist on this runner" >&2 + exit 1 + fi + fi + mkdir -p "$OUTPUT_DIR" + echo "Resolved DB_PATH: $DB_PATH" + echo "Resolved IMAGES_ZIP_PATH: ${IMAGES_ZIP_PATH:-(skipped)}" + echo "Resolved OUTPUT_DIR: $OUTPUT_DIR" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up JDK (for the kdoc-to-json / kotlin-stdlib-docs Gradle builds) + uses: actions/setup-java@v4 + with: + distribution: temurin + # kdoc-to-json's own Gradle wrapper is pinned to Gradle 9.1.0, which + # needs JDK 17+. Bump this if the kotlin checkout's own wrapper + # (invoked by build-stdlib-json-docs.sh against kotlin-stdlib-docs) + # turns out to need something newer - verify on first real run. + java-version: '17' + + - name: Install system dependencies + run: | + sudo apt-get update -y + # brotli: the CLI, not the Python package. populate_db.py's + # DictionaryCompressor and sync_kdoc_json_to_db.py shell out to it because + # no Python binding exposes a custom dictionary (ADFA-5153). + sudo apt-get install -y pngquant unzip sqlite3 brotli + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + # markdown-it-py: ProcessKotlinWebsiteJSON's own requirement (see + # its README); scour/cairosvg are in requirements.txt already. + pip install markdown-it-py + + - name: Copy documentation.db from local disk + run: | + cp "$DB_PATH" documentation.db + sqlite3 documentation.db "SELECT 1;" > /dev/null + echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV" + + - name: 'Notify Slack: build started' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Grabbing baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi + + - name: Clone kotlin-web-site + if: ${{ !inputs.skip_website_docs }} + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_WEB_SITE_REF" ] && ARGS+=(--branch "$KOTLIN_WEB_SITE_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin-web-site.git kotlin-web-site + + - name: Copy Writerside image export from local disk + if: ${{ !inputs.skip_website_docs }} + run: cp "$IMAGES_ZIP_PATH" webHelpImages.zip + + - name: 'Step 1/5: find_missing_assets.py (source QA report)' + if: ${{ !inputs.skip_website_docs }} + run: | + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py \ + kotlin-web-site/docs missing-assets-report.md + + - name: Write missing-assets report to output_dir + if: ${{ !inputs.skip_website_docs }} + run: cp missing-assets-report.md "$OUTPUT_DIR/missing-assets-report.md" + + - name: 'Step 2/5: populate_db.py (convert docs, prune blacklist, insert into db)' + if: ${{ !inputs.skip_website_docs }} + run: | + # BLACKLISTED_ELEMENT_TITLES is defined once in this job's env: block + # (ADFA-4737); the verification step below reads the same variable. + mapfile -t BLACKLIST <<< "$BLACKLISTED_ELEMENT_TITLES" + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \ + kotlin-web-site/docs \ + ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json \ + webHelpImages.zip \ + documentation.db \ + --blacklisted-element-titles "${BLACKLIST[@]}" + + - name: 'Step 3/5: insert_optimized_media.py (re-optimize + reinsert images)' + if: ${{ !inputs.skip_website_docs }} + run: | + # --webp requires an "image/webp" ContentTypes row. The current + # production database already has one, so this is normally a no-op; + # it stays for older copies that predate it (idempotent either way). + sqlite3 documentation.db \ + "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" + mkdir -p media + unzip -q webHelpImages.zip -d media + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py \ + media documentation.db \ + --jpeg-quality 85 --webp --webp-quality 90 --verbose + + - name: Clone kotlin (for kotlin-stdlib-docs) + if: ${{ !inputs.skip_stdlib_docs }} + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_REF" ] && ARGS+=(--branch "$KOTLIN_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin.git kotlin-repo + + - name: 'Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON)' + if: ${{ !inputs.skip_stdlib_docs }} + id: stdlib_docs + run: | + ARGS=() + [ -n "$KOTLIN_LIBS_VERSION" ] && ARGS+=(--kotlin-libs-version "$KOTLIN_LIBS_VERSION") + [ -n "$KOTLIN_LIBS_REPO" ] && ARGS+=(--kotlin-libs-repo "$KOTLIN_LIBS_REPO") + OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh \ + "${ARGS[@]}" kotlin-repo stdlib-json-build)" + echo "Generated JSON docs at $OUTPUT" + echo "all_libs_dir=$OUTPUT" >> "$GITHUB_OUTPUT" + + - name: 'Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content)' + if: ${{ !inputs.skip_stdlib_docs }} + run: | + python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \ + "${{ steps.stdlib_docs.outputs.all_libs_dir }}" --db documentation.db + + - name: Summary + run: | + python3 - documentation.db <<'PYEOF' + import sqlite3 + import sys + + conn = sqlite3.connect(sys.argv[1]) + + def count(where, params=()): + return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] + + print(f"Database: {sys.argv[1]}") + print(f" k/html/* rows: {count('path LIKE ?', ('k/html/%',))}") + print(f" k/html/images/* rows: {count('path LIKE ?', ('k/html/images/%',))}") + print(f" k/html/images/*.webp rows: {count('path LIKE ?', ('k/html/images/%.webp%',))}") + print(f" k/kotlin-stdlib/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-stdlib/%', 'k/kotlin-stdlib'))}") + print(f" k/kotlin-reflect/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-reflect/%', 'k/kotlin-reflect'))}") + print(f" k/kotlin-test/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-test/%', 'k/kotlin-test'))}") + conn.close() + PYEOF + + - name: Blacklist pruning verification + if: ${{ !inputs.skip_website_docs }} + run: | + # Same BLACKLISTED_ELEMENT_TITLES the populate_db.py step above + # applied - read from the job env rather than restated here, so this + # check can't silently verify a different list than the one used. + mapfile -t BLACKLIST <<< "$BLACKLISTED_ELEMENT_TITLES" + python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \ + "${BLACKLIST[@]}" <<'PYEOF' + import sqlite3 + import sys + import xml.etree.ElementTree as ET + from pathlib import Path + + process_dir, docs_root, db_path, *blacklist_raw = sys.argv[1:] + sys.path.insert(0, process_dir) + import populate_db # noqa: E402 + + root = ET.parse(Path(docs_root) / "kr.tree").getroot() + blacklisted_paths = {populate_db.parse_blacklist_path(raw) for raw in blacklist_raw} + blacklisted_stems, unmatched_paths = populate_db.prune_blacklisted_elements(root, blacklisted_paths) + + conn = sqlite3.connect(db_path) + leftover = [] + for stem in sorted(blacklisted_stems): + path = f"k/html/{stem}.html" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (path,)).fetchone(): + leftover.append(path) + conn.close() + + print(f"Blacklisted toc-element path(s) checked: {len(blacklisted_paths)}") + for path in sorted(blacklisted_paths): + status = "unmatched (no such element in kr.tree)" if path in unmatched_paths else "matched" + print(f" {' > '.join(path)}: {status}") + print(f"Topic page(s) expected removed: {len(blacklisted_stems)}") + + if unmatched_paths: + print(f"FAIL: {len(unmatched_paths)} blacklist path(s) never matched a .") + sys.exit(1) + if leftover: + print(f"FAIL: {len(leftover)} blacklisted page(s) still present in the database:") + for path in leftover: + print(f" {path}") + sys.exit(1) + + print(f"PASS: all {len(blacklisted_stems)} blacklisted topic page(s) confirmed absent from {db_path}.") + PYEOF + + - name: Write built database to output_dir + run: | + cp documentation.db "$OUTPUT_DIR/documentation-db-${{ github.run_number }}.db" + echo "Wrote $OUTPUT_DIR/documentation-db-${{ github.run_number }}.db" + + - name: Write updated database back to db_path + if: ${{ !inputs.dry_run }} + run: | + cp documentation.db "$DB_PATH" + echo "Wrote updated documentation.db back to $DB_PATH" + + # if: always() - the baton must be dropped even when the build fails, + # otherwise the channel shows it held forever by a dead run, which is the + # exact failure this convention exists to prevent. Deliberately NOT gated + # on dry_run: "build started" above is ungated, and since dry_run defaults + # to true an asymmetric gate meant every ordinary run grabbed the baton + # and never dropped it. The message reports the outcome instead. + - name: 'Notify Slack: build complete' + if: always() + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + JOB_STATUS: ${{ job.status }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + # The baton is always dropped, so the text has to say what actually + # happened rather than always claiming an update. + if [ "$JOB_STATUS" != "success" ]; then + TEXT="Kotlin documentation build FAILED ($JOB_STATUS) - database unchanged. Dropping baton" + elif [ "$DRY_RUN" = "true" ]; then + TEXT="Kotlin documentation dry run complete - database unchanged. Dropping baton" + else + TEXT="Updated Kotlin documentation. Dropping baton" + fi + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data "$(printf '{"text": "%s"}' "$TEXT")" \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi diff --git a/.github/workflows/build-kotlin-docs.yaml b/.github/workflows/build-kotlin-docs.yaml index c6341556..ade77985 100644 --- a/.github/workflows/build-kotlin-docs.yaml +++ b/.github/workflows/build-kotlin-docs.yaml @@ -76,11 +76,39 @@ on: Ignored if skip_website_docs is true. required: false default: '' + kotlin_libs_version: + description: >- + Version of the published kotlin-stdlib/-reflect/-test artifacts to + document (Gradle -PdeployVersion). kotlin_big extracts the real + binaries at this version rather than requiring a local build of the + whole kotlin repo, which is what the checkout's own + defaultSnapshotVersion would otherwise demand. Keep this in step with + kotlin_ref. Set empty to fall back to that snapshot default, which + only resolves if you have built the kotlin repo yourself. + required: false + default: '2.4.10' + kotlin_libs_repo: + description: >- + Maven repository to resolve those artifacts from (Gradle + -PkotlinLibsRepo). kotlin_big already declares mavenCentral(), so a + released kotlin_libs_version needs nothing here; set it to point at a + private or snapshot repository instead. + required: false + default: '' skip_website_docs: description: 'Skip the kotlin-web-site steps and only refresh kotlin-stdlib/-reflect/-test JSON content.' required: false default: false type: boolean + skip_stdlib_docs: + description: >- + Skip the kotlin-stdlib/-reflect/-test steps (cloning JetBrains/kotlin, + the Dokka JSON build, and the sync into the database) and only refresh + the kotlin-web-site content. The mirror image of skip_website_docs - + setting both leaves nothing for the run to do and is rejected. + required: false + default: false + type: boolean dry_run: description: >- If true, build and verify everything but do NOT upload the result @@ -108,12 +136,34 @@ jobs: # Drive). Leave both empty ('') for normal operation. TEST_DB_FILE_ID: '' TEST_IMAGES_ZIP_FILE_ID: '' + SKIP_WEBSITE_DOCS: ${{ inputs.skip_website_docs }} + KOTLIN_LIBS_VERSION: ${{ inputs.kotlin_libs_version }} + KOTLIN_LIBS_REPO: ${{ inputs.kotlin_libs_repo }} + SKIP_STDLIB_DOCS: ${{ inputs.skip_stdlib_docs }} + # The ADFA-4737 blacklist, defined once and consumed by both the + # populate_db.py step and the verification step that checks its effect. + # Previously spelled out separately in each, which let the verification + # drift onto a different list than the one actually applied and still + # report PASS. run_e2e_pipeline_test.sh already had it right (a single + # BLACKLIST array expanded at both call sites); this matches that. + # One entry per line, read back with `mapfile -t`. Re-derive these from + # kotlin-web-site/docs/kr.tree if its nav structure has changed. + # "|-" (not "|") so there's no trailing blank line to become a 4th, + # empty array element. + BLACKLISTED_ELEMENT_TITLES: |- + Development\/Web development + Interoperability\/Swift/Objective-C and C interop + Interoperability\/JavaScript interop steps: - name: Checkout OfflineDocumentationTools uses: actions/checkout@v4 - name: Resolve Google Drive file IDs run: | + if [ "$SKIP_WEBSITE_DOCS" = "true" ] && [ "$SKIP_STDLIB_DOCS" = "true" ]; then + echo "Error: skip_website_docs and skip_stdlib_docs are both true - that skips every step that changes the database, leaving nothing for this run to do" >&2 + exit 1 + fi DB_FILE_ID="${TEST_DB_FILE_ID:-$DB_FILE_ID_SECRET}" IMG_FILE_ID="${TEST_IMAGES_ZIP_FILE_ID:-${IMAGES_ZIP_FILE_ID_INPUT:-$IMAGES_ZIP_FILE_ID_SECRET}}" if [ -z "$DB_FILE_ID" ]; then @@ -151,11 +201,11 @@ jobs: - name: Install Python dependencies run: | pip install -r requirements.txt - # markdown-it-py/scour/cairosvg: ProcessKotlinWebsiteJSON's own - # requirements (see its README), not in the root requirements.txt. + # markdown-it-py: ProcessKotlinWebsiteJSON's own requirement (see + # its README); scour/cairosvg are in requirements.txt already. # google-api-python-client & friends: Drive download/upload, same # libraries check-tools/download_database.py already depends on. - pip install markdown-it-py scour cairosvg \ + pip install markdown-it-py \ google-api-python-client google-auth-httplib2 google-auth-oauthlib - name: Authenticate to Google Cloud using Workload Identity Federation @@ -226,24 +276,22 @@ jobs: - name: 'Step 2/5: populate_db.py (convert docs, prune blacklist, insert into db)' if: ${{ !inputs.skip_website_docs }} run: | - # Same three blacklist entries as run_e2e_pipeline_test.sh - # (ADFA-4737) - re-derive these from kotlin-web-site/docs/kr.tree - # if its nav structure has changed since this was written. + # BLACKLISTED_ELEMENT_TITLES is defined once in this job's env: block + # (ADFA-4737); the verification step below reads the same variable. + mapfile -t BLACKLIST <<< "$BLACKLISTED_ELEMENT_TITLES" python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \ kotlin-web-site/docs \ ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json \ webHelpImages.zip \ documentation.db \ - --blacklisted-element-titles \ - 'Development\/Web development' \ - 'Interoperability\/Swift/Objective-C and C interop' \ - 'Interoperability\/JavaScript interop' + --blacklisted-element-titles "${BLACKLIST[@]}" - name: 'Step 3/5: insert_optimized_media.py (re-optimize + reinsert images)' if: ${{ !inputs.skip_website_docs }} run: | - # --webp requires an "image/webp" ContentTypes row, which this - # database doesn't ship with by default (idempotent). + # --webp requires an "image/webp" ContentTypes row. The current + # production database already has one, so this is normally a no-op; + # it stays for older copies that predate it (idempotent either way). sqlite3 documentation.db \ "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" mkdir -p media @@ -253,19 +301,26 @@ jobs: --jpeg-quality 85 --webp --webp-quality 90 --verbose - name: Clone kotlin (for kotlin-stdlib-docs) + if: ${{ !inputs.skip_stdlib_docs }} run: | ARGS=(--depth 1) [ -n "$KOTLIN_REF" ] && ARGS+=(--branch "$KOTLIN_REF") git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin.git kotlin-repo - name: 'Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON)' + if: ${{ !inputs.skip_stdlib_docs }} id: stdlib_docs run: | - OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh kotlin-repo stdlib-json-build)" + ARGS=() + [ -n "$KOTLIN_LIBS_VERSION" ] && ARGS+=(--kotlin-libs-version "$KOTLIN_LIBS_VERSION") + [ -n "$KOTLIN_LIBS_REPO" ] && ARGS+=(--kotlin-libs-repo "$KOTLIN_LIBS_REPO") + OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh \ + "${ARGS[@]}" kotlin-repo stdlib-json-build)" echo "Generated JSON docs at $OUTPUT" echo "all_libs_dir=$OUTPUT" >> "$GITHUB_OUTPUT" - name: 'Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content)' + if: ${{ !inputs.skip_stdlib_docs }} run: | python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \ "${{ steps.stdlib_docs.outputs.all_libs_dir }}" --db documentation.db @@ -294,10 +349,12 @@ jobs: - name: Blacklist pruning verification if: ${{ !inputs.skip_website_docs }} run: | + # Same BLACKLISTED_ELEMENT_TITLES the populate_db.py step above + # applied - read from the job env rather than restated here, so this + # check can't silently verify a different list than the one used. + mapfile -t BLACKLIST <<< "$BLACKLISTED_ELEMENT_TITLES" python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \ - 'Development\/Web development' \ - 'Interoperability\/Swift/Objective-C and C interop' \ - 'Interoperability\/JavaScript interop' <<'PYEOF' + "${BLACKLIST[@]}" <<'PYEOF' import sqlite3 import sys import xml.etree.ElementTree as ET @@ -367,15 +424,32 @@ jobs: print(f"Uploaded new revision of {file_id}: {updated}") PYEOF + # if: always() - the baton must be dropped even when the build fails, + # otherwise the channel shows it held forever by a dead run, which is the + # exact failure this convention exists to prevent. Deliberately NOT gated + # on dry_run: "build started" above is ungated, and since dry_run defaults + # to true an asymmetric gate meant every ordinary run grabbed the baton + # and never dropped it. The message reports the outcome instead. - name: 'Notify Slack: build complete' - if: ${{ !inputs.dry_run }} + if: always() env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + JOB_STATUS: ${{ job.status }} + DRY_RUN: ${{ inputs.dry_run }} run: | + # The baton is always dropped, so the text has to say what actually + # happened rather than always claiming an update. + if [ "$JOB_STATUS" != "success" ]; then + TEXT="Kotlin documentation build FAILED ($JOB_STATUS) - database unchanged. Dropping baton" + elif [ "$DRY_RUN" = "true" ]; then + TEXT="Kotlin documentation dry run complete - database unchanged. Dropping baton" + else + TEXT="Updated Kotlin documentation. Dropping baton" + fi if [ -z "$SLACK_WEBHOOK_URL" ]; then echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 else curl -sS -X POST -H 'Content-type: application/json' \ - --data '{"text": "Updated Kotlin documentation. Dropping baton"}' \ + --data "$(printf '{"text": "%s"}' "$TEXT")" \ "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 fi diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml new file mode 100644 index 00000000..326690d8 --- /dev/null +++ b/.github/workflows/python-tests.yaml @@ -0,0 +1,70 @@ +name: Python tests + +# Runs the repo's pytest suites on every push and PR. +# +# Added because nothing here executed them: the Kotlin-docs pipeline ships +# ~1,300 lines of regression tests covering permanent, silent data-loss paths +# (a migration deleting an unrelated page, an image optimizer collapsing two +# sources onto one output, a chunk chain reassembling truncated), and until now +# they only ran when someone remembered to run them locally. Tests that never +# run in CI rot, and these are exactly the ones whose failure is invisible +# without them. +# +# The suites are separate because their dependencies are: docdb-studio and +# check-tools each own a pyproject.toml + uv.lock, while ProcessKotlinWebsiteJSON +# runs against the root requirements.txt. + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: python-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + pytest: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install system dependencies + run: | + sudo apt-get update -y + # pngquant: optimize_media.py shells out to it, and its tests exercise + # the real binary rather than mocking it. + # brotli: the CLI, not the Python package - DictionaryCompressor uses + # it because no Python binding exposes a custom dictionary. + # zstd: train_dictionary uses `zstd --train-fastcover` to build the + # shared dictionary the migration tests need. + sudo apt-get install -y pngquant brotli zstd + + - name: Install Python dependencies + run: | + pip install -r requirements.txt pytest + + - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs' + run: | + python -m pytest \ + ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON \ + scripts/sync_kotlin_stdlib_docs \ + -q + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: docdb-studio + working-directory: docdb-studio + run: uv run --frozen -- python -m pytest -q diff --git a/.gitignore b/.gitignore index 59cc3acb..794d4897 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,11 @@ __pycache__/ *$py.class *.db *.sqlite +run_e2e_pipeline_test.local.sh +grep_content_blobs.local.py + +# Timestamped safety backups written by populate_db.py / +# insert_optimized_media.py / sync_kdoc_json_to_db.py before they modify a +# database ("*.db" above does not match these - the timestamp comes last). +*.db.backup-* +*.db.bak.* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..73e32deb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,224 @@ +# CLAUDE.md + +Guidance for Claude (and anyone else) working in this repository. + +## What this repository is + +App Dev For All builds **Code on the Go**, an Android IDE aimed at users with no or limited +internet access +(code: [appdevforall/CodeOnTheGo](https://github.com/appdevforall/CodeOnTheGo)). To support that, +Java/Kotlin/Android API documentation is bundled into the app as a single SQLite file — the +**documentation database** — rather than fetched from the web. + +The documentation database serves two distinct features in the IDE: + +1. **Tooltips (Tier 1/2).** When a user selects a keyword/symbol in the code editor, a dialog + shows short (Tier 1) and detailed (Tier 2) tooltip text if the selection matches an entry in + the DB. This lookup happens elsewhere in the CodeOnTheGo Android code (not in this repo, and + not in `WebServer.kt` — see below). +2. **Content pages (Tier 3).** From a tooltip, the user can click through to a full documentation + page. Those pages (and other static content — HTML, images, PDFs) are served over HTTP by + **`WebServer.kt`** + ([CodeOnTheGo/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt](https://github.com/appdevforall/CodeOnTheGo/blob/stage/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt)), + which runs inside the app and reads directly from the `Content` (and, as of recently, + `Templates`/`Bookshelf`/`BookCategories`) tables of the same database. + +**This repository (`OfflineDocumentationTools`) is the collection of offline tools that build and +edit that database** — it contains no part of the production Android app itself. + +> **Alex's standing caveat, worth repeating at the top of every session:** nothing in this +> repository is guaranteed to work against the *current* production database. The schema has moved +> forward (in the app / by hand) faster than the tooling in this repo has been updated. See +> "Schema: current vs. what this repo expects" below — that gap is the most important thing to +> understand before making changes here. + +## Schema: current vs. what this repo expects + +> ### Schema 2.0.0: every Brotli row uses a shared dictionary +> +> As of 2026-08-20 `~/documentation.db` reports **2.0.0** in `DocumentationDatabaseVersion` +> ("Add CompressionDictionary table", David) — a deliberately *incompatible* major bump. Every +> `brotli` Content row is compressed against the shared 256 KiB raw LZ77 dictionary held in +> `CompressionDictionary` (id 1). Measured on that database: **0 of 24 sampled brotli rows decode +> with plain Brotli; all 24 require `brotli -D`.** A dictionary stream and a plain one are not +> interchangeable, so anything reading or writing `Content` has to go through the dictionary. +> +> No Python Brotli binding exposes a custom dictionary (the `brotli` package has no such parameter; +> `brotlicffi` dropped `BrotliDecoderSetCustomDictionary` in 1.2), so all three writers shell out to +> the **`brotli` CLI** — which therefore has to be installed wherever the pipeline runs. Both +> workflows install it. +> +> | Writer | Dictionary handling | +> |---|---| +> | `ProcessKotlinWebsiteJSON/populate_db.py` | `DictionaryCompressor` + `load_or_create_dictionary`; trains one only if the table is absent/empty, and **never** retrains. | +> | `ProcessKotlinWebsiteJSON/insert_optimized_media.py` | Reads the existing dictionary via `load_dictionary`; never creates one. | +> | `scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py` | `DictionaryBrotli` + `load_compression_dictionary`; falls back to plain Brotli only for pre-2.0.0 databases. | +> +> Retraining an existing dictionary would orphan every row already compressed against the old one, +> which is why all three only ever read what is already stored. +> `migrate_content_to_dictionary_brotli.py` recompresses any remaining plain-Brotli rows. +> +> Note also that `image/webp` (id 26) and `video/quicktime` (id 28) now exist in `ContentTypes`, so +> the workflows' `INSERT OR IGNORE ... image/webp` step is a no-op against current copies. + +The schema below is what `~/documentation.db` (Alex's current production copy) actually contains, +as of 2026-08-05 — predating the 2.0.0 bump described above, which additionally adds the +`CompressionDictionary` and `DocumentationDatabaseVersion` tables: + +```sql +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL); +CREATE TABLE TooltipButtonNumbers (id INTEGER UNIQUE); -- manually assigned display order +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (languageID) REFERENCES Languages(id), FOREIGN KEY (contentTypeID) REFERENCES ContentTypes(id), + UNIQUE('path') +); +CREATE TABLE Tooltips ( + id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL, + summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag), + FOREIGN KEY(categoryId) REFERENCES TooltipCategories(id) +); +CREATE TABLE TooltipButtons ( + tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT, + FOREIGN KEY(tooltipId) REFERENCES Tooltips(id), FOREIGN KEY(buttonNumberId) REFERENCES TooltipButtonNumbers(id) +); +CREATE TABLE LastChange (documentationSet TEXT, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, who TEXT); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE('name')); +CREATE TABLE BookCategories (id INTEGER PRIMARY KEY AUTOINCREMENT, category STRING, description STRING DEFAULT '', UNIQUE('category')); +CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT '', description STRING DEFAULT '', + bookCategoryID INTEGER, FOREIGN KEY (bookCategoryID) REFERENCES BookCategories(id), UNIQUE(title, bookCategoryId)); +-- Triggers keep Bookshelf in sync when a .pdf row is added to/removed from Content. +CREATE TABLE PUCC_Students (...), PUCC_Classes (...), PUCC_Sections (...), PUCC_Professors (...), + PUCC_StudentAssignments (...), PUCC_ProfessorAssignments (...) +-- Unrelated to documentation tooling (confirmed by Alex) — ignore, leave as-is, do not +-- document or maintain further in this repo. +``` + +**`Templates`, `BookCategories`, and `Bookshelf` are not documentation cruft — `WebServer.kt` +actively depends on them.** Its `/pr/bs` endpoint builds a JSON "bookshelf" payload straight from +`Content` + `Bookshelf` + `BookCategories`, looks up a template named `'bookshelf'` in `Templates`, +and renders it with the Pebble template engine. More generally, any `Content` row with a non-zero +`templateId` gets its stored (decompressed) content run through the matching row in `Templates` as +a Pebble template before being served. This is a real, current feature of the shipped server, not +a placeholder. Note this repo does now write to `Templates`: `populate_db.py` upserts `page.peb` +and `nav.peb` there (and points every page it inserts at them via `templateId`), so the table is +no longer read-only from this side. + +**Nothing that currently builds or writes to the database in this repository knows about any of +that — and that's expected.** `Templates`/`Bookshelf`/`BookCategories` are populated by a separate +plugin system, not by anything in this repo: App Dev For All supports plugins that write into the +documentation database, including the bookshelf feature specifically — +[appdevforall/bookshelf-plugin](https://github.com/appdevforall/bookshelf-plugin). So the absence +of any `Templates`/`Bookshelf`/`BookCategories` handling here is not a gap to fill; it's out of +scope for this repo. (A repo-wide search for `Templates`, `Bookshelf`, `BookCategories`, or `PUCC` +turns up zero matches outside `WebServer.kt` itself, which is consistent with that division of +responsibility. `templateId` itself is a different story - `populate_db.py` and +`insert_optimized_media.py` both read/write it directly, since it's a plain column on `Content` +they populate; it's only the `Templates` table and the plugin system that reference it that stay +out of scope.) Concretely, relative to the schema above: + +| Piece | What it thinks the schema is | Consequence | +| --- | --- | --- | +| `scripts/DocumentationDatabase.py` (used by `scripts/ingest.py`, and hence by `.github/workflows/publish-doc-db.yaml`) | `Content` / `Languages` / `ContentTypes` only, plus an optional `ide_tooltip_table`. Its constructor explicitly **raises `ValueError`** if it opens a DB containing any table outside that whitelist. | **This will refuse to open the current production `documentation.db` at all** — it will list `Tooltips`, `TooltipCategories`, `TooltipButtons`, `TooltipButtonNumbers`, `LastChange`, `Templates`, `BookCategories`, `Bookshelf`, and every `PUCC_*` table as "unexpected." This is the single biggest blocker to reusing this script as-is. | +| `docdb-studio/SCHEMA.md` / `AGENTS.md` (states the schema is "locked," no migrations) | `Content` (no `templateId`, no `UNIQUE(path)`), `Tooltips`, `TooltipButtons`, `TooltipCategories`, `TooltipButtonNumbers`, `LastChange` (with a *different* shape: `documentationSet`/`changeTime`/`who` — this part does match current), plus a legacy `ide_tooltip_table`. Missing `templateId`, `Templates`, `BookCategories`, `Bookshelf`, `PUCC_*`. | Closest of the three documented schemas to reality, but still out of date. `docdb_studio.py`'s own "never change the schema" policy is itself now stale, since the live schema has already changed underneath it. | +| `check-tools/README.md`'s embedded schema (and by extension the mental model behind `check-tools/db_health_checker.py`) | `Content` (no `templateId`, no `UNIQUE(path)`), `Tooltips`, `TooltipButtons`, `TooltipCategories`, `TooltipButtonNumbers`, and a *third* variant of `LastChange` (`now`/`who`). No `Templates`/`Bookshelf`/`BookCategories`/`PUCC_*`. | The health checker's required-table check still passes (it only checks that its known tables exist, not that no others do). Since `Templates`/`Bookshelf`/`BookCategories` are out of scope for this repo (see above), this is not being treated as something to fix right now. | + +There also appear to be **two unrelated tooltip storage formats** in this repo's history, and it's +worth being deliberate about which one is current: + +- The **normalized** format (`Tooltips` + `TooltipCategories` + `TooltipButtons` + + `TooltipButtonNumbers`) — this is what's in the live schema above, what `docdb-studio` edits, + what `check-tools/db_health_checker.py` validates, and what `scripts/TooltipManager.py` + dumps/rebuilds via CSV. +- A **legacy flat** format, a single `ide_tooltip_table(tooltipCategory, tooltipTag, + tooltipSummary, tooltipDetail, tooltipButtons)` table (button data packed as a JSON string in + one column) — written by `scripts/tooltips.py` (`TooltipDatabase`, driven by + `scripts/import_tooltips.py` from `SourceDocs/Tooltips/tooltips.xlsx`) and by + `scripts/load_android_data.py` (fed by pickle files that `scripts/android_tooltips.py` / + `scripts/java_tooltips.py` scrape from Android/Java HTML doc trees). **`ide_tooltip_table` does + not exist in the current production schema at all.** + +**`ide_tooltip_table` is officially dead (confirmed by Alex).** That means the entire chain that +targets it — `scripts/tooltips.py`, `scripts/import_tooltips.py`, `scripts/android_tooltips.py`, +`scripts/java_tooltips.py`, `scripts/android_html_page.py`, and `scripts/load_android_data.py` — is +**deprecated legacy code**. It's left in the repo for reference/history, but none of it should be +extended or relied on, and none of it writes to a table the shipped app or `docdb-studio` actually +uses. Any future Android/Java tooltip work should target the normalized `Tooltips` / +`TooltipCategories` / `TooltipButtons` / `TooltipButtonNumbers` tables instead (the same ones +`docdb-studio` and `scripts/TooltipManager.py` already use for Kotlin tooltips). + +## Repository tour + +- **`docdb-studio/`** — a Flet (Flutter-for-Python) desktop GUI for browsing/editing `Tooltips` / + `TooltipCategories` / `TooltipButtons` and importing `Content`. Has its own `CLAUDE.md`, + `AGENTS.md`, `SCHEMA.md`, and a real pytest suite. Actively maintained (most recent commits in + the repo touch this tool), but per the table above, its documented schema is behind the live one. + That's an accepted state, not an active problem: schema evolution happens outside + `docdb-studio` (and outside this repo, e.g. via plugins — see below), and `docdb-studio` is + expected to catch up after the fact rather than lead. Its `AGENTS.md`/`SCHEMA.md` "never migrate + the schema" language should be read as "don't migrate it from in here," not as a claim that the + schema never changes. +- **`check-tools/`** — `db_health_checker.py` (schema/integrity/referential checks against the + *old* normalized schema) plus `download_database.py`, a working Google Drive downloader + authenticated via GCP Workload Identity Federation (no long-lived keys). Wired into + `.github/workflows/docdb-regression-test.yaml`, which runs it daily against the production DB on + Drive. +- **`scripts/`** — the original CLI toolbox. Live/current: `DocumentationDatabase.py` (Content + ingestion — see whitelist issue above), `ingest.py` (thin CLI over it, used by + `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), + `create_empty_database.py`, `list_database_documents.py`. **Deprecated/dead** (target the + removed `ide_tooltip_table` — see above, kept for reference only): `tooltips.py`, + `import_tooltips.py`, `android_tooltips.py`, `java_tooltips.py`, `android_html_page.py`, + `load_android_data.py`. +- **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates + `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli + decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and + knows nothing about compression-aware content types, templates, or fragmentation. **This is not + what ships in the app** — treat it as historical/reference only, not as documentation of current + server behavior. `WebServer.kt` is the real thing. +- **`Dokka-plugin-kdoc2json/`** — the Dokka `JsonRenderer`/`ModelMapper`/`LinkPostProcessor` plugin, + its test suite, and the `kotlin-stdlib-docs` build scripts, merged to `main` via `fix/ADFA-4514` + (`4c6b8aef`). Consumed by `Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh` and + `scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py` (ADFA-4739) to generate and load + kotlin-stdlib/-reflect/-test JSON docs. +- **`ProcessDocs/`** — HTML-processing pipelines that predate the "build docs as JSON" goal: + `ProcessKotlinDocs/` (turns Kotlin's HTML doc export into a self-contained HTML set + table of + contents, used by `.github/workflows/automate-kotlin.yaml`), `ProcessAndroidDevSite/`, `AndroidDocs/` + (holds `android-tooltips.pkl`, the pickle consumed by the now-deprecated `load_android_data.py`), + `ProcessPDFs/`. +- **`SourceDocs/`** — raw inputs: `KotlinDocs/html`, `JavaDocs/html` + `java_keywords.html`, + `Tooltips/tooltips.xlsx`, `KotlinDocs/kotlin-spec.pdf`. +- **`DocumentationAnalysis/`, `DocAnalysis/`, `png_optimization/`, `androidxtooltips/`** — Jupyter + notebooks and one-off scripts for doc-set size analysis, image/PNG compression experiments, and a + one-time AndroidX tooltip import (ADFA-1419). Not part of the critical build path. +- **`.github/workflows/`** — `automate-kotlin.yaml` (tag-triggered, builds the + Kotlin HTML doc bundle as a GitHub release asset), `publish-doc-db.yaml` (tag-triggered, runs the + `scripts/ingest.py` pipeline and releases the resulting `.sqlite`), `docdb-regression-test.yaml` + (daily cron, downloads the production DB from Google Drive via WIF and runs + `check-tools/main.py` against it), `build-kotlin-docs.yaml` and its local-filesystem counterpart + `build-kotlin-docs-local.yaml` (manual dispatch, the five-step Kotlin docs pipeline — these two + are the only ones with Slack notifications), and `python-tests.yaml` (push/PR, runs the pytest + suites). + +## Decisions log + +Settled with Alex on 2026-08-05, folded into the sections above; recorded here so the reasoning +isn't lost: + +- `ide_tooltip_table` and everything that targets it are dead. Treat as deprecated, not as a gap. +- `Templates`/`Bookshelf`/`BookCategories` are populated by App Dev For All's plugin system + (e.g. [bookshelf-plugin](https://github.com/appdevforall/bookshelf-plugin)), not by this repo. + Not a gap to fill here. +- `PUCC_*` tables are unrelated to documentation tooling. Ignore; leave as-is. +- `docdb-studio`'s schema is expected to lag the live schema and catch up after the fact; that's + fine, no urgent update needed. +- `check-tools/db_health_checker.py` is not being extended with `Templates`/`Bookshelf` checks + right now — deliberately out of scope for the moment. + +The one piece of this document that still describes an *active* problem rather than a settled +scope boundary is `scripts/DocumentationDatabase.py`'s hard failure on unrecognized tables (see the +table above) — that will need to be addressed before `scripts/ingest.py` / +`publish-doc-db.yaml` can run against a current-schema database. diff --git a/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh b/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh new file mode 100755 index 00000000..ff3089e3 --- /dev/null +++ b/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Builds the kotlin-stdlib/kotlin-test/kotlin-reflect API docs as JSON via the +# kdoc-to-json Dokka plugin, against a full kotlin/ (https://github.com/JetBrains/kotlin) +# repo checkout - freshly compiling and publishing the plugin from source +# first, so every run picks up whatever's currently in +# Dokka-plugin-kdoc2json/kdoc-to-json/src, not a jar left over from an +# earlier run. +# +# Only generates the JSON output (dokkaGenerateModuleJson), not the default +# HTML - JSON/latest/all-libs is the only thing this project's pipeline +# (sync_kdoc_json_to_db.py) consumes. Use build-kotlin-stdlib.sh directly, +# against libraries/tools/kotlin-stdlib-docs, if you also want the HTML +# comparison output that test_kotlin_stdlib.sh checks against. +# +# The target kotlin-stdlib-docs project's build.gradle.kts is swapped out +# for this directory's own (JSON-plugin-enabled) copy for the duration of +# the build, then restored automatically on exit - the kotlin checkout is +# left exactly as it was found, whether the build succeeds or fails. +# +# kotlin_big (a subproject of kotlin-stdlib-docs) extracts the actual +# kotlin-stdlib/-reflect/-test binaries it documents from a Maven repo. Left to +# its own devices it looks for "/build/repo" at the checkout's +# own defaultSnapshotVersion - i.e. artifacts that only exist if you have built +# the entire kotlin repo locally first, and that are published nowhere public. +# Against a plain `git clone --depth 1` that resolves to nothing and the build +# fails before generating any docs. --kotlin-libs-version / --kotlin-libs-repo +# point it at already-published artifacts instead, which is hours of CI cheaper +# than building Kotlin just to document it. +# +# Only the final output path is written to stdout; every other message goes +# to stderr, so this composes as: +# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh )" +# +# Usage: +# build-stdlib-json-docs.sh [options] [output-dir] +# +# Options: +# --kotlin-libs-version V Version of the kotlin-stdlib/-reflect/-test +# artifacts to document (Gradle -PdeployVersion). +# Should match 's checked-out ref. +# Default: unset, i.e. the checkout's own +# defaultSnapshotVersion, which needs a local build +# of the kotlin repo to exist. +# --kotlin-libs-repo URL Maven repo to resolve them from (Gradle +# -PkotlinLibsRepo). Default: unset. kotlin_big +# already declares mavenCentral(), so a released +# --kotlin-libs-version needs no repo override; this +# is for a private or snapshot repo. +set -euo pipefail + +log() { echo "$@" >&2; } + +usage() { log "Usage: $0 [--kotlin-libs-version V] [--kotlin-libs-repo URL] [output-dir]"; } + +KOTLIN_LIBS_VERSION="" +KOTLIN_LIBS_REPO="" +POSITIONAL=() +while [ $# -gt 0 ]; do + case "$1" in + --kotlin-libs-version) KOTLIN_LIBS_VERSION="$2"; shift 2 ;; + --kotlin-libs-repo) KOTLIN_LIBS_REPO="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + --) shift; POSITIONAL+=("$@"); break ;; + -*) log "error: unrecognized option '$1'"; usage; exit 1 ;; + *) POSITIONAL+=("$1"); shift ;; + esac +done +set -- ${POSITIONAL[@]+"${POSITIONAL[@]}"} + +if [ $# -lt 1 ]; then + usage + exit 1 +fi + +KOTLIN_ROOT="$(cd "$1" && pwd)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_DIR="$(cd "$SCRIPT_DIR/../../kdoc-to-json" && pwd)" +STDLIB_DOCS_DIR="$KOTLIN_ROOT/libraries/tools/kotlin-stdlib-docs" +OUTPUT_ROOT="$(mkdir -p "${2:-$SCRIPT_DIR/build-output}" && cd "${2:-$SCRIPT_DIR/build-output}" && pwd)" +JSON_OUTPUT_DIR="$OUTPUT_ROOT/json" + +# Passed through only when set, so an unset value falls through to the +# build's own default rather than overriding it with an empty string. +ARTIFACT_ARGS=() +[ -n "$KOTLIN_LIBS_VERSION" ] && ARTIFACT_ARGS+=("-PdeployVersion=$KOTLIN_LIBS_VERSION") +[ -n "$KOTLIN_LIBS_REPO" ] && ARTIFACT_ARGS+=("-PkotlinLibsRepo=$KOTLIN_LIBS_REPO") + +if [ ! -f "$KOTLIN_ROOT/gradle.properties" ]; then + log "error: '$KOTLIN_ROOT' doesn't look like a kotlin repo checkout (missing gradle.properties)." + exit 1 +fi +if [ ! -f "$STDLIB_DOCS_DIR/settings.gradle.kts" ] || [ ! -x "$STDLIB_DOCS_DIR/gradlew" ]; then + log "error: '$STDLIB_DOCS_DIR' doesn't look like a kotlin-stdlib-docs project (missing settings.gradle.kts or gradlew)." + exit 1 +fi +if [ ! -x "$PLUGIN_DIR/gradlew" ]; then + log "error: kdoc-to-json plugin project not found at '$PLUGIN_DIR' (missing gradlew)." + exit 1 +fi + +# The JSON-plugin-enabled build.gradle.kts we're about to install reads +# dokka_version as a plain Gradle project property (-Pdokka_version=...) +# rather than through this repo's own version catalog, so it has to be +# supplied explicitly - pulled from the same catalog entry the rest of the +# kotlin repo's Dokka usage is pinned to, so it never drifts out of sync. +# "|| true": under `set -e` a command substitution whose pipeline exits +# non-zero kills the script outright, so a catalog with no 'dokka =' entry +# (a kotlin ref that renamed the key) aborted "Step 4/5" with exit 1 and no +# output at all - the friendly message below was unreachable. +DOKKA_VERSION="$(grep -m1 '^dokka[[:space:]]*=' "$KOTLIN_ROOT/gradle/libs.versions.toml" | sed -E 's/^dokka[[:space:]]*=[[:space:]]*"([^"]*)".*/\1/' || true)" +if [ -z "$DOKKA_VERSION" ]; then + log "error: couldn't find a 'dokka = \"...\"' entry in $KOTLIN_ROOT/gradle/libs.versions.toml" + exit 1 +fi + +log "==> [1/2] Building and publishing a fresh copy of the kdoc-to-json plugin..." +# Sent to stderr (fd 2), not left on stdout - a caller doing +# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh ...)" must only capture the +# final path this script echoes, not gradlew's own build console output. +( cd "$PLUGIN_DIR" && ./gradlew clean publishToMavenLocal ) >&2 + +log "==> Installing kdoc-to-json-enabled build.gradle.kts into $STDLIB_DOCS_DIR" +ORIGINAL_BUILD_GRADLE="$(mktemp)" +cp "$STDLIB_DOCS_DIR/build.gradle.kts" "$ORIGINAL_BUILD_GRADLE" +restore_build_gradle() { + cp "$ORIGINAL_BUILD_GRADLE" "$STDLIB_DOCS_DIR/build.gradle.kts" + rm -f "$ORIGINAL_BUILD_GRADLE" +} +# INT/TERM/HUP as well as EXIT: a bare EXIT trap doesn't run on an untrapped +# fatal signal, so Ctrl-C during the long Gradle build left the swapped-in +# build.gradle.kts sitting in the developer's kotlin clone and orphaned the +# mktemp copy - contradicting this script's promise above that the checkout is +# left exactly as it was found. +trap restore_build_gradle EXIT INT TERM HUP +cp "$SCRIPT_DIR/build.gradle.kts" "$STDLIB_DOCS_DIR/build.gradle.kts" + +log "==> [2/2] Generating JSON documentation via kdoc-to-json (dokka $DOKKA_VERSION)..." +log " stdlib artifacts: ${KOTLIN_LIBS_VERSION:-(checkout default: needs a local kotlin build)}" \ + "from ${KOTLIN_LIBS_REPO:-(mavenCentral + checkout default repo)}" +# --refresh-dependencies forces Gradle to re-resolve the just-published +# SNAPSHOT jar from mavenLocal() rather than serving a same-GAV copy it +# cached from an earlier run of this same script. +( cd "$STDLIB_DOCS_DIR" && ./gradlew dokkaGenerateModuleJson \ + "-PdocsBuildDir=$JSON_OUTPUT_DIR" \ + "-Pdokka_version=$DOKKA_VERSION" \ + ${ARTIFACT_ARGS[@]+"${ARTIFACT_ARGS[@]}"} \ + --refresh-dependencies ) >&2 + +ALL_LIBS_DIR="$JSON_OUTPUT_DIR/latest/all-libs" +if [ ! -d "$ALL_LIBS_DIR" ]; then + log "error: expected output at '$ALL_LIBS_DIR' but it wasn't created." + exit 1 +fi + +log "==> Done." +echo "$ALL_LIBS_DIR" diff --git a/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts b/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts index db4c7298..ecb4a0b9 100644 --- a/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts +++ b/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts @@ -46,7 +46,14 @@ allprojects { // 3. Maven Central (Keep this for standard standard stable libraries like Gson/Coroutines) mavenCentral() - // ALL REMOTE JETBRAINS SNAPSHOT SERVERS HAVE BEEN REMOVED! + // 4. Dokka's own dev-snapshot server - required by plugins:dokka-samples-transformer-plugin + // and plugins:dokka-version-filter-plugin (both included by kotlin-stdlib-docs' + // settings.gradle.kts and pulled onto the build graph by its dokka-convention plugin), + // which pin to a Dokka dev build rather than a Maven Central release. Same property + + // default kotlin-stdlib-docs' own settings.gradle.kts uses, so this only ever points + // wherever that project already expects it to. + maven(url = providers.gradleProperty("dokka_repository") + .getOrElse("https://redirector.kotlinlang.org/maven/dokka-dev")) } // --- ADDED THIS EXCLUSION BLOCK --- @@ -54,6 +61,39 @@ allprojects { configurations.all { exclude(group = "org.jetbrains.dokka", module = "kotlin-playground-samples-plugin") } + + // kotlin-stdlib-docs' own two Dokka plugin subprojects + // (plugins:dokka-samples-transformer-plugin, plugins:dokka-version-filter-plugin) + // each hardcode `kotlin { jvmToolchain(8) }`. Both are pulled onto the build + // graph by the dokka-convention plugin, and dokkaGenerateModuleJson below + // depends on dokkaGeneratePublicationHtml, so their dependencies have to + // resolve even though this build only ever wants JSON out. Gradle then needs a + // JDK 8 toolchain, and with no toolchain download repository configured it + // fails during task-graph resolution before a single doc is generated: + // > Failed to calculate the value of task + // ':plugins:dokka-samples-transformer-plugin:compileJava' property 'javaCompiler'. + // > Cannot find a Java installation on your machine ... matching: + // {languageVersion=8, ...}. Toolchain download repositories have not been configured. + // That only survives on a runner that happens to have an EOL JDK 8 lying + // around for auto-detection to find; it fails on any that doesn't, which + // includes the act container build-kotlin-docs-local.yaml is written for. + // These two plugins are only ever loaded in-process by Dokka, under the same + // JVM this build is already running on, so compiling them for that JVM instead + // is sufficient - and it means the pipeline needs exactly one JDK (the 17 that + // .github/workflows/build-kotlin-docs*.yaml installs), not two. + // + // Deliberately keyed off the running JVM rather than a hardcoded 17: whatever + // JDK Gradle is on is guaranteed to be present, so this can never ask for a + // toolchain that isn't installed, even if the workflow's java-version moves. + // Registered via plugins.withId + afterEvaluate so it runs after each + // subproject's own jvmToolchain(8) call rather than being overwritten by it. + plugins.withId("org.jetbrains.kotlin.jvm") { + afterEvaluate { + extensions.findByType(JavaPluginExtension::class.java)?.toolchain { + languageVersion.set(JavaLanguageVersion.of(JavaVersion.current().majorVersion)) + } + } + } } val isTeamcityBuild = project.hasProperty("teamcity.version") || @@ -70,8 +110,38 @@ val defaultSnapshotVersion: String by rootProperties val kotlinLanguageVersion: String by rootProperties val githubRevision = if (isTeamcityBuild) project.property("githubRevision") else "master" -val artifactsVersion by extra(if (isTeamcityBuild) project.property("deployVersion") as String else defaultSnapshotVersion) -val artifactsRepo by extra(if (isTeamcityBuild) project.property("kotlinLibsRepo") as String else "$kotlin_root/build/repo") + +// Where kotlin_big pulls the kotlin-stdlib/-reflect/-test binaries it extracts +// and documents from, and at what version. +// +// Upstream only honours -PkotlinLibsRepo/-PdeployVersion under TeamCity, and +// otherwise hardcodes "$kotlin_root/build/repo" at defaultSnapshotVersion +// (2.5.255-SNAPSHOT here) - i.e. the artifacts a *local build of the kotlin +// repo itself* would have published. A plain `git clone --depth 1` has no such +// build output, and that snapshot version is published nowhere public, so +// :kotlin_big:extractStdlibCommonMain fails to resolve and the whole docs build +// dies before generating anything: +// > Could not find org.jetbrains.kotlin:kotlin-stdlib:2.5.255-SNAPSHOT +// Building the kotlin repo just to get them is hours of CI for artifacts that +// already exist on Maven Central, so accept both as ordinary Gradle properties +// regardless of TeamCity. kotlin_big already declares mavenCentral() alongside +// artifactsRepo, so a released version resolves with no repo override at all - +// -PkotlinLibsRepo is there for a private/snapshot repo (and to keep the pair +// symmetric with upstream's own TeamCity path). +// +// Blank is treated as unset so a workflow input that defaults to '' falls +// through to the upstream behaviour rather than resolving against an empty URL. +fun overrideOrNull(name: String): String? = + (findProperty(name) as String?)?.takeIf { it.isNotBlank() } + +val artifactsVersion by extra( + overrideOrNull("deployVersion") + ?: if (isTeamcityBuild) project.property("deployVersion") as String else defaultSnapshotVersion +) +val artifactsRepo by extra( + overrideOrNull("kotlinLibsRepo") + ?: if (isTeamcityBuild) project.property("kotlinLibsRepo") as String else "$kotlin_root/build/repo" +) val dokka_version: String by project println("# Parameters summary:") diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md index 67cb27da..2062ec1f 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md @@ -2,38 +2,43 @@ Converts a `kotlin-web-site/docs` checkout (JetBrains Writerside-flavored Markdown) into the JSON block schema this project's templating engine -renders. +renders, then (optionally) builds the sidebar nav and loads everything +straight into a `documentation.db`-schema SQLite database. -This PR (ADFA-5039) covers only [`md_to_json.py`](md_to_json.py) — the -conversion step itself. Building the sidebar nav from `kr.tree` -(`build_nav.py`), QA-ing the source tree for broken links/images -(`find_missing_assets.py`), and loading any of this into `documentation.db` -(`populate_db.py`, `insert_optimized_media.py`) are a separate ticket -(ADFA-4739) and land in a later PR. +## Scripts + +| Script | Purpose | +|---|---| +| [`md_to_json.py`](md_to_json.py) | Converts every `topics/**/*.md` page into one JSON file. Writes `theme.json` and copies `images/` into the output directory. See "Page JSON schema" below. | +| [`build_nav.py`](build_nav.py) | Builds `nav.json`/`nav.html` sidebar navigation from `kr.tree`, resolving each `` against `md_to_json.py`'s output. | +| [`find_missing_assets.py`](find_missing_assets.py) | QA pass: reports cross-page links, images, and `` targets in the source tree that don't resolve to anything. Reuses `md_to_json.py`'s own resolution logic, so it flags exactly what would end up broken on the rendered site. | +| [`populate_db.py`](populate_db.py) | The database path: converts the docs tree the same way `md_to_json.py` does, builds nav the same way `build_nav.py` does, and inserts pages + nav + images + CSS/JS directly into `documentation.db` (replacing everything under `k/html/` and `assets/`). Supports pruning whole `kr.tree` subtrees via `--blacklisted-element-titles`. | +| [`optimize_media.py`](optimize_media.py) | Standalone media optimizer: downscales/recompresses a directory of images (pngquant, Pillow, Scour/cairosvg for SVG) into a mirrored output directory. | +| [`insert_optimized_media.py`](insert_optimized_media.py) | Runs `optimize_media.py`'s pipeline over a directory of raw media, then replaces the corresponding `k/html/images/*` rows in an existing database, rewriting any page that referenced a renamed file and deleting anything left unreferenced. | ## Requirements -- Python 3.10+ -- `markdown-it-py` (now in the repo's root `requirements.txt`) +- Python 3.10+ and [`uv`](https://docs.astral.sh/uv/getting-started/installation/) — every command below is run as `uv run --with-requirements /requirements.txt + + + + +{# + Recursive block renderer. Macros only see the variables passed to them, so + every block that can nest other blocks (blockquote, note/tip/warning, list + items, table cells, tabs) passes its children back through renderBlock(). + Macros defined in a template are directly visible to themselves and to each + other within that same template, so no self-import is needed for recursion. +#} +{% macro renderBlock(b) %} +{% if b.type == "heading" %} +{{ b.html|raw }} + +{% elseif b.type == "paragraph" %} +

{{ b.html|raw }}

+ +{% elseif b.type == "code" %} +
{{ b.code }}
+ +{% elseif b.type == "blockquote" %} +
+{% if b.attrs.title %}

{{ b.attrs.title }}

{% endif %} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %}
+ +{% elseif b.type == "note" or b.type == "tip" or b.type == "warning" %} +
+{% if b.attrs.title %}

{{ b.attrs.title }}

{% endif %} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %}
+ +{% elseif b.type == "list" %} +{% if b.ordered %}
    {% else %}
      {% endif %} +{% for item in b.items %}
    • {% for child in item.blocks %}{{ renderBlock(child) }}{% endfor %}
      • +{% endfor %}{% if b.ordered %}
{% else %}{% endif %} + +{% elseif b.type == "table" %} + +{% if b.headers is not empty %} +{% for h in b.headers %}{% endfor %} +{% endif %} + +{% for row in b.rows %}{% for cell in row %}{% endfor %} +{% endfor %} +
{{ h|raw }}
{{ cell|raw }}
+ +{% elseif b.type == "image" %} +{{ b.alt|default('') }} + +{% elseif b.type == "hr" %} +
+ +{% elseif b.type == "tabs" and b.tabs is not empty %} +{# + Tab switching + the group-key syncing (e.g. picking "Groovy" in one + Kotlin/Groovy/Maven tabs block switches every other tabs block sharing the + same data-group on the page, matching Writerside's data-sync-tabs + behavior) is implemented in assets/tabs.js. md_to_json.py's + _finalize_container always gives every "tabs" block a non-empty "tabs" + list (synthesizing one from code-block languages, or dropping the wrapper + entirely, when the source had no children) - the "b.tabs is not + empty" guard here is just a defensive backstop against any other producer + of this JSON schema making the same mistake, not something this pipeline + itself still needs. +#} +
+
+ {% for tab in b.tabs %}{% set tabKey = tab.attrs["group-key"]|default(tab.title)|default(loop.index) %} + {% endfor %}
+ {% for tab in b.tabs %}{% set tabKey = tab.attrs["group-key"]|default(tab.title)|default(loop.index) %}
+ {% for child in tab.blocks %}{{ renderBlock(child) }} + {% endfor %}
+ {% endfor %} +
+ +{% elseif b.type == "html" %} +{{ b.html|raw }} + +{% elseif b.type == "tab" %} +{# A lone not wrapped in (e.g. seen in eap.json's HTML-table + compatibility layout); render its children rather than dropping them. #} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %} + +{% else %} +{% if b.html %}{{ b.html|raw }}{% endif %} + +{% endif %} +{% endmacro %} diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py new file mode 100644 index 00000000..dba7f948 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Tests for renumber_misnumbered_fragments.py (ADFA-5171). + +Run directly: python3 test_renumber_misnumbered_fragments.py +""" +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from populate_db import CHUNK_SIZE +from renumber_misnumbered_fragments import repair + +SCHEMA_SQL = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + languageID INTEGER NOT NULL, + content BLOB NOT NULL, + contentTypeID INTEGER NOT NULL, + templateId INTEGER, + UNIQUE(path) +); +""" + + +def chunk_bytes(n: int, fill: bytes) -> bytes: + return (fill * (n // len(fill) + 1))[:n] + + +class RenumberMisnumberedFragmentsTest(unittest.TestCase): + def setUp(self): + fd, path = tempfile.mkstemp(suffix=".db") + Path(path).unlink(missing_ok=True) + self.db_path = Path(path) + self.conn = sqlite3.connect(self.db_path) + self.conn.executescript(SCHEMA_SQL) + self.conn.execute("INSERT INTO Languages (value) VALUES ('en-US')") + self.conn.execute("INSERT INTO ContentTypes (value, compression) VALUES ('image/gif', 'none')") + self.conn.commit() + + def tearDown(self): + self.conn.close() + self.db_path.unlink(missing_ok=True) + + def insert(self, path: str, content: bytes): + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID) VALUES (?, 1, ?, 1)", + (path, content), + ) + + def all_paths(self) -> set: + return {row[0] for row in self.conn.execute("SELECT path FROM Content")} + + def content_at(self, path: str) -> bytes: + return self.conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0] + + def test_renumbers_chain_starting_at_minus_2(self): + base = "a/devsite/media/size-range.gif" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-3", chunk_bytes(CHUNK_SIZE, b"C")) + self.insert(f"{base}-4", chunk_bytes(CHUNK_SIZE, b"D")) + self.insert(f"{base}-5", b"E" * 100) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 1) + self.assertEqual(stats["fragments_moved"], 4) + self.assertEqual(stats["chains_gapped"], 0) + self.assertEqual( + self.all_paths(), + {base, f"{base}-1", f"{base}-2", f"{base}-3", f"{base}-4"}, + ) + self.assertEqual(self.content_at(f"{base}-1"), chunk_bytes(CHUNK_SIZE, b"B")) + self.assertEqual(self.content_at(f"{base}-2"), chunk_bytes(CHUNK_SIZE, b"C")) + self.assertEqual(self.content_at(f"{base}-3"), chunk_bytes(CHUNK_SIZE, b"D")) + self.assertEqual(self.content_at(f"{base}-4"), b"E" * 100) + + def test_renumbers_zero_based_chain(self): + """A chain numbered from -0 shifts *up*, where renaming in ascending + order would land on a slot still occupied and trip UNIQUE(path) - + rolling back every other repair in the same pass. It is as broken as a + -2 chain: WebServer.kt probes "-1", finds it, and serves the chain with + "-0" silently dropped.""" + base = "a/devsite/media/zero-based.gif" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-0", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-1", chunk_bytes(CHUNK_SIZE, b"C")) + self.insert(f"{base}-2", b"D" * 100) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 1) + self.assertEqual(stats["chains_gapped"], 0) + self.assertEqual(self.all_paths(), {base, f"{base}-1", f"{base}-2", f"{base}-3"}) + # order preserved: -0 -> -1, -1 -> -2, -2 -> -3 + self.assertEqual(self.content_at(f"{base}-1"), chunk_bytes(CHUNK_SIZE, b"B")) + self.assertEqual(self.content_at(f"{base}-2"), chunk_bytes(CHUNK_SIZE, b"C")) + self.assertEqual(self.content_at(f"{base}-3"), b"D" * 100) + + def test_zero_based_chain_does_not_block_other_repairs(self): + """One chain tripping UNIQUE(path) used to roll back the whole run.""" + zero_based = "a/devsite/media/zero.gif" + self.insert(zero_based, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{zero_based}-0", b"B" * 100) + two_based = "a/devsite/media/two.gif" + self.insert(two_based, chunk_bytes(CHUNK_SIZE, b"C")) + self.insert(f"{two_based}-2", b"D" * 100) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 2) + self.assertEqual(self.all_paths(), + {zero_based, f"{zero_based}-1", two_based, f"{two_based}-1"}) + + def test_single_orphaned_continuation(self): + base = "j/html/api/index-all.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", b"tail" * 10) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 1) + self.assertEqual(stats["fragments_moved"], 1) + self.assertEqual(self.all_paths(), {base, f"{base}-1"}) + self.assertEqual(self.content_at(f"{base}-1"), b"tail" * 10) + + def test_correctly_numbered_chain_untouched(self): + base = "k/html/already-fine.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-1", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-2", b"tail") + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["fragments_moved"], 0) + self.assertEqual(self.all_paths(), {base, f"{base}-1", f"{base}-2"}) + + def test_idempotent_second_run(self): + base = "a/devsite/media/size-range.gif" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", b"tail") + self.conn.commit() + + repair(self.conn) + self.conn.commit() + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["fragments_moved"], 0) + + def test_exact_size_file_with_no_continuation_left_alone(self): + path = "k/html/exactly-one-mb.bin" + self.insert(path, chunk_bytes(CHUNK_SIZE, b"A")) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["chains_gapped"], 0) + self.assertEqual(self.all_paths(), {path}) + + def test_chain_with_real_gap_reported_and_left_untouched(self): + base = "k/html/actually-missing-a-chunk.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-4", b"tail") # -3 is genuinely missing + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["chains_gapped"], 1) + self.assertEqual(self.all_paths(), {base, f"{base}-2", f"{base}-4"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py new file mode 100644 index 00000000..33c79c32 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py @@ -0,0 +1,73 @@ +"""Regression test for find_missing_assets.py's exit-code behavior (PR #24 +review). Runs the script as a subprocess against the real md_to_json.py +(merged from ADFA-5039); a non-UTF-8 .md file gives convert_file a genuine +reason to raise, matching the pattern md_to_json.py's own test suite uses +for its equivalent main()-exit-code tests. +""" +import subprocess +import sys +from pathlib import Path + +import find_missing_assets as fma + + +def _write_minimal_docs_root(tmp_path, *, with_failure=False): + docs_root = tmp_path / "docs" + (docs_root / "topics").mkdir(parents=True) + (docs_root / "topics" / "good.md").write_text("# Good\n\nHello.\n", encoding="utf-8") + if with_failure: + # Not valid UTF-8 - convert_file's read_text(encoding="utf-8") raises. + (docs_root / "topics" / "bad.md").write_bytes(b"\xff\xfe not utf-8") + return docs_root + + +def _run(*args): + script = Path(__file__).resolve().parent.parent / "find_missing_assets.py" + return subprocess.run([sys.executable, str(script), *map(str, args)], capture_output=True, text=True) + + +def test_exits_zero_and_reports_zero_failures_when_nothing_fails(tmp_path): + docs_root = _write_minimal_docs_root(tmp_path) + report = tmp_path / "report.md" + result = _run(docs_root, report) + assert result.returncode == 0 + assert "0 file(s) failed to scan" in report.read_text(encoding="utf-8") + + +def test_exits_nonzero_when_a_file_fails_to_scan(tmp_path): + """A per-file scan failure used to be printed to stderr and otherwise + ignored - the report still claimed a clean summary and the process + still exited 0, so a totally broken corpus was indistinguishable from a + clean one (this is the pre-flight gate run before populate_db.py). The + bad file is scanned by two independent passes (the main conversion loop + and find_include_warnings' own scan), so it counts twice.""" + docs_root = _write_minimal_docs_root(tmp_path, with_failure=True) + report = tmp_path / "report.md" + result = _run(docs_root, report) + assert result.returncode == 1 + text = report.read_text(encoding="utf-8") + assert "2 file(s) failed to scan" in text + assert "incomplete" in text.lower() + + +def test_allow_failures_exits_zero_despite_failure(tmp_path): + docs_root = _write_minimal_docs_root(tmp_path, with_failure=True) + report = tmp_path / "report.md" + result = _run(docs_root, report, "--allow-failures") + assert result.returncode == 0 + + +def test_find_include_warnings_reports_failure_instead_of_raising(tmp_path): + """find_include_warnings had its own unguarded read_text(encoding="utf-8") + outside the main loop's try/except - a non-UTF-8 file crashed the whole + process with an uncaught traceback, bypassing --allow-failures entirely + rather than being counted as a scan failure like every other file-read + in this script.""" + topics_dir = tmp_path / "topics" + topics_dir.mkdir() + (topics_dir / "good.md").write_text("no includes here\n", encoding="utf-8") + (topics_dir / "bad.md").write_bytes(b"\xff\xfe not utf-8") + + warnings, failed = fma.find_include_warnings(topics_dir) + assert warnings == [] + assert failed == 1 diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_insert_optimized_media.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_insert_optimized_media.py new file mode 100644 index 00000000..71a8e657 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_insert_optimized_media.py @@ -0,0 +1,150 @@ +"""Regression tests for insert_optimized_media.py's destructive paths. + +Every test here covers a way this script could delete a row it shouldn't - +the LIKE-wildcard over-match in delete_content, and the "no pages to check +against" case that would otherwise wipe the whole image corpus. +""" +import shutil +import sqlite3 + +import pytest + +from insert_optimized_media import ( + IMAGES_URL_PREFIX, + collect_referenced_media, + delete_content, + delete_unreferenced_media, +) +from optimize_media import Logger +from populate_db import DictionaryCompressor + +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, + compression TEXT NOT NULL); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + languageID INTEGER NOT NULL, + content BLOB NOT NULL, + contentTypeID INTEGER NOT NULL, + templateId INTEGER NOT NULL DEFAULT 0, + UNIQUE(path) +); +""" + +PAGE_TYPE_ID = 12 +# Content is dictionary-compressed from schema 2.0.0 on (ADFA-5153), and +# insert_optimized_media reads pages back through that dictionary, so these +# fixtures have to speak it too. +DICTIONARY = bytes(range(256)) * 64 +needs_brotli_cli = pytest.mark.skipif(shutil.which("brotli") is None, reason="brotli CLI not installed") + +pytestmark = needs_brotli_cli + + +@pytest.fixture +def compressor(): + instance = DictionaryCompressor(DICTIONARY) + yield instance + instance.close() + + +@pytest.fixture +def conn(): + connection = sqlite3.connect(":memory:") + connection.executescript(SCHEMA) + connection.execute("INSERT INTO Languages (id, value) VALUES (1, 'en-US')") + connection.execute("INSERT INTO ContentTypes (id, value, compression) VALUES (?, 'text/html', 'brotli')", + (PAGE_TYPE_ID,)) + yield connection + connection.close() + + +def add_row(conn, path, blob=b"x", template_id=0): + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, 1, ?, ?, ?)", + (path, blob, PAGE_TYPE_ID, template_id), + ) + + +def paths(conn): + return {row[0] for row in conn.execute("SELECT path FROM Content")} + + +def page_blob(compressor, *image_names): + """A stored page blob referencing each image the way md_to_json bakes it + in: an HTML src="..." attribute inside JSON, so the filename is followed + by an escaped quote.""" + srcs = "".join(f'' for name in image_names) + return compressor.compress(f'{{"blocks":[{{"html":"{srcs}"}}]}}'.encode("utf-8")) + + +class TestDeleteContent: + def test_removes_the_row_and_its_chunk_fragments(self, conn): + add_row(conn, "k/html/images/big.png") + add_row(conn, "k/html/images/big.png-1") + add_row(conn, "k/html/images/big.png-2") + add_row(conn, "k/html/images/other.png") + + delete_content(conn, "k/html/images/big.png") + + assert paths(conn) == {"k/html/images/other.png"} + + def test_underscore_is_not_treated_as_a_wildcard(self, conn): + # "_" in a LIKE pattern matches any single character, so an unescaped + # "k/html/_nav.html-%" would also match "k/html/Xnav.html-1" and take + # an unrelated page's chunk fragment with it. + add_row(conn, "k/html/_nav.html") + add_row(conn, "k/html/_nav.html-1") + add_row(conn, "k/html/Xnav.html") + add_row(conn, "k/html/Xnav.html-1") + + delete_content(conn, "k/html/_nav.html") + + assert paths(conn) == {"k/html/Xnav.html", "k/html/Xnav.html-1"} + + def test_percent_is_not_treated_as_a_wildcard(self, conn): + add_row(conn, "k/html/images/100%.png") + add_row(conn, "k/html/images/100%.png-1") + add_row(conn, "k/html/images/100-other.png-1") + + delete_content(conn, "k/html/images/100%.png") + + assert paths(conn) == {"k/html/images/100-other.png-1"} + + +class TestDeleteUnreferencedMedia: + def test_removes_only_images_no_page_references(self, conn, compressor): + add_row(conn, "k/html/page.html", page_blob(compressor, "kept.png"), template_id=2) + add_row(conn, "k/html/images/kept.png") + add_row(conn, "k/html/images/orphan.png") + add_row(conn, "k/html/images/orphan.png-1") + + removed = delete_unreferenced_media(conn, PAGE_TYPE_ID, Logger(None), compressor) + + assert removed == 1 + assert paths(conn) == {"k/html/page.html", "k/html/images/kept.png"} + + def test_refuses_to_run_when_no_page_references_any_image(self, conn, compressor): + # populate_db.py hasn't written its pages yet (or was skipped): the + # reference scan comes back empty and every stored image looks like + # garbage. Deleting the whole corpus is never what was meant. + add_row(conn, "k/html/images/a.png") + add_row(conn, "k/html/images/b.png") + + with pytest.raises(RuntimeError, match="no page references any image"): + delete_unreferenced_media(conn, PAGE_TYPE_ID, Logger(None), compressor) + + assert paths(conn) == {"k/html/images/a.png", "k/html/images/b.png"} + + def test_empty_database_is_not_an_error(self, conn, compressor): + assert delete_unreferenced_media(conn, PAGE_TYPE_ID, Logger(None), compressor) == 0 + + def test_untemplated_rows_are_not_scanned_for_references(self, conn, compressor): + # templateId 0 marks a raw asset, not a page; only page/nav rows carry + # the JSON that image references live in. + add_row(conn, "k/html/page.html", page_blob(compressor, "kept.png"), template_id=2) + add_row(conn, "k/html/images/kept.png") + + assert collect_referenced_media(conn, PAGE_TYPE_ID, compressor) == {"kept.png"} diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_review_findings.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_review_findings.py new file mode 100644 index 00000000..181d4d66 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_review_findings.py @@ -0,0 +1,206 @@ +"""Regression tests for the PR #24 review findings (F01-F15). + +Each test constructs the specific input the reviewer identified as untested - +the shapes nothing else in the suite feeds these functions. Every one of them +fails against the code as it stood before the corresponding fix, which is the +only reason they are worth having: the two criticals in particular were silent, +exit-0 data loss that truthful-looking statistics actively concealed. +""" +import json +import random +import sqlite3 +import sys + +import brotli +import pytest +from PIL import Image + +import optimize_media as om +from build_nav import load_page_index +from insert_optimized_media import reassemble_content +from migrate_content_to_dictionary_brotli import is_chunked_base, load_base_rows, write_item +from populate_db import CHUNK_SIZE +from renumber_misnumbered_fragments import find_chains, find_fragment_paths + +SCHEMA_SQL = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, + compression TEXT NOT NULL); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + languageID INTEGER NOT NULL, + content BLOB NOT NULL, + contentTypeID INTEGER NOT NULL, + templateId INTEGER, + UNIQUE(path) +); +""" + + +def _new_stats(): + return {"raster": 0, "svg": 0, "svg_rasterized": 0, "copied": 0, "errors": 0, + "original_bytes": 0, "optimized_bytes": 0} + + +@pytest.fixture +def conn(): + connection = sqlite3.connect(":memory:") + connection.executescript(SCHEMA_SQL) + connection.execute("INSERT INTO Languages (value) VALUES ('en-US')") + connection.execute("INSERT INTO ContentTypes (value, compression) VALUES ('text/html', 'brotli')") + yield connection + connection.close() + + +def _insert(conn, path, blob): + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, 1, ?, 1, 0)", + (path, blob), + ) + + +# --- F01: two sources colliding on a rewritten extension --------------------- + +def test_sources_differing_only_by_extension_both_survive(tmp_path): + """logo.png + logo.jpg both become logo.webp, and the loser used to be + silently gone - with errors 0 and both pages repointed at the survivor.""" + src, out = tmp_path / "in", tmp_path / "out" + src.mkdir(), out.mkdir() + Image.new("RGB", (40, 30), (200, 30, 30)).save(src / "logo.png") + Image.new("RGB", (40, 30), (30, 30, 200)).save(src / "logo.jpg") + + cfg = dict(om.BUILTIN_DEFAULTS) | {"webp": True} + renamed = om.optimize_directory(src, out, cfg=cfg, pngquant_path=om.find_pngquant(), + logger=om.Logger(sys.stdout), stats=_new_stats()) + + assert len(list(out.iterdir())) == 2, "one source was clobbered by the other" + # Both renames are reported, so a caller rewriting stored URLs follows them. + assert set(renamed) == {"logo.png", "logo.jpg"} + assert len(set(renamed.values())) == 2, "both sources still map to one output" + + +def test_three_way_extension_collision_all_survive(tmp_path): + src, out = tmp_path / "in", tmp_path / "out" + src.mkdir(), out.mkdir() + for name, colour in (("logo.png", (1, 1, 1)), ("logo.jpg", (2, 2, 2)), ("logo.gif", (3, 3, 3))): + Image.new("RGB", (20, 20), colour).save(src / name) + + cfg = dict(om.BUILTIN_DEFAULTS) | {"webp": True} + om.optimize_directory(src, out, cfg=cfg, pngquant_path=om.find_pngquant(), + logger=om.Logger(sys.stdout), stats=_new_stats()) + + assert len(list(out.iterdir())) == 3 + + +def test_non_colliding_names_keep_their_own_stems(tmp_path): + """The de-confliction must not rename anything that didn't collide.""" + src, out = tmp_path / "in", tmp_path / "out" + src.mkdir(), out.mkdir() + Image.new("RGB", (20, 20), (5, 5, 5)).save(src / "alpha.png") + Image.new("RGB", (20, 20), (6, 6, 6)).save(src / "beta.png") + + om.optimize_directory(src, out, cfg=dict(om.BUILTIN_DEFAULTS), pngquant_path=om.find_pngquant(), + logger=om.Logger(sys.stdout), stats=_new_stats()) + + assert sorted(p.name for p in out.iterdir()) == ["alpha.png", "beta.png"] + + +# --- F07: optimizing a directory into itself --------------------------------- + +def test_optimizing_into_the_input_directory_is_refused(tmp_path): + """Used to destroy the originals in place; the only error raised was a + copy2 SameFileError on the first non-image file, long after the damage.""" + Image.new("RGB", (400, 300), (10, 200, 10)).save(tmp_path / "logo.png") + before = (tmp_path / "logo.png").read_bytes() + + with pytest.raises(ValueError, match="input directory"): + om.optimize_directory(tmp_path, tmp_path, cfg=dict(om.BUILTIN_DEFAULTS), + pngquant_path=om.find_pngquant(), logger=om.Logger(sys.stdout), + stats=_new_stats()) + + assert (tmp_path / "logo.png").read_bytes() == before + + +# --- F02: an unrelated "X-1" page alongside "X" ------------------------------- + +def test_independent_page_named_like_a_fragment_is_not_a_continuation(conn): + _insert(conn, "k/html/guide.html", brotli.compress(b"the base page, comfortably under one chunk")) + _insert(conn, "k/html/guide.html-1", brotli.compress(b"a wholly unrelated page")) + + scanned = {row[0] for row in load_base_rows(conn)} + assert "k/html/guide.html-1" in scanned, "victim was invisible to the migration entirely" + + +def test_write_item_does_not_delete_an_unrelated_lookalike_page(conn): + victim = brotli.compress(b"a wholly unrelated page") + _insert(conn, "k/html/guide.html", brotli.compress(b"the base page")) + _insert(conn, "k/html/guide.html-1", victim) + + write_item(conn, "k/html/guide.html", 1, 1, 0, brotli.compress(b"rewritten base page")) + + row = conn.execute("SELECT content FROM Content WHERE path = 'k/html/guide.html-1'").fetchone() + assert row is not None, "unrelated page deleted as a surplus fragment" + assert row[0] == victim, "unrelated page overwritten" + + +def test_genuinely_chunked_base_still_owns_its_continuations(conn): + """The length gate must not break real chunking - including an ADFA-5171 + chain numbered from -2, which has no -1 at all.""" + _insert(conn, "k/html/big.html", b"x" * CHUNK_SIZE) + _insert(conn, "k/html/big.html-2", b"y" * 10) + + scanned = {row[0] for row in load_base_rows(conn)} + assert "k/html/big.html-2" not in scanned, "real continuation treated as its own page" + assert is_chunked_base({"k/html/big.html": CHUNK_SIZE}, "k/html/big.html") + assert not is_chunked_base({"k/html/guide.html": 42}, "k/html/guide.html") + + +# --- F06: reassembly of a chain numbered from -2 ------------------------------ + +def test_reassemble_content_handles_a_chain_numbered_from_two(conn): + """Probing "-1" first returned a truncated stream for the exact shape + renumber_misnumbered_fragments.py exists to repair.""" + tail = b"z" * 20 + _insert(conn, "k/html/images/big.png", b"a" * CHUNK_SIZE) + _insert(conn, "k/html/images/big.png-2", tail) + + assembled = reassemble_content(conn, "k/html/images/big.png", b"a" * CHUNK_SIZE) + assert assembled == b"a" * CHUNK_SIZE + tail + + +# --- F08: a chain with an interior gap --------------------------------------- + +def test_chain_with_interior_gap_is_reported_as_gapped(conn): + """p-1, p-2, p-4 starts at 1, so it used to short-circuit as healthy and be + reported as "0 chain(s) had a real gap" on a truncated page.""" + _insert(conn, "k/html/page.html", b"a" * CHUNK_SIZE) + for n in (1, 2, 4): + _insert(conn, f"k/html/page.html-{n}", b"a" * (CHUNK_SIZE if n != 4 else 10)) + + misnumbered, gapped = find_chains(conn, find_fragment_paths(conn)) + + assert [path for path, _f in gapped] == ["k/html/page.html"] + assert misnumbered == [] + + +def test_contiguous_chain_from_one_is_left_alone(conn): + _insert(conn, "k/html/page.html", b"a" * CHUNK_SIZE) + _insert(conn, "k/html/page.html-1", b"a" * 10) + + misnumbered, gapped = find_chains(conn, find_fragment_paths(conn)) + assert misnumbered == [] and gapped == [] + + +# --- F03: build_nav reading its own nav.json --------------------------------- + +def test_load_page_index_skips_the_generated_nav_json(tmp_path): + """The documented invocation passes output_dir as the scan dir, so a second + run read its own nav.json - a top-level array - and died on list.get.""" + (tmp_path / "page.json").write_text(json.dumps({"id": "k/html/a", "title": "A"}), encoding="utf-8") + (tmp_path / "nav.json").write_text(json.dumps([{"id": "k/html/a", "children": []}]), encoding="utf-8") + + stem_to_id, id_to_title = load_page_index(tmp_path) + + assert stem_to_id == {"a": "k/html/a"} + assert id_to_title == {"k/html/a": "A"} diff --git a/ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh b/ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh new file mode 100755 index 00000000..bbbcf8df --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh @@ -0,0 +1,242 @@ +#!/usr/bin/env bash +# End-to-end test of the Kotlin website JSON/DB pipeline (ADFA-4737): convert +# docs + prune blacklist into the database, re-optimize/reinsert media, +# generate fresh kotlin-stdlib/-reflect/-test JSON docs via a freshly-built +# kdoc-to-json plugin, then sync them into the database. Operates on a +# scratch copy of documentation.db so the real database is never touched. +# Re-run freely; each run recopies the source db from scratch. +# +# Before running: fill in every value below for your machine. +# The script refuses to start if any are left unfilled or don't exist on disk. +set -euo pipefail + +if ! command -v uv >/dev/null 2>&1; then + echo "error: uv is required - see https://docs.astral.sh/uv/getting-started/installation/" >&2 + exit 1 +fi + +# --- Repo-relative paths - auto-detected, no edits needed --------------- +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PROCESS_DIR="$REPO_ROOT/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON" +SYNC_SCRIPT="$REPO_ROOT/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py" +UV_RUN=(uv run --with-requirements "$REPO_ROOT/requirements.txt") + +# config.json, templates/, and assets/ are staged directly in $PROCESS_DIR +# (this repo) rather than pulled from anyone's local machine - populate_db.py +# looks these up next to its own script location: +# config.json - theming config (broken-ext-link-color, menu-no-link-color) +# templates/*.peb - page.peb / nav.peb, upserted into the Templates table +# assets/* - docs.css / tabs.js / sidebar.js, inserted at assets/ +CONFIG_JSON="$PROCESS_DIR/config.json" + +# --- Machine-specific paths - fill these in -------------------------------- + +# DOCS_ROOT: the "docs" subdirectory of a Writerside checkout of the official +# Kotlin website. Get it from https://github.com/JetBrains/kotlin-web-site - +# clone that repo and point this at "/kotlin-web-site/docs" (the +# directory directly containing kr.tree, topics/, images/, v.list). +DOCS_ROOT="" + +# IMAGES_ZIP: Writerside's own image export for that same docs project (e.g. +# "webHelpImages.zip"). Produced by running IntelliJ IDEA's Writerside plugin +# build/export action against DOCS_ROOT's parent Writerside project; the zip +# is written next to kr.tree once that build finishes. +IMAGES_ZIP="" + +# STDLIB_DOCS_DIR: the "libraries/tools/kotlin-stdlib-docs" directory inside +# a full clone of https://github.com/JetBrains/kotlin (not the kotlin repo +# root itself - this exact subdirectory). Step 4 below builds a fresh copy +# of the kdoc-to-json plugin from this repo's Dokka-plugin-kdoc2json/ and +# runs it against this checkout to produce kotlin-stdlib/-reflect/-test JSON +# docs (common + jvm source sets only) - no separate manual doc-generation +# step needed. +STDLIB_DOCS_DIR="" + +# SOURCE_DB: the runtime "documentation.db" SQLite database this project's +# offline documentation app/server reads from (see docdb-studio/ and +# check-tools/ in this repo for tooling that operates on the same file). Must +# already have its schema populated (Languages, ContentTypes, Templates +# tables) - point this at your own working copy. +SOURCE_DB="" + +TEST_DB="$(dirname "$SOURCE_DB")/documentation.test.db" + +JPEG_QUALITY=85 +WEBP_QUALITY=90 + +# Which published kotlin-stdlib/-reflect/-test artifacts step 4/5 documents. +# kotlin_big (inside kotlin-stdlib-docs) extracts the real binaries from a +# Maven repo; left unset it looks for "/build/repo" at the +# checkout's own defaultSnapshotVersion, i.e. artifacts that only exist if you +# have built the whole kotlin repo locally. Naming a released version instead +# resolves them straight from Maven Central. Keep it in step with the ref your +# STDLIB_DOCS_DIR checkout is on; set empty to use the checkout's own default. +KOTLIN_LIBS_VERSION=2.4.10 +# Maven repo to resolve them from. Empty is fine for a released +# KOTLIN_LIBS_VERSION (kotlin_big already declares mavenCentral()); set this +# only to point at a private or snapshot repository. +KOTLIN_LIBS_REPO="" + +# Full toc-title path (top-level -> ... -> target), joined with "\/" per +# populate_db.py's --blacklisted-element-titles convention. These are the +# concrete cases named in ADFA-4737; add more "path" entries here to prune +# additional sections. Re-derive these from your own DOCS_ROOT/kr.tree if the +# site's navigation structure has changed since this was written. +BLACKLIST=( + 'Development\/Web development' + 'Interoperability\/Swift/Objective-C and C interop' + 'Interoperability\/JavaScript interop' +) + +# --- Fail fast on unfilled placeholders or missing paths ------------------- +require_path() { + local name="$1" value="$2" + if [[ "$value" == "<"*">" ]]; then + echo "error: $name is still a placeholder ('$value') - edit this script and fill in your local path." >&2 + exit 1 + fi + if [[ ! -e "$value" ]]; then + echo "error: $name points to '$value', which does not exist." >&2 + exit 1 + fi +} +require_path DOCS_ROOT "$DOCS_ROOT" +require_path IMAGES_ZIP "$IMAGES_ZIP" +require_path STDLIB_DOCS_DIR "$STDLIB_DOCS_DIR" +require_path SOURCE_DB "$SOURCE_DB" + +WORKDIR="$(mktemp -d /tmp/adfa4737-e2e.XXXXXX)" +cleanup() { rm -rf "$WORKDIR"; } +trap cleanup EXIT + +echo "== Copying $SOURCE_DB -> $TEST_DB ==" +rm -f "$TEST_DB" +cp "$SOURCE_DB" "$TEST_DB" + +echo +echo "== Step 1/5: find_missing_assets.py (source QA report) ==" +REPORT_PATH="$WORKDIR/missing-assets-report.md" +"${UV_RUN[@]}" "$PROCESS_DIR/find_missing_assets.py" "$DOCS_ROOT" "$REPORT_PATH" +echo "Report written to $REPORT_PATH" + +echo +echo "== Step 2/5: populate_db.py (convert docs, prune blacklist, insert into test db) ==" +( cd "$PROCESS_DIR" && "${UV_RUN[@]}" populate_db.py "$DOCS_ROOT" "$CONFIG_JSON" "$IMAGES_ZIP" "$TEST_DB" \ + --blacklisted-element-titles "${BLACKLIST[@]}" ) + +echo +echo "== Step 3/5: insert_optimized_media.py (re-optimize + reinsert k/html/images/*) ==" +# --webp requires an "image/webp" ContentTypes row, which this database +# doesn't ship with (see insert_optimized_media.py's own module docstring) - +# add it (idempotent) before running. +sqlite3 "$TEST_DB" "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" + +# insert_optimized_media.py addresses images by bare filename, matching +# populate_db.py's own flat k/html/images/ convention - so its input +# media_dir needs to be a directory of files with those same basenames. +# The images actually inserted above came from IMAGES_ZIP, so extract that +# same zip here rather than pointing at DOCS_ROOT/images (the raw, unoptimized +# Writerside source tree - a different, much larger set of files). +MEDIA_DIR="$WORKDIR/media" +mkdir -p "$MEDIA_DIR" +unzip -q "$IMAGES_ZIP" -d "$MEDIA_DIR" + +"${UV_RUN[@]}" "$PROCESS_DIR/insert_optimized_media.py" "$MEDIA_DIR" "$TEST_DB" \ + --jpeg-quality "$JPEG_QUALITY" --webp --webp-quality "$WEBP_QUALITY" --verbose + +echo +echo "== Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON) ==" +# STDLIB_DOCS_DIR is .../kotlin/libraries/tools/kotlin-stdlib-docs; the +# kotlin repo root (needed to locate gradle/libs.versions.toml and to +# resolve kotlin_root inside the injected build.gradle.kts) is exactly three +# levels up, matching that build.gradle.kts's own "../../../" convention. +KOTLIN_ROOT="$(cd "$STDLIB_DOCS_DIR/../../.." && pwd)" +STDLIB_ARGS=() +[ -n "$KOTLIN_LIBS_VERSION" ] && STDLIB_ARGS+=(--kotlin-libs-version "$KOTLIN_LIBS_VERSION") +[ -n "$KOTLIN_LIBS_REPO" ] && STDLIB_ARGS+=(--kotlin-libs-repo "$KOTLIN_LIBS_REPO") +STDLIB_ALL_LIBS="$("$REPO_ROOT/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh" \ + ${STDLIB_ARGS[@]+"${STDLIB_ARGS[@]}"} "$KOTLIN_ROOT" "$WORKDIR/stdlib-json")" +echo "Generated JSON docs at $STDLIB_ALL_LIBS" + +echo +echo "== Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content) ==" +"${UV_RUN[@]}" "$SYNC_SCRIPT" "$STDLIB_ALL_LIBS" --db "$TEST_DB" + +echo +echo "== Summary ==" +"${UV_RUN[@]}" python3 - "$TEST_DB" <<'PYEOF' +import sqlite3 +import sys + +db_path = sys.argv[1] +conn = sqlite3.connect(db_path) + + +def count(where, params=()): + return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] + + +print(f"Database: {db_path}") +print(f" k/html/* rows: {count('path LIKE ?', ('k/html/%',))}") +print(f" k/html/images/* rows: {count('path LIKE ?', ('k/html/images/%',))}") +print(f" k/html/images/*.webp rows: {count('path LIKE ?', ('k/html/images/%.webp%',))}") +print(f" k/kotlin-stdlib/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-stdlib/%', 'k/kotlin-stdlib'))}") +print(f" k/kotlin-reflect/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-reflect/%', 'k/kotlin-reflect'))}") +print(f" k/kotlin-test/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-test/%', 'k/kotlin-test'))}") + +conn.close() +PYEOF + +echo +echo "== Blacklist pruning verification ==" +# Recomputes, from the same kr.tree and BLACKLIST used above, exactly which +# topic stems populate_db.py's own prune_blacklisted_elements() decided to +# exclude - then confirms none of those pages made it into the database. +# Reusing that real pruning logic (rather than guessing at path patterns) +# means this check stays correct if BLACKLIST or kr.tree's structure change. +"${UV_RUN[@]}" python3 - "$PROCESS_DIR" "$DOCS_ROOT" "$TEST_DB" "${BLACKLIST[@]}" <<'PYEOF' +import sqlite3 +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +process_dir, docs_root, db_path, *blacklist_raw = sys.argv[1:] +sys.path.insert(0, process_dir) +import populate_db # noqa: E402 + +root = ET.parse(Path(docs_root) / "kr.tree").getroot() +blacklisted_paths = {populate_db.parse_blacklist_path(raw) for raw in blacklist_raw} +blacklisted_stems, unmatched_paths = populate_db.prune_blacklisted_elements(root, blacklisted_paths) + +conn = sqlite3.connect(db_path) +leftover = [] +for stem in sorted(blacklisted_stems): + path = f"k/html/{stem}.html" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (path,)).fetchone(): + leftover.append(path) +conn.close() + +print(f"Blacklisted toc-element path(s) checked: {len(blacklisted_paths)}") +for path in sorted(blacklisted_paths): + status = "unmatched (no such element in kr.tree)" if path in unmatched_paths else "matched" + print(f" {' > '.join(path)}: {status}") +print(f"Topic page(s) expected removed: {len(blacklisted_stems)}") + +if unmatched_paths: + print(f"FAIL: {len(unmatched_paths)} blacklist path(s) never matched a - " + "check BLACKLIST against this DOCS_ROOT's kr.tree.") + sys.exit(1) +if leftover: + print(f"FAIL: {len(leftover)} blacklisted page(s) still present in the database:") + for path in leftover: + print(f" {path}") + sys.exit(1) + +print(f"PASS: all {len(blacklisted_stems)} blacklisted topic page(s) confirmed absent from {db_path}.") +PYEOF + +echo +echo "Done. Backups (populate_db.py, insert_optimized_media.py, and sync_kdoc_json_to_db.py" +echo "each make their own) live alongside $TEST_DB as documentation.test.db.backup-* and" +echo "documentation.test.db.bak.*" +echo "The real database at $SOURCE_DB was never opened for writing." diff --git a/docdb-studio/docdb_studio.py b/docdb-studio/docdb_studio.py index 3a9991d9..89268baf 100644 --- a/docdb-studio/docdb_studio.py +++ b/docdb-studio/docdb_studio.py @@ -1348,7 +1348,7 @@ def _find_brotli_cli() -> str: return path -def get_compression_dictionary(db_path: Path) -> bytes | None: +def get_compression_dictionary(db_path: Path, *, strict: bool = False) -> bytes | None: """Returns db_path's CompressionDictionary bytes (see ADFA-5153), or None if it doesn't have one yet. @@ -1357,7 +1357,15 @@ def get_compression_dictionary(db_path: Path) -> bytes | None: writes the same file; caching that as None would downgrade the whole session to plain Brotli, so imports would write plain rows into a dictionary database and reads of existing rows would fail. On any such error this returns None for this - one call and retries on the next.""" + one call and retries on the next. + + `strict=True` raises instead of returning None on that indeterminate case. + Callers that are about to *write* must pass it: a None they cannot tell apart + from "no dictionary" makes them store a plain-Brotli row in a dictionary + database, which nothing detects afterwards - the row simply fails to decode + later. Read paths can afford the lenient answer, because decoding a + dictionary row without the dictionary raises loudly rather than returning + wrong bytes.""" if db_path in _dictionary_cache: return _dictionary_cache[db_path] dictionary_data: bytes | None = None @@ -1373,6 +1381,12 @@ def get_compression_dictionary(db_path: Path) -> bytes | None: if data_row is not None: dictionary_data = data_row[0] except sqlite3.OperationalError as exc: + if strict: + raise RuntimeError( + f"could not determine whether {db_path} has a shared compression dictionary ({exc}); " + "refusing to guess, because writing a plain-Brotli row into a dictionary database " + "produces content that cannot be decoded later" + ) from exc print(f"warning: could not read {db_path}'s compression dictionary ({exc}); " f"not caching that, will retry", file=sys.stderr) return None @@ -1399,7 +1413,12 @@ def compress_for_storage(data: bytes, compression: str, db_path: Path) -> bytes: that migration. Anything else passes through unchanged.""" if compression != "brotli": return data - dictionary_data = get_compression_dictionary(db_path) + # strict: a lock-induced None here would be read as "no dictionary" and + # silently store a plain row in a dictionary database (see ADFA-5153). + # Reachable from import_content_files itself, whose phase-1 orphan DELETE + # holds a write transaction on one connection while this opens another + # against the same file. + dictionary_data = get_compression_dictionary(db_path, strict=True) if dictionary_data is None: return brotli.compress(data) dict_path = _dictionary_temp_path(db_path, dictionary_data) @@ -1442,9 +1461,27 @@ def decompress_brotli(data: bytes, db_path: Path) -> bytes: if dictionary_data is None: return brotli.decompress(data) dict_path = _dictionary_temp_path(db_path, dictionary_data) + try: + # BrotliCliMissing subclasses brotli.error, not OSError, so resolving the + # CLI inside the argv list below let it escape the except: a *plain* row + # in a dictionary database became unreadable without the CLI, even though + # brotli.decompress handles it fine and did before. Those are exactly the + # rows the fallback at the end exists for - plugin-contributed content and + # partly-migrated databases. + # + # Only that case falls through, though. If plain decoding also fails the + # row really is dictionary-compressed and the CLI really is required, so + # re-raise and let the call sites print where to get it - telling someone + # to install brotli is right there and useless on a plain row. + brotli_cli = _find_brotli_cli() + except BrotliCliMissing as cli_missing: + try: + return brotli.decompress(data) + except brotli.error: + raise cli_missing from None try: result = subprocess.run( - [_find_brotli_cli(), "-d", "-D", str(dict_path), "-c"], + [brotli_cli, "-d", "-D", str(dict_path), "-c"], input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, ) except OSError as exc: diff --git a/requirements.txt b/requirements.txt index c5fc3aa0..9d7c018c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,6 @@ brotli Pillow openpyxl>=3.1.0 tqdm-loggable>=0.1.0 +scour +cairosvg markdown-it-py>=2.0 diff --git a/run-build-kotlin-docs-with-act.sh b/run-build-kotlin-docs-with-act.sh new file mode 100755 index 00000000..0a09469f --- /dev/null +++ b/run-build-kotlin-docs-with-act.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +# Runs the "Build Kotlin Docs (Local)" GitHub Actions workflow +# (.github/workflows/build-kotlin-docs-local.yaml) locally via act +# (https://github.com/nektos/act). +# +# This drives the *local* workflow, not build-kotlin-docs.yaml. The Drive +# workflow authenticates to Google Cloud with Workload Identity Federation, +# and WIF validates the OIDC token's issuer against GitHub's own token +# endpoint for a specific repo and run - act cannot mint a token GCP will +# accept, so that workflow can never get past its auth step locally no matter +# what secrets you supply. build-kotlin-docs-local.yaml exists precisely to +# be runnable here: it reads its documentation.db / webHelpImages.zip from +# disk and writes its outputs back to disk, and is otherwise step-for-step +# identical to the Drive workflow (same find_missing_assets -> populate_db -> +# insert_optimized_media -> build-stdlib-json-docs -> sync_kdoc_json_to_db, +# same ADFA-4737 blacklist, same verification). +# +# Requires: +# - act (https://github.com/nektos/act#installation) on PATH +# - a running Docker daemon (act executes each step inside a container) +# +# Secrets: none are required. SLACK_WEBHOOK_URL is the only secret this +# workflow reads, and it is optional - the two "Notify Slack" steps print a +# skip notice and continue when it is unset. Export it if you want to see +# them actually fire ("build complete" additionally needs --live, since it is +# gated on dry_run being false). GitHub never exposes a stored secret's value +# through any API or CLI, so if you do want the real webhook you have to +# supply your own copy of the value. +# +# Inputs are host paths, bind-mounted into the job container at fixed +# locations and passed to the workflow as those in-container paths (a +# GitHub-hosted runner has no access to your disk, so the workflow only ever +# sees the mounted paths). Note this means the host paths must live somewhere +# your container runtime is allowed to share - under $HOME is safe for both +# colima and Docker Desktop; /tmp on macOS often is not. +# +# Usage: +# ./run-build-kotlin-docs-with-act.sh --db-path PATH [options] [-- ] +# +# Options: +# --db-path PATH Host path to the input documentation.db (required). +# With --live this file is overwritten in place. +# --images-zip-path PATH Host path to Writerside's webHelpImages.zip. +# Required unless --skip-website-docs. +# --output-dir PATH Host directory for outputs - the missing-assets +# report and a run-numbered copy of the built +# database. Created if absent. +# (default: ./build-kotlin-docs-output) +# --live dry_run=false: write the rebuilt database back +# over --db-path when the run finishes. Also +# required for the "build complete" Slack +# notification to fire. Default is dry_run=true. +# --skip-website-docs skip_website_docs=true (default: false) +# --skip-stdlib-docs skip_stdlib_docs=true (default: false). Skips +# cloning JetBrains/kotlin and the Dokka JSON +# build - by far the slowest part of a run, and +# the half you don't need when iterating on the +# kotlin-web-site content. +# --kotlin-web-site-ref REF kotlin_web_site_ref input (default: '') +# --kotlin-ref REF kotlin_ref input (default: '') +# --kotlin-libs-version V Version of the published kotlin-stdlib/-reflect/ +# -test artifacts to document. Defaults to the +# workflow's own default; pass '' to fall back to +# the kotlin checkout's snapshot version, which +# only resolves if you built the kotlin repo. +# --kotlin-libs-repo URL Maven repo to resolve them from (default: '', +# i.e. mavenCentral, which is enough for a +# released --kotlin-libs-version). +# +# Every workflow input is passed explicitly on every run, including the ones +# whose YAML "default:" would cover them. act does not apply +# workflow_dispatch input defaults - an input you don't pass arrives empty - +# and for dry_run that inverts the intended behaviour: "${{ !inputs.dry_run }}" +# on an empty value is true, so the step that writes the database back over +# --db-path would run. Passing all of them keeps a local run's semantics +# identical to a real dispatch. +# +# On Apple Silicon act warns about container architecture; append +# `-- --container-architecture linux/arm64` if you want to silence it (the +# default works). +set -euo pipefail + +if ! command -v act >/dev/null 2>&1; then + echo "error: act is required - see https://github.com/nektos/act#installation" >&2 + exit 1 +fi + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKFLOW="$REPO_ROOT/.github/workflows/build-kotlin-docs-local.yaml" + +# Where the host paths below get bind-mounted inside the job container, and +# therefore what the workflow itself is told its inputs are. +CONTAINER_DB_PATH="/mnt/act-inputs/documentation.db" +CONTAINER_IMAGES_ZIP_PATH="/mnt/act-inputs/webHelpImages.zip" +CONTAINER_OUTPUT_DIR="/mnt/act-output" + +DB_PATH="" +IMAGES_ZIP_PATH="" +OUTPUT_DIR="$REPO_ROOT/build-kotlin-docs-output" +KOTLIN_WEB_SITE_REF="" +KOTLIN_REF="" +# Mirrors build-kotlin-docs-local.yaml's own default. Restated here because act +# does not apply workflow_dispatch defaults (see the note above); passing the +# input unconditionally is what keeps a local run equivalent to a real one. +KOTLIN_LIBS_VERSION="2.4.10" +KOTLIN_LIBS_REPO="" +SKIP_WEBSITE_DOCS="false" +SKIP_STDLIB_DOCS="false" +DRY_RUN="true" + +EXTRA_ACT_ARGS=() + +while [ $# -gt 0 ]; do + case "$1" in + --db-path) DB_PATH="$2"; shift 2 ;; + --images-zip-path) IMAGES_ZIP_PATH="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + --live) DRY_RUN="false"; shift ;; + --skip-website-docs) SKIP_WEBSITE_DOCS="true"; shift ;; + --skip-stdlib-docs) SKIP_STDLIB_DOCS="true"; shift ;; + --kotlin-web-site-ref) KOTLIN_WEB_SITE_REF="$2"; shift 2 ;; + --kotlin-ref) KOTLIN_REF="$2"; shift 2 ;; + --kotlin-libs-version) KOTLIN_LIBS_VERSION="$2"; shift 2 ;; + --kotlin-libs-repo) KOTLIN_LIBS_REPO="$2"; shift 2 ;; + --) shift; EXTRA_ACT_ARGS+=("$@"); break ;; + *) echo "error: unrecognized argument '$1'" >&2; exit 1 ;; + esac +done + +if [ "$SKIP_WEBSITE_DOCS" = "true" ] && [ "$SKIP_STDLIB_DOCS" = "true" ]; then + echo "error: --skip-website-docs and --skip-stdlib-docs together skip every step that" >&2 + echo "error: changes the database, leaving nothing for the run to do." >&2 + exit 1 +fi + +if [ -z "$DB_PATH" ]; then + echo "error: --db-path is required (host path to the documentation.db to build against)" >&2 + exit 1 +fi +if [ ! -f "$DB_PATH" ]; then + echo "error: --db-path '$DB_PATH' does not exist or is not a file" >&2 + exit 1 +fi +DB_PATH="$(cd "$(dirname "$DB_PATH")" && pwd)/$(basename "$DB_PATH")" + +if [ "$SKIP_WEBSITE_DOCS" != "true" ]; then + if [ -z "$IMAGES_ZIP_PATH" ]; then + echo "error: --images-zip-path is required unless --skip-website-docs is passed." >&2 + echo "error: Writerside's webHelpImages.zip is only produced by IntelliJ IDEA's" >&2 + echo "error: Writerside plugin - there is no headless way to generate it. See the" >&2 + echo "error: KNOWN LIMITATION note at the top of $WORKFLOW." >&2 + exit 1 + fi + if [ ! -f "$IMAGES_ZIP_PATH" ]; then + echo "error: --images-zip-path '$IMAGES_ZIP_PATH' does not exist or is not a file" >&2 + exit 1 + fi + IMAGES_ZIP_PATH="$(cd "$(dirname "$IMAGES_ZIP_PATH")" && pwd)/$(basename "$IMAGES_ZIP_PATH")" +fi + +mkdir -p "$OUTPUT_DIR" +OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)" + +# One -v per input. Mounting the files individually (rather than their parent +# directories) keeps the container's view to exactly what the run needs, and +# lets --db-path and --images-zip-path live in unrelated places on the host. +# +# act takes --container-options as one string and splits it with shell-style +# quoting rules, so each mount spec is emitted double-quoted: an unquoted +# join would break the moment a host path contained a space. +CONTAINER_OPTIONS="" +add_mount() { CONTAINER_OPTIONS+=" -v \"$1:$2\""; } +add_mount "$DB_PATH" "$CONTAINER_DB_PATH" +add_mount "$OUTPUT_DIR" "$CONTAINER_OUTPUT_DIR" +WORKFLOW_IMAGES_ZIP_PATH="" +if [ "$SKIP_WEBSITE_DOCS" != "true" ]; then + add_mount "$IMAGES_ZIP_PATH" "$CONTAINER_IMAGES_ZIP_PATH" + WORKFLOW_IMAGES_ZIP_PATH="$CONTAINER_IMAGES_ZIP_PATH" +fi + +if [ "$DRY_RUN" = "true" ]; then + echo "note: dry_run=true - '$DB_PATH' will NOT be modified; the built database is" >&2 + echo "note: written to '$OUTPUT_DIR' only. The 'build started' Slack notification" >&2 + echo "note: still fires (if SLACK_WEBHOOK_URL is set) but 'build complete' is gated" >&2 + echo "note: on dry_run=false. Pass --live to write back and see it." >&2 +else + echo "WARNING: --live - '$DB_PATH' will be OVERWRITTEN in place when the run finishes." >&2 +fi + +# The workflow reads SLACK_WEBHOOK_URL and tolerates it being unset, so pass +# it through when it's in the environment and stay silent when it isn't. +# +# SECRET_ARGS and EXTRA_ACT_ARGS are expanded below as +# ${arr[@]+"${arr[@]}"} rather than plain "${arr[@]}": macOS still ships bash +# 3.2, where `set -u` treats an empty array's "${arr[@]}" as an unbound +# variable and aborts. Both arrays are empty on a normal run. +SECRET_ARGS=() +if [ -n "${SLACK_WEBHOOK_URL:-}" ]; then + SECRETS_FILE="$(mktemp)" + trap 'rm -f "$SECRETS_FILE"' EXIT + printf 'SLACK_WEBHOOK_URL=%s\n' "$SLACK_WEBHOOK_URL" > "$SECRETS_FILE" + SECRET_ARGS=(--secret-file "$SECRETS_FILE") +fi + +echo "== Running $WORKFLOW via act ==" +echo " db_path $DB_PATH -> $CONTAINER_DB_PATH" +echo " images_zip_path ${IMAGES_ZIP_PATH:-(skipped)}${IMAGES_ZIP_PATH:+ -> $CONTAINER_IMAGES_ZIP_PATH}" +echo " output_dir $OUTPUT_DIR -> $CONTAINER_OUTPUT_DIR" +echo " dry_run=$DRY_RUN skip_website_docs=$SKIP_WEBSITE_DOCS skip_stdlib_docs=$SKIP_STDLIB_DOCS" +if [ "$SKIP_STDLIB_DOCS" != "true" ]; then + echo " stdlib artifacts ${KOTLIN_LIBS_VERSION:-(kotlin checkout default)} from ${KOTLIN_LIBS_REPO:-(mavenCentral)}" +fi + +# --container-daemon-socket - : act otherwise bind-mounts the host's Docker +# socket into the job container so steps can run Docker themselves. Nothing in +# this workflow does, and the mount outright fails on runtimes whose socket +# isn't a plain bind-mountable file - under colima it aborts the run with +# "error while creating mount source path ...: operation not supported". +act workflow_dispatch \ + -W "$WORKFLOW" \ + -P ubuntu-latest=catthehacker/ubuntu:act-latest \ + --container-daemon-socket - \ + --container-options "$CONTAINER_OPTIONS" \ + --input db_path="$CONTAINER_DB_PATH" \ + --input images_zip_path="$WORKFLOW_IMAGES_ZIP_PATH" \ + --input output_dir="$CONTAINER_OUTPUT_DIR" \ + --input kotlin_web_site_ref="$KOTLIN_WEB_SITE_REF" \ + --input kotlin_ref="$KOTLIN_REF" \ + --input kotlin_libs_version="$KOTLIN_LIBS_VERSION" \ + --input kotlin_libs_repo="$KOTLIN_LIBS_REPO" \ + --input skip_website_docs="$SKIP_WEBSITE_DOCS" \ + --input skip_stdlib_docs="$SKIP_STDLIB_DOCS" \ + --input dry_run="$DRY_RUN" \ + ${SECRET_ARGS[@]+"${SECRET_ARGS[@]}"} \ + ${EXTRA_ACT_ARGS[@]+"${EXTRA_ACT_ARGS[@]}"} diff --git a/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py b/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py index d1585de6..c50818c3 100755 --- a/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py +++ b/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py @@ -11,19 +11,35 @@ - If that file exists, re-compress it (matching the row's existing ContentTypes.compression) and overwrite the row's `content` blob only -- `path`, `languageID`, `contentTypeID`, and `templateId` are left untouched. - - If it doesn't exist, delete the row. - -Any TooltipButtons row whose `uri` (ignoring a trailing "#fragment") matches one -of the deleted Content paths is now a dead link. Its entire parent Tooltips + A result too large for a single row is split into "-1", "-2", + ... continuation rows instead (see CHUNK_SIZE), the same fragmentation + populate_db.py writes and WebServer.kt reads back; that case replaces the + base row rather than updating it in place, so its `id` changes, but the + four columns above still carry over unchanged. + - If it doesn't exist, delete the row (and any continuation rows it owns). + +Existing "-N" continuation rows are not treated as pages of their own: +they belong to their base row and are rewritten or removed along with it. + +Any TooltipButtons row whose `uri` (ignoring a "?query" and/or "#fragment", +matching docdb-studio's own URI normalizer) matches one of the deleted Content +paths is now a dead link. Its entire parent Tooltips record -- along with all of that tooltip's other TooltipButtons rows, dead or not -- is deleted too, since TooltipButtons has no ON DELETE CASCADE and a dangling tooltipId would otherwise be left behind. -A timestamped backup of the database is made before anything is modified. +A timestamped backup of the database is made before anything is modified - taken +after the prechecks, so a run that refuses to proceed doesn't leave one behind. +The database is VACUUMed at the end to reclaim the space freed by the rewrite. + +Pages present in the plugin output with no existing Content row are reported but +not inserted: this script only ever updates or deletes rows it found in the +database. """ import argparse import atexit import os +import re import shutil import sqlite3 import subprocess @@ -41,14 +57,107 @@ # and the only signal would be "Done: updated 0, deleted N" on a gutted database. MAX_DELETE_FRACTION = 0.5 +# Must match WebServer.kt's "contentChunkSize" (1024 * 1024) and +# populate_db.py's CHUNK_SIZE exactly. The server decides a row is fragmented +# purely by its content being exactly this many bytes, then keeps requesting +# "-1", "-2", ... until it gets a shorter fragment or a missing +# row - so anything written here that exceeds it has to be split the same way +# populate_db.py splits it, or the server will serve a truncated page. +CHUNK_SIZE = 1024 * 1024 + def backup_database(db_path): + """Writes a timestamped backup beside db_path. Uses SQLite's own VACUUM + INTO rather than a file copy: it takes a read transaction for the + duration, so the result is always an internally consistent database even + if something else is mid-write (a plain copy of a live, or WAL-mode, + database can be torn). Same approach populate_db.py uses.""" timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") backup_path = f"{db_path}.bak.{timestamp}" - shutil.copy2(db_path, backup_path) + conn = sqlite3.connect(db_path) + try: + conn.execute("VACUUM INTO ?", (backup_path,)) + finally: + conn.close() return backup_path +def is_fragment_path(path, all_paths): + """True when path is a "-" chunk continuation row whose base is + also present - the fragmentation convention populate_db.py writes and + WebServer.kt reads. Ambiguous only for a real page literally named + "-", which the plugin output never produces.""" + base, sep, suffix = path.rpartition("-") + return sep == "-" and suffix.isdigit() and base in all_paths + + +FRAGMENT_SUFFIX_RE = re.compile(r"^(.*)-(\d+)$") + + +def fragment_paths(conn, path): + """Every "-" continuation row present, ordered by N. + + Deliberately mirrors populate_db.fragment_chain, including why it works + this way. Probing "-1" and stopping at the first gap misses a chain + numbered from -2 (the ADFA-5171 case) and silently leaves those rows + behind. The LIKE pattern instead over-matches on purpose - "_" is a + single-character wildcard and "-%" doesn't constrain the tail to digits - + and the regex re-check below is what makes the result exact. Never build + a DELETE straight off that pattern: deleting a row that merely resembles + a continuation is permanent.""" + chain = [] + for (candidate,) in conn.execute("SELECT path FROM Content WHERE path LIKE ?", (f"{path}-%",)).fetchall(): + match = FRAGMENT_SUFFIX_RE.match(candidate) + if match and match.group(1) == path: + chain.append((int(match.group(2)), candidate)) + chain.sort(key=lambda item: item[0]) + return [candidate for _number, candidate in chain] + + +def delete_content_with_fragments(cur, content_id, path): + """Deletes a Content row along with any chunk continuation rows it owns.""" + cur.execute("DELETE FROM Content WHERE id = ?", (content_id,)) + for fragment_path in fragment_paths(cur, path): + cur.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) + + +def write_content(cur, content_id, path, blob, language_id, content_type_id, template_id, chunked_log): + """Replaces the content stored at `path` with `blob`, honouring the + CHUNK_SIZE fragmentation contract. + + The common case (blob fits in one row) is an in-place UPDATE, which keeps + the row's id stable; any stale fragments left over from a previous, + larger version of the page are removed. An oversized blob can't be stored + that way at all - the server would only ever serve the first row - so the + row is replaced by a fresh base row plus "-1", "-2", ... + continuations, each carrying the original row's languageID/contentTypeID/ + templateId. Appends (path, total size, chunk count) to chunked_log for + anything that needed more than one row.""" + stale = fragment_paths(cur, path) + + if len(blob) <= CHUNK_SIZE: + cur.execute("UPDATE Content SET content = ? WHERE id = ?", (blob, content_id)) + for fragment_path in stale: + cur.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) + return + + cur.execute("DELETE FROM Content WHERE id = ?", (content_id,)) + for fragment_path in stale: + cur.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) + + insert = ("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "VALUES (?, ?, ?, ?, ?)") + cur.execute(insert, (path, language_id, blob[:CHUNK_SIZE], content_type_id, template_id)) + fragment_number = 1 + offset = CHUNK_SIZE + while offset < len(blob): + cur.execute(insert, (f"{path}-{fragment_number}", language_id, blob[offset:offset + CHUNK_SIZE], + content_type_id, template_id)) + offset += CHUNK_SIZE + fragment_number += 1 + chunked_log.append((path, len(blob), fragment_number)) # fragment_number == total chunk count here + + def relative_target_path(content_path): """'k/kotlin-stdlib/kotlin.text/index.html' -> 'kotlin-stdlib/kotlin.text/index.json' 'k/kotlin-stdlib/package-list' -> 'kotlin-stdlib/package-list' (no extension to swap)""" @@ -111,6 +220,49 @@ def compress_for(compression, raw_bytes, path, compressor=None): raise ValueError(f"Unknown compression '{compression}' needed for {path}") +def content_path_for_source(plugin_output_root, source_file): + """Inverse of relative_target_path: the Content.path a Dokka output file + would populate. '/kotlin-stdlib/kotlin.text/index.json' -> + 'k/kotlin-stdlib/kotlin.text/index.html'.""" + rel = os.path.relpath(source_file, plugin_output_root) + rel = rel.replace(os.sep, "/") + if rel.endswith(".json"): + rel = rel[: -len(".json")] + ".html" + return "k/" + rel + + +def unmatched_source_pages(plugin_output_root, known_paths): + """Dokka output files that map to no existing Content row, sorted. + + This script can only UPDATE or DELETE: its loop iterates rows read from the + database, so a page Dokka newly emits has nothing to match and is silently + dropped. Bumping --kotlin-libs-version to a release that adds stdlib API is + exactly that case, and the refreshed pages link to those missing pages, so + every such link 404s in the app. Reporting them is the minimum; actually + inserting them needs contentTypeID/templateId decisions this script has no + basis to make on its own.""" + found = [] + for dirpath, _dirnames, filenames in os.walk(plugin_output_root): + for name in filenames: + if not (name.endswith(".json") or name == "package-list"): + continue + candidate = content_path_for_source(plugin_output_root, os.path.join(dirpath, name)) + if candidate not in known_paths: + found.append(candidate) + return sorted(found) + + +def _content_path_for_uri(uri): + """The Content.path a TooltipButtons.uri addresses: everything before the + first '?' or '#'. Mirrors docdb-studio's _uri_path_for_content_lookup.""" + u = uri or "" + if "?" in u: + u = u.split("?", 1)[0] + if "#" in u: + u = u.split("#", 1)[0] + return u + + def cleanup_orphaned_tooltips(cur, deleted_paths, dry_run): """Delete any Tooltips (and all their TooltipButtons) that reference a now-deleted Content path via a TooltipButtons.uri. Returns (tooltips_removed, @@ -129,8 +281,14 @@ def cleanup_orphaned_tooltips(cur, deleted_paths, dry_run): f"SELECT tooltipId, uri FROM TooltipButtons WHERE {where_clause}", params ).fetchall() + # Strip ?query as well as #fragment, matching this repo's canonical + # normalizer (docdb-studio's _uri_path_for_content_lookup). Splitting on + # "#" alone left a "...html?v=2" button pointing at a row this run just + # deleted - exactly the dangling state this cleanup exists to prevent, and + # one docdb-studio's "Validate URIs" audit then flags. orphaned_tooltip_ids = sorted( - {tooltip_id for tooltip_id, uri in candidate_buttons if uri.split("#", 1)[0] in deleted_path_set} + {tooltip_id for tooltip_id, uri in candidate_buttons + if _content_path_for_uri(uri) in deleted_path_set} ) if not orphaned_tooltip_ids: return 0, 0 @@ -170,9 +328,6 @@ def main(): if args.dry_run: print("Dry run: no backup will be made and no changes will be written.") - else: - backup_path = backup_database(args.db) - print(f"Backed up database to: {backup_path}") conn = sqlite3.connect(args.db) cur = conn.cursor() @@ -184,16 +339,27 @@ def main(): for prefix in PREFIXES: params.extend([prefix, prefix + "/%"]) - rows = cur.execute( - f"SELECT id, path, contentTypeID FROM Content WHERE {where_clause}", params + all_rows = cur.execute( + f"SELECT id, path, contentTypeID, languageID, templateId FROM Content WHERE {where_clause}", params ).fetchall() - print(f"Found {len(rows)} existing Content record(s) under {PREFIXES}.") + # A chunked page is stored as a base row plus "-1", "-2", ... + # continuation rows (see CHUNK_SIZE). Those fragments are part of their + # base row's content, not pages in their own right - handled wholesale by + # write_content below - so drop them from the work list. Left in, each + # would be looked up as its own source file, never found (there's no + # "index.html-1" in the plugin output), and counted as a deletion. + all_paths = {row[1] for row in all_rows} + rows = [row for row in all_rows if not is_fragment_path(row[1], all_paths)] + fragments = len(all_rows) - len(rows) + + print(f"Found {len(rows)} existing Content record(s) under {PREFIXES}" + f"{f' (plus {fragments} chunk continuation row(s))' if fragments else ''}.") updated = 0 deleted = 0 deleted_paths = [] - unknown_types = set() + chunked_log = [] dictionary_data = load_compression_dictionary(conn) compressor = DictionaryBrotli(dictionary_data) if dictionary_data else None @@ -206,8 +372,8 @@ def main(): # Resolve every source file before touching anything, so a wholesale miss # aborts instead of deleting the rows one at a time (see MAX_DELETE_FRACTION). - missing = [path for _id, path, _type in rows - if not os.path.isfile(os.path.join(args.plugin_output_root, relative_target_path(path)))] + missing = [row[1] for row in rows + if not os.path.isfile(os.path.join(args.plugin_output_root, relative_target_path(row[1])))] if rows and len(missing) >= max(1, int(len(rows) * MAX_DELETE_FRACTION)): print( f"error: {len(missing)} of {len(rows)} matched Content rows resolve to no file under " @@ -219,9 +385,27 @@ def main(): conn.close() sys.exit(1) + # Reported, not inserted - see unmatched_source_pages. Printed before the + # transaction so it shows up even on a dry run. + unmatched = unmatched_source_pages(args.plugin_output_root, {row[0] for row in all_rows}) + if unmatched: + print( + f"warning: {len(unmatched)} page(s) in {args.plugin_output_root!r} have no Content row and " + f"will NOT be inserted; links to them will 404. Examples: {', '.join(unmatched[:3])}", + file=sys.stderr, + ) + + # Backed up only once every check that can still refuse to run has passed - + # the MAX_DELETE_FRACTION precheck above is the last of them. Taking it + # earlier meant a run that correctly aborted on a layout mismatch still left + # a full-size copy of the database behind, for nothing. + if not args.dry_run: + backup_path = backup_database(args.db) + print(f"Backed up database to: {backup_path}") + try: conn.execute("BEGIN") - for content_id, path, content_type_id in rows: + for content_id, path, content_type_id, language_id, template_id in rows: rel_target = relative_target_path(path) source_file = os.path.join(args.plugin_output_root, rel_target) @@ -229,34 +413,44 @@ def main(): with open(source_file, "rb") as f: raw_bytes = f.read() + # An unresolvable contentTypeID means this row's declared + # type isn't in ContentTypes at all, so there's no way to + # know whether the server will try to decompress what gets + # written here. Guessing "uncompressed" and committing anyway + # is how a row ends up serving bytes that contradict its own + # declared type - fail instead. compression = compression_by_type.get(content_type_id) if compression is None: - unknown_types.add(content_type_id) - compression = "none" + raise RuntimeError( + f"{path} has contentTypeID {content_type_id}, which has no row in ContentTypes; " + "cannot tell how its content should be compressed. Fix the database's ContentTypes " + "table (or this row's contentTypeID) and re-run." + ) new_blob = compress_for(compression, raw_bytes, path, compressor) if args.dry_run: - print(f" [UPDATE] {path} <- {rel_target}") + chunks = -(-len(new_blob) // CHUNK_SIZE) or 1 + print(f" [UPDATE] {path} <- {rel_target}" + f"{f' ({len(new_blob):,} bytes -> {chunks} chunks)' if chunks > 1 else ''}") else: - cur.execute("UPDATE Content SET content = ? WHERE id = ?", (new_blob, content_id)) + write_content(cur, content_id, path, new_blob, language_id, content_type_id, template_id, + chunked_log) updated += 1 else: if args.dry_run: print(f" [DELETE] {path} (no matching {rel_target})") else: - cur.execute("DELETE FROM Content WHERE id = ?", (content_id,)) + delete_content_with_fragments(cur, content_id, path) deleted += 1 deleted_paths.append(path) tooltips_removed, buttons_removed = cleanup_orphaned_tooltips(cur, deleted_paths, args.dry_run) - if unknown_types: - print( - f"WARNING: contentTypeID(s) {sorted(unknown_types)} not found in ContentTypes; " - "treated as uncompressed.", - file=sys.stderr, - ) + if chunked_log: + print(f"Chunked {len(chunked_log)} file(s) over {CHUNK_SIZE:,} bytes:") + for path, total_size, chunk_count in chunked_log: + print(f" {path}: {total_size:,} bytes -> {chunk_count} chunks") if args.dry_run: conn.rollback() @@ -277,6 +471,22 @@ def main(): finally: conn.close() + # SQLite only ever moves freed pages onto its internal freelist; the file + # itself never shrinks. This script rewrites every k/kotlin-stdlib* blob and + # deletes Content, Tooltips and TooltipButtons rows, so a run that replaces + # large pages with smaller ones leaves the difference as dead space. It is + # step 5/5 of the pipeline, so nothing downstream reclaims it and the bloat + # ships in the on-device database. Same trailing VACUUM as populate_db.py, + # insert_optimized_media.py and renumber_misnumbered_fragments.py, on its + # own connection because SQLite refuses to VACUUM inside a transaction. + if not args.dry_run: + print("Vacuuming database to reclaim freed space...") + vacuum_conn = sqlite3.connect(args.db) + try: + vacuum_conn.execute("VACUUM") + finally: + vacuum_conn.close() + if __name__ == "__main__": main() diff --git a/scripts/sync_kotlin_stdlib_docs/test_sync_kdoc_json_to_db.py b/scripts/sync_kotlin_stdlib_docs/test_sync_kdoc_json_to_db.py new file mode 100644 index 00000000..c5c18cf8 --- /dev/null +++ b/scripts/sync_kotlin_stdlib_docs/test_sync_kdoc_json_to_db.py @@ -0,0 +1,269 @@ +"""Regression tests for sync_kdoc_json_to_db.py. + +Covers the ways this script could write content the server cannot read back: +ignoring the CHUNK_SIZE fragmentation contract, over-matching when deleting +continuation rows, and guessing at a compression policy it can't determine. +Dictionary compression itself (ADFA-5153) is exercised end-to-end here too, +since a plain-Brotli row in a dictionary database is unreadable. +""" +import shutil +import sqlite3 +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from sync_kdoc_json_to_db import ( # noqa: E402 + CHUNK_SIZE, + DictionaryBrotli, + backup_database, + compress_for, + delete_content_with_fragments, + fragment_paths, + is_fragment_path, + load_compression_dictionary, + relative_target_path, + write_content, +) + +SCHEMA = """ +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, + compression TEXT NOT NULL); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + languageID INTEGER NOT NULL, + content BLOB NOT NULL, + contentTypeID INTEGER NOT NULL, + templateId INTEGER NOT NULL DEFAULT 0, + UNIQUE(path) +); +""" + +HTML_TYPE_ID = 12 +needs_brotli_cli = pytest.mark.skipif(shutil.which("brotli") is None, reason="brotli CLI not installed") + + +@pytest.fixture +def conn(): + connection = sqlite3.connect(":memory:") + connection.executescript(SCHEMA) + connection.execute("INSERT INTO ContentTypes (id, value, compression) VALUES (?, 'text/html', 'brotli')", + (HTML_TYPE_ID,)) + yield connection + connection.close() + + +def add_row(conn, path, blob=b"old", template_id=7, language_id=1): + return conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + (path, language_id, blob, HTML_TYPE_ID, template_id), + ).lastrowid + + +def rows(conn): + return dict(conn.execute("SELECT path, content FROM Content")) + + +class TestRelativeTargetPath: + def test_html_becomes_json(self): + assert relative_target_path("k/kotlin-stdlib/kotlin.text/index.html") == "kotlin-stdlib/kotlin.text/index.json" + + def test_extensionless_path_is_unchanged(self): + assert relative_target_path("k/kotlin-stdlib/package-list") == "kotlin-stdlib/package-list" + + +class TestIsFragmentPath: + def test_recognises_a_continuation_row(self): + assert is_fragment_path("k/kotlin-stdlib/x.html-1", {"k/kotlin-stdlib/x.html", "k/kotlin-stdlib/x.html-1"}) + + def test_a_base_row_is_not_a_fragment(self): + assert not is_fragment_path("k/kotlin-stdlib/x.html", {"k/kotlin-stdlib/x.html"}) + + def test_trailing_digits_without_a_base_are_not_a_fragment(self): + assert not is_fragment_path("k/kotlin-stdlib/part-2", {"k/kotlin-stdlib/part-2"}) + + def test_non_numeric_suffix_is_not_a_fragment(self): + assert not is_fragment_path("k/kotlin-stdlib/all-types", {"k/kotlin-stdlib/all", "k/kotlin-stdlib/all-types"}) + + +class TestFragmentPaths: + def test_finds_the_chain_in_order(self, conn): + add_row(conn, "k/kotlin-stdlib/x.html") + for n in (2, 1, 3): + add_row(conn, f"k/kotlin-stdlib/x.html-{n}") + assert fragment_paths(conn, "k/kotlin-stdlib/x.html") == [ + "k/kotlin-stdlib/x.html-1", "k/kotlin-stdlib/x.html-2", "k/kotlin-stdlib/x.html-3", + ] + + def test_finds_a_chain_that_starts_at_two(self, conn): + # ADFA-5171: probing "-1" first and stopping at the gap would miss + # these entirely and leave them behind as orphans. + add_row(conn, "k/kotlin-stdlib/x.html") + add_row(conn, "k/kotlin-stdlib/x.html-2") + add_row(conn, "k/kotlin-stdlib/x.html-3") + assert fragment_paths(conn, "k/kotlin-stdlib/x.html") == [ + "k/kotlin-stdlib/x.html-2", "k/kotlin-stdlib/x.html-3", + ] + + def test_underscore_in_a_path_is_not_treated_as_a_wildcard(self, conn): + add_row(conn, "k/kotlin-stdlib/a_b.html") + add_row(conn, "k/kotlin-stdlib/a_b.html-1") + add_row(conn, "k/kotlin-stdlib/aXb.html-1") + assert fragment_paths(conn, "k/kotlin-stdlib/a_b.html") == ["k/kotlin-stdlib/a_b.html-1"] + + def test_lookalike_suffixes_are_excluded(self, conn): + add_row(conn, "k/kotlin-stdlib/x.html") + add_row(conn, "k/kotlin-stdlib/x.html-notanumber") + assert fragment_paths(conn, "k/kotlin-stdlib/x.html") == [] + + +class TestWriteContent: + def test_small_blob_updates_in_place_and_keeps_the_row_id(self, conn): + row_id = add_row(conn, "k/kotlin-stdlib/x.html") + write_content(conn, row_id, "k/kotlin-stdlib/x.html", b"new", 1, HTML_TYPE_ID, 7, []) + assert rows(conn) == {"k/kotlin-stdlib/x.html": b"new"} + assert conn.execute("SELECT id FROM Content").fetchone()[0] == row_id + + def test_small_blob_clears_stale_fragments_from_a_previous_larger_version(self, conn): + row_id = add_row(conn, "k/kotlin-stdlib/x.html") + add_row(conn, "k/kotlin-stdlib/x.html-1") + add_row(conn, "k/kotlin-stdlib/x.html-2") + write_content(conn, row_id, "k/kotlin-stdlib/x.html", b"new", 1, HTML_TYPE_ID, 7, []) + assert set(rows(conn)) == {"k/kotlin-stdlib/x.html"} + + def test_oversized_blob_is_split_across_continuation_rows(self, conn): + row_id = add_row(conn, "k/kotlin-stdlib/big.html") + blob = b"z" * (CHUNK_SIZE * 2 + 17) + chunked_log = [] + write_content(conn, row_id, "k/kotlin-stdlib/big.html", blob, 1, HTML_TYPE_ID, 7, chunked_log) + + stored = rows(conn) + assert set(stored) == {"k/kotlin-stdlib/big.html", "k/kotlin-stdlib/big.html-1", "k/kotlin-stdlib/big.html-2"} + # The server detects fragmentation by the base row being exactly + # CHUNK_SIZE, then reads on until a short row. + assert len(stored["k/kotlin-stdlib/big.html"]) == CHUNK_SIZE + assert len(stored["k/kotlin-stdlib/big.html-1"]) == CHUNK_SIZE + assert len(stored["k/kotlin-stdlib/big.html-2"]) == 17 + assert (stored["k/kotlin-stdlib/big.html"] + stored["k/kotlin-stdlib/big.html-1"] + + stored["k/kotlin-stdlib/big.html-2"]) == blob + assert chunked_log == [("k/kotlin-stdlib/big.html", len(blob), 3)] + + def test_fragments_inherit_the_base_row_columns(self, conn): + row_id = add_row(conn, "k/kotlin-stdlib/big.html", template_id=9) + write_content(conn, row_id, "k/kotlin-stdlib/big.html", b"z" * (CHUNK_SIZE + 1), 1, HTML_TYPE_ID, 9, []) + for language_id, content_type_id, template_id in conn.execute( + "SELECT languageID, contentTypeID, templateId FROM Content" + ): + assert (language_id, content_type_id, template_id) == (1, HTML_TYPE_ID, 9) + + def test_exactly_chunk_size_stays_a_single_row(self, conn): + row_id = add_row(conn, "k/kotlin-stdlib/x.html") + write_content(conn, row_id, "k/kotlin-stdlib/x.html", b"z" * CHUNK_SIZE, 1, HTML_TYPE_ID, 7, []) + assert set(rows(conn)) == {"k/kotlin-stdlib/x.html"} + + +class TestDeleteContentWithFragments: + def test_removes_the_base_row_and_its_fragments_only(self, conn): + row_id = add_row(conn, "k/kotlin-stdlib/x.html") + add_row(conn, "k/kotlin-stdlib/x.html-1") + add_row(conn, "k/kotlin-stdlib/y.html") + delete_content_with_fragments(conn, row_id, "k/kotlin-stdlib/x.html") + assert set(rows(conn)) == {"k/kotlin-stdlib/y.html"} + + def test_does_not_delete_lookalike_rows(self, conn): + row_id = add_row(conn, "k/kotlin-stdlib/a_b.html") + add_row(conn, "k/kotlin-stdlib/a_b.html-1") + add_row(conn, "k/kotlin-stdlib/aXb.html-1") + delete_content_with_fragments(conn, row_id, "k/kotlin-stdlib/a_b.html") + assert set(rows(conn)) == {"k/kotlin-stdlib/aXb.html-1"} + + +class TestCompressFor: + def test_unknown_compression_is_an_error(self): + with pytest.raises(ValueError, match="Unknown compression"): + compress_for("lzma", b"data", "k/kotlin-stdlib/x.html") + + def test_none_passes_bytes_through(self): + assert compress_for("none", b"data", "p") == b"data" + + def test_brotli_without_a_dictionary_is_plain_brotli(self): + import brotli + assert brotli.decompress(compress_for("brotli", b"data" * 50, "p")) == b"data" * 50 + + +class TestLoadCompressionDictionary: + def test_returns_none_when_the_table_predates_schema_2(self, conn): + assert load_compression_dictionary(conn) is None + + def test_returns_none_for_an_empty_dictionary_table(self, conn): + conn.execute("CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL)") + assert load_compression_dictionary(conn) is None + + def test_returns_the_stored_bytes(self, conn): + conn.execute("CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL)") + conn.execute("INSERT INTO CompressionDictionary (id, data) VALUES (1, ?)", (b"dictionary-bytes",)) + assert load_compression_dictionary(conn) == b"dictionary-bytes" + + +@needs_brotli_cli +class TestDictionaryBrotli: + """DictionaryBrotli only compresses - this script never reads content back - + so these decode through the `brotli` CLI directly, the same way the server + ultimately does.""" + + DICTIONARY = bytes(range(256)) * 64 + + @staticmethod + def cli_decompress(blob, dictionary, tmp_path): + dict_path = tmp_path / "dictionary.bin" + dict_path.write_bytes(dictionary) + import subprocess + result = subprocess.run( + [shutil.which("brotli"), "-d", "-D", str(dict_path), "-c"], + input=blob, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + assert result.returncode == 0, result.stderr.decode(errors="replace") + return result.stdout + + def test_round_trips_through_the_dictionary(self, tmp_path): + payload = b'{"id":"k/kotlin-stdlib/x","blocks":[]}' * 20 + blob = DictionaryBrotli(self.DICTIONARY).compress(payload) + assert self.cli_decompress(blob, self.DICTIONARY, tmp_path) == payload + + def test_dictionary_output_is_not_readable_as_plain_brotli(self): + # The whole reason this matters: the two encodings are not + # interchangeable, so writing plain Brotli into a dictionary database + # produces rows the server cannot decode. + import brotli + blob = DictionaryBrotli(self.DICTIONARY).compress(b"kotlin stdlib documentation payload" * 40) + with pytest.raises(Exception): + brotli.decompress(blob) + + def test_compress_for_uses_the_dictionary_when_given_one(self, tmp_path): + compressor = DictionaryBrotli(self.DICTIONARY) + payload = b"payload" * 100 + blob = compress_for("brotli", payload, "p", compressor) + assert self.cli_decompress(blob, self.DICTIONARY, tmp_path) == payload + + +class TestBackupDatabase: + def test_backup_is_a_readable_database_not_a_file_copy(self, tmp_path): + db_path = tmp_path / "documentation.db" + setup = sqlite3.connect(db_path) + setup.executescript(SCHEMA) + setup.execute("INSERT INTO ContentTypes (value, compression) VALUES ('text/html', 'brotli')") + setup.commit() + setup.close() + + backup_path = backup_database(str(db_path)) + + assert Path(backup_path).is_file() + restored = sqlite3.connect(backup_path) + try: + assert restored.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + assert restored.execute("SELECT value FROM ContentTypes").fetchone()[0] == "text/html" + finally: + restored.close()