From 3056fabbd887a2d91fd4ee1e694291b94b41970b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 21 Nov 2026 20:01:47 +0100 Subject: [PATCH 1/2] test(e2e): give the panel measurement 90 minutes, so a slow EBI download no longer fails a batch On a dispatched run, prs-3 waits up to 30 minutes for the panel download and then gives pgsc_calc 35 minutes. When EBI served the 7.4 GB panel in 31 minutes instead of 13, the case passed the runner's 60-minute limit. It now raises its own limit to 90 minutes, as the Locityper case does. --- docs/testing.md | 2 +- tests/e2e/prs-3-panel-measure.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index edf9ecd8b..451649ec9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -121,7 +121,7 @@ What it covers: - **Index times** (`run-all-sample-qc-and-stale-index-1-index-times.sh`): every index the direct `nextflow run` and the launcher published is not older than its data file, and samtools and bcftools print no stale-index warning on the BAM and the VCF. Then a short VCF-only run with three planted pairs checks `main.nf`'s completion handler: it sets the index times of the run's own files after the last copy, and leaves an index older than the run as it was. A copied pair with its index set an hour older is the negative control. - **The Nextflow report's tables** (`paralogs-report-section-1-nf-report-tables.sh`): `HTML_REPORT` runs for real on the CYP2D6 depth check, the cDNA read check and the phased compound-het pairs that earlier cases wrote, and its reports must show the depth-check line, the cDNA column with its flagged count and the pairs by phase; the same run without the three tables must show none of them. A stub run of every tool then checks that `main.nf` passes the cDNA table to each `HTML_REPORT` task that got ClinVar hits, against a copy of the pipeline with that join removed. -- **Locityper, step 53** (`ci-e2e-2-locityper.sh`): `setup.sh --locityper-data` builds the database for the HLA-DQB1 row of `assets/locityper_loci.bed` from a chr6 and chr20 reference cut from the fixture's, and `scripts/53-locityper.sh` genotypes the fixture's reads aligned to it, with chr20 as the background region. The fixture has no reads over FCGR or KIR, and HLA-DRB1 is left out for time: with it the case took 59 minutes, 35 of them jellyfish's k-mer count and 23 `locityper augment`. With HLA-DQB1 alone `augment` takes about a minute. Jellyfish does not depend on the loci: with its hash fixed at 3G entries it took 36 minutes (2,289 s for the case, fork run 42325306077), and with the hash sized to the reference it takes 36 seconds (142 s for the case, fork run 42421585308). The case keeps a limit of 90 minutes (`# case-timeout: 5400`; every other case keeps 60) as room for a slow runner. The locus must come out called, with two panel haplotypes, a quality, reads and the authors' filter, in the columns `og locityper report` writes. Red controls: the step stops without the database, without the BAM's index, and with a database whose `DONE` names another reference. +- **Locityper, step 53** (`ci-e2e-2-locityper.sh`): `setup.sh --locityper-data` builds the database for the HLA-DQB1 row of `assets/locityper_loci.bed` from a chr6 and chr20 reference cut from the fixture's, and `scripts/53-locityper.sh` genotypes the fixture's reads aligned to it, with chr20 as the background region. The fixture has no reads over FCGR or KIR, and HLA-DRB1 is left out for time: with it the case took 59 minutes, 35 of them jellyfish's k-mer count and 23 `locityper augment`. With HLA-DQB1 alone `augment` takes about a minute. Jellyfish does not depend on the loci: with its hash fixed at 3G entries it took 36 minutes (2,289 s for the case, fork run 42325306077), and with the hash sized to the reference it takes 36 seconds (142 s for the case, fork run 42421585308). The case keeps a limit of 90 minutes (`# case-timeout: 5400`) as room for a slow runner. So does `prs-3-panel-measure.sh`, whose wait for the panel download (up to 30 minutes when EBI is slow) and pgsc_calc's 35 minutes can pass 60. Every other case keeps 60. The locus must come out called, with two panel haplotypes, a quality, reads and the authors' filter, in the columns `og locityper report` writes. Red controls: the step stops without the database, without the BAM's index, and with a database whose `DONE` names another reference. - **The SV rule regions** (`followups-ci-3-sv-rule-regions.sh`): `scripts/ci/build-sv-rule-regions.sh --check` rebuilds `assets/sv_rule_regions.tsv` from its four pinned downloads and must find the committed file byte for byte, with its row counts (2,409 coding tandem repeats). Red control: `--check` on a copy with the MHC row's end moved by one must exit 1 and show that row. The downloads come from the `catalogs` data cache entry. diff --git a/tests/e2e/prs-3-panel-measure.sh b/tests/e2e/prs-3-panel-measure.sh index 8e6e76a1b..27a294dc3 100644 --- a/tests/e2e/prs-3-panel-measure.sh +++ b/tests/e2e/prs-3-panel-measure.sh @@ -25,6 +25,10 @@ # used until then. # Case prs-2 projects HG002 onto the small synthetic panel on every run. # Removes the panel and everything else it made. +# Its own limit, above the runner's 3600 s: the wait for the download (up to +# 30 minutes, when EBI is slow), the site list, the target and pgsc_calc's 35 +# minutes add up to about 80. +# case-timeout: 5400 . "$(dirname "$0")/lib.sh" measure_on() { [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ]; }; } From 057ddd94c7c22ee17d36a9c726d439119f5ee998 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 21 Nov 2026 20:16:23 +0100 Subject: [PATCH 2/2] test(e2e): skip the panel measurement on a traced run, and say what it costs today A traced run puts every case in one leg with a 355-minute limit, and its time table keeps prs-3 at 0 s, so the 35 to 80 minute measurement does not belong there. PGSC_MEASURE=1 still forces it. The case header and docs/25-prs.md no longer cite the build-fixture job's 150 minutes or say the download outlasts a pull request's run. --- docs/25-prs.md | 2 +- tests/e2e/prs-1-agreement.sh | 8 ++++---- tests/e2e/prs-3-panel-measure.sh | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/25-prs.md b/docs/25-prs.md index 8ef1b061d..496488a1b 100644 --- a/docs/25-prs.md +++ b/docs/25-prs.md @@ -179,7 +179,7 @@ The panel is pgsc_calc's 1000 Genomes database, `pgsc_1000G_v1.tar.zst` (`PGSC_P | Memory | 7.3 to 7.6 GB at the peak for the whole machine; the largest task, 6.9 to 7.1 GB (plink2 scoring the panel's own samples, and the panel's variant filter) | | Time | 20 to 30 minutes for the run, on 4 CPUs | -That run scored PGS000018 alone and fits a 16 GB machine; the default scores need more ([Memory](#memory)). It does not fit the 14 GB of disk a GitHub-hosted runner guarantees, and the download alone outlasts the pull request test budget, so the panel stays opt-in (`setup.sh --ancestry-panel`), and the tests on every pull request project the fixture sample onto the PGS Catalog's 268 MB synthetic panel instead (`tests/e2e/prs-2-ancestry.sh`). The 1000 Genomes run above repeats on an E2E run dispatched on a branch other than main (or with `PGSC_MEASURE=1`): it adds about 40 minutes, which a dispatched run of main keeps for its other cases. +That run scored PGS000018 alone and fits a 16 GB machine; the default scores need more ([Memory](#memory)). It does not fit the 14 GB of disk a GitHub-hosted runner guarantees, and its download takes 13 to 31 minutes on a runner, so the panel stays opt-in (`setup.sh --ancestry-panel`), and the tests on every pull request project the fixture sample onto the PGS Catalog's 268 MB synthetic panel instead (`tests/e2e/prs-2-ancestry.sh`). The 1000 Genomes run above repeats on an E2E run dispatched on a branch other than main and not traced (or with `PGSC_MEASURE=1`): it adds about 40 minutes, which a dispatched run of main keeps for its other cases. ## Interpreting Results diff --git a/tests/e2e/prs-1-agreement.sh b/tests/e2e/prs-1-agreement.sh index 1cd31e37a..da0df96ba 100644 --- a/tests/e2e/prs-1-agreement.sh +++ b/tests/e2e/prs-1-agreement.sh @@ -20,12 +20,12 @@ # match log, and its sum less the autosomal sum above equals plink2's # --score of those rows on the same genotypes with the sex set. # Writes nothing other cases read; removes what it made. When case prs-3 -# measures (a run dispatched on a branch other than main) it also starts that -# case's download of the 1000 Genomes panel in the background, so the 7.4 GB -# arrive while cases prs-1 and prs-2 run. +# measures (a run dispatched on a branch other than main, not traced) it +# also starts that case's download of the 1000 Genomes panel in the +# background, so the 7.4 GB arrive while cases prs-1 and prs-2 run. . "$(dirname "$0")/lib.sh" -if [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ]; }; then +if [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ] && [ "${E2E_TRACE:-false}" != true ]; }; then rm -f "${E2E_WORK}/panel-download.status" # Its own session, so the case's timeout does not stop it; it ends on its own. setsid nohup timeout 2700 bash "$(dirname "$0")/prs-3-panel-measure.sh" --download \ diff --git a/tests/e2e/prs-3-panel-measure.sh b/tests/e2e/prs-3-panel-measure.sh index 27a294dc3..484dfa150 100644 --- a/tests/e2e/prs-3-panel-measure.sh +++ b/tests/e2e/prs-3-panel-measure.sh @@ -2,10 +2,10 @@ # What the 1000 Genomes ancestry panel costs on a GitHub-hosted runner, for # docs/25-prs.md: the download, the disk at its fullest and the memory at its # highest while pgsc_calc projects a genome-wide target onto the panel and -# scores it. Runs on an E2E run dispatched on a branch other than main, or -# with PGSC_MEASURE=1: it adds about 40 minutes, more than the monthly, pull -# request and main dispatched runs have left in their 150 (a dispatched run -# that carried it hit that limit twice). +# scores it. Runs on an E2E run dispatched on a branch other than main and not +# traced, or with PGSC_MEASURE=1: it adds 35 to 80 minutes to its shard, which +# the monthly, pull request and main runs do not pay. A traced run skips it, so +# the time table keeps its row at 0 s and the one traced leg stays in its limit. # # --download (started in the background by case prs-1, so the download # runs while other cases do) fetches the panel with 8 byte @@ -31,7 +31,7 @@ # case-timeout: 5400 . "$(dirname "$0")/lib.sh" -measure_on() { [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ]; }; } +measure_on() { [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ] && [ "${E2E_TRACE:-false}" != true ]; }; } RES=https://ftp.ebi.ac.uk/pub/databases/spot/pgs/resources DIR="${GENOME_DIR}/reference/pgsc_calc" PANEL="${DIR}/${PGSC_PANEL}.tar.zst" @@ -69,7 +69,7 @@ if [ "${1:-}" = --download ]; then fi if ! measure_on; then - NO_CHECKS_REASON="the 1000 Genomes panel measurement runs on an E2E run dispatched on a branch other than main, or with PGSC_MEASURE=1" + NO_CHECKS_REASON="the 1000 Genomes panel measurement runs on an E2E run dispatched on a branch other than main and not traced, or with PGSC_MEASURE=1" finish fi command -v nextflow >/dev/null || { fail "nextflow is not on PATH"; finish; }