Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions .github/RUNNERS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# Runner label catalog

CI runner labels live in one place: [`.github/runners.yaml`](./runners.yaml).
This lets a runner-image migration (OS bump, hosted-image retirement) update a
single file instead of grepping every workflow.

## How it works

`runners.yaml` is the source of truth. A generated reusable workflow,
[`.github/workflows/reusable-runner-names.yml`](./workflows/reusable-runner-names.yml),
exports each catalog entry as a job output. Callers pull the label from that
output instead of hardcoding it, because `runs-on:` is evaluated before any step
runs, so a composite action cannot supply the label β€” a reusable workflow's
outputs can.

A catalog value is one of:

- a **scalar** label (`windows-2022`) β€” consumed as
`runs-on: ${{ needs.runner_names.outputs.<key> }}`
- a **composite label set** (`[self-hosted, Linux, X64]`) β€” exported as a JSON
array string and consumed as
`runs-on: ${{ fromJSON(needs.runner_names.outputs.<key>) }}`

Rolling `-latest` aliases (`ubuntu-latest`, `macos-latest`, `windows-latest`)
are intentionally left hardcoded β€” they are GitHub aliases, not fleet labels.
`matrix.os` values (and `matrix.os == '...'` conditionals) are frozen logical
identities and are not managed by the catalog.

## Wiring a job

```yaml
jobs:
runner_names:
permissions:
contents: read
uses: ./.github/workflows/reusable-runner-names.yml

my-job:
needs: runner_names
runs-on: ${{ needs.runner_names.outputs.windows_2022 }}
steps: ...
```

## Changing a label

1. Edit `.github/runners.yaml`.
2. Regenerate: `node .github/scripts/sync-runner-names.mjs`
3. Test: `node --test .github/scripts/test/runner-names.test.mjs`

CI enforces both invariants via
[`runner-names-validate.yml`](./workflows/runner-names-validate.yml).
17 changes: 17 additions & 0 deletions .github/runners.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Specialized GitHub Actions runner labels for qvac-ext-stable-diffusion.cpp CI.
# Edit this file, then run: node .github/scripts/sync-runner-names.mjs
#
# ubuntu-latest / macos-latest / windows-latest are intentionally omitted; keep
# them hardcoded as GitHub's rolling aliases.
#
# FORMAT CONTRACT: a flat `key: value # comment` list, one entry per line. The
# parser (.github/scripts/lib/runner-names.mjs) is line-based, not YAML. A value
# is EITHER a bare single label (`windows-2022`) OR a flow array of bare tokens
# (`[self-hosted, Linux, X64]`). Do not quote values or nest maps (the parser
# fails loudly).
#
# SCOPE: this catalog governs only where jobs RUN (runs-on / matrix.runner). It
# does NOT manage `matrix.os` identities or `matrix.os == '...'` conditionals.

# GitHub-hosted (pinned images)
windows_2022: windows-2022 # Windows Server 2022 hosted x64
255 changes: 255 additions & 0 deletions .github/scripts/lib/runner-names.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
/**
* Shared helpers for the qvac-ext-stable-diffusion.cpp runner-label catalog.
*
* Source of truth: .github/runners.yaml
* Generated consumer: .github/workflows/reusable-runner-names.yml
*
* A catalog entry is either a scalar label or an ordered array of labels
* (composite self-hosted set). Array entries are exported as a JSON-array
* string and consumed with `fromJSON(...)` in runs-on.
*/
import { readdirSync, readFileSync } from 'node:fs'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'

const here = dirname(fileURLToPath(import.meta.url))
export const repoRoot = resolve(here, '../../..')

export const RUNNERS_YAML = '.github/runners.yaml'
export const REUSABLE_WORKFLOW = '.github/workflows/reusable-runner-names.yml'
export const REUSABLE_USES = './.github/workflows/reusable-runner-names.yml'

// Workflows wired to the catalog and validated. Orchestration-only files that
// use nothing but ubuntu-latest / the org reusable (check-approvals,
// security-baseline) stay out of scope.
const ADDON_WORKFLOWS = new Set(['build.yml'])

const KEY = '[a-z][a-z0-9_]*'
const COMMENT = '(?:#\\s*(.*\\S))?'
const SCALAR_RE = new RegExp(`^(${KEY}):\\s+([^#\\[\\s'"][^#]*?)\\s*${COMMENT}\\s*$`)
const ARRAY_RE = new RegExp(`^(${KEY}):\\s+\\[([^\\]]+)\\]\\s*${COMMENT}\\s*$`)

export function parseRunnersYaml(source) {
const runners = []
const seenKeys = new Set()
const seenTargets = new Set()

for (const line of source.split(/\r?\n/)) {
const trimmed = line.trim()
if (!trimmed || trimmed.startsWith('#')) continue

const arrayMatch = line.match(ARRAY_RE)
const scalarMatch = arrayMatch ? null : line.match(SCALAR_RE)
if (!arrayMatch && !scalarMatch) {
throw new Error(`invalid runners.yaml line (expected "key: label" or "key: [a, b]"): ${JSON.stringify(line)}`)
}

let entry
if (arrayMatch) {
const [, key, body, description = ''] = arrayMatch
const labels = body.split(',').map((token) => token.trim()).filter(Boolean)
if (labels.length === 0) throw new Error(`empty runner array: ${JSON.stringify(line)}`)
for (const label of labels) {
if (/["'{}[\]]/.test(label)) {
throw new Error(`runner label must be bare, not quoted/structured: ${JSON.stringify(line)}`)
}
}
entry = { key, kind: 'array', labels, description }
} else {
const [, key, rawLabel, description = ''] = scalarMatch
const label = rawLabel.trim()
if (/["'{}[\]]/.test(label)) {
throw new Error(`runner label must be bare, not quoted/structured: ${JSON.stringify(line)}`)
}
entry = { key, kind: 'scalar', label, description }
}

const target = targetKeyString(entry)
if (seenKeys.has(entry.key)) throw new Error(`duplicate runner key: ${entry.key}`)
if (seenTargets.has(target)) throw new Error(`duplicate runner target: ${target}`)
seenKeys.add(entry.key)
seenTargets.add(target)
runners.push(entry)
}

if (runners.length === 0) throw new Error('runners.yaml has no entries')
return runners
}

/** Canonical string for a target, used for duplicate detection + matching. */
export function targetKeyString(entry) {
return entry.kind === 'array' ? `[${entry.labels.join(',')}]` : entry.label
}

/** The output value the reusable exports: scalar label, or compact JSON array. */
export function outputValue(entry) {
return entry.kind === 'array' ? JSON.stringify(entry.labels) : entry.label
}

/** How a caller references the output in `runs-on:`. */
export function runsOnExpression(entry) {
const ref = `needs.runner_names.outputs.${entry.key}`
return entry.kind === 'array' ? `\${{ fromJSON(${ref}) }}` : `\${{ ${ref} }}`
}

export function loadRunners() {
return parseRunnersYaml(readFileSync(join(repoRoot, RUNNERS_YAML), 'utf8'))
}

export function listAddonWorkflows() {
const directory = join(repoRoot, '.github/workflows')
return readdirSync(directory)
.filter((name) => ADDON_WORKFLOWS.has(name))
.map((name) => `.github/workflows/${name}`)
.sort()
}

export function renderReusableWorkflow(runners) {
const outputsBlock = runners
.map((entry) => {
const description = entry.description || `Runner target ${targetKeyString(entry)}`
return [
` ${entry.key}:`,
` description: ${yamlDoubleQuoted(description)}`,
` value: \${{ jobs.export.outputs.${entry.key} }}`,
].join('\n')
})
.join('\n')

const jobOutputs = runners
.map((entry) => ` ${entry.key}: \${{ steps.export.outputs.${entry.key} }}`)
.join('\n')

const exportLines = runners
.map((entry) => ` echo '${entry.key}=${outputValue(entry)}' >> "$GITHUB_OUTPUT"`)
.join('\n')

return `# AUTO-GENERATED by .github/scripts/sync-runner-names.mjs
# Source of truth: ${RUNNERS_YAML}
# Do not edit this file by hand.

name: Runner names

on:
workflow_call:
outputs:
${outputsBlock}

permissions:
contents: read

jobs:
export:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
${jobOutputs}
steps:
- name: Export runner labels
id: export
run: |
${exportLines}
`
}

export function assertReusableMatchesCatalog(runners, reusableSource) {
const expected = renderReusableWorkflow(runners)
if (normalizeNewlines(reusableSource) !== normalizeNewlines(expected)) {
throw new Error(`${REUSABLE_WORKFLOW} is out of date. Run: node .github/scripts/sync-runner-names.mjs`)
}
}

const LABEL_BOUNDARY_BEFORE = `(^|[\\s"'\\[,])`
const LABEL_BOUNDARY_AFTER = `([\\s"'\\],]|$)`

function escapeLabel(label) {
return label.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')
}

function normalizeArrayTokens(body) {
return body.split(',').map((token) => token.trim().replace(/^['"]|['"]$/g, '')).filter(Boolean)
}

function isCommentLine(line) {
return /^\s*#/.test(line)
}

/**
* Flags catalog targets hardcoded in `runs-on:` where they select the machine.
* Handles scalar forms (bare / quoted / inside a `${{ }}` expression) and the
* composite flow-array form. Frozen logical identities are intentionally NOT
* flagged: `matrix.os` values and `matrix.<x> == '<label>'` conditionals (the
* label there names an identity, not where the job runs β€” the runs-on value is
* still driven by the catalog outputs on the same line).
*/
export function findHardcodedLabelViolations(relativePath, source, runners) {
const findings = []
const lines = source.split(/\r?\n/)
const scalars = runners.filter((entry) => entry.kind === 'scalar')
const arrays = runners.filter((entry) => entry.kind === 'array')

for (let i = 0; i < lines.length; i++) {
const line = lines[i]
if (isCommentLine(line)) continue

const assign = line.match(/^\s+runs-on:\s*(.*?)\s*$/)
if (!assign) continue
const value = assign[1]

// Composite flow array: compare token sets against catalog array entries.
const arrayValue = value.match(/^\[([^\]]+)\]$/)
if (arrayValue) {
const tokens = normalizeArrayTokens(arrayValue[1])
const match = arrays.find((entry) => sameSet(entry.labels, tokens))
if (match) findings.push({ file: relativePath, line: i + 1, target: targetKeyString(match), text: line.trim() })
continue
}

// `${{ }}` expression: flag a quoted scalar label used as a VALUE. Strip
// `== '...'` / `!= '...'` operands first so logical-id conditionals like
// `matrix.os == 'macos-12'` are not mistaken for a hardcoded runner label.
if (value.includes('${{')) {
const scan = value.replace(/(==|!=)\s*(['"])[^'"]*\2/g, '')
const hit = scalars.find((entry) => new RegExp(`['"]${escapeLabel(entry.label)}['"]`).test(scan))
if (hit) findings.push({ file: relativePath, line: i + 1, target: hit.label, text: line.trim() })
continue
}

// Bare / quoted scalar.
const hit = scalars.find((entry) =>
new RegExp(`${LABEL_BOUNDARY_BEFORE}${escapeLabel(entry.label)}${LABEL_BOUNDARY_AFTER}`).test(value),
)
if (hit) findings.push({ file: relativePath, line: i + 1, target: hit.label, text: line.trim() })
}

return findings
}

export function hasRunnerNamesJob(source) {
return /^\s+runner_names:\s*$/m.test(source) && source.includes(`uses: ${REUSABLE_USES}`)
}

export function findMissingRunnerNamesNeeds(relativePath, source) {
if (!source.includes('needs.runner_names.outputs')) return []
if (hasRunnerNamesJob(source)) return []
return [{ file: relativePath, message: `references needs.runner_names.outputs but has no runner_names job using ${REUSABLE_USES}` }]
}

function sameSet(a, b) {
if (a.length !== b.length) return false
const sortedA = [...a].sort()
const sortedB = [...b].sort()
return sortedA.every((value, index) => value === sortedB[index])
}

function yamlDoubleQuoted(value) {
return `"${value.replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`
}

function normalizeNewlines(text) {
return text.replaceAll('\r\n', '\n')
}

export function readRepoFile(relativePath) {
return readFileSync(join(repoRoot, relativePath), 'utf8')
}
15 changes: 15 additions & 0 deletions .github/scripts/sync-runner-names.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env node
/**
* Regenerate .github/workflows/reusable-runner-names.yml from .github/runners.yaml.
*
* After editing the catalog:
* node .github/scripts/sync-runner-names.mjs
* node --test .github/scripts/test/runner-names.test.mjs
*/
import { writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { REUSABLE_WORKFLOW, RUNNERS_YAML, loadRunners, renderReusableWorkflow, repoRoot } from './lib/runner-names.mjs'

const runners = loadRunners()
writeFileSync(join(repoRoot, REUSABLE_WORKFLOW), renderReusableWorkflow(runners), 'utf8')
console.log(`wrote ${REUSABLE_WORKFLOW} (${runners.length} targets from ${RUNNERS_YAML})`)
47 changes: 47 additions & 0 deletions .github/scripts/validate-runner-names.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env node
/**
* Fail if CI workflows hardcode catalog runner targets or drift from the
* generated reusable-runner-names workflow.
*
* Usage: node .github/scripts/validate-runner-names.mjs
*/
import {
REUSABLE_WORKFLOW,
assertReusableMatchesCatalog,
findHardcodedLabelViolations,
findMissingRunnerNamesNeeds,
listAddonWorkflows,
loadRunners,
readRepoFile,
} from './lib/runner-names.mjs'

function main() {
const runners = loadRunners()
const errors = []

try {
assertReusableMatchesCatalog(runners, readRepoFile(REUSABLE_WORKFLOW))
} catch (error) {
errors.push(error.message)
}

for (const file of listAddonWorkflows()) {
const source = readRepoFile(file)
for (const finding of findHardcodedLabelViolations(file, source, runners)) {
errors.push(`${finding.file}:${finding.line} hardcodes runner target ${finding.target}: ${finding.text}`)
}
for (const finding of findMissingRunnerNamesNeeds(file, source)) {
errors.push(`${finding.file}: ${finding.message}`)
}
}

if (errors.length > 0) {
console.error(`validate-runner-names: ${errors.length} finding(s):`)
for (const error of errors) console.error(` ${error}`)
process.exit(1)
}

console.log(`validate-runner-names: ok (${runners.length} targets, ${listAddonWorkflows().length} workflow(s))`)
}

main()
Loading
Loading