diff --git a/.gitignore b/.gitignore index 9277cdf090f0..397525073668 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,11 @@ _UpgradeReport_Files/ tools/*/*.i tools/*/*.i.tmp +# === Rules for pgo artifacts === +*.profdata +*.profraw +*.gcda + # === Rules for release artifacts === /*.tar.* /*.pkg diff --git a/common.gypi b/common.gypi index 71712d308472..1a2ca605f33b 100644 --- a/common.gypi +++ b/common.gypi @@ -12,6 +12,7 @@ 'msvs_multi_core_compile': '0', # we do enable multicore compiles, but not using the V8 way 'enable_pgo_generate%': '0', 'enable_pgo_use%': '0', + 'pgo_profile%': '', 'clang_profile_lib%': '', 'python%': 'python', 'emulator%': [], @@ -193,6 +194,7 @@ }], ['clang==1', { 'lto': ' -flto ', # Clang + 'pgo_use': '-fprofile-use=<(pgo_profile)', }, { 'lto': ' -flto=4 -ffat-lto-objects ', # GCC }], @@ -247,6 +249,34 @@ },], ], },], + ['OS=="mac"', { + 'conditions': [ + ['enable_pgo_generate=="true"', { + 'xcode_settings': { + 'OTHER_CFLAGS': ['<(pgo_generate)'], + }, + 'target_conditions': [ + ['_type!="static_library"', { + 'xcode_settings': { + 'OTHER_LDFLAGS': ['<(pgo_generate)'], + }, + }], + ], + }], + ['enable_pgo_use=="true"', { + 'xcode_settings': { + 'OTHER_CFLAGS': ['<(pgo_use)'], + }, + 'target_conditions': [ + ['_type!="static_library"', { + 'xcode_settings': { + 'OTHER_LDFLAGS': ['<(pgo_use)'], + }, + }], + ], + }], + ], + }], ['OS=="win"', { 'conditions': [ ['enable_lto=="true"', { diff --git a/configure.py b/configure.py index 8b3332a461f4..4972b66a349e 100755 --- a/configure.py +++ b/configure.py @@ -208,14 +208,15 @@ dest="enable_pgo_generate", default=None, help="Enable profiling with pgo of a binary. This feature is only available " - "on linux with gcc and g++ 5.4.1 or newer and on windows.") + "on linux with GCC or Clang, on macOS with Clang, and on windows.") parser.add_argument("--enable-pgo-use", action="store_true", dest="enable_pgo_use", default=None, help="Enable use of the profile generated with --enable-pgo-generate. This " - "feature is only available on linux with gcc and g++ 5.4.1 or newer and on windows.") + "feature is only available on linux with GCC or Clang, on macOS " + "with Clang, and on windows.") parser.add_argument("--enable-lto", action="store_true", @@ -2004,18 +2005,9 @@ def configure_node(o): else: o['variables']['node_enable_v8_vtunejit'] = 'false' - if (flavor != 'linux' and flavor != 'win') and (options.enable_pgo_generate or options.enable_pgo_use): + if flavor not in ('linux', 'mac', 'win') and (options.enable_pgo_generate or options.enable_pgo_use): raise Exception( - 'The pgo option is supported only on linux and windows.') - - if flavor == 'linux': - if options.enable_pgo_generate or options.enable_pgo_use: - version_checked = (5, 4, 1) - if not gcc_version_ge(version_checked): - version_checked_str = ".".join(map(str, version_checked)) - raise Exception( - 'The options --enable-pgo-generate and --enable-pgo-use ' - f'are supported for gcc and gxx {version_checked_str} or newer only.') + 'The pgo option is supported only on linux, macOS, and windows.') if options.enable_pgo_generate and options.enable_pgo_use: raise Exception( @@ -2024,6 +2016,24 @@ def configure_node(o): '--enable-pgo-generate first, profile node, and then recompile ' 'with --enable-pgo-use') + if flavor in ('linux', 'mac'): + if options.enable_pgo_generate or options.enable_pgo_use: + clang_compilers = [try_check_compiler(compiler, language)[1] + for compiler, language in ((CC, 'c'), (CXX, 'c++'))] + if all(clang_compilers): + profile = os.path.abspath('node.profdata') + if options.enable_pgo_use and not os.path.isfile(profile): + raise Exception( + f'PGO profile not found: {profile}. Run llvm-profdata merge first.') + o['variables']['pgo_profile'] = profile + elif flavor == 'mac' or any(clang_compilers): + raise Exception('PGO requires both CC and CXX to use Clang on macOS ' + 'or the same compiler family on linux.') + elif not gcc_version_ge((5, 4, 1)): + raise Exception( + 'The options --enable-pgo-generate and --enable-pgo-use ' + 'require gcc and gxx 5.4.1 or newer.') + o['variables']['enable_pgo_generate'] = b(options.enable_pgo_generate) o['variables']['enable_pgo_use'] = b(options.enable_pgo_use) diff --git a/tools/pgo/README.md b/tools/pgo/README.md index 234f20e638ac..956535217928 100644 --- a/tools/pgo/README.md +++ b/tools/pgo/README.md @@ -15,11 +15,11 @@ The process has three phases: ## Platform Support -| Platform | Supported toolchains | Driver | -| -------- | -------------------- | ------------------------- | -| Windows | Clang-CL | `vcbuild.bat` + `pgo.ps1` | -| Linux | GCC | `configure` + `make` | -| macOS | — | — | +| Platform | Supported toolchains | Driver | +| -------- | -------------------- | ------------------------------------- | +| Windows | Clang-CL | `vcbuild.bat` + `pgo.py` | +| Linux | GCC, Clang | `configure` + `make`, `pgo.py` for Clang | +| macOS | Clang | `configure` + `make` + `pgo.py` | The two supported flows differ in how profile data is collected. Clang writes one `.profraw` file per process, which must be merged into a single @@ -27,8 +27,6 @@ one `.profraw` file per process, which must be merged into a single into `.gcda` files next to each object file as each process exits, so there is no merge step. -Clang on Linux and macOS are not supported yet. - ## Quick Start: Windows From a VS Developer Command Prompt, at the repo root: @@ -38,25 +36,25 @@ From a VS Developer Command Prompt, at the repo root: vcbuild.bat pgo-generate # Step 2: Run workloads to collect profile data -powershell -ExecutionPolicy Bypass -File .\tools\pgo\pgo.ps1 +python tools\pgo\pgo.py # Step 3: Build the optimized binary vcbuild.bat pgo-use ``` -`pgo.ps1` expects the instrumented binary at `Release\node.exe` (produced by +`pgo.py` expects the instrumented binary at `Release\node.exe` (produced by step 1) and writes `node.profdata` to the repo root (consumed by step 3). -The script is unsigned, so the default execution policy refuses to run it -without `-ExecutionPolicy Bypass`. Use `pwsh` in place of `powershell` on -PowerShell 7. +The script finds `llvm-profdata` in the Visual Studio LLVM toolset, then +`PATH`. Set `LLVM_PROFDATA` to the matching tool when using a different +Clang installation. ```powershell # Optionally set a longer training duration (default: 15s per script) -powershell -ExecutionPolicy Bypass -File .\tools\pgo\pgo.ps1 -Duration 30 +python tools\pgo\pgo.py --duration=30 ``` -## Quick Start: Linux +## Quick Start: Linux with GCC ```bash # Step 1: Build the instrumented binary @@ -88,6 +86,42 @@ updates from the worker threads and the libuv thread pool race with each other, and GCC treats the resulting inconsistent profile as an error unless told to smooth it out. +## Quick Start: Linux and macOS with Clang + +From the repo root: + +```bash +# Step 1: Build the instrumented binary +./configure --ninja --enable-pgo-generate +make + +# Step 2: Run workloads to collect profile data +python3 tools/pgo/pgo.py + +# Step 3: Build the optimized binary +./configure --ninja --enable-pgo-use +make +``` + +`pgo.py` expects the instrumented binary at `out/Release/node` (produced by +step 1) and writes `node.profdata` to the repo root (consumed by step 3). + +The script finds `llvm-profdata` through `xcrun` on macOS and `PATH` on Linux. +Set `LLVM_PROFDATA` to the matching tool when using a different Clang +installation. + +```bash +# Optionally set a longer training duration (default: 15s per script) +python3 tools/pgo/pgo.py --duration=30 +``` + +## Clang Profile Collection + +On all platforms, `pgo.py` collects raw profiles in a fresh directory and +replaces `node.profdata` after a successful merge. It removes raw profiles +after success and preserves them if training or merging fails. Training +failures stop the script so the workloads can be fixed before trying again. + ## Training Scripts All scripts use only Node.js built-in modules (no npm dependencies). @@ -110,7 +144,7 @@ Each script is run as a separate process via `fork()`. ### Running the Orchestrator Directly The orchestrator can also be invoked directly (e.g. for testing individual -workloads). When used with `pgo.ps1`, this is handled automatically. +workloads). When used with `pgo.py`, this is handled automatically. ```bash # Run all scripts @@ -131,7 +165,7 @@ automatically from the `--duration` flag (in seconds). ``` tools/pgo/ -├── pgo.ps1 # Windows training driver (collect + merge) +├── pgo.py # Clang training driver (collect + merge) ├── pgo-run-all.js # Training orchestrator ├── pgo-http-server.js # HTTP server + client workload ├── pgo-json.js # JSON parse/stringify workload diff --git a/tools/pgo/pgo.ps1 b/tools/pgo/pgo.ps1 deleted file mode 100644 index 1cfedddd9e69..000000000000 --- a/tools/pgo/pgo.ps1 +++ /dev/null @@ -1,169 +0,0 @@ -# PGO (Profile-Guided Optimization) training script for Node.js (Clang / LLVM) -# -# Runs PGO training workloads against an instrumented Node.js binary -# (Release\node.exe) and merges the resulting .profraw files into -# node.profdata for use with -fprofile-use. -# -# Usage (from a VS Developer Command Prompt, at the repo root): -# powershell -ExecutionPolicy Bypass -File .\tools\pgo\pgo.ps1 -# powershell -ExecutionPolicy Bypass -File .\tools\pgo\pgo.ps1 -Duration 30 -# -# The script is unsigned, so the default execution policy blocks it without -# -ExecutionPolicy Bypass. Default duration is 15s per workload. -# -# Prerequisites: -# - Release\node.exe must be an instrumented build (built with pgo-generate) -# - llvm-profdata must be available (shipped with VS LLVM toolset) -# -# Output: -# - node.profdata in the repo root (ready for vcbuild.bat pgo-use) - -param( - [int]$Duration = 15 -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -# The instrumented binary and the merged profile both live at the repo root, -# two levels up from tools\pgo. common.gypi reads node.profdata from there. -$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path - -# --------------------------------------------------------------------------- -# Locate llvm-profdata shipped with Visual Studio's LLVM toolset -# --------------------------------------------------------------------------- - -function Find-LlvmProfdata { - # vcbuild.bat uses %VCINSTALLDIR%\Tools\Llvm\x64\bin for clang.exe - same spot for profdata - $vcInstallDir = $env:VCINSTALLDIR - - if ($vcInstallDir) { - $candidate = Join-Path $vcInstallDir "Tools\Llvm\x64\bin\llvm-profdata.exe" - if (Test-Path $candidate) { - return $candidate - } - } - - # Fallback: try VS 2022 / 2026 default install locations - $vsPaths = @( - "${env:ProgramFiles}\Microsoft Visual Studio\2026\Enterprise\VC\Tools\Llvm\x64\bin", - "${env:ProgramFiles}\Microsoft Visual Studio\2026\Community\VC\Tools\Llvm\x64\bin", - "${env:ProgramFiles}\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm\x64\bin", - "${env:ProgramFiles}\Microsoft Visual Studio\2022\Community\VC\Tools\Llvm\x64\bin" - ) - foreach ($dir in $vsPaths) { - $candidate = Join-Path $dir "llvm-profdata.exe" - if (Test-Path $candidate) { - return $candidate - } - } - - # Last resort: PATH - $fromPath = Get-Command llvm-profdata -ErrorAction SilentlyContinue - if ($fromPath) { - return $fromPath.Source - } - - return $null -} - -# --------------------------------------------------------------------------- -# Validate prerequisites -# --------------------------------------------------------------------------- - -$instrumentedNode = Join-Path $repoRoot "Release\node.exe" -if (-not (Test-Path $instrumentedNode)) { - Write-Error "Instrumented binary not found: $instrumentedNode`nBuild with: vcbuild.bat pgo-generate" - exit 1 -} - -$pgoRunAll = Join-Path $PSScriptRoot "pgo-run-all.js" -if (-not (Test-Path $pgoRunAll)) { - Write-Error "PGO training script not found: $pgoRunAll" - exit 1 -} - -$llvmProfdata = Find-LlvmProfdata -if (-not $llvmProfdata) { - Write-Error "llvm-profdata not found. Install the LLVM toolset via Visual Studio Installer." - exit 1 -} - -# --------------------------------------------------------------------------- -# STEP 1 – Run workloads with the instrumented binary to collect profiles -# --------------------------------------------------------------------------- - -Write-Host "`n=== STEP 1: Collect PGO profiles ===" -ForegroundColor Cyan - -# Directory that will receive .profraw files from the instrumented binary. -# %p (PID) and %m (module hash) keep concurrent/fork'd processes from colliding. -$profileDir = Join-Path $repoRoot "pgo-profiles" - -if (Test-Path $profileDir) { - Remove-Item -Recurse -Force $profileDir -} -New-Item -ItemType Directory -Path $profileDir | Out-Null - -$env:LLVM_PROFILE_FILE = Join-Path $profileDir "node-%p-%m.profraw" - -Write-Host "Instrumented node : $instrumentedNode" -Write-Host "Profile output : $($env:LLVM_PROFILE_FILE)" -Write-Host "Duration per script: ${Duration}s" -Write-Host "" - -$sw = [System.Diagnostics.Stopwatch]::StartNew() -$proc = Start-Process ` - -FilePath $instrumentedNode ` - -ArgumentList "`"$pgoRunAll`" --verbose --duration=$Duration" ` - -Wait -PassThru -NoNewWindow -$sw.Stop() -Write-Host ("PGO training completed in {0}m {1}s (exit code: {2})" -f ` - $sw.Elapsed.Minutes, $sw.Elapsed.Seconds, $proc.ExitCode) -if ($proc.ExitCode -ne 0) { - Write-Warning "PGO training exited with code $($proc.ExitCode) - continuing with merge" -} - -# Remove the env var so subsequent builds are not affected -Remove-Item Env:\LLVM_PROFILE_FILE -ErrorAction SilentlyContinue - -# --------------------------------------------------------------------------- -# STEP 2 – Merge .profraw files -> node.profdata -# --------------------------------------------------------------------------- - -Write-Host "`n=== STEP 2: Merge profile data ===" -ForegroundColor Cyan - -Write-Host "Using llvm-profdata: $llvmProfdata" - -$profrawFiles = Get-ChildItem -Path $profileDir -Filter "*.profraw" -ErrorAction SilentlyContinue -if ($profrawFiles.Count -eq 0) { - Write-Error "No .profraw files found in '$profileDir'. The instrumented binary may not have generated profile data." - exit 1 -} - -$totalSize = ($profrawFiles | Measure-Object -Property Length -Sum).Sum -$totalSizeMB = [math]::Round($totalSize / 1MB, 1) -Write-Host "Found $($profrawFiles.Count) .profraw file(s), ${totalSizeMB} MB total" - -$profdata = Join-Path $repoRoot "node.profdata" -$mergeArgs = @("merge", "--output=$profdata") + ($profrawFiles | Select-Object -ExpandProperty FullName) - -$mergeStopwatch = [System.Diagnostics.Stopwatch]::StartNew() -& $llvmProfdata @mergeArgs -$mergeExitCode = $LASTEXITCODE -$mergeStopwatch.Stop() - -if ($mergeExitCode -ne 0) { - Write-Error "llvm-profdata merge failed (exit code $mergeExitCode)" - exit $mergeExitCode -} - -$profdataSize = [math]::Round((Get-Item $profdata).Length / 1MB, 1) -Write-Host "Merge completed in $([math]::Round($mergeStopwatch.Elapsed.TotalSeconds, 1))s" - -# Clean up .profraw files now that they've been merged -Remove-Item -Recurse -Force $profileDir -Write-Host "Removed $($profrawFiles.Count) .profraw file(s) (${totalSizeMB} MB reclaimed)" - -Write-Host "`n=== PGO training complete ===" -ForegroundColor Green -Write-Host " Profile data: $profdata (${profdataSize} MB)" -Write-Host " Next step: vcbuild.bat pgo-use" diff --git a/tools/pgo/pgo.py b/tools/pgo/pgo.py new file mode 100644 index 000000000000..7847b769843a --- /dev/null +++ b/tools/pgo/pgo.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Train a Clang PGO build and merge its profiles into node.profdata.""" + +import argparse +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +def find_llvm_profdata(): + override = os.environ.get('LLVM_PROFDATA') + if override: + tool = shutil.which(override) + elif sys.platform == 'darwin': + tool = subprocess.check_output( + ['xcrun', '--find', 'llvm-profdata'], text=True).strip() + else: + candidates = [] + if sys.platform == 'win32': + vc_install = os.environ.get('VCINSTALLDIR') + if vc_install: + candidates.append(Path(vc_install) / 'Tools/Llvm/x64/bin/llvm-profdata.exe') + program_files = os.environ.get('ProgramFiles') + if program_files: + candidates.extend( + Path(program_files) / 'Microsoft Visual Studio' / version / edition / + 'VC/Tools/Llvm/x64/bin/llvm-profdata.exe' + for version in ('2026', '2022') + for edition in ('Enterprise', 'Community', 'Professional', 'BuildTools')) + tool = next((str(path) for path in candidates if path.is_file()), None) + if tool is None: + tool = shutil.which('llvm-profdata') + if not tool: + raise ValueError('llvm-profdata not found. Set LLVM_PROFDATA to the tool ' + 'from your Clang toolchain.') + return tool + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--duration', type=int, default=15, + help='seconds per workload (default: 15)') + args = parser.parse_args() + if args.duration <= 0: + parser.error('--duration must be a positive integer') + + repo_root = Path(__file__).resolve().parents[2] + if sys.platform == 'win32': + node = repo_root / 'Release/node.exe' + build = 'vcbuild.bat pgo-generate' + rebuild = 'vcbuild.bat pgo-use' + else: + node = repo_root / 'out/Release/node' + build = './configure --ninja --enable-pgo-generate && make' + rebuild = './configure --ninja --enable-pgo-use && make' + if not node.is_file(): + raise ValueError(f'Instrumented binary not found: {node}\nBuild with: {build}') + llvm_profdata = find_llvm_profdata() + + profile_dir = Path(tempfile.mkdtemp(prefix='pgo-profiles.', dir=repo_root)) + env = os.environ.copy() + env['LLVM_PROFILE_FILE'] = str(profile_dir / 'node-%m-%p.profraw') + + print(f'Training {node} for {args.duration}s per workload', flush=True) + print(f'Profile directory: {profile_dir}', flush=True) + try: + subprocess.run([str(node), str(repo_root / 'tools/pgo/pgo-run-all.js'), + f'--duration={args.duration}', '--verbose'], + cwd=repo_root, env=env, check=True) + profiles = list(profile_dir.glob('*.profraw')) + if not profiles: + raise ValueError(f'No .profraw files found in {profile_dir}. ' + 'Build Node with PGO instrumentation using Clang.') + + print(f'Merging {len(profiles)} profiles with {llvm_profdata}', flush=True) + merged = profile_dir / 'node.profdata' + subprocess.run([llvm_profdata, 'merge', '-o', str(merged), + *map(str, profiles)], check=True) + merged.replace(repo_root / 'node.profdata') + except (OSError, ValueError, subprocess.CalledProcessError): + print(f'PGO failed. Collected profiles are in: {profile_dir}', file=sys.stderr) + raise + shutil.rmtree(profile_dir) + print(f'Profile data: {repo_root / "node.profdata"}') + print(f'Next step: {rebuild}') + + +if __name__ == '__main__': + main()