diff --git a/bazelisk.py b/bazelisk.py index e182c643..82d3f395 100755 --- a/bazelisk.py +++ b/bazelisk.py @@ -310,7 +310,23 @@ def normalized_machine_arch_name(): return machine -def determine_url(version, is_commit, bazel_filename): +def parse_base_urls(): + """Returns a list of base URLs to try for downloading Bazel. + + BAZELISK_BASE_URLS (comma-separated) takes precedence over BAZELISK_BASE_URL. + Supports http://, https://, and file:// schemes. + Returns an empty list if neither variable is set (upstream is used). + """ + urls_value = get_env_or_config("BAZELISK_BASE_URLS") + if urls_value is not None: + return [u.strip() for u in urls_value.split(",") if u.strip()] + single_url = get_env_or_config("BAZELISK_BASE_URL") + if single_url is not None: + return [single_url] + return [] + + +def determine_url(version, is_commit, bazel_filename, base_url=None): if is_commit: sys.stderr.write("Using unreleased version at commit {}\n".format(version)) # No need to validate the platform thanks to determine_bazel_filename(). @@ -323,9 +339,8 @@ def determine_url(version, is_commit, bazel_filename): # Example: '0.19.1' -> ('0.19.1', None), '0.20.0rc1' -> ('0.20.0', 'rc1') (version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups() - bazelisk_base_url = get_env_or_config("BAZELISK_BASE_URL") - if bazelisk_base_url is not None: - return "{}/{}{}/{}".format(bazelisk_base_url, version, rc if rc else "", bazel_filename) + if base_url is not None: + return "{}/{}{}/{}".format(base_url, version, rc if rc else "", bazel_filename) else: return "https://releases.bazel.build/{}/{}/{}".format( version, rc if rc else "release", bazel_filename @@ -341,7 +356,6 @@ def trim_suffix(string, suffix): def download_bazel_into_directory(version, is_commit, directory): bazel_filename = determine_bazel_filename(version) - bazel_url = determine_url(version, is_commit, bazel_filename) filename_suffix = determine_executable_filename_suffix() bazel_directory_name = trim_suffix(bazel_filename, filename_suffix) @@ -350,7 +364,28 @@ def download_bazel_into_directory(version, is_commit, directory): destination_path = os.path.join(destination_dir, "bazel" + filename_suffix) if not os.path.exists(destination_path): - download(bazel_url, destination_path) + base_urls = parse_base_urls() + if base_urls: + errors = [] + for base_url in base_urls: + url = determine_url(version, is_commit, bazel_filename, base_url) + try: + download(url, destination_path) + break + except Exception as e: + sys.stderr.write("Could not download Bazel from {}: {}\n".format(url, e)) + errors.append("{}: {}".format(url, e)) + if os.path.exists(destination_path): + os.remove(destination_path) + else: + raise Exception( + "Could not download Bazel from any URL in BAZELISK_BASE_URLS: {}".format( + "; ".join(errors) + ) + ) + else: + bazel_url = determine_url(version, is_commit, bazel_filename) + download(bazel_url, destination_path) os.chmod(destination_path, 0o755) sha256_path = destination_path + ".sha256" @@ -389,8 +424,12 @@ def download_bazel_into_directory(version, is_commit, directory): def download(url, destination_path): sys.stderr.write("Downloading {}...\n".format(url)) request = Request(url) - if get_env_or_config("BAZELISK_BASE_URL") is not None: - parts = urlparse(url) + parts = urlparse(url) + using_custom_url = ( + get_env_or_config("BAZELISK_BASE_URL") is not None + or get_env_or_config("BAZELISK_BASE_URLS") is not None + ) + if using_custom_url and parts.scheme != "file": creds = None try: creds = netrc.netrc().hosts.get(parts.netloc) diff --git a/core/core.go b/core/core.go index 47b1d848..38976758 100644 --- a/core/core.go +++ b/core/core.go @@ -411,6 +411,25 @@ func parseBazelForkAndVersion(bazelForkAndVersion string) (string, string, error return bazelFork, bazelVersion, nil } +// parseBaseURLs returns the list of base URLs to try for downloading Bazel. +// BAZELISK_BASE_URLS (comma-separated) takes precedence over BAZELISK_BASE_URL (single URL). +func parseBaseURLs(config config.Config) []string { + if urls := config.Get(BaseURLsEnv); urls != "" { + var result []string + for _, u := range strings.Split(urls, ",") { + u = strings.TrimSpace(u) + if u != "" { + result = append(result, u) + } + } + return result + } + if u := config.Get(BaseURLEnv); u != "" { + return []string{u} + } + return nil +} + func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Repositories, config config.Config) (string, error) { bazelFork, bazelVersion, err := parseBazelForkAndVersion(bazelVersionString) if err != nil { @@ -422,17 +441,14 @@ func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Reposi return "", fmt.Errorf("could not resolve the version '%s' to an actual version number: %v", bazelVersion, err) } - bazelForkOrURL := dirForURL(config.Get(BaseURLEnv)) - if len(bazelForkOrURL) == 0 { - bazelForkOrURL = bazelFork - } + baseURLs := parseBaseURLs(config) - bazelPath, err := downloadBazelIfNecessary(resolvedBazelVersion, bazeliskHome, bazelForkOrURL, repos, config, downloader) + bazelPath, err := downloadBazelIfNecessary(resolvedBazelVersion, bazeliskHome, bazelFork, baseURLs, repos, config, downloader) return bazelPath, err } // downloadBazelIfNecessary returns a path to a bazel which can be run, which may have been cached. -// The directory it returns may depend on version and bazeliskHome, but does not depend on bazelForkOrURLDirName. +// The directory it returns may depend on version and bazeliskHome, but does not depend on where the binary was downloaded from. // This is important, as the directory may be added to $PATH, and varying the path for equivalent files may cause unnecessary repository rule cache invalidations. // Where a file was downloaded from shouldn't affect cache behaviour of Bazel invocations. // @@ -440,7 +456,7 @@ func downloadBazel(bazelVersionString string, bazeliskHome string, repos *Reposi // // downloads/metadata/[fork-or-url]/bazel-[version-os-etc] is a text file containing a hex sha256 of the contents of the downloaded bazel file. // downloads/sha256/[sha256]/bin/bazel[extension] contains the bazel with a particular sha256. -func downloadBazelIfNecessary(version string, bazeliskHome string, bazelForkOrURLDirName string, repos *Repositories, config config.Config, downloader DownloadFunc) (string, error) { +func downloadBazelIfNecessary(version string, bazeliskHome string, bazelFork string, baseURLs []string, repos *Repositories, config config.Config, downloader DownloadFunc) (string, error) { pathSegment, err := platforms.DetermineBazelFilename(version, false, config) if err != nil { return "", fmt.Errorf("could not determine path segment to use for Bazel binary: %v", err) @@ -448,16 +464,27 @@ func downloadBazelIfNecessary(version string, bazeliskHome string, bazelForkOrUR destFile := "bazel" + platforms.DetermineExecutableFilenameSuffix() - mappingPath := filepath.Join(bazeliskHome, "downloads", "metadata", bazelForkOrURLDirName, pathSegment) - digestFromMappingFile, err := os.ReadFile(mappingPath) - if err == nil { - pathToBazelInCAS := filepath.Join(bazeliskHome, "downloads", "sha256", string(digestFromMappingFile), "bin", destFile) - if _, err := os.Stat(pathToBazelInCAS); err == nil { - return pathToBazelInCAS, nil + // Build the list of cache directory names to check, in priority order. + // Each base URL gets its own metadata dir; if no base URLs are set, use the fork name. + cacheDirNames := make([]string, 0, len(baseURLs)+1) + for _, u := range baseURLs { + cacheDirNames = append(cacheDirNames, dirForURL(u)) + } + if len(cacheDirNames) == 0 { + cacheDirNames = append(cacheDirNames, bazelFork) + } + + for _, cacheDirName := range cacheDirNames { + mappingPath := filepath.Join(bazeliskHome, "downloads", "metadata", cacheDirName, pathSegment) + if digestFromMappingFile, err := os.ReadFile(mappingPath); err == nil { + pathToBazelInCAS := filepath.Join(bazeliskHome, "downloads", "sha256", string(digestFromMappingFile), "bin", destFile) + if _, err := os.Stat(pathToBazelInCAS); err == nil { + return pathToBazelInCAS, nil + } } } - pathToBazelInCAS, downloadedDigest, err := downloadBazelToCAS(version, bazeliskHome, repos, config, downloader) + pathToBazelInCAS, downloadedDigest, successfulURL, err := downloadBazelToCAS(version, bazeliskHome, baseURLs, repos, config, downloader) if err != nil { return "", fmt.Errorf("failed to download bazel: %w", err) } @@ -469,6 +496,15 @@ func downloadBazelIfNecessary(version string, bazeliskHome string, bazelForkOrUR } } + // Write metadata under the cache dir of the URL that actually provided the binary. + // For standard repos (no base URLs), use the fork name. + var writeCacheDirName string + if successfulURL != "" { + writeCacheDirName = dirForURL(successfulURL) + } else { + writeCacheDirName = bazelFork + } + mappingPath := filepath.Join(bazeliskHome, "downloads", "metadata", writeCacheDirName, pathSegment) if err := atomicWriteFile(mappingPath, []byte(downloadedDigest), 0644); err != nil { return "", fmt.Errorf("failed to write mapping file after downloading bazel: %w", err) } @@ -525,43 +561,58 @@ func lockedRenameIfDstAbsent(src, dst string) error { return os.Rename(src, dst) } -func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories, config config.Config, downloader DownloadFunc) (string, string, error) { +// downloadBazelToCAS downloads a Bazel binary to the content-addressable store and returns its path, +// sha256 digest, and the base URL that provided the binary (empty if downloaded via standard repos). +func downloadBazelToCAS(version string, bazeliskHome string, baseURLs []string, repos *Repositories, config config.Config, downloader DownloadFunc) (string, string, string, error) { downloadsDir := filepath.Join(bazeliskHome, "downloads") temporaryDownloadDir := filepath.Join(downloadsDir, "_tmp") casDir := filepath.Join(bazeliskHome, "downloads", "sha256") tmpDestFileBytes := make([]byte, 32) if _, err := rand.Read(tmpDestFileBytes); err != nil { - return "", "", fmt.Errorf("failed to generate temporary file name: %w", err) + return "", "", "", fmt.Errorf("failed to generate temporary file name: %w", err) } tmpDestFile := fmt.Sprintf("%x", tmpDestFileBytes) var tmpDestPath string + var successfulURL string var err error - baseURL := config.Get(BaseURLEnv) + formatURL := config.Get(FormatURLEnv) - if baseURL != "" && formatURL != "" { - return "", "", fmt.Errorf("cannot set %s and %s at once", BaseURLEnv, FormatURLEnv) + if len(baseURLs) > 0 && formatURL != "" { + return "", "", "", fmt.Errorf("cannot set %s or %s and %s at once", BaseURLEnv, BaseURLsEnv, FormatURLEnv) } else if formatURL != "" { tmpDestPath, err = repos.DownloadFromFormatURL(config, formatURL, version, temporaryDownloadDir, tmpDestFile) - } else if baseURL != "" { - tmpDestPath, err = repos.DownloadFromBaseURL(baseURL, version, temporaryDownloadDir, tmpDestFile, config) + } else if len(baseURLs) > 0 { + var errs []string + for _, u := range baseURLs { + tmpDestPath, err = repos.DownloadFromBaseURL(u, version, temporaryDownloadDir, tmpDestFile, config) + if err == nil { + successfulURL = u + break + } + log.Printf("Could not download Bazel from %s: %v", u, err) + errs = append(errs, fmt.Sprintf("%s: %v", u, err)) + } + if successfulURL == "" { + return "", "", "", fmt.Errorf("could not download Bazel from any URL in %s: %s", BaseURLsEnv, strings.Join(errs, "; ")) + } } else { tmpDestPath, err = downloader(temporaryDownloadDir, tmpDestFile) } if err != nil { - return "", "", fmt.Errorf("failed to download bazel: %w", err) + return "", "", "", fmt.Errorf("failed to download bazel: %w", err) } f, err := os.Open(tmpDestPath) if err != nil { - return "", "", fmt.Errorf("failed to open downloaded bazel to digest it: %w", err) + return "", "", "", fmt.Errorf("failed to open downloaded bazel to digest it: %w", err) } h := sha256.New() if _, err := io.Copy(h, f); err != nil { f.Close() - return "", "", fmt.Errorf("cannot compute sha256 of %s after download: %v", tmpDestPath, err) + return "", "", "", fmt.Errorf("cannot compute sha256 of %s after download: %v", tmpDestPath, err) } f.Close() actualSha256 := strings.ToLower(fmt.Sprintf("%x", h.Sum(nil))) @@ -570,24 +621,24 @@ func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories pathToBazelInCAS := filepath.Join(casDir, actualSha256, "bin", bazelInCASBasename) dirForBazelInCAS := filepath.Dir(pathToBazelInCAS) if err := os.MkdirAll(dirForBazelInCAS, 0755); err != nil { - return "", "", fmt.Errorf("failed to MkdirAll parent of %s: %w", pathToBazelInCAS, err) + return "", "", "", fmt.Errorf("failed to MkdirAll parent of %s: %w", pathToBazelInCAS, err) } tmpPathFile, err := os.CreateTemp(dirForBazelInCAS, bazelInCASBasename+".tmp") if err != nil { - return "", "", fmt.Errorf("failed to create temporary file in %s: %w", dirForBazelInCAS, err) + return "", "", "", fmt.Errorf("failed to create temporary file in %s: %w", dirForBazelInCAS, err) } tmpPathFile.Close() defer os.Remove(tmpPathFile.Name()) tmpPathInCorrectDirectory := tmpPathFile.Name() if err := os.Rename(tmpDestPath, tmpPathInCorrectDirectory); err != nil { - return "", "", fmt.Errorf("failed to move %s to %s: %w", tmpDestPath, tmpPathInCorrectDirectory, err) + return "", "", "", fmt.Errorf("failed to move %s to %s: %w", tmpDestPath, tmpPathInCorrectDirectory, err) } if err := lockedRenameIfDstAbsent(tmpPathInCorrectDirectory, pathToBazelInCAS); err != nil { - return "", "", fmt.Errorf("failed to move %s to %s: %w", tmpPathInCorrectDirectory, pathToBazelInCAS, err) + return "", "", "", fmt.Errorf("failed to move %s to %s: %w", tmpPathInCorrectDirectory, pathToBazelInCAS, err) } - return pathToBazelInCAS, actualSha256, nil + return pathToBazelInCAS, actualSha256, successfulURL, nil } func copyFile(src, dst string, perm os.FileMode) error { diff --git a/core/core_test.go b/core/core_test.go index 567be609..92468915 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -888,3 +888,65 @@ func TestRunBazeliskWithStderrRedirection(t *testing.T) { t.Error("stdout content should not appear in stderr") } } + +func TestParseBaseURLs(t *testing.T) { + tests := []struct { + name string + env map[string]string + want []string + }{ + { + name: "neither env var set", + env: map[string]string{}, + want: nil, + }, + { + name: "single URL via BAZELISK_BASE_URL", + env: map[string]string{BaseURLEnv: "https://example.com/bazel"}, + want: []string{"https://example.com/bazel"}, + }, + { + name: "single URL via BAZELISK_BASE_URLS", + env: map[string]string{BaseURLsEnv: "https://example.com/bazel"}, + want: []string{"https://example.com/bazel"}, + }, + { + name: "multiple URLs via BAZELISK_BASE_URLS", + env: map[string]string{BaseURLsEnv: "file:///opt/cache,https://corp.example.com,https://releases.bazel.build"}, + want: []string{"file:///opt/cache", "https://corp.example.com", "https://releases.bazel.build"}, + }, + { + name: "BAZELISK_BASE_URLS with spaces around commas", + env: map[string]string{BaseURLsEnv: "file:///opt/cache , https://corp.example.com"}, + want: []string{"file:///opt/cache", "https://corp.example.com"}, + }, + { + name: "BAZELISK_BASE_URLS takes precedence over BAZELISK_BASE_URL", + env: map[string]string{ + BaseURLEnv: "https://old.example.com", + BaseURLsEnv: "https://new.example.com", + }, + want: []string{"https://new.example.com"}, + }, + { + name: "BAZELISK_BASE_URLS ignores empty entries", + env: map[string]string{BaseURLsEnv: ",https://example.com,,"}, + want: []string{"https://example.com"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := config.Static(tc.env) + got := parseBaseURLs(cfg) + if len(got) != len(tc.want) { + t.Fatalf("parseBaseURLs() = %v, want %v", got, tc.want) + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("parseBaseURLs()[%d] = %q, want %q", i, got[i], tc.want[i]) + } + } + }) + } +} diff --git a/core/repositories.go b/core/repositories.go index 8427953d..488e60a8 100644 --- a/core/repositories.go +++ b/core/repositories.go @@ -15,6 +15,11 @@ const ( // BaseURLEnv is the name of the environment variable that stores the base URL for downloads. BaseURLEnv = "BAZELISK_BASE_URL" + // BaseURLsEnv is the name of the environment variable that stores a comma-separated list of base URLs for downloads. + // URLs are tried in order; the first successful download wins. Supports http://, https://, and file:// schemes. + // If set, takes precedence over BaseURLEnv. + BaseURLsEnv = "BAZELISK_BASE_URLS" + // FormatURLEnv is the name of the environment variable that stores the format string to generate URLs for downloads. FormatURLEnv = "BAZELISK_FORMAT_URL" ) diff --git a/httputil/httputil.go b/httputil/httputil.go index 9f15a93d..989134df 100644 --- a/httputil/httputil.go +++ b/httputil/httputil.go @@ -17,6 +17,7 @@ import ( "path/filepath" "regexp" "strconv" + "strings" "time" netrc "github.com/bgentry/go-netrc/netrc" @@ -193,6 +194,7 @@ func tryFindNetrcFileCreds(host string) (string, error) { } // DownloadBinary downloads a file from the given URL into the specified location, marks it executable and returns its full path. +// Supports http://, https://, and file:// URL schemes. For file:// URLs, returns NotFound if the file does not exist. func DownloadBinary(originURL, destDir, destFile string, config config.Config, verifySignature bool) (string, error) { err := os.MkdirAll(destDir, 0755) if err != nil { @@ -200,6 +202,32 @@ func DownloadBinary(originURL, destDir, destFile string, config config.Config, v } destinationPath := filepath.Join(destDir, destFile) + // Normalize file:// URLs before parsing. On Windows, a raw path like + // file://C:\foo is not a valid URL (C is parsed as host, :\foo as invalid + // port). Convert to file:///C:/foo so url.Parse succeeds. + if strings.HasPrefix(originURL, "file://") { + rest := originURL[len("file://"):] + if len(rest) > 0 && rest[0] != '/' { + originURL = "file:///" + filepath.ToSlash(rest) + } + } + + u, err := url.Parse(originURL) + if err != nil { + return "", err + } + + if u.Scheme == "file" { + // Convert the URL path back to an OS-native path. + // url.Parse("file:///C:/foo") gives u.Path = "/C:/foo" on all platforms; + // on Windows we strip the leading separator before the drive letter. + localPath := filepath.FromSlash(u.Path) + if len(localPath) > 2 && (localPath[0] == '/' || localPath[0] == '\\') && localPath[2] == ':' { + localPath = localPath[1:] + } + return copyFromLocalDisk(localPath, destinationPath) + } + if _, err := os.Stat(destinationPath); err != nil { tmpfile, err := os.CreateTemp(destDir, "download") if err != nil { @@ -212,12 +240,6 @@ func DownloadBinary(originURL, destDir, destFile string, config config.Config, v } }() - u, err := url.Parse(originURL) - if err != nil { - // originURL supposed to be valid - return "", err - } - log.Printf("Downloading %s...", originURL) var auth string = "" @@ -297,6 +319,30 @@ func DownloadBinary(originURL, destDir, destFile string, config config.Config, v return destinationPath, nil } +// copyFromLocalDisk copies a local file to destinationPath, marking it executable. +// Returns NotFound if the source file does not exist. +func copyFromLocalDisk(srcPath, destinationPath string) (string, error) { + if _, err := os.Stat(srcPath); os.IsNotExist(err) { + return "", NotFound + } + src, err := os.Open(srcPath) + if err != nil { + return "", fmt.Errorf("could not open %s: %v", srcPath, err) + } + defer src.Close() + + dst, err := os.OpenFile(destinationPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return "", fmt.Errorf("could not create %s: %v", destinationPath, err) + } + defer dst.Close() + + if _, err := io.Copy(dst, src); err != nil { + return "", fmt.Errorf("could not copy %s to %s: %v", srcPath, destinationPath, err) + } + return destinationPath, nil +} + // ContentMerger is a function that merges multiple HTTP payloads into a single message. type ContentMerger func([][]byte) ([]byte, error) diff --git a/httputil/httputil_test.go b/httputil/httputil_test.go index 1a3f2281..5d8a920f 100644 --- a/httputil/httputil_test.go +++ b/httputil/httputil_test.go @@ -3,10 +3,14 @@ package httputil import ( "errors" "net/http" + "os" + "path/filepath" "strconv" "strings" "testing" "time" + + "github.com/bazelbuild/bazelisk/config" ) var ( @@ -251,3 +255,41 @@ func TestNoRetryOnPermanentError(t *testing.T) { t.Fatalf("Expected no retries for permanent error, but got %d", clock.TimesSlept()) } } + +func TestDownloadBinaryFromFileURL(t *testing.T) { + srcDir := t.TempDir() + srcFile := filepath.Join(srcDir, "bazel") + content := []byte("fake bazel binary content") + if err := os.WriteFile(srcFile, content, 0755); err != nil { + t.Fatalf("Could not write source file: %v", err) + } + + destDir := t.TempDir() + originURL := "file://" + srcFile + + got, err := DownloadBinary(originURL, destDir, "bazel", config.Null(), false) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + gotContent, err := os.ReadFile(got) + if err != nil { + t.Fatalf("Could not read destination file: %v", err) + } + if string(gotContent) != string(content) { + t.Fatalf("Expected content %q, got %q", content, gotContent) + } +} + +func TestDownloadBinaryFromFileURLNotFound(t *testing.T) { + destDir := t.TempDir() + originURL := "file:///nonexistent/path/to/bazel" + + _, err := DownloadBinary(originURL, destDir, "bazel", config.Null(), false) + if err == nil { + t.Fatal("Expected error for missing file, got nil") + } + if !errors.Is(err, NotFound) { + t.Fatalf("Expected NotFound error, got: %v", err) + } +} diff --git a/scripts/populate_bazel_cache.ps1 b/scripts/populate_bazel_cache.ps1 new file mode 100644 index 00000000..fcaa099b --- /dev/null +++ b/scripts/populate_bazel_cache.ps1 @@ -0,0 +1,172 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Pre-populate a bazelisk download cache with Bazel binaries. + +.DESCRIPTION + Downloads Bazel binaries (and their .sha256 files) into a local directory + laid out as: + // + + This matches what bazelisk expects when BAZELISK_BASE_URL is set to point + at the cache root, e.g.: + $env:BAZELISK_BASE_URL = "file:///C:/bazel-cache" + +.PARAMETER CacheRoot + Root directory for the bazelisk cache. Required. + +.PARAMETER Versions + One or more Bazel versions to download, e.g. "7.4.1","8.0.0". Required. + +.PARAMETER Oses + One or more target OSes: linux, darwin, windows. + Defaults to the host OS (windows). + +.PARAMETER Archs + One or more target architectures: x86_64, arm64. + Defaults to the host architecture. + +.PARAMETER NoJdk + Also download bazel_nojdk variants. Off by default. + +.EXAMPLE + .\populate_bazel_cache.ps1 ` + -CacheRoot C:\bazel-cache ` + -Versions 7.4.1,8.0.0 ` + -Oses linux,windows ` + -Archs x86_64,arm64 +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory)][string] $CacheRoot, + [Parameter(Mandatory)][string[]] $Versions, + [string[]] $Oses = @(), + [string[]] $Archs = @(), + [switch] $NoJdk +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$BazelBaseUrl = "https://github.com/bazelbuild/bazel/releases/download" + +# ------------------------------------------------------------------------------ +function Get-HostOs { + if ($IsLinux) { return "linux" } + if ($IsMacOS) { return "darwin" } + return "windows" +} + +function Get-HostArch { + switch ($env:PROCESSOR_ARCHITECTURE) { + "AMD64" { return "x86_64" } + "ARM64" { return "arm64" } + default { + # Fallback for 32-bit host or unusual environments + throw "Unsupported architecture: $($env:PROCESSOR_ARCHITECTURE)" + } + } +} + +function Invoke-Download { + param([string]$Url, [string]$Dest) + try { + Invoke-WebRequest -Uri $Url -OutFile $Dest -UseBasicParsing ` + -ErrorAction Stop + } catch { + throw "Failed to download ${Url}: $_" + } +} + +function Get-Sha256 { + param([string]$File) + return (Get-FileHash -Path $File -Algorithm SHA256).Hash.ToLower() +} + +# ------------------------------------------------------------------------------ +if ($Oses.Count -eq 0) { $Oses = @(Get-HostOs) } +if ($Archs.Count -eq 0) { $Archs = @(Get-HostArch) } + +$Flavors = @("bazel") +if ($NoJdk) { $Flavors += "bazel_nojdk" } + +$errors = 0 + +foreach ($version in $Versions) { + foreach ($os in $Oses) { + foreach ($arch in $Archs) { + foreach ($flavor in $Flavors) { + + $suffix = if ($os -eq "windows") { ".exe" } else { "" } + $filename = "${flavor}-${version}-${os}-${arch}${suffix}" + $destDir = Join-Path $CacheRoot $version + $binDest = Join-Path $destDir $filename + $shaDest = "${binDest}.sha256" + + if ((Test-Path $binDest) -and (Test-Path $shaDest)) { + Write-Host " [skip] $filename (already cached)" + continue + } + + New-Item -ItemType Directory -Force -Path $destDir | Out-Null + + $binUrl = "${BazelBaseUrl}/${version}/${filename}" + $shaUrl = "${binUrl}.sha256" + + Write-Host "Downloading $filename..." + + $tmpBin = Join-Path $destDir (".tmp." + [System.IO.Path]::GetRandomFileName()) + $tmpSha = Join-Path $destDir (".tmp." + [System.IO.Path]::GetRandomFileName()) + + try { + try { + Invoke-Download $binUrl $tmpBin + } catch { + Write-Error " ERROR: $_" + $errors++ + continue + } + + try { + Invoke-Download $shaUrl $tmpSha + } catch { + Write-Error " ERROR: $_" + $errors++ + continue + } + + # The .sha256 file contains the hex digest, optionally + # followed by a filename. Take only the first token. + $expected = (Get-Content $tmpSha -Raw).Trim().Split()[0].ToLower() + $actual = Get-Sha256 $tmpBin + + if ($expected -ne $actual) { + Write-Error (" ERROR: SHA256 mismatch for ${filename}`n" + + " expected: $expected`n" + + " actual: $actual") + $errors++ + continue + } + + Write-Host " verified: $actual" + + Move-Item -Force $tmpBin $binDest + Move-Item -Force $tmpSha $shaDest + + } finally { + # Clean up temp files if they weren't successfully moved. + if (Test-Path $tmpBin) { Remove-Item -Force $tmpBin } + if (Test-Path $tmpSha) { Remove-Item -Force $tmpSha } + } + } + } + } +} + +Write-Host "" +if ($errors -gt 0) { + Write-Error "Completed with $errors error(s). Cache may be incomplete." + exit 1 +} +Write-Host "Done. Cache populated at $CacheRoot" diff --git a/scripts/populate_bazel_cache.sh b/scripts/populate_bazel_cache.sh new file mode 100755 index 00000000..4b0a4a24 --- /dev/null +++ b/scripts/populate_bazel_cache.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# populate_bazel_cache.sh — Pre-populate a bazelisk download cache with Bazel binaries. +# +# The cache layout matches what bazelisk expects when BAZELISK_BASE_URL is set +# to point at the cache root: +# // +# +# Usage example (in a Dockerfile): +# RUN ./populate_bazel_cache.sh \ +# --cache-root /opt/bazel-cache \ +# --version 7.4.1 --version 8.0.0 \ +# --os linux --arch x86_64 +# ENV BAZELISK_BASE_URL=file:///opt/bazel-cache + +set -euo pipefail + +BAZEL_BASE_URL="https://github.com/bazelbuild/bazel/releases/download" + +# ------------------------------------------------------------------------------ +usage() { + cat < --version [options] + +Required: + --cache-root Root directory for the bazelisk cache + --version Bazel version to download (repeatable) + +Optional: + --os Target OS: linux, darwin, windows + (repeatable; default: host OS) + --arch Target arch: x86_64, arm64 + (repeatable; default: host arch) + --nojdk Also download bazel_nojdk variants + -h, --help Show this help message + +Example: + $(basename "$0") --cache-root /opt/bazel-cache \\ + --version 7.4.1 --version 8.0.0 \\ + --os linux --os darwin --arch x86_64 --arch arm64 +EOF +} + +# ------------------------------------------------------------------------------ +detect_os() { + case "$(uname -s)" in + Linux) echo "linux" ;; + Darwin) echo "darwin" ;; + MINGW*|MSYS*|CYGWIN*) echo "windows" ;; + *) echo "ERROR: Unsupported OS: $(uname -s)" >&2; exit 1 ;; + esac +} + +detect_arch() { + case "$(uname -m)" in + x86_64|amd64) echo "x86_64" ;; + arm64|aarch64) echo "arm64" ;; + *) echo "ERROR: Unsupported arch: $(uname -m)" >&2; exit 1 ;; + esac +} + +# ------------------------------------------------------------------------------ +sha256_of_file() { + local file="$1" + if command -v sha256sum &>/dev/null; then + sha256sum "$file" | awk '{print $1}' + elif command -v shasum &>/dev/null; then + shasum -a 256 "$file" | awk '{print $1}' + else + echo "ERROR: no sha256 tool found (need sha256sum or shasum)" >&2 + exit 1 + fi +} + +download_file() { + local url="$1" + local dest="$2" + if command -v curl &>/dev/null; then + curl -fsSL --retry 3 --retry-delay 2 -o "$dest" "$url" + elif command -v wget &>/dev/null; then + wget -q --tries=3 -O "$dest" "$url" + else + echo "ERROR: no download tool found (need curl or wget)" >&2 + exit 1 + fi +} + +# ------------------------------------------------------------------------------ +CACHE_ROOT="" +VERSIONS=() +OSES=() +ARCHS=() +NOJDK=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --cache-root) CACHE_ROOT="$2"; shift 2 ;; + --version) VERSIONS+=("$2"); shift 2 ;; + --os) OSES+=("$2"); shift 2 ;; + --arch) ARCHS+=("$2"); shift 2 ;; + --nojdk) NOJDK=true; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: Unknown option: $1" >&2; echo; usage >&2; exit 1 ;; + esac +done + +if [[ -z "$CACHE_ROOT" ]]; then + echo "ERROR: --cache-root is required" >&2; echo; usage >&2; exit 1 +fi +if [[ ${#VERSIONS[@]} -eq 0 ]]; then + echo "ERROR: at least one --version is required" >&2; echo; usage >&2; exit 1 +fi + +[[ ${#OSES[@]} -eq 0 ]] && OSES=("$(detect_os)") +[[ ${#ARCHS[@]} -eq 0 ]] && ARCHS=("$(detect_arch)") + +FLAVORS=("bazel") +[[ "$NOJDK" == "true" ]] && FLAVORS+=("bazel_nojdk") + +# ------------------------------------------------------------------------------ +errors=0 + +for version in "${VERSIONS[@]}"; do + for os in "${OSES[@]}"; do + for arch in "${ARCHS[@]}"; do + for flavor in "${FLAVORS[@]}"; do + + suffix="" + [[ "$os" == "windows" ]] && suffix=".exe" + + filename="${flavor}-${version}-${os}-${arch}${suffix}" + dest_dir="${CACHE_ROOT}/${version}" + bin_dest="${dest_dir}/${filename}" + sha_dest="${bin_dest}.sha256" + + if [[ -f "$bin_dest" && -f "$sha_dest" ]]; then + echo " [skip] ${filename} (already cached)" + continue + fi + + mkdir -p "$dest_dir" + + bin_url="${BAZEL_BASE_URL}/${version}/${filename}" + sha_url="${bin_url}.sha256" + + echo "Downloading ${filename}..." + + tmp_bin="$(mktemp "${dest_dir}/.tmp.XXXXXX")" + tmp_sha="$(mktemp "${dest_dir}/.tmp.XXXXXX")" + # Clean up temp files on any exit from this iteration. + trap 'rm -f "$tmp_bin" "$tmp_sha"' RETURN + + if ! download_file "$bin_url" "$tmp_bin"; then + echo " ERROR: failed to download $bin_url" >&2 + errors=$((errors + 1)) + continue + fi + + if ! download_file "$sha_url" "$tmp_sha"; then + echo " ERROR: failed to download $sha_url" >&2 + errors=$((errors + 1)) + continue + fi + + expected="$(awk '{print $1}' "$tmp_sha")" + actual="$(sha256_of_file "$tmp_bin")" + + if [[ "$expected" != "$actual" ]]; then + echo " ERROR: SHA256 mismatch for ${filename}" >&2 + echo " expected: $expected" >&2 + echo " actual: $actual" >&2 + errors=$((errors + 1)) + continue + fi + + echo " verified: $actual" + + [[ "$os" != "windows" ]] && chmod +x "$tmp_bin" + + mv "$tmp_bin" "$bin_dest" + mv "$tmp_sha" "$sha_dest" + + trap - RETURN + done + done + done +done + +echo +if [[ $errors -gt 0 ]]; then + echo "Completed with $errors error(s). Cache may be incomplete." >&2 + exit 1 +fi +echo "Done. Cache populated at ${CACHE_ROOT}"