Retry link checks on 429/503 to fix flaky ci

README.md

@@ -92,6 +92,8 @@ unityxrefmaps \
 
 Providing an `xrefmap.yml` file will check each `href` element to see if it is a valid URL.
 
+Links answering 429 (Too Many Requests) or 503 (Service Unavailable) are retried, waiting as asked by the `Retry-After` header or with an exponential backoff. `--maxAttempts` sets the maximum number of requests per link (default: 5).
+
 Example:
 
 ```bash

UnityXrefMaps.Tests/CommandTests.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.CommandLine;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
@@ -14,6 +15,12 @@ namespace UnityXrefMaps.Tests
 {
     public class CommandTests(ITestOutputHelper output) : IAsyncLifetime, IAsyncDisposable
     {
+        /// <summary>
+        /// docs.unity3d.com rate-limits GitHub runners longer than the default attempts cover (1+2+4+8 s of
+        /// backoff); 8 attempts wait up to 1+2+4+8+16+32+60 s per link.
+        /// </summary>
+        private const int TestMaxAttempts = 8;
+
         private string? _repositoryDirectoryPath;
         private string? _xrefDirectoryPath;
         private string? _docFxFilePath;
@@ -145,7 +152,9 @@ await buildCommand
 
                 string[] testArgs = [
                     "--xrefPath",
-                    xrefFilePath
+                    xrefFilePath,
+                    "--maxAttempts",
+                    TestMaxAttempts.ToString(CultureInfo.InvariantCulture)
                 ];
 
                 Assert.Equal(
@@ -274,7 +283,9 @@ await buildCommand
 
                 string[] testArgs = [
                     "--xrefPath",
-                    xrefFilePath
+                    xrefFilePath,
+                    "--maxAttempts",
+                    TestMaxAttempts.ToString(CultureInfo.InvariantCulture)
                 ];
 
                 Assert.Equal(

UnityXrefMaps.Tests/UtilsTests.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Net;
+using System.Net.Sockets;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace UnityXrefMaps.Tests
+{
+    public class UtilsTests
+    {
+        private const int RateLimitedRequestCount = 2;
+
+        [Theory]
+        [InlineData(RateLimitedRequestCount + 1, true)]
+        [InlineData(RateLimitedRequestCount, false)]
+        public async Task TestUriExists_RateLimited_RetriesUpToMaxAttempts(int maxAttempts, bool expected)
+        {
+            int port = GetFreePort();
+
+            using var listener = new HttpListener();
+            listener.Prefixes.Add($"http://127.0.0.1:{port}/");
+            listener.Start();
+
+            int requestCount = 0;
+
+            // Answers 429 (Retry-After: 0) RateLimitedRequestCount times, then 200.
+            Task server = Task.Run(async () =>
+            {
+                try
+                {
+                    while (true)
+                    {
+                        HttpListenerContext context = await listener.GetContextAsync();
+
+                        if (Interlocked.Increment(ref requestCount) <= RateLimitedRequestCount)
+                        {
+                            context.Response.StatusCode = (int)HttpStatusCode.TooManyRequests;
+                            context.Response.Headers.Add("Retry-After", "0");
+                        }
+
+                        context.Response.Close();
+                    }
+                }
+                catch (HttpListenerException)
+                {
+                    // Listener stopped.
+                }
+                catch (ObjectDisposedException)
+                {
+                    // Listener disposed.
+                }
+            }, TestContext.Current.CancellationToken);
+
+            bool exists = await Utils.TestUriExists(
+                $"http://127.0.0.1:{port}/",
+                maxAttempts,
+                NullLogger.Instance,
+                TestContext.Current.CancellationToken);
+
+            listener.Stop();
+            await server;
+
+            Assert.Equal(expected, exists);
+            Assert.Equal(maxAttempts, requestCount);
+        }
+
+        private static int GetFreePort()
+        {
+            using var tcpListener = new TcpListener(IPAddress.Loopback, 0);
+            tcpListener.Start();
+
+            return ((IPEndPoint)tcpListener.LocalEndpoint).Port;
+        }
+    }
+}

UnityXrefMaps/Commands/TestCommand.cs

@@ -13,7 +13,22 @@ public TestCommand(ILoggerFactory loggerFactory) : base("test", $"Check that the
             Required = true
         };
 
+        Option<int> maxAttemptsOption = new("--maxAttempts")
+        {
+            Description = "The maximum number of requests sent for a link when the server answers 429 or 503.",
+            DefaultValueFactory = _ => Constants.DefaultMaxAttempts
+        };
+
+        maxAttemptsOption.Validators.Add(result =>
+        {
+            if (result.GetValueOrDefault<int>() < 1)
+            {
+                result.AddError("--maxAttempts must be at least 1.");
+            }
+        });
+
         Options.Add(xrefPathOption);
+        Options.Add(maxAttemptsOption);
 
         SetAction(async (parseResult, cancellationToken) =>
         {
@@ -24,12 +39,13 @@ public TestCommand(ILoggerFactory loggerFactory) : base("test", $"Check that the
             ILogger logger = loggerFactory.CreateLogger<BuildCommand>();
 
             string? xrefPath = parseResult.GetValue(xrefPathOption);
+            int maxAttempts = parseResult.GetValue(maxAttemptsOption);
 
             XrefMap xrefMap = await xrefMapService.Load(xrefPath!, cancellationToken);
 
             foreach (XrefMapReference reference in xrefMap.References!)
             {
-                if (!await Utils.TestUriExists(reference.Href, logger, cancellationToken))
+                if (!await Utils.TestUriExists(reference.Href, maxAttempts, logger, cancellationToken))
                 {
                     result = false;
 

UnityXrefMaps/Constants.cs

@@ -2,6 +2,11 @@ namespace UnityXrefMaps;
 
 internal static class Constants
 {
+    /// <summary>
+    /// The default maximum number of requests sent for a single URI when the server answers 429 or 503.
+    /// </summary>
+    public const int DefaultMaxAttempts = 5;
+
     /// <summary>
     /// Gets the default URL of the online API documentation of Unity.
     /// </summary>

UnityXrefMaps/Utils.cs

@@ -15,6 +15,21 @@ internal static class Utils
     /// </summary>
     private static readonly HttpClient s_httpClient = new();
 
+    /// <summary>
+    /// Delay waited before the first retry when the server sends no Retry-After header.
+    /// </summary>
+    private static readonly TimeSpan s_initialRetryDelay = TimeSpan.FromSeconds(1);
+
+    /// <summary>
+    /// Upper bound of the delay waited between two requests, whatever the server asks in Retry-After.
+    /// </summary>
+    private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(60);
+
+    /// <summary>
+    /// Factor applied to the delay after each retry when the server sends no Retry-After header.
+    /// </summary>
+    private const double RetryBackoffFactor = 2;
+
     /// <summary>
     /// Copy a source file to a destination file. Intermediate folders will be automatically created.
     /// </summary>
@@ -64,16 +79,26 @@ public static async Task<int> RunCommand(string command, string arguments, Actio
 
     /// <summary>
     /// Requests the specified URI with <see cref="s_httpClient"/> and returns if the response status code is in the
-    /// range 200-299.
+    /// range 200-299. Rate-limited (429) or unavailable (503) responses are retried, honoring Retry-After or
+    /// backing off exponentially.
     /// </summary>
     /// <param name="uri">The URI to request.</param>
+    /// <param name="maxAttempts">The maximum number of requests sent when the server answers 429 or 503.</param>
     /// <returns><c>true</c> if the response status code is in the range 200-299.</returns>
-    public static async Task<bool> TestUriExists(string? uri, ILogger logger, CancellationToken cancellationToken = default)
+    public static async Task<bool> TestUriExists(string? uri, int maxAttempts, ILogger logger, CancellationToken cancellationToken = default)
     {
         try
         {
             HttpResponseMessage response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken);
 
+            for (int attempt = 1; attempt < maxAttempts && IsTransient(response.StatusCode); attempt++)
+            {
+                await Task.Delay(GetRetryDelay(response, attempt), cancellationToken);
+
+                response.Dispose();
+                response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken);
+            }
+
             if (!response.IsSuccessStatusCode && response.StatusCode != System.Net.HttpStatusCode.NotFound)
             {
                 logger.LogError("HTTP response code on {Uri} is {StatusCode}", uri, response.StatusCode);
@@ -88,4 +113,23 @@ public static async Task<bool> TestUriExists(string? uri, ILogger logger, Cancel
             return false;
         }
     }
+
+    private static bool IsTransient(System.Net.HttpStatusCode statusCode) =>
+        statusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.ServiceUnavailable;
+
+    /// <summary>
+    /// Returns the delay asked by the Retry-After header, or an exponential backoff from
+    /// <see cref="s_initialRetryDelay"/> when absent, capped at <see cref="s_maxRetryDelay"/>.
+    /// </summary>
+    private static TimeSpan GetRetryDelay(HttpResponseMessage response, int attempt)
+    {
+        // Math.Min in seconds: Math.Pow may reach infinity on high attempts, TimeSpan would overflow.
+        TimeSpan delay = response.Headers.RetryAfter?.Delta
+            ?? (response.Headers.RetryAfter?.Date - DateTimeOffset.UtcNow)
+            ?? TimeSpan.FromSeconds(Math.Min(
+                s_initialRetryDelay.TotalSeconds * Math.Pow(RetryBackoffFactor, attempt - 1),
+                s_maxRetryDelay.TotalSeconds));
+
+        return delay < TimeSpan.Zero ? TimeSpan.Zero : delay > s_maxRetryDelay ? s_maxRetryDelay : delay;
+    }
 }