diff --git a/README.md b/README.md
index 4345fd6..b63ca3f 100644
--- a/README.md
+++ b/README.md
@@ -92,6 +92,8 @@ unityxrefmaps \
Providing an `xrefmap.yml` file will check each `href` element to see if it is a valid URL.
+Links answering 429 (Too Many Requests) or 503 (Service Unavailable) are retried, waiting as asked by the `Retry-After` header or with an exponential backoff. `--maxAttempts` sets the maximum number of requests per link (default: 5).
+
Example:
```bash
diff --git a/UnityXrefMaps.Tests/CommandTests.cs b/UnityXrefMaps.Tests/CommandTests.cs
index 2f248d0..1c8f7ba 100644
--- a/UnityXrefMaps.Tests/CommandTests.cs
+++ b/UnityXrefMaps.Tests/CommandTests.cs
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.CommandLine;
+using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
@@ -14,6 +15,12 @@ namespace UnityXrefMaps.Tests
{
public class CommandTests(ITestOutputHelper output) : IAsyncLifetime, IAsyncDisposable
{
+ ///
+ /// docs.unity3d.com rate-limits GitHub runners longer than the default attempts cover (1+2+4+8 s of
+ /// backoff); 8 attempts wait up to 1+2+4+8+16+32+60 s per link.
+ ///
+ private const int TestMaxAttempts = 8;
+
private string? _repositoryDirectoryPath;
private string? _xrefDirectoryPath;
private string? _docFxFilePath;
@@ -145,7 +152,9 @@ await buildCommand
string[] testArgs = [
"--xrefPath",
- xrefFilePath
+ xrefFilePath,
+ "--maxAttempts",
+ TestMaxAttempts.ToString(CultureInfo.InvariantCulture)
];
Assert.Equal(
@@ -274,7 +283,9 @@ await buildCommand
string[] testArgs = [
"--xrefPath",
- xrefFilePath
+ xrefFilePath,
+ "--maxAttempts",
+ TestMaxAttempts.ToString(CultureInfo.InvariantCulture)
];
Assert.Equal(
diff --git a/UnityXrefMaps.Tests/UtilsTests.cs b/UnityXrefMaps.Tests/UtilsTests.cs
new file mode 100644
index 0000000..30b6a45
--- /dev/null
+++ b/UnityXrefMaps.Tests/UtilsTests.cs
@@ -0,0 +1,76 @@
+using System;
+using System.Net;
+using System.Net.Sockets;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace UnityXrefMaps.Tests
+{
+ public class UtilsTests
+ {
+ private const int RateLimitedRequestCount = 2;
+
+ [Theory]
+ [InlineData(RateLimitedRequestCount + 1, true)]
+ [InlineData(RateLimitedRequestCount, false)]
+ public async Task TestUriExists_RateLimited_RetriesUpToMaxAttempts(int maxAttempts, bool expected)
+ {
+ int port = GetFreePort();
+
+ using var listener = new HttpListener();
+ listener.Prefixes.Add($"http://127.0.0.1:{port}/");
+ listener.Start();
+
+ int requestCount = 0;
+
+ // Answers 429 (Retry-After: 0) RateLimitedRequestCount times, then 200.
+ Task server = Task.Run(async () =>
+ {
+ try
+ {
+ while (true)
+ {
+ HttpListenerContext context = await listener.GetContextAsync();
+
+ if (Interlocked.Increment(ref requestCount) <= RateLimitedRequestCount)
+ {
+ context.Response.StatusCode = (int)HttpStatusCode.TooManyRequests;
+ context.Response.Headers.Add("Retry-After", "0");
+ }
+
+ context.Response.Close();
+ }
+ }
+ catch (HttpListenerException)
+ {
+ // Listener stopped.
+ }
+ catch (ObjectDisposedException)
+ {
+ // Listener disposed.
+ }
+ }, TestContext.Current.CancellationToken);
+
+ bool exists = await Utils.TestUriExists(
+ $"http://127.0.0.1:{port}/",
+ maxAttempts,
+ NullLogger.Instance,
+ TestContext.Current.CancellationToken);
+
+ listener.Stop();
+ await server;
+
+ Assert.Equal(expected, exists);
+ Assert.Equal(maxAttempts, requestCount);
+ }
+
+ private static int GetFreePort()
+ {
+ using var tcpListener = new TcpListener(IPAddress.Loopback, 0);
+ tcpListener.Start();
+
+ return ((IPEndPoint)tcpListener.LocalEndpoint).Port;
+ }
+ }
+}
diff --git a/UnityXrefMaps/Commands/TestCommand.cs b/UnityXrefMaps/Commands/TestCommand.cs
index 6f0db0d..76b2934 100644
--- a/UnityXrefMaps/Commands/TestCommand.cs
+++ b/UnityXrefMaps/Commands/TestCommand.cs
@@ -13,7 +13,22 @@ public TestCommand(ILoggerFactory loggerFactory) : base("test", $"Check that the
Required = true
};
+ Option maxAttemptsOption = new("--maxAttempts")
+ {
+ Description = "The maximum number of requests sent for a link when the server answers 429 or 503.",
+ DefaultValueFactory = _ => Constants.DefaultMaxAttempts
+ };
+
+ maxAttemptsOption.Validators.Add(result =>
+ {
+ if (result.GetValueOrDefault() < 1)
+ {
+ result.AddError("--maxAttempts must be at least 1.");
+ }
+ });
+
Options.Add(xrefPathOption);
+ Options.Add(maxAttemptsOption);
SetAction(async (parseResult, cancellationToken) =>
{
@@ -24,12 +39,13 @@ public TestCommand(ILoggerFactory loggerFactory) : base("test", $"Check that the
ILogger logger = loggerFactory.CreateLogger();
string? xrefPath = parseResult.GetValue(xrefPathOption);
+ int maxAttempts = parseResult.GetValue(maxAttemptsOption);
XrefMap xrefMap = await xrefMapService.Load(xrefPath!, cancellationToken);
foreach (XrefMapReference reference in xrefMap.References!)
{
- if (!await Utils.TestUriExists(reference.Href, logger, cancellationToken))
+ if (!await Utils.TestUriExists(reference.Href, maxAttempts, logger, cancellationToken))
{
result = false;
diff --git a/UnityXrefMaps/Constants.cs b/UnityXrefMaps/Constants.cs
index c4d0004..9abb0a5 100644
--- a/UnityXrefMaps/Constants.cs
+++ b/UnityXrefMaps/Constants.cs
@@ -2,6 +2,11 @@ namespace UnityXrefMaps;
internal static class Constants
{
+ ///
+ /// The default maximum number of requests sent for a single URI when the server answers 429 or 503.
+ ///
+ public const int DefaultMaxAttempts = 5;
+
///
/// Gets the default URL of the online API documentation of Unity.
///
diff --git a/UnityXrefMaps/Utils.cs b/UnityXrefMaps/Utils.cs
index 6e49723..d9985f4 100644
--- a/UnityXrefMaps/Utils.cs
+++ b/UnityXrefMaps/Utils.cs
@@ -15,6 +15,21 @@ internal static class Utils
///
private static readonly HttpClient s_httpClient = new();
+ ///
+ /// Delay waited before the first retry when the server sends no Retry-After header.
+ ///
+ private static readonly TimeSpan s_initialRetryDelay = TimeSpan.FromSeconds(1);
+
+ ///
+ /// Upper bound of the delay waited between two requests, whatever the server asks in Retry-After.
+ ///
+ private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(60);
+
+ ///
+ /// Factor applied to the delay after each retry when the server sends no Retry-After header.
+ ///
+ private const double RetryBackoffFactor = 2;
+
///
/// Copy a source file to a destination file. Intermediate folders will be automatically created.
///
@@ -64,16 +79,26 @@ public static async Task RunCommand(string command, string arguments, Actio
///
/// Requests the specified URI with and returns if the response status code is in the
- /// range 200-299.
+ /// range 200-299. Rate-limited (429) or unavailable (503) responses are retried, honoring Retry-After or
+ /// backing off exponentially.
///
/// The URI to request.
+ /// The maximum number of requests sent when the server answers 429 or 503.
/// true if the response status code is in the range 200-299.
- public static async Task TestUriExists(string? uri, ILogger logger, CancellationToken cancellationToken = default)
+ public static async Task TestUriExists(string? uri, int maxAttempts, ILogger logger, CancellationToken cancellationToken = default)
{
try
{
HttpResponseMessage response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken);
+ for (int attempt = 1; attempt < maxAttempts && IsTransient(response.StatusCode); attempt++)
+ {
+ await Task.Delay(GetRetryDelay(response, attempt), cancellationToken);
+
+ response.Dispose();
+ response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken);
+ }
+
if (!response.IsSuccessStatusCode && response.StatusCode != System.Net.HttpStatusCode.NotFound)
{
logger.LogError("HTTP response code on {Uri} is {StatusCode}", uri, response.StatusCode);
@@ -88,4 +113,23 @@ public static async Task TestUriExists(string? uri, ILogger logger, Cancel
return false;
}
}
+
+ private static bool IsTransient(System.Net.HttpStatusCode statusCode) =>
+ statusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.ServiceUnavailable;
+
+ ///
+ /// Returns the delay asked by the Retry-After header, or an exponential backoff from
+ /// when absent, capped at .
+ ///
+ private static TimeSpan GetRetryDelay(HttpResponseMessage response, int attempt)
+ {
+ // Math.Min in seconds: Math.Pow may reach infinity on high attempts, TimeSpan would overflow.
+ TimeSpan delay = response.Headers.RetryAfter?.Delta
+ ?? (response.Headers.RetryAfter?.Date - DateTimeOffset.UtcNow)
+ ?? TimeSpan.FromSeconds(Math.Min(
+ s_initialRetryDelay.TotalSeconds * Math.Pow(RetryBackoffFactor, attempt - 1),
+ s_maxRetryDelay.TotalSeconds));
+
+ return delay < TimeSpan.Zero ? TimeSpan.Zero : delay > s_maxRetryDelay ? s_maxRetryDelay : delay;
+ }
}