diff --git a/README.md b/README.md index 4345fd6..b63ca3f 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ unityxrefmaps \ Providing an `xrefmap.yml` file will check each `href` element to see if it is a valid URL. +Links answering 429 (Too Many Requests) or 503 (Service Unavailable) are retried, waiting as asked by the `Retry-After` header or with an exponential backoff. `--maxAttempts` sets the maximum number of requests per link (default: 5). + Example: ```bash diff --git a/UnityXrefMaps.Tests/CommandTests.cs b/UnityXrefMaps.Tests/CommandTests.cs index 2f248d0..1c8f7ba 100644 --- a/UnityXrefMaps.Tests/CommandTests.cs +++ b/UnityXrefMaps.Tests/CommandTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.CommandLine; +using System.Globalization; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -14,6 +15,12 @@ namespace UnityXrefMaps.Tests { public class CommandTests(ITestOutputHelper output) : IAsyncLifetime, IAsyncDisposable { + /// + /// docs.unity3d.com rate-limits GitHub runners longer than the default attempts cover (1+2+4+8 s of + /// backoff); 8 attempts wait up to 1+2+4+8+16+32+60 s per link. + /// + private const int TestMaxAttempts = 8; + private string? _repositoryDirectoryPath; private string? _xrefDirectoryPath; private string? _docFxFilePath; @@ -145,7 +152,9 @@ await buildCommand string[] testArgs = [ "--xrefPath", - xrefFilePath + xrefFilePath, + "--maxAttempts", + TestMaxAttempts.ToString(CultureInfo.InvariantCulture) ]; Assert.Equal( @@ -274,7 +283,9 @@ await buildCommand string[] testArgs = [ "--xrefPath", - xrefFilePath + xrefFilePath, + "--maxAttempts", + TestMaxAttempts.ToString(CultureInfo.InvariantCulture) ]; Assert.Equal( diff --git a/UnityXrefMaps.Tests/UtilsTests.cs b/UnityXrefMaps.Tests/UtilsTests.cs new file mode 100644 index 0000000..30b6a45 --- /dev/null +++ b/UnityXrefMaps.Tests/UtilsTests.cs @@ -0,0 +1,76 @@ +using System; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; + +namespace UnityXrefMaps.Tests +{ + public class UtilsTests + { + private const int RateLimitedRequestCount = 2; + + [Theory] + [InlineData(RateLimitedRequestCount + 1, true)] + [InlineData(RateLimitedRequestCount, false)] + public async Task TestUriExists_RateLimited_RetriesUpToMaxAttempts(int maxAttempts, bool expected) + { + int port = GetFreePort(); + + using var listener = new HttpListener(); + listener.Prefixes.Add($"http://127.0.0.1:{port}/"); + listener.Start(); + + int requestCount = 0; + + // Answers 429 (Retry-After: 0) RateLimitedRequestCount times, then 200. + Task server = Task.Run(async () => + { + try + { + while (true) + { + HttpListenerContext context = await listener.GetContextAsync(); + + if (Interlocked.Increment(ref requestCount) <= RateLimitedRequestCount) + { + context.Response.StatusCode = (int)HttpStatusCode.TooManyRequests; + context.Response.Headers.Add("Retry-After", "0"); + } + + context.Response.Close(); + } + } + catch (HttpListenerException) + { + // Listener stopped. + } + catch (ObjectDisposedException) + { + // Listener disposed. + } + }, TestContext.Current.CancellationToken); + + bool exists = await Utils.TestUriExists( + $"http://127.0.0.1:{port}/", + maxAttempts, + NullLogger.Instance, + TestContext.Current.CancellationToken); + + listener.Stop(); + await server; + + Assert.Equal(expected, exists); + Assert.Equal(maxAttempts, requestCount); + } + + private static int GetFreePort() + { + using var tcpListener = new TcpListener(IPAddress.Loopback, 0); + tcpListener.Start(); + + return ((IPEndPoint)tcpListener.LocalEndpoint).Port; + } + } +} diff --git a/UnityXrefMaps/Commands/TestCommand.cs b/UnityXrefMaps/Commands/TestCommand.cs index 6f0db0d..76b2934 100644 --- a/UnityXrefMaps/Commands/TestCommand.cs +++ b/UnityXrefMaps/Commands/TestCommand.cs @@ -13,7 +13,22 @@ public TestCommand(ILoggerFactory loggerFactory) : base("test", $"Check that the Required = true }; + Option maxAttemptsOption = new("--maxAttempts") + { + Description = "The maximum number of requests sent for a link when the server answers 429 or 503.", + DefaultValueFactory = _ => Constants.DefaultMaxAttempts + }; + + maxAttemptsOption.Validators.Add(result => + { + if (result.GetValueOrDefault() < 1) + { + result.AddError("--maxAttempts must be at least 1."); + } + }); + Options.Add(xrefPathOption); + Options.Add(maxAttemptsOption); SetAction(async (parseResult, cancellationToken) => { @@ -24,12 +39,13 @@ public TestCommand(ILoggerFactory loggerFactory) : base("test", $"Check that the ILogger logger = loggerFactory.CreateLogger(); string? xrefPath = parseResult.GetValue(xrefPathOption); + int maxAttempts = parseResult.GetValue(maxAttemptsOption); XrefMap xrefMap = await xrefMapService.Load(xrefPath!, cancellationToken); foreach (XrefMapReference reference in xrefMap.References!) { - if (!await Utils.TestUriExists(reference.Href, logger, cancellationToken)) + if (!await Utils.TestUriExists(reference.Href, maxAttempts, logger, cancellationToken)) { result = false; diff --git a/UnityXrefMaps/Constants.cs b/UnityXrefMaps/Constants.cs index c4d0004..9abb0a5 100644 --- a/UnityXrefMaps/Constants.cs +++ b/UnityXrefMaps/Constants.cs @@ -2,6 +2,11 @@ namespace UnityXrefMaps; internal static class Constants { + /// + /// The default maximum number of requests sent for a single URI when the server answers 429 or 503. + /// + public const int DefaultMaxAttempts = 5; + /// /// Gets the default URL of the online API documentation of Unity. /// diff --git a/UnityXrefMaps/Utils.cs b/UnityXrefMaps/Utils.cs index 6e49723..d9985f4 100644 --- a/UnityXrefMaps/Utils.cs +++ b/UnityXrefMaps/Utils.cs @@ -15,6 +15,21 @@ internal static class Utils /// private static readonly HttpClient s_httpClient = new(); + /// + /// Delay waited before the first retry when the server sends no Retry-After header. + /// + private static readonly TimeSpan s_initialRetryDelay = TimeSpan.FromSeconds(1); + + /// + /// Upper bound of the delay waited between two requests, whatever the server asks in Retry-After. + /// + private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(60); + + /// + /// Factor applied to the delay after each retry when the server sends no Retry-After header. + /// + private const double RetryBackoffFactor = 2; + /// /// Copy a source file to a destination file. Intermediate folders will be automatically created. /// @@ -64,16 +79,26 @@ public static async Task RunCommand(string command, string arguments, Actio /// /// Requests the specified URI with and returns if the response status code is in the - /// range 200-299. + /// range 200-299. Rate-limited (429) or unavailable (503) responses are retried, honoring Retry-After or + /// backing off exponentially. /// /// The URI to request. + /// The maximum number of requests sent when the server answers 429 or 503. /// true if the response status code is in the range 200-299. - public static async Task TestUriExists(string? uri, ILogger logger, CancellationToken cancellationToken = default) + public static async Task TestUriExists(string? uri, int maxAttempts, ILogger logger, CancellationToken cancellationToken = default) { try { HttpResponseMessage response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken); + for (int attempt = 1; attempt < maxAttempts && IsTransient(response.StatusCode); attempt++) + { + await Task.Delay(GetRetryDelay(response, attempt), cancellationToken); + + response.Dispose(); + response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken); + } + if (!response.IsSuccessStatusCode && response.StatusCode != System.Net.HttpStatusCode.NotFound) { logger.LogError("HTTP response code on {Uri} is {StatusCode}", uri, response.StatusCode); @@ -88,4 +113,23 @@ public static async Task TestUriExists(string? uri, ILogger logger, Cancel return false; } } + + private static bool IsTransient(System.Net.HttpStatusCode statusCode) => + statusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.ServiceUnavailable; + + /// + /// Returns the delay asked by the Retry-After header, or an exponential backoff from + /// when absent, capped at . + /// + private static TimeSpan GetRetryDelay(HttpResponseMessage response, int attempt) + { + // Math.Min in seconds: Math.Pow may reach infinity on high attempts, TimeSpan would overflow. + TimeSpan delay = response.Headers.RetryAfter?.Delta + ?? (response.Headers.RetryAfter?.Date - DateTimeOffset.UtcNow) + ?? TimeSpan.FromSeconds(Math.Min( + s_initialRetryDelay.TotalSeconds * Math.Pow(RetryBackoffFactor, attempt - 1), + s_maxRetryDelay.TotalSeconds)); + + return delay < TimeSpan.Zero ? TimeSpan.Zero : delay > s_maxRetryDelay ? s_maxRetryDelay : delay; + } }