From a46d6e2f502e1e01f374822f064104879de906c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 17:52:53 +0000 Subject: [PATCH 1/4] fix(test): retry link checks on 429/503 with Retry-After backoff docs.unity3d.com rate-limits HEAD requests, making the test command report valid links as invalid and CommandTests fail intermittently. Retry up to 5 attempts, honoring Retry-After or backing off 1/2/4/8s (capped at 60s). No log per retry: tests assert zero log records. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015PW29oEyQ3Jh7amkT1sjAi --- UnityXrefMaps/Utils.cs | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/UnityXrefMaps/Utils.cs b/UnityXrefMaps/Utils.cs index 6e49723..85dcc01 100644 --- a/UnityXrefMaps/Utils.cs +++ b/UnityXrefMaps/Utils.cs @@ -62,9 +62,20 @@ public static async Task RunCommand(string command, string arguments, Actio return process.ExitCode; } + /// + /// Maximum number of requests sent for a single URI when the server answers 429 or 503. + /// + private const int MaxAttempts = 5; + + /// + /// Upper bound of the delay waited between two requests, whatever the server asks in Retry-After. + /// + private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(60); + /// /// Requests the specified URI with and returns if the response status code is in the - /// range 200-299. + /// range 200-299. Rate-limited (429) or unavailable (503) responses are retried up to + /// times, honoring Retry-After or backing off exponentially. /// /// The URI to request. /// true if the response status code is in the range 200-299. @@ -74,6 +85,14 @@ public static async Task TestUriExists(string? uri, ILogger logger, Cancel { HttpResponseMessage response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken); + for (int attempt = 1; attempt < MaxAttempts && IsTransient(response.StatusCode); attempt++) + { + await Task.Delay(GetRetryDelay(response, attempt), cancellationToken); + + response.Dispose(); + response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken); + } + if (!response.IsSuccessStatusCode && response.StatusCode != System.Net.HttpStatusCode.NotFound) { logger.LogError("HTTP response code on {Uri} is {StatusCode}", uri, response.StatusCode); @@ -88,4 +107,20 @@ public static async Task TestUriExists(string? uri, ILogger logger, Cancel return false; } } + + private static bool IsTransient(System.Net.HttpStatusCode statusCode) => + statusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.ServiceUnavailable; + + /// + /// Returns the delay asked by the Retry-After header, or 1s, 2s, 4s... when absent, capped at + /// . + /// + private static TimeSpan GetRetryDelay(HttpResponseMessage response, int attempt) + { + TimeSpan delay = response.Headers.RetryAfter?.Delta + ?? (response.Headers.RetryAfter?.Date - DateTimeOffset.UtcNow) + ?? TimeSpan.FromSeconds(1 << (attempt - 1)); + + return delay < TimeSpan.Zero ? TimeSpan.Zero : delay > s_maxRetryDelay ? s_maxRetryDelay : delay; + } } From 162b5c7b50243126e44a07f7bedfc9fdfe66c1cc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 18:08:14 +0000 Subject: [PATCH 2/4] feat(test): add --maxAttempts option for link check retries Default 5 (Constants.DefaultMaxAttempts); values below 1 are rejected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015PW29oEyQ3Jh7amkT1sjAi --- UnityXrefMaps/Commands/TestCommand.cs | 18 +++++++++++++++++- UnityXrefMaps/Constants.cs | 5 +++++ UnityXrefMaps/Utils.cs | 14 +++++--------- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/UnityXrefMaps/Commands/TestCommand.cs b/UnityXrefMaps/Commands/TestCommand.cs index 6f0db0d..76b2934 100644 --- a/UnityXrefMaps/Commands/TestCommand.cs +++ b/UnityXrefMaps/Commands/TestCommand.cs @@ -13,7 +13,22 @@ public TestCommand(ILoggerFactory loggerFactory) : base("test", $"Check that the Required = true }; + Option maxAttemptsOption = new("--maxAttempts") + { + Description = "The maximum number of requests sent for a link when the server answers 429 or 503.", + DefaultValueFactory = _ => Constants.DefaultMaxAttempts + }; + + maxAttemptsOption.Validators.Add(result => + { + if (result.GetValueOrDefault() < 1) + { + result.AddError("--maxAttempts must be at least 1."); + } + }); + Options.Add(xrefPathOption); + Options.Add(maxAttemptsOption); SetAction(async (parseResult, cancellationToken) => { @@ -24,12 +39,13 @@ public TestCommand(ILoggerFactory loggerFactory) : base("test", $"Check that the ILogger logger = loggerFactory.CreateLogger(); string? xrefPath = parseResult.GetValue(xrefPathOption); + int maxAttempts = parseResult.GetValue(maxAttemptsOption); XrefMap xrefMap = await xrefMapService.Load(xrefPath!, cancellationToken); foreach (XrefMapReference reference in xrefMap.References!) { - if (!await Utils.TestUriExists(reference.Href, logger, cancellationToken)) + if (!await Utils.TestUriExists(reference.Href, maxAttempts, logger, cancellationToken)) { result = false; diff --git a/UnityXrefMaps/Constants.cs b/UnityXrefMaps/Constants.cs index c4d0004..9abb0a5 100644 --- a/UnityXrefMaps/Constants.cs +++ b/UnityXrefMaps/Constants.cs @@ -2,6 +2,11 @@ namespace UnityXrefMaps; internal static class Constants { + /// + /// The default maximum number of requests sent for a single URI when the server answers 429 or 503. + /// + public const int DefaultMaxAttempts = 5; + /// /// Gets the default URL of the online API documentation of Unity. /// diff --git a/UnityXrefMaps/Utils.cs b/UnityXrefMaps/Utils.cs index 85dcc01..955ecf6 100644 --- a/UnityXrefMaps/Utils.cs +++ b/UnityXrefMaps/Utils.cs @@ -62,11 +62,6 @@ public static async Task RunCommand(string command, string arguments, Actio return process.ExitCode; } - /// - /// Maximum number of requests sent for a single URI when the server answers 429 or 503. - /// - private const int MaxAttempts = 5; - /// /// Upper bound of the delay waited between two requests, whatever the server asks in Retry-After. /// @@ -74,18 +69,19 @@ public static async Task RunCommand(string command, string arguments, Actio /// /// Requests the specified URI with and returns if the response status code is in the - /// range 200-299. Rate-limited (429) or unavailable (503) responses are retried up to - /// times, honoring Retry-After or backing off exponentially. + /// range 200-299. Rate-limited (429) or unavailable (503) responses are retried, honoring Retry-After or + /// backing off exponentially. /// /// The URI to request. + /// The maximum number of requests sent when the server answers 429 or 503. /// true if the response status code is in the range 200-299. - public static async Task TestUriExists(string? uri, ILogger logger, CancellationToken cancellationToken = default) + public static async Task TestUriExists(string? uri, int maxAttempts, ILogger logger, CancellationToken cancellationToken = default) { try { HttpResponseMessage response = await s_httpClient.SendAsync(new(HttpMethod.Head, uri), cancellationToken); - for (int attempt = 1; attempt < MaxAttempts && IsTransient(response.StatusCode); attempt++) + for (int attempt = 1; attempt < maxAttempts && IsTransient(response.StatusCode); attempt++) { await Task.Delay(GetRetryDelay(response, attempt), cancellationToken); From 06f4651751225d872ec945a2b141bd77dc748e93 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 18:25:40 +0000 Subject: [PATCH 3/4] fix(test): bound retry backoff, test retries, document --maxAttempts - Backoff uses named constants and Math.Min instead of a bit shift, which overflowed from attempt 32 (negative delay clamped to zero). - UtilsTests: local HttpListener answers 429 twice, then 200; checks request count and result for maxAttempts 2 and 3. - README: document retries and --maxAttempts. - Move retry fields next to s_httpClient. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015PW29oEyQ3Jh7amkT1sjAi --- README.md | 2 + UnityXrefMaps.Tests/UtilsTests.cs | 76 +++++++++++++++++++++++++++++++ UnityXrefMaps/Utils.cs | 29 ++++++++---- 3 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 UnityXrefMaps.Tests/UtilsTests.cs diff --git a/README.md b/README.md index 4345fd6..b63ca3f 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,8 @@ unityxrefmaps \ Providing an `xrefmap.yml` file will check each `href` element to see if it is a valid URL. +Links answering 429 (Too Many Requests) or 503 (Service Unavailable) are retried, waiting as asked by the `Retry-After` header or with an exponential backoff. `--maxAttempts` sets the maximum number of requests per link (default: 5). + Example: ```bash diff --git a/UnityXrefMaps.Tests/UtilsTests.cs b/UnityXrefMaps.Tests/UtilsTests.cs new file mode 100644 index 0000000..30b6a45 --- /dev/null +++ b/UnityXrefMaps.Tests/UtilsTests.cs @@ -0,0 +1,76 @@ +using System; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; + +namespace UnityXrefMaps.Tests +{ + public class UtilsTests + { + private const int RateLimitedRequestCount = 2; + + [Theory] + [InlineData(RateLimitedRequestCount + 1, true)] + [InlineData(RateLimitedRequestCount, false)] + public async Task TestUriExists_RateLimited_RetriesUpToMaxAttempts(int maxAttempts, bool expected) + { + int port = GetFreePort(); + + using var listener = new HttpListener(); + listener.Prefixes.Add($"http://127.0.0.1:{port}/"); + listener.Start(); + + int requestCount = 0; + + // Answers 429 (Retry-After: 0) RateLimitedRequestCount times, then 200. + Task server = Task.Run(async () => + { + try + { + while (true) + { + HttpListenerContext context = await listener.GetContextAsync(); + + if (Interlocked.Increment(ref requestCount) <= RateLimitedRequestCount) + { + context.Response.StatusCode = (int)HttpStatusCode.TooManyRequests; + context.Response.Headers.Add("Retry-After", "0"); + } + + context.Response.Close(); + } + } + catch (HttpListenerException) + { + // Listener stopped. + } + catch (ObjectDisposedException) + { + // Listener disposed. + } + }, TestContext.Current.CancellationToken); + + bool exists = await Utils.TestUriExists( + $"http://127.0.0.1:{port}/", + maxAttempts, + NullLogger.Instance, + TestContext.Current.CancellationToken); + + listener.Stop(); + await server; + + Assert.Equal(expected, exists); + Assert.Equal(maxAttempts, requestCount); + } + + private static int GetFreePort() + { + using var tcpListener = new TcpListener(IPAddress.Loopback, 0); + tcpListener.Start(); + + return ((IPEndPoint)tcpListener.LocalEndpoint).Port; + } + } +} diff --git a/UnityXrefMaps/Utils.cs b/UnityXrefMaps/Utils.cs index 955ecf6..d9985f4 100644 --- a/UnityXrefMaps/Utils.cs +++ b/UnityXrefMaps/Utils.cs @@ -15,6 +15,21 @@ internal static class Utils /// private static readonly HttpClient s_httpClient = new(); + /// + /// Delay waited before the first retry when the server sends no Retry-After header. + /// + private static readonly TimeSpan s_initialRetryDelay = TimeSpan.FromSeconds(1); + + /// + /// Upper bound of the delay waited between two requests, whatever the server asks in Retry-After. + /// + private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(60); + + /// + /// Factor applied to the delay after each retry when the server sends no Retry-After header. + /// + private const double RetryBackoffFactor = 2; + /// /// Copy a source file to a destination file. Intermediate folders will be automatically created. /// @@ -62,11 +77,6 @@ public static async Task RunCommand(string command, string arguments, Actio return process.ExitCode; } - /// - /// Upper bound of the delay waited between two requests, whatever the server asks in Retry-After. - /// - private static readonly TimeSpan s_maxRetryDelay = TimeSpan.FromSeconds(60); - /// /// Requests the specified URI with and returns if the response status code is in the /// range 200-299. Rate-limited (429) or unavailable (503) responses are retried, honoring Retry-After or @@ -108,14 +118,17 @@ private static bool IsTransient(System.Net.HttpStatusCode statusCode) => statusCode is System.Net.HttpStatusCode.TooManyRequests or System.Net.HttpStatusCode.ServiceUnavailable; /// - /// Returns the delay asked by the Retry-After header, or 1s, 2s, 4s... when absent, capped at - /// . + /// Returns the delay asked by the Retry-After header, or an exponential backoff from + /// when absent, capped at . /// private static TimeSpan GetRetryDelay(HttpResponseMessage response, int attempt) { + // Math.Min in seconds: Math.Pow may reach infinity on high attempts, TimeSpan would overflow. TimeSpan delay = response.Headers.RetryAfter?.Delta ?? (response.Headers.RetryAfter?.Date - DateTimeOffset.UtcNow) - ?? TimeSpan.FromSeconds(1 << (attempt - 1)); + ?? TimeSpan.FromSeconds(Math.Min( + s_initialRetryDelay.TotalSeconds * Math.Pow(RetryBackoffFactor, attempt - 1), + s_maxRetryDelay.TotalSeconds)); return delay < TimeSpan.Zero ? TimeSpan.Zero : delay > s_maxRetryDelay ? s_maxRetryDelay : delay; } From b0648d377c030f54ab02e868418b39c422ba60ed Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 18:52:29 +0000 Subject: [PATCH 4/4] test: pass --maxAttempts 8 to link checks in CommandTests CI on 06f4651 still saw 2 links answer 429 after the default 5 attempts (~15 s of backoff) on GitHub runners. 8 attempts allow up to 123 s per link; the tool default stays 5. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015PW29oEyQ3Jh7amkT1sjAi --- UnityXrefMaps.Tests/CommandTests.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/UnityXrefMaps.Tests/CommandTests.cs b/UnityXrefMaps.Tests/CommandTests.cs index 2f248d0..1c8f7ba 100644 --- a/UnityXrefMaps.Tests/CommandTests.cs +++ b/UnityXrefMaps.Tests/CommandTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.CommandLine; +using System.Globalization; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -14,6 +15,12 @@ namespace UnityXrefMaps.Tests { public class CommandTests(ITestOutputHelper output) : IAsyncLifetime, IAsyncDisposable { + /// + /// docs.unity3d.com rate-limits GitHub runners longer than the default attempts cover (1+2+4+8 s of + /// backoff); 8 attempts wait up to 1+2+4+8+16+32+60 s per link. + /// + private const int TestMaxAttempts = 8; + private string? _repositoryDirectoryPath; private string? _xrefDirectoryPath; private string? _docFxFilePath; @@ -145,7 +152,9 @@ await buildCommand string[] testArgs = [ "--xrefPath", - xrefFilePath + xrefFilePath, + "--maxAttempts", + TestMaxAttempts.ToString(CultureInfo.InvariantCulture) ]; Assert.Equal( @@ -274,7 +283,9 @@ await buildCommand string[] testArgs = [ "--xrefPath", - xrefFilePath + xrefFilePath, + "--maxAttempts", + TestMaxAttempts.ToString(CultureInfo.InvariantCulture) ]; Assert.Equal(