diff --git a/src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs b/src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs index a7b0a04..2480180 100644 --- a/src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs +++ b/src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs @@ -111,6 +111,7 @@ public static IServiceCollection AddPkgDiffBackgroundJobs(this IServiceCollectio services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddSingleton(); services.AddScoped(); diff --git a/src/backend/PkgDiff.API/Extensions/StartupTasks.cs b/src/backend/PkgDiff.API/Extensions/StartupTasks.cs index cc16a0b..1029d2b 100644 --- a/src/backend/PkgDiff.API/Extensions/StartupTasks.cs +++ b/src/backend/PkgDiff.API/Extensions/StartupTasks.cs @@ -84,6 +84,9 @@ public static async Task ScheduleRecurringJobsAsync(this WebApplication app) "autopilot-blog-generator", job => job.RunAutoPilotJobAsync(), await ReadAutoPilotCronAsync(app)); + + // A no-op unless the release-note parser changed since the notes were stored + BackgroundJob.Enqueue(job => job.ExecuteAsync()); } private static async Task ReadAutoPilotCronAsync(WebApplication app) diff --git a/src/backend/PkgDiff.Application/Jobs/ReleaseNoteReparseJob.cs b/src/backend/PkgDiff.Application/Jobs/ReleaseNoteReparseJob.cs new file mode 100644 index 0000000..da8de5f --- /dev/null +++ b/src/backend/PkgDiff.Application/Jobs/ReleaseNoteReparseJob.cs @@ -0,0 +1,76 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using PkgDiff.Application.Interfaces; +using PkgDiff.Domain.Entities; +using PkgDiff.Domain.Interfaces; +using PkgDiff.Shared.Constants; +using PkgDiff.Shared.ReleaseNotes; + +namespace PkgDiff.Application.Jobs; + +// Stored categories are built when notes are scraped; a parser change re-parses them once, a package per job +public class ReleaseNoteReparseJob +{ + // Raise when the release-note parser changes what it returns + public const string ParserVersion = "2"; + + private readonly IUnitOfWork _unitOfWork; + private readonly IAnalyzerService _analyzer; + private readonly IJobScheduler _jobs; + private readonly ILogger _logger; + + public ReleaseNoteReparseJob( + IUnitOfWork unitOfWork, + IAnalyzerService analyzer, + IJobScheduler jobs, + ILogger logger) + { + _unitOfWork = unitOfWork; + _analyzer = analyzer; + _jobs = jobs; + _logger = logger; + } + + public async Task ExecuteAsync() + { + var setting = (await _unitOfWork.SystemSettings.FindAsync(s => s.Key == SettingKeys.ReleaseNoteParserVersion)).FirstOrDefault(); + if (setting?.Value == ParserVersion) + return; + + var packageIds = await _unitOfWork.ToListAsync(_unitOfWork.CachedPackages.Query().Select(p => p.Id)); + foreach (var packageId in packageIds) + _jobs.Enqueue(job => job.ReparsePackageAsync(packageId)); + + if (setting is null) + { + var definition = SettingKeys.Find(SettingKeys.ReleaseNoteParserVersion)!; + await _unitOfWork.SystemSettings.AddAsync(new SystemSettings + { + Key = definition.Key, + Value = ParserVersion, + Description = definition.Description, + DataType = definition.DataType, + Category = definition.Category, + IsSecret = definition.IsSecret + }); + } + else + { + setting.Value = ParserVersion; + } + await _unitOfWork.SaveChangesAsync(); + + _logger.LogInformation("Re-parsing the stored release notes of {Count} packages for parser {Version}", packageIds.Count, ParserVersion); + } + + public async Task ReparsePackageAsync(int cachedPackageId) + { + var notes = await _unitOfWork.CachedReleaseNotes.FindAsync(n => n.CachedPackageId == cachedPackageId && !n.IsEmpty); + foreach (var note in notes) + { + note.ParsedNotesJson = JsonSerializer.Serialize( + _analyzer.ParseReleaseNotes(CompiledReleaseNotes.Split(note.ReleaseNotes).Body)); + } + await _unitOfWork.SaveChangesAsync(); + } +} diff --git a/src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs b/src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs index c35fd6a..a15b8ad 100644 --- a/src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs +++ b/src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs @@ -325,6 +325,7 @@ private async Task> ProcessVersionsAsync( var versionInfoList = new List(); int consecutiveEmpty = 0; int current = 0; + int dateLookups = 0; foreach (var version in versionsToProcess) { @@ -357,7 +358,10 @@ private async Task> ProcessVersionsAsync( { releaseNotes = string.Empty; releaseNotesSource = registryClient.SourceLabel; - publishedDate = await registryClient.GetPublishedDateAsync(packageId, versionString); + // Some registries answer a date with the whole package document; the scrape stores the rest + publishedDate = dateLookups++ < maxConsecutiveEmpty + ? await registryClient.GetPublishedDateAsync(packageId, versionString) + : null; isFromCache = false; _logger.LogDebug("[PackageAnalysis] No cache found for {PackageId} v{Version}, triggering background scrape.", packageId, versionString); @@ -367,7 +371,9 @@ private async Task> ProcessVersionsAsync( var versionInfo = CreateVersionInfo(version, releaseNotes, releaseNotesSource, publishedDate, cachedNote, isFromCache); versionInfoList.Add(versionInfo); - if (versionInfo.IsEmpty) + // Only a version whose notes were looked up and not found counts; one not scraped yet may still have some + var lookedUp = !string.IsNullOrEmpty(cachedNote?.ReleaseNotesSource); + if (versionInfo.IsEmpty && lookedUp) { consecutiveEmpty++; if (consecutiveEmpty >= maxConsecutiveEmpty) @@ -376,7 +382,7 @@ private async Task> ProcessVersionsAsync( break; } } - else + else if (!versionInfo.IsEmpty) { consecutiveEmpty = 0; } diff --git a/src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs b/src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs index 4792958..998f601 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs @@ -9,7 +9,7 @@ public class AnalyzerService : IAnalyzerService // Ordered most specific first private static readonly (string[] Prefixes, ChangeCategory Category)[] LinePrefixRules = [ - (["breaking change", "breaking"], ChangeCategory.BreakingChange), + (["breaking change", "breaking", "bc break"], ChangeCategory.BreakingChange), (["new feature", "feature", "added", "feat"], ChangeCategory.NewFeature), (["security fix", "security"], ChangeCategory.Security), (["bug fix", "bugfix", "fixed", "fix", "bug"], ChangeCategory.BugFix), @@ -18,9 +18,22 @@ private static readonly (string[] Prefixes, ChangeCategory Category)[] LinePrefi (["change", "changed", "update", "updated", "improvement", "enhanced", "enhancement"], ChangeCategory.Other), ]; + // A heading names the category of the lines below it, until the next heading + private static readonly (string[] Keywords, ChangeCategory Category)[] SectionRules = + [ + (["breaking", "bc break", "bc-break", "incompatible"], ChangeCategory.BreakingChange), + (["contributor", "thank", "acknowledg", "full changelog", "upgrade", "install"], ChangeCategory.Other), + (["security"], ChangeCategory.Security), + (["deprecat", "removed", "removal"], ChangeCategory.Deprecation), + (["bug", "fixed", "fixes", "fix"], ChangeCategory.BugFix), + (["feature", "added", "new"], ChangeCategory.NewFeature), + (["performance", "perf"], ChangeCategory.Performance), + (["changed", "changes", "improvement", "enhancement", "other", "misc", "internal", "dependencies"], ChangeCategory.Other), + ]; + private static readonly (string[] Keywords, ChangeCategory Category)[] KeywordRules = [ - (["breaking", "incompatible"], ChangeCategory.BreakingChange), + (["breaking", "incompatible", "bc break"], ChangeCategory.BreakingChange), (["security", "vulnerability", "cve"], ChangeCategory.Security), (["deprecat", "obsolete"], ChangeCategory.Deprecation), (["performance", "optimize", "faster"], ChangeCategory.Performance), @@ -33,6 +46,7 @@ public List ParseReleaseNotes(string releaseNotes) var entries = new List(); var lines = releaseNotes.Split('\n', StringSplitOptions.RemoveEmptyEntries); + ChangeCategory? section = null; foreach (var rawLine in lines) { @@ -40,6 +54,30 @@ public List ParseReleaseNotes(string releaseNotes) if (string.IsNullOrWhiteSpace(line)) continue; + var heading = HeadingText(line); + if (heading is not null && SectionOf(heading) is { } named) + { + section = named; + continue; + } + + // "### 3.0.0" ends the section; a sentence that merely ends in ":" does not + if (heading is not null && !IsColonHeading(line)) + { + section = null; + } + else if (section is { } current) + { + var (lineCategory, lineText) = DetectAndStripPrefix(StripListMarker(line)); + if (!string.IsNullOrWhiteSpace(lineText)) + entries.Add(new ReleaseNoteEntry + { + Category = lineCategory == ChangeCategory.Other ? current : lineCategory, + Text = lineText.Trim() + }); + continue; + } + var cleaned = StripListMarker(line); var (category, text) = DetectAndStripPrefix(cleaned); @@ -163,6 +201,34 @@ private static (ChangeCategory Category, string Text) DetectAndStripPrefix(strin return (ChangeCategory.Other, line); } + private const int MaxColonHeadingLength = 60; + + // "## Fixed", "**Breaking changes**" or a short "Noteworthy BC Breaks:" line on its own + private static string? HeadingText(string line) + { + if (line.StartsWith('#')) + return line.TrimStart('#').Trim(); + + if (line.Length > 4 && line.StartsWith("**") && (line.EndsWith("**") || line.EndsWith("**:"))) + return line.Trim('*', ':', ' '); + + return IsColonHeading(line) ? line.TrimEnd(':').Trim() : null; + } + + private static bool IsColonHeading(string line) => + line.EndsWith(':') && line.Length <= MaxColonHeadingLength && StripListMarker(line) == line; + + private static ChangeCategory? SectionOf(string heading) + { + var lower = heading.ToLowerInvariant(); + foreach (var (keywords, category) in SectionRules) + { + if (keywords.Any(lower.Contains)) + return category; + } + return null; + } + private static string StripListMarker(string line) { var trimmed = line.TrimStart(); diff --git a/src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs b/src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs index fbc5a5c..b959ae2 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs @@ -421,16 +421,20 @@ private static IEnumerable TagCandidates(string? packageId, string versi yield return $"{name}/v{version}"; yield return $"{name}/{version}"; yield return $"{name}_v{version}"; + yield return $"{name}-parent-{version}"; } yield return $"release-{version}"; yield return $"release/{version}"; yield return $"rel_{version.Replace('.', '_')}"; + yield return $"parent-{version}"; + yield return $"r{version}"; + yield return $"v_{version}"; } [System.Text.RegularExpressions.GeneratedRegex(@"^[vV]?\d")] private static partial System.Text.RegularExpressions.Regex PlainVersionTag(); - // 6.4.0.Final is tagged 6.4.0 - [System.Text.RegularExpressions.GeneratedRegex(@"[.-](?:final|release|ga)$", System.Text.RegularExpressions.RegexOptions.IgnoreCase)] + // 6.4.0.Final and 33.0.0-jre are tagged 6.4.0 and v33.0.0 + [System.Text.RegularExpressions.GeneratedRegex(@"[.-](?:final|release|ga|jre|android)$", System.Text.RegularExpressions.RegexOptions.IgnoreCase)] private static partial System.Text.RegularExpressions.Regex ReleaseSuffix(); } diff --git a/src/backend/PkgDiff.Shared/Constants/SettingKeys.cs b/src/backend/PkgDiff.Shared/Constants/SettingKeys.cs index 11cb00e..6c59ebf 100644 --- a/src/backend/PkgDiff.Shared/Constants/SettingKeys.cs +++ b/src/backend/PkgDiff.Shared/Constants/SettingKeys.cs @@ -15,6 +15,7 @@ public static class SettingKeys public const string MaxConsecutiveEmptyVersions = "MaxConsecutiveEmptyVersions"; public const string ScrapingTimeout = "ScrapingTimeout"; public const string HangfireWorkerCount = "Hangfire.WorkerCount"; + public const string ReleaseNoteParserVersion = "Analyzer.ParserVersion"; public const string AutoPilotEnabled = "AutoPilot:Enabled"; public const string AutoPilotLlmModel = "AutoPilot:LlmModel"; @@ -51,6 +52,7 @@ public static class SettingKeys new SettingDefinition(MaxConsecutiveEmptyVersions, "10", "Stop after N consecutive empty release notes", TypeInt, CategoryGeneral, false), new SettingDefinition(ScrapingTimeout, "30", "Scraping timeout in seconds", TypeInt, CategoryGeneral, false), new SettingDefinition(HangfireWorkerCount, "10", "Number of Hangfire background workers (informational; the server does not read this yet)", TypeInt, CategoryGeneral, false), + new SettingDefinition(ReleaseNoteParserVersion, "1", "Release-note parser the stored categories were built with (set by the system)", TypeInt, CategoryGeneral, false), new SettingDefinition(AutoPilotEnabled, "true", "Enable/disable AutoPilot blog generation job", TypeBool, CategoryAutoPilot, false), new SettingDefinition(AutoPilotLlmModel, "qwen2:0.5b", "Ollama model name used by AutoPilot for blog generation", TypeString, CategoryAutoPilot, false),