Merge pull request #16 from pkgdiff-dev/gsd/quick-261021-analysis-fixes

fix(analysis): categorise notes by their section and stop early only on looked-up versions

Author: fleizean

src/backend/PkgDiff.API/Extensions/HostingServiceCollectionExtensions.cs

@@ -111,6 +111,7 @@ public static IServiceCollection AddPkgDiffBackgroundJobs(this IServiceCollectio
         services.AddScoped<PackageAnalysisJob>();
         services.AddScoped<PackageRefreshJob>();
         services.AddScoped<DatabaseCleanupJob>();
+        services.AddScoped<ReleaseNoteReparseJob>();
 
         services.AddSingleton<HangfireDashboardAccess>();
         services.AddScoped<IAdminRoleChecker, IdentityAdminRoleChecker>();

src/backend/PkgDiff.API/Extensions/StartupTasks.cs

@@ -84,6 +84,9 @@ public static async Task ScheduleRecurringJobsAsync(this WebApplication app)
             "autopilot-blog-generator",
             job => job.RunAutoPilotJobAsync(),
             await ReadAutoPilotCronAsync(app));
+
+        // A no-op unless the release-note parser changed since the notes were stored
+        BackgroundJob.Enqueue<ReleaseNoteReparseJob>(job => job.ExecuteAsync());
     }
 
     private static async Task<string> ReadAutoPilotCronAsync(WebApplication app)

src/backend/PkgDiff.Application/Jobs/ReleaseNoteReparseJob.cs

@@ -0,0 +1,76 @@
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using PkgDiff.Application.Interfaces;
+using PkgDiff.Domain.Entities;
+using PkgDiff.Domain.Interfaces;
+using PkgDiff.Shared.Constants;
+using PkgDiff.Shared.ReleaseNotes;
+
+namespace PkgDiff.Application.Jobs;
+
+// Stored categories are built when notes are scraped; a parser change re-parses them once, a package per job
+public class ReleaseNoteReparseJob
+{
+    // Raise when the release-note parser changes what it returns
+    public const string ParserVersion = "2";
+
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly IAnalyzerService _analyzer;
+    private readonly IJobScheduler _jobs;
+    private readonly ILogger<ReleaseNoteReparseJob> _logger;
+
+    public ReleaseNoteReparseJob(
+        IUnitOfWork unitOfWork,
+        IAnalyzerService analyzer,
+        IJobScheduler jobs,
+        ILogger<ReleaseNoteReparseJob> logger)
+    {
+        _unitOfWork = unitOfWork;
+        _analyzer = analyzer;
+        _jobs = jobs;
+        _logger = logger;
+    }
+
+    public async Task ExecuteAsync()
+    {
+        var setting = (await _unitOfWork.SystemSettings.FindAsync(s => s.Key == SettingKeys.ReleaseNoteParserVersion)).FirstOrDefault();
+        if (setting?.Value == ParserVersion)
+            return;
+
+        var packageIds = await _unitOfWork.ToListAsync(_unitOfWork.CachedPackages.Query().Select(p => p.Id));
+        foreach (var packageId in packageIds)
+            _jobs.Enqueue<ReleaseNoteReparseJob>(job => job.ReparsePackageAsync(packageId));
+
+        if (setting is null)
+        {
+            var definition = SettingKeys.Find(SettingKeys.ReleaseNoteParserVersion)!;
+            await _unitOfWork.SystemSettings.AddAsync(new SystemSettings
+            {
+                Key = definition.Key,
+                Value = ParserVersion,
+                Description = definition.Description,
+                DataType = definition.DataType,
+                Category = definition.Category,
+                IsSecret = definition.IsSecret
+            });
+        }
+        else
+        {
+            setting.Value = ParserVersion;
+        }
+        await _unitOfWork.SaveChangesAsync();
+
+        _logger.LogInformation("Re-parsing the stored release notes of {Count} packages for parser {Version}", packageIds.Count, ParserVersion);
+    }
+
+    public async Task ReparsePackageAsync(int cachedPackageId)
+    {
+        var notes = await _unitOfWork.CachedReleaseNotes.FindAsync(n => n.CachedPackageId == cachedPackageId && !n.IsEmpty);
+        foreach (var note in notes)
+        {
+            note.ParsedNotesJson = JsonSerializer.Serialize(
+                _analyzer.ParseReleaseNotes(CompiledReleaseNotes.Split(note.ReleaseNotes).Body));
+        }
+        await _unitOfWork.SaveChangesAsync();
+    }
+}

src/backend/PkgDiff.Application/Services/PackageAnalysisService.cs

@@ -325,6 +325,7 @@ private async Task<List<VersionInfo>> ProcessVersionsAsync(
         var versionInfoList = new List<VersionInfo>();
         int consecutiveEmpty = 0;
         int current = 0;
+        int dateLookups = 0;
 
         foreach (var version in versionsToProcess)
         {
@@ -357,7 +358,10 @@ private async Task<List<VersionInfo>> ProcessVersionsAsync(
             {
                 releaseNotes = string.Empty;
                 releaseNotesSource = registryClient.SourceLabel;
-                publishedDate = await registryClient.GetPublishedDateAsync(packageId, versionString);
+                // Some registries answer a date with the whole package document; the scrape stores the rest
+                publishedDate = dateLookups++ < maxConsecutiveEmpty
+                    ? await registryClient.GetPublishedDateAsync(packageId, versionString)
+                    : null;
                 isFromCache = false;
 
                 _logger.LogDebug("[PackageAnalysis] No cache found for {PackageId} v{Version}, triggering background scrape.", packageId, versionString);
@@ -367,7 +371,9 @@ private async Task<List<VersionInfo>> ProcessVersionsAsync(
             var versionInfo = CreateVersionInfo(version, releaseNotes, releaseNotesSource, publishedDate, cachedNote, isFromCache);
             versionInfoList.Add(versionInfo);
 
-            if (versionInfo.IsEmpty)
+            // Only a version whose notes were looked up and not found counts; one not scraped yet may still have some
+            var lookedUp = !string.IsNullOrEmpty(cachedNote?.ReleaseNotesSource);
+            if (versionInfo.IsEmpty && lookedUp)
             {
                 consecutiveEmpty++;
                 if (consecutiveEmpty >= maxConsecutiveEmpty)
@@ -376,7 +382,7 @@ private async Task<List<VersionInfo>> ProcessVersionsAsync(
                     break;
                 }
             }
-            else
+            else if (!versionInfo.IsEmpty)
             {
                 consecutiveEmpty = 0;
             }

src/backend/PkgDiff.Infrastructure/Services/AnalyzerService.cs

@@ -9,7 +9,7 @@ public class AnalyzerService : IAnalyzerService
     // Ordered most specific first
     private static readonly (string[] Prefixes, ChangeCategory Category)[] LinePrefixRules =
     [
-        (["breaking change", "breaking"], ChangeCategory.BreakingChange),
+        (["breaking change", "breaking", "bc break"], ChangeCategory.BreakingChange),
         (["new feature", "feature", "added", "feat"], ChangeCategory.NewFeature),
         (["security fix", "security"], ChangeCategory.Security),
         (["bug fix", "bugfix", "fixed", "fix", "bug"], ChangeCategory.BugFix),
@@ -18,9 +18,22 @@ private static readonly (string[] Prefixes, ChangeCategory Category)[] LinePrefi
         (["change", "changed", "update", "updated", "improvement", "enhanced", "enhancement"], ChangeCategory.Other),
     ];
 
+    // A heading names the category of the lines below it, until the next heading
+    private static readonly (string[] Keywords, ChangeCategory Category)[] SectionRules =
+    [
+        (["breaking", "bc break", "bc-break", "incompatible"], ChangeCategory.BreakingChange),
+        (["contributor", "thank", "acknowledg", "full changelog", "upgrade", "install"], ChangeCategory.Other),
+        (["security"], ChangeCategory.Security),
+        (["deprecat", "removed", "removal"], ChangeCategory.Deprecation),
+        (["bug", "fixed", "fixes", "fix"], ChangeCategory.BugFix),
+        (["feature", "added", "new"], ChangeCategory.NewFeature),
+        (["performance", "perf"], ChangeCategory.Performance),
+        (["changed", "changes", "improvement", "enhancement", "other", "misc", "internal", "dependencies"], ChangeCategory.Other),
+    ];
+
     private static readonly (string[] Keywords, ChangeCategory Category)[] KeywordRules =
     [
-        (["breaking", "incompatible"], ChangeCategory.BreakingChange),
+        (["breaking", "incompatible", "bc break"], ChangeCategory.BreakingChange),
         (["security", "vulnerability", "cve"], ChangeCategory.Security),
         (["deprecat", "obsolete"], ChangeCategory.Deprecation),
         (["performance", "optimize", "faster"], ChangeCategory.Performance),
@@ -33,13 +46,38 @@ public List<ReleaseNoteEntry> ParseReleaseNotes(string releaseNotes)
 
         var entries = new List<ReleaseNoteEntry>();
         var lines = releaseNotes.Split('\n', StringSplitOptions.RemoveEmptyEntries);
+        ChangeCategory? section = null;
 
         foreach (var rawLine in lines)
         {
             var line = rawLine.Trim();
             if (string.IsNullOrWhiteSpace(line))
                 continue;
 
+            var heading = HeadingText(line);
+            if (heading is not null && SectionOf(heading) is { } named)
+            {
+                section = named;
+                continue;
+            }
+
+            // "### 3.0.0" ends the section; a sentence that merely ends in ":" does not
+            if (heading is not null && !IsColonHeading(line))
+            {
+                section = null;
+            }
+            else if (section is { } current)
+            {
+                var (lineCategory, lineText) = DetectAndStripPrefix(StripListMarker(line));
+                if (!string.IsNullOrWhiteSpace(lineText))
+                    entries.Add(new ReleaseNoteEntry
+                    {
+                        Category = lineCategory == ChangeCategory.Other ? current : lineCategory,
+                        Text = lineText.Trim()
+                    });
+                continue;
+            }
+
             var cleaned = StripListMarker(line);
 
             var (category, text) = DetectAndStripPrefix(cleaned);
@@ -163,6 +201,34 @@ private static (ChangeCategory Category, string Text) DetectAndStripPrefix(strin
         return (ChangeCategory.Other, line);
     }
 
+    private const int MaxColonHeadingLength = 60;
+
+    // "## Fixed", "**Breaking changes**" or a short "Noteworthy BC Breaks:" line on its own
+    private static string? HeadingText(string line)
+    {
+        if (line.StartsWith('#'))
+            return line.TrimStart('#').Trim();
+
+        if (line.Length > 4 && line.StartsWith("**") && (line.EndsWith("**") || line.EndsWith("**:")))
+            return line.Trim('*', ':', ' ');
+
+        return IsColonHeading(line) ? line.TrimEnd(':').Trim() : null;
+    }
+
+    private static bool IsColonHeading(string line) =>
+        line.EndsWith(':') && line.Length <= MaxColonHeadingLength && StripListMarker(line) == line;
+
+    private static ChangeCategory? SectionOf(string heading)
+    {
+        var lower = heading.ToLowerInvariant();
+        foreach (var (keywords, category) in SectionRules)
+        {
+            if (keywords.Any(lower.Contains))
+                return category;
+        }
+        return null;
+    }
+
     private static string StripListMarker(string line)
     {
         var trimmed = line.TrimStart();

src/backend/PkgDiff.Infrastructure/Services/GitHubScraperService.cs

@@ -421,16 +421,20 @@ private static IEnumerable<string> TagCandidates(string? packageId, string versi
             yield return $"{name}/v{version}";
             yield return $"{name}/{version}";
             yield return $"{name}_v{version}";
+            yield return $"{name}-parent-{version}";
         }
         yield return $"release-{version}";
         yield return $"release/{version}";
         yield return $"rel_{version.Replace('.', '_')}";
+        yield return $"parent-{version}";
+        yield return $"r{version}";
+        yield return $"v_{version}";
     }
 
     [System.Text.RegularExpressions.GeneratedRegex(@"^[vV]?\d")]
     private static partial System.Text.RegularExpressions.Regex PlainVersionTag();
 
-    // 6.4.0.Final is tagged 6.4.0
-    [System.Text.RegularExpressions.GeneratedRegex(@"[.-](?:final|release|ga)$", System.Text.RegularExpressions.RegexOptions.IgnoreCase)]
+    // 6.4.0.Final and 33.0.0-jre are tagged 6.4.0 and v33.0.0
+    [System.Text.RegularExpressions.GeneratedRegex(@"[.-](?:final|release|ga|jre|android)$", System.Text.RegularExpressions.RegexOptions.IgnoreCase)]
     private static partial System.Text.RegularExpressions.Regex ReleaseSuffix();
 }

src/backend/PkgDiff.Shared/Constants/SettingKeys.cs

@@ -15,6 +15,7 @@ public static class SettingKeys
     public const string MaxConsecutiveEmptyVersions = "MaxConsecutiveEmptyVersions";
     public const string ScrapingTimeout = "ScrapingTimeout";
     public const string HangfireWorkerCount = "Hangfire.WorkerCount";
+    public const string ReleaseNoteParserVersion = "Analyzer.ParserVersion";
 
     public const string AutoPilotEnabled = "AutoPilot:Enabled";
     public const string AutoPilotLlmModel = "AutoPilot:LlmModel";
@@ -51,6 +52,7 @@ public static class SettingKeys
         new SettingDefinition(MaxConsecutiveEmptyVersions, "10", "Stop after N consecutive empty release notes", TypeInt, CategoryGeneral, false),
         new SettingDefinition(ScrapingTimeout, "30", "Scraping timeout in seconds", TypeInt, CategoryGeneral, false),
         new SettingDefinition(HangfireWorkerCount, "10", "Number of Hangfire background workers (informational; the server does not read this yet)", TypeInt, CategoryGeneral, false),
+        new SettingDefinition(ReleaseNoteParserVersion, "1", "Release-note parser the stored categories were built with (set by the system)", TypeInt, CategoryGeneral, false),
 
         new SettingDefinition(AutoPilotEnabled, "true", "Enable/disable AutoPilot blog generation job", TypeBool, CategoryAutoPilot, false),
         new SettingDefinition(AutoPilotLlmModel, "qwen2:0.5b", "Ollama model name used by AutoPilot for blog generation", TypeString, CategoryAutoPilot, false),