Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions src/backend/PkgDiff.API/Extensions/DatabaseSeeder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,22 @@ public static async Task SeedDataAsync(IServiceProvider serviceProvider)
}
}

var packagistSpellings = PkgDiff.Shared.Constants.RegistryNames.SpellingsOf(PkgDiff.Shared.Constants.RegistryNames.Packagist);
if (!await context.Registries.AnyAsync(r => packagistSpellings.Contains(r.Value.ToLower())))
{
var displayOrder = (await context.Registries.MaxAsync(r => (int?)r.DisplayOrder) ?? 0) + 1;
context.Registries.Add(new Registry
{
Value = PkgDiff.Shared.Constants.RegistryNames.Packagist,
Label = "Packagist",
Icon = "/icons/packagist.svg",
Disabled = false,
DisplayOrder = displayOrder
});
await context.SaveChangesAsync();
logger.LogInformation("Registry {Registry} added", PkgDiff.Shared.Constants.RegistryNames.Packagist);
}

var expectedPatterns = new[]
{
new { Id = 1, Pattern = "dev", Description = "Development builds (e.g., 1.0.0-dev-123)" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,15 @@ public static IServiceCollection AddPkgDiffOutboundHttp(this IServiceCollection
client.MaxResponseContentBufferSize = RegistryMaxResponseBytes;
});

// Packagist asks API clients to name themselves with a way to reach them
services.AddHttpClient(HttpClientNames.PackagistRepository, client =>
{
client.BaseAddress = new Uri("https://repo.packagist.org/");
client.Timeout = FixedHostTimeout;
client.MaxResponseContentBufferSize = RegistryMaxResponseBytes;
client.DefaultRequestHeaders.Add("User-Agent", "PkgDiff/1.0 (+https://pkgdiff.dev)");
});

services.AddHttpClient<IPubDevService, PubDevService>(HttpClientNames.PubDev, client =>
{
client.BaseAddress = new Uri("https://pub.dev/");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public static IServiceCollection AddBusinessServices(this IServiceCollection ser
services.AddScoped<INugetService, NugetService>();
services.AddScoped<INpmService, NpmService>();
services.AddScoped<IPyPIService, PyPIService>();
services.AddScoped<IPackagistService, PackagistService>();
// IPubDevService is the PubDev typed client registered by AddPkgDiffOutboundHttp
services.AddScoped<IScraperService, ScraperService>();
services.AddSingleton<GitHubScrapeCache>();
Expand All @@ -44,6 +45,7 @@ public static IServiceCollection AddBusinessServices(this IServiceCollection ser
services.AddScoped<IRegistryClient, NpmRegistryClient>();
services.AddScoped<IRegistryClient, PyPIRegistryClient>();
services.AddScoped<IRegistryClient, PubDevRegistryClient>();
services.AddScoped<IRegistryClient, PackagistRegistryClient>();

services.AddScoped<IRegistryServiceFactory, RegistryServiceFactory>();
services.AddScoped<IPackageCacheService, PackageCacheService>();
Expand Down
10 changes: 10 additions & 0 deletions src/backend/PkgDiff.Application/Interfaces/IPackagistService.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
using PkgDiff.Shared.DTOs;

namespace PkgDiff.Application.Interfaces;

public interface IPackagistService
{
Task<IEnumerable<string>> GetPackageVersionsAsync(string packageId, bool includePrerelease = false);
Task<DateTime?> GetPackagePublishedDateAsync(string packageId, string version);
Task<PackageMetadata> GetPackageMetadataAsync(string packageId);
}
128 changes: 128 additions & 0 deletions src/backend/PkgDiff.Application/Versioning/ComposerVersion.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
using System.Text.RegularExpressions;

namespace PkgDiff.Application.Versioning;

// A Packagist version as Composer orders it: dev < alpha < beta < RC < stable < patch
public sealed class ComposerVersion : IComparable<ComposerVersion>, IComparable
{
private const int MaxLength = 100;
private const int MaxReleaseSegments = 4;
private const int StableRank = 4;

private static readonly Regex Pattern = new(
@"^v?
(?<release>[0-9]+(?:\.[0-9]+)*)
(?:[._-]?(?<stability>stable|beta|b|rc|alpha|a|patch|pl|p)(?<number>[.-]?[0-9]+(?:[.-][0-9]+)*)?)?
(?<dev>[.-]?dev)?
(?:\+[0-9A-Za-z.-]+)?$",
RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.CultureInvariant,
TimeSpan.FromMilliseconds(250));

private readonly long[] _release;
private readonly int _rank;
private readonly long[] _number;
private readonly bool _dev;

private ComposerVersion(long[] release, int rank, long[] number, bool dev)
{
_release = release;
_rank = rank;
_number = number;
_dev = dev;
}

public bool IsPrerelease => _rank < StableRank || _dev;

public static bool TryParse(string? value, out ComposerVersion version)
{
version = null!;
if (string.IsNullOrWhiteSpace(value) || value.Length > MaxLength)
return false;

Match match;
try
{
match = Pattern.Match(value.Trim());
}
catch (RegexMatchTimeoutException)
{
return false;
}

if (!match.Success)
return false;

var segments = match.Groups["release"].Value.Split('.');
if (segments.Length > MaxReleaseSegments || !TryNumbers(segments, out var release))
return false;

var number = Array.Empty<long>();
if (match.Groups["number"].Success
&& !TryNumbers(match.Groups["number"].Value.Split(new[] { '.', '-' }, StringSplitOptions.RemoveEmptyEntries), out number))
return false;

var dev = match.Groups["dev"].Success;
var rank = match.Groups["stability"].Success
? RankOf(match.Groups["stability"].Value)
: dev ? 0 : StableRank;

version = new ComposerVersion(release, rank, number, dev && match.Groups["stability"].Success);
return true;
}

public int CompareTo(ComposerVersion? other)
{
if (other is null)
return 1;

var result = CompareNumbers(_release, other._release);
if (result != 0)
return result;

result = _rank.CompareTo(other._rank);
if (result != 0)
return result;

result = CompareNumbers(_number, other._number);
if (result != 0)
return result;

// 1.0.0-beta1-dev comes before 1.0.0-beta1
return other._dev.CompareTo(_dev);
}

public int CompareTo(object? obj) => CompareTo(obj as ComposerVersion);

private static int RankOf(string stability) => stability.ToLowerInvariant() switch
{
"alpha" or "a" => 1,
"beta" or "b" => 2,
"rc" => 3,
"stable" => StableRank,
_ => 5
};

private static bool TryNumbers(string[] parts, out long[] numbers)
{
numbers = new long[parts.Length];
for (var i = 0; i < parts.Length; i++)
{
if (!long.TryParse(parts[i], out numbers[i]))
return false;
}
return true;
}

// Trailing zeros do not count: 1.0 and 1.0.0.0 are the same release
private static int CompareNumbers(long[] left, long[] right)
{
var length = Math.Max(left.Length, right.Length);
for (var i = 0; i < length; i++)
{
var result = (i < left.Length ? left[i] : 0).CompareTo(i < right.Length ? right[i] : 0);
if (result != 0)
return result;
}
return 0;
}
}
13 changes: 11 additions & 2 deletions src/backend/PkgDiff.Application/Versioning/PackageVersion.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ namespace PkgDiff.Application.Versioning;

public sealed class PackageVersion : IComparable<PackageVersion>, IEquatable<PackageVersion>
{
// Either a Pep440Version or a NuGetVersion; versions of one package always share the kind
// A Pep440Version, ComposerVersion or NuGetVersion; versions of one package always share the kind
private readonly IComparable _key;

private PackageVersion(string value, bool isPrerelease, IComparable key)
Expand All @@ -15,7 +15,7 @@ private PackageVersion(string value, bool isPrerelease, IComparable key)
_key = key;
}

// The spelling the registry is asked with: its own for npm, pub.dev and PyPI, NuGetVersion.ToString() for NuGet
// The spelling the registry is asked with: its own for npm, pub.dev, PyPI and Packagist, NuGetVersion.ToString() for NuGet
public string Value { get; }

public bool IsPrerelease { get; }
Expand All @@ -37,6 +37,15 @@ public static bool TryParse(VersionScheme scheme, string? value, [NotNullWhen(tr
return true;
}

if (scheme == VersionScheme.Composer)
{
if (!ComposerVersion.TryParse(value, out var composer))
return false;

version = new PackageVersion(value, composer.IsPrerelease, composer);
return true;
}

if (!NuGetVersion.TryParse(value, out var semantic))
return false;

Expand Down
5 changes: 4 additions & 1 deletion src/backend/PkgDiff.Application/Versioning/VersionScheme.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,8 @@ public enum VersionScheme
SemVer,

// PEP 440: PyPI
Pep440
Pep440,

// Composer's rules: Packagist
Composer
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ public static class HttpClientNames
public const string NpmRegistry = "NpmRegistry";
public const string PyPIRegistry = "PyPIRegistry";
public const string PubDev = "PubDev";
public const string PackagistRepository = "PackagistRepository";
public const string HealthProbe = "HealthProbe";
public const string Osv = "Osv";
public const string DepsDev = "DepsDev";
Expand Down
81 changes: 81 additions & 0 deletions src/backend/PkgDiff.Infrastructure/Services/ManifestParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ public ManifestParseResult Parse(string registry, string content, string? fileNa
RegistryNames.NuGet => ParseNuGet(content),
RegistryNames.PyPI => ParsePyPI(content, fileName),
RegistryNames.PubDev => ParsePub(content),
RegistryNames.Packagist => ParseComposer(content),
_ => throw new ArgumentException("Unknown registry.")
};
result.Registry = canonical;
Expand Down Expand Up @@ -587,6 +588,86 @@ private static AuditDependency FromPubConstraint(string name, string? requiremen
};
}

private static ManifestParseResult ParseComposer(string content)
{
var root = ParseJson(content, "a composer.json or composer.lock");

if (root.TryGetProperty("packages", out var packages) && packages.ValueKind == JsonValueKind.Array)
{
var lockResult = new ManifestParseResult { Format = "composer.lock" };
ReadComposerLock(packages, null, lockResult);
if (root.TryGetProperty("packages-dev", out var devPackages) && devPackages.ValueKind == JsonValueKind.Array)
ReadComposerLock(devPackages, Dev, lockResult);
return lockResult;
}

if (!root.TryGetProperty("require", out _) && !root.TryGetProperty("require-dev", out _))
throw new ArgumentException("This JSON has neither require nor packages: paste a composer.json or a composer.lock.");

var result = new ManifestParseResult { Format = "composer.json" };
foreach (var (section, scope) in new[] { ("require", (string?)null), ("require-dev", Dev) })
{
if (!root.TryGetProperty(section, out var dependencies) || dependencies.ValueKind != JsonValueKind.Object)
continue;
foreach (var dependency in dependencies.EnumerateObject())
{
// php, ext-*, lib-* and composer-plugin-api name the platform, not a package
if (!dependency.Name.Contains('/'))
continue;
var requirement = dependency.Value.ValueKind == JsonValueKind.String ? dependency.Value.GetString() : null;
result.Dependencies.Add(FromComposerConstraint(dependency.Name.ToLowerInvariant(), requirement, scope, result.Warnings));
}
}
return result;
}

private static void ReadComposerLock(JsonElement packages, string? scope, ManifestParseResult result)
{
foreach (var package in packages.EnumerateArray())
{
var name = Text(package, "name");
var version = Text(package, "version");
if (name is null || !name.Contains('/'))
continue;
if (version is null || version.StartsWith("dev-", StringComparison.OrdinalIgnoreCase))
{
result.Warnings.Add($"{name}: \"{version}\" is a branch, not a release; its latest version is checked.");
result.Dependencies.Add(new AuditDependency { Name = name.ToLowerInvariant(), Requirement = version, Scope = scope });
continue;
}
result.Dependencies.Add(Exact(name.ToLowerInvariant(), StripComposerV(version), scope));
}
}

private static AuditDependency FromComposerConstraint(string name, string? requirement, string? scope, List<string> warnings)
{
var value = requirement?.Trim() ?? string.Empty;
if (value.Length == 0 || value == "*")
return new AuditDependency { Name = name, Requirement = requirement, Scope = scope };

if (value.Contains("dev-", StringComparison.OrdinalIgnoreCase))
{
warnings.Add($"{name}: \"{value}\" is a branch, not a release; its latest version is checked.");
return new AuditDependency { Name = name, Requirement = requirement, Scope = scope };
}

// "^1.0 || ^2.0" allows several ranges; the first one is checked, without its @stability flag
var first = (value.Split('|', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault() ?? value).Trim().Split('@')[0];
var exact = SemVerExact().IsMatch(first);
var match = SemVerStart().Match(first);
return new AuditDependency
{
Name = name,
Requirement = requirement,
Scope = scope,
IsExact = exact,
Version = match.Success ? Complete(match.Value) : null
};
}

private static string StripComposerV(string version) =>
version.Length > 1 && (version[0] == 'v' || version[0] == 'V') && char.IsDigit(version[1]) ? version[1..] : version;

private static JsonElement ParseJson(string content, string expected)
{
try
Expand Down
1 change: 1 addition & 0 deletions src/backend/PkgDiff.Infrastructure/Services/OsvClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ private static string SeverityOf(Vulnerability vuln)
RegistryNames.Npm => "npm",
RegistryNames.PyPI => "PyPI",
RegistryNames.PubDev => "Pub",
RegistryNames.Packagist => "Packagist",
_ => null
};

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
using Microsoft.Extensions.Logging;
using PkgDiff.Application.Interfaces;
using PkgDiff.Application.Versioning;
using PkgDiff.Domain.Interfaces;
using PkgDiff.Shared.Constants;
using PkgDiff.Shared.DTOs;

namespace PkgDiff.Infrastructure.Services;

public class PackagistRegistryClient : ListedVersionsRegistryClient
{
private readonly IPackagistService _packagistService;
private readonly GitHubFirstScrapingStrategy _githubFirstStrategy;

public PackagistRegistryClient(
IPackagistService packagistService,
GitHubFirstScrapingStrategy githubFirstStrategy,
IVersionFilterService versionFilterService,
ILogger<PackagistRegistryClient> logger)
: base(versionFilterService, logger)
{
_packagistService = packagistService;
_githubFirstStrategy = githubFirstStrategy;
}

public override string Id => RegistryNames.Packagist;

public override string SourceLabel => "Packagist";

public override VersionScheme VersionScheme => VersionScheme.Composer;

public override bool PublishesReleaseNotes => false;

protected override Task<IEnumerable<string>> ListVersionsAsync(string packageId, bool includePrerelease) =>
_packagistService.GetPackageVersionsAsync(packageId, includePrerelease);

public override Task<DateTime?> GetPublishedDateAsync(string packageId, string version) =>
_packagistService.GetPackagePublishedDateAsync(packageId, version);

protected override Task<PackageMetadata> FetchMetadataAsync(string packageId) =>
_packagistService.GetPackageMetadataAsync(packageId);

public override Task<string?> GetOwnReleaseNotesAsync(string packageId, string version) =>
Task.FromResult<string?>(null);

public override IReleaseNotesScrapingStrategy GetScrapingStrategy(string? githubRepoUrl) => _githubFirstStrategy;
}
Loading