diff --git a/dev/design/net-async-http-uat-blockers.md b/dev/design/net-async-http-uat-blockers.md new file mode 100644 index 0000000000..06bb5b0a27 --- /dev/null +++ b/dev/design/net-async-http-uat-blockers.md @@ -0,0 +1,50 @@ +# Net::Async::HTTP UAT Blockers + +**Status:** Implemented and validated; PR #1213 is based on `master` and ready +for its final CI gate. + +## Scope + +The Net::Async::HTTP 0.50 UAT run identified three compatibility failures that +are separate from the refcount-owner ledger work: + +1. `HTTP::Cookies` sent `Cookie2: \\$Version="1"`, retaining a literal + escape before the RFC 2965 version marker. +2. `Compress::Raw::Zlib::Deflate` accepted `WANT_GZIP` but emitted raw deflate + bytes without the required RFC 1952 header and trailer. Net::Async::HTTP's + content decoder correctly rejected those bytes as an invalid gzip header. +3. The bundled `Compress::Bzip2` lacked its streaming + `bzinflateInit`/`inflateInit` and `bzdeflateInit` facade over the bundled + raw bzip2 implementation, preventing Net::Async::HTTP from decoding bzip2 + content. + +## Completed Work + +- [x] Corrected the Cookie2 literal in `src/main/perl/lib/HTTP/Cookies.pm`. +- [x] Added gzip header/trailer emission, CRC32, size bookkeeping, and reset + handling to `CompressRawZlib` for `WANT_GZIP` streams. +- [x] Added `Compress::Bzip2` streaming deflate and inflate facades, including + the `bzinflate`, `inflate`, and `bzerror` stream methods required by the + CPAN API. +- [x] Added system-Perl-validated regressions: + - `unit/http_cookies_cookie2.t` + - `unit/compress_raw_zlib_gzip_wrapper.t` + - `unit/compress_bzip2_streaming_writer.t` + - `unit/compress_bzip2_streaming_inflater.t` +- [x] All four regressions pass on JVM and interpreter; full `make` passes. +- [x] Net::Async::HTTP `t/09cookies.t` and `t/18content-coding.t` pass on the + stacked artifact; the latter passes all gzip, deflate, and bzip2 assertions + on both backends. +- [x] Full `jcpan -t Net::Async::HTTP` validation passes on the stacked + artifact: 41 files and 554 tests, with only the expected optional and + platform-specific skips. + +## Next Steps + +1. Wait for the `master`-targeted CI checks for PR #1213, then merge it. +2. The prerequisite socket `fileno` support was delivered by PR #1204, merged + as `e20a49bdcd06f1a42fc0a30492be6f414b0e8111`. + +## Open Questions + +None. diff --git a/docs/about/changelog.md b/docs/about/changelog.md index de92c1d734..1155e60813 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -10,6 +10,8 @@ Release history of PerlOnJava. See [Roadmap](roadmap.md) for future plans. global destruction. - Release interpreter hash-slice RHS staging owners after their durable hash slots are created, restoring Net::Async::HTTP connection refcounts. +- Correct bundled `HTTP::Cookies` Cookie2 quoting and restore gzip and bzip2 + content-coding wrappers used by `Net::Async::HTTP`. - Add Mojolicious 9.49 support through `jcpan`; 109 files and 4,194 tests pass in 955 seconds with only upstream developer/optional-feature skips. - Make Catalyst::Runtime pass 199 supported files and 3,774 assertions in diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/CompressRawZlib.java b/src/main/java/org/perlonjava/runtime/perlmodule/CompressRawZlib.java index 2baf651aaf..94b2b69f10 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/CompressRawZlib.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/CompressRawZlib.java @@ -312,7 +312,7 @@ public static RuntimeList deflateInit(RuntimeArray args, int ctx) { actualWbits = -wbits; } else if (wbits > 15) { // Gzip mode - nowrap = true; // We'll handle gzip header/trailer in Perl + nowrap = true; // Emit the RFC 1952 wrapper around raw deflate bytes. gzipMode = true; actualWbits = wbits - 16; } else { @@ -344,6 +344,9 @@ public static RuntimeList deflateInit(RuntimeArray args, int ctx) { self.put("_adler32", new RuntimeScalar(1L)); self.put("_dict_adler", new RuntimeScalar(dictAdler)); self.put("_msg", new RuntimeScalar()); + self.put("_gzip_mode", new RuntimeScalar(gzipMode ? 1 : 0)); + self.put("_gzip_header_emitted", new RuntimeScalar(0)); + self.put("_gzip_crc32", new RuntimeScalar(0L)); RuntimeScalar ref = self.createReference(); ReferenceOperators.bless(ref, new RuntimeScalar("Compress::Raw::Zlib::deflateStream")); @@ -488,6 +491,11 @@ public static RuntimeList ds_deflate(RuntimeArray args, int ctx) { int flags = self.get("_flags").getInt(); byte[] input = getInputBytes(inputScalar); + if (self.get("_gzip_mode").getBoolean()) { + self.put("_gzip_crc32", new RuntimeScalar(crc32WithSeed( + input, self.get("_gzip_crc32").getLong() & 0xFFFFFFFFL))); + } + // Track CRC/Adler of uncompressed data if ((flags & FLAG_CRC) != 0) { long crc = crc32WithSeed(input, self.get("_crc32").getLong() & 0xFFFFFFFFL); @@ -587,6 +595,8 @@ public static RuntimeList ds_deflateReset(RuntimeArray args, int ctx) { self.put("_total_out", new RuntimeScalar(0)); self.put("_crc32", new RuntimeScalar(0L)); self.put("_adler32", new RuntimeScalar(1L)); + self.put("_gzip_header_emitted", new RuntimeScalar(0)); + self.put("_gzip_crc32", new RuntimeScalar(0L)); self.put("_msg", new RuntimeScalar()); return new RuntimeScalar(Z_OK).getList(); } @@ -1568,6 +1578,23 @@ private static void setScalarBytes(RuntimeScalar scalar, String value) { */ private static void writeDeflateOutput(RuntimeHash self, RuntimeScalar outputRef, ByteArrayOutputStream baos, int flags, boolean finishing) { + if (self.get("_gzip_mode").getBoolean()) { + ByteArrayOutputStream wrapped = new ByteArrayOutputStream(baos.size() + 18); + if (!self.get("_gzip_header_emitted").getBoolean()) { + // RFC 1952 fixed gzip header: deflate method, no flags, no mtime. + wrapped.writeBytes(new byte[] { + 0x1f, (byte) 0x8b, 8, 0, 0, 0, 0, 0, 0, (byte) 0xff + }); + self.put("_gzip_header_emitted", new RuntimeScalar(1)); + } + wrapped.writeBytes(baos.toByteArray()); + if (finishing) { + writeLittleEndian32(wrapped, self.get("_gzip_crc32").getLong()); + writeLittleEndian32(wrapped, self.get("_total_in").getLong()); + } + baos = wrapped; + } + RuntimeScalar bitsScalar = self.get("_prime_bits"); int primeBits = bitsScalar != null ? bitsScalar.getInt() : 0; if (primeBits <= 0 || primeBits >= 8) { @@ -1598,6 +1625,13 @@ private static void writeDeflateOutput(RuntimeHash self, RuntimeScalar outputRef writeOutput(outputRef, shifted, flags); } + private static void writeLittleEndian32(ByteArrayOutputStream output, long value) { + output.write((int) (value & 0xff)); + output.write((int) ((value >>> 8) & 0xff)); + output.write((int) ((value >>> 16) & 0xff)); + output.write((int) ((value >>> 24) & 0xff)); + } + private static void writeOutput(RuntimeScalar outputRef, ByteArrayOutputStream baos, int flags) { String outStr = baos.toString(StandardCharsets.ISO_8859_1); RuntimeScalar outScalar; diff --git a/src/main/perl/lib/Compress/Bzip2.pm b/src/main/perl/lib/Compress/Bzip2.pm index 9bc3a468d8..b0bd7dcf82 100644 --- a/src/main/perl/lib/Compress/Bzip2.pm +++ b/src/main/perl/lib/Compress/Bzip2.pm @@ -34,7 +34,7 @@ our %EXPORT_TAGS = ( BZ_STREAM_END BZ_UNEXPECTED_EOF ) ], 'utilities' => [ qw( - memBzip memBunzip bzip2 bzunzip + memBzip memBunzip bzip2 bzunzip bzinflateInit bzdeflateInit ) ], 'bzip1' => [ qw(bzopen bzclose bzread bzreadline bzwrite bzeof bzerror) ], 'gzip' => [ qw(bzopen bzclose bzread bzreadline bzwrite bzeof bzerror) ], @@ -43,6 +43,70 @@ our @EXPORT_OK = ( map { @$_ } values %EXPORT_TAGS ); $EXPORT_TAGS{'all'} = [ @EXPORT_OK ]; our @EXPORT = qw(); +# Compress::Bzip2 exposes a small streaming writer API in addition to its +# one-shot helpers. The Java raw backend already owns the buffered compressor; +# this wrapper adapts its output-parameter API to Compress::Bzip2's +# return-bytes convention. +sub bzdeflateInit { + require Compress::Raw::Bzip2; + my $raw = Compress::Raw::Bzip2->new(1); + return unless $raw; + return bless { raw => $raw }, 'Compress::Bzip2::bzdeflateStream'; +} + +# Compress::Bzip2 also exposes its bzip2 reader through the historical +# Compress::Zlib-compatible inflateInit API. Keep the facade here so callers +# such as Net::Async::HTTP can use the bundled raw streaming implementation +# without depending on its output-parameter interface. +sub bzinflateInit { + require Compress::Raw::Bzip2; + my $raw = Compress::Raw::Bunzip2->new(1); + return unless $raw; + return bless { raw => $raw }, 'Compress::Bzip2::bzinflateStream'; +} + +sub inflateInit { + return bzinflateInit(@_); +} + +package Compress::Bzip2::bzdeflateStream; + +sub bzdeflate { + my ($self, $input) = @_; + my $output = ''; + $self->{raw}->bzdeflate($input, $output); + return $output; +} + +sub bzclose { + my ($self) = @_; + my $output = ''; + $self->{raw}->bzclose($output); + return $output; +} + +package Compress::Bzip2::bzinflateStream; + +sub bzinflate { + my ($self, $input) = @_; + my $output = ''; + my $status = $self->{raw}->bzinflate($input, $output); + return wantarray ? (undef, $status) : undef if $status < 0; + return wantarray ? ($output, $status) : $output; +} + +sub inflate { + goto &bzinflate; +} + +sub bzerror { + return $_[0]->{raw}->status; +} + +sub gzerror { + goto &bzerror; +} + package Compress::Bzip2::bzFile; sub read { diff --git a/src/main/perl/lib/HTTP/Cookies.pm b/src/main/perl/lib/HTTP/Cookies.pm index 758cac814a..e6936782a8 100644 --- a/src/main/perl/lib/HTTP/Cookies.pm +++ b/src/main/perl/lib/HTTP/Cookies.pm @@ -63,7 +63,7 @@ sub add_cookie_header { push @values, "\$Version=$version"; } elsif (!$self->{hide_cookie2}) { - $request->header(Cookie2 => '\$Version="1"'); + $request->header(Cookie2 => '$Version="1"'); } } diff --git a/src/test/resources/unit/compress_bzip2_streaming_inflater.t b/src/test/resources/unit/compress_bzip2_streaming_inflater.t new file mode 100644 index 0000000000..b2913b7577 --- /dev/null +++ b/src/test/resources/unit/compress_bzip2_streaming_inflater.t @@ -0,0 +1,17 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use Test::More; +use Compress::Bzip2 qw(); + +my $payload = "bzip2 streaming inflater payload\n"; +my $compressor = Compress::Bzip2::bzdeflateInit(); +my $compressed = $compressor->bzdeflate($payload) . $compressor->bzclose; +my $inflater = Compress::Bzip2::inflateInit(); +ok($inflater, 'inflateInit creates a Compress::Zlib-compatible bzip2 inflater'); + +my $output = $inflater->bzinflate(\$compressed); +is($output, $payload, 'bzinflate returns the decompressed payload'); + +done_testing; diff --git a/src/test/resources/unit/compress_bzip2_streaming_writer.t b/src/test/resources/unit/compress_bzip2_streaming_writer.t new file mode 100644 index 0000000000..4fca41439e --- /dev/null +++ b/src/test/resources/unit/compress_bzip2_streaming_writer.t @@ -0,0 +1,19 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use Test::More; +use Compress::Bzip2 qw(); + +my $payload = "bzip2 streaming payload\n"; +my $compressor = Compress::Bzip2::bzdeflateInit(); +ok($compressor, 'bzdeflateInit creates a streaming compressor'); + +my $compressed = $compressor->bzdeflate($payload); +$compressed .= $compressor->bzclose; + +ok(length($compressed), 'streaming compressor returns bzip2 bytes at close'); +is(Compress::Bzip2::memBunzip($compressed), $payload, + 'streaming bzip2 output round-trips through memBunzip'); + +done_testing; diff --git a/src/test/resources/unit/compress_raw_zlib_gzip_wrapper.t b/src/test/resources/unit/compress_raw_zlib_gzip_wrapper.t new file mode 100644 index 0000000000..34312cbaa5 --- /dev/null +++ b/src/test/resources/unit/compress_raw_zlib_gzip_wrapper.t @@ -0,0 +1,30 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use Test::More; +use Compress::Raw::Zlib qw(WANT_GZIP Z_STREAM_END); + +my $payload = "gzip wrapper payload\n"; +my $deflater = Compress::Raw::Zlib::Deflate->new( + -WindowBits => WANT_GZIP(), + -AppendOutput => 1, +); +my $wire = ''; +$deflater->deflate($payload, $wire); +$deflater->flush($wire); + +is(substr($wire, 0, 3), "\x1f\x8b\x08", + 'WANT_GZIP emits an RFC 1952 gzip header'); + +my $inflater = Compress::Raw::Zlib::Inflate->new( + -ConsumeInput => 0, + -WindowBits => WANT_GZIP(), +); +my $output = ''; +my $status = $inflater->inflate($wire, $output); + +is(0 + $status, 0 + Z_STREAM_END(), 'gzip stream reaches end'); +is($output, $payload, 'gzip stream round-trips through raw zlib API'); + +done_testing; diff --git a/src/test/resources/unit/http_cookies_cookie2.t b/src/test/resources/unit/http_cookies_cookie2.t new file mode 100644 index 0000000000..f8069bdaaf --- /dev/null +++ b/src/test/resources/unit/http_cookies_cookie2.t @@ -0,0 +1,18 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use Test::More; +use HTTP::Cookies; +use HTTP::Request; + +my $cookies = HTTP::Cookies->new; +$cookies->set_cookie(0, 'X_TEST', 'MyCookie', '/', 'myhost.local'); + +my $request = HTTP::Request->new(POST => 'http://myhost/'); +$cookies->add_cookie_header($request); + +is($request->header('Cookie2'), '$Version="1"', + 'Cookie2 advertises RFC 2965 support without a literal escape'); + +done_testing;