From c89d9f77ab617fed94402c2f4e207402dfe0afe6 Mon Sep 17 00:00:00 2001 From: kafkiansky Date: Tue, 17 Nov 2026 14:00:30 +0300 Subject: [PATCH 1/2] perf: decode and encode varints with native ints instead of bcmath --- composer.json | 3 +- src/Internal/Buffer/ReadBuffer.php | 18 ++-- src/Internal/Buffer/Reader.php | 46 ++++++++-- src/Internal/Buffer/buffer.php | 9 +- src/Internal/Serde/DeserializeEnum.php | 20 ++--- src/Internal/Serde/SerdeBool.php | 8 +- src/Internal/Serde/SerdeFixed64.php | 11 ++- src/Internal/Serde/SerdeInt32.php | 22 ++--- src/Internal/Serde/SerdeInt64.php | 40 +-------- src/Internal/Serde/SerdeSFixed64.php | 11 ++- src/Internal/Serde/SerdeSInt32.php | 18 +--- src/Internal/Serde/SerdeSInt64.php | 11 ++- src/Internal/Serde/SerdeString.php | 11 ++- src/Internal/Serde/SerdeUint32.php | 13 +-- src/Internal/Serde/SerdeVarint.php | 5 +- src/Internal/Serde/SerializeEnum.php | 3 +- src/Internal/Wire/wire.php | 74 ++++++++++------ src/Tag.php | 15 +++- tests/BufferTest.php | 26 ++---- tests/VarintTest.php | 118 +++++++++++++++++++++++++ 20 files changed, 300 insertions(+), 182 deletions(-) create mode 100644 tests/VarintTest.php diff --git a/composer.json b/composer.json index 4f5b549..78d1c73 100644 --- a/composer.json +++ b/composer.json @@ -21,8 +21,7 @@ "php": "^8.4", "ext-bcmath": "*", "psr/simple-cache": "^3.0", - "thesis/endian": "^0.3.3", - "thesis/varint": "^0.2.0" + "thesis/endian": "^0.3.3" }, "require-dev": { "ext-filter": "*", diff --git a/src/Internal/Buffer/ReadBuffer.php b/src/Internal/Buffer/ReadBuffer.php index 0e20d6d..d82e58a 100644 --- a/src/Internal/Buffer/ReadBuffer.php +++ b/src/Internal/Buffer/ReadBuffer.php @@ -5,6 +5,7 @@ namespace Thesis\Protobuf\Internal\Buffer; use Thesis\Protobuf\Exception\BufferUnderflow; +use Thesis\Protobuf\Exception\MalformedVarint; /** * @internal @@ -12,14 +13,21 @@ interface ReadBuffer extends \Countable { /** - * Peek should not move the cursor or remove data from the buffer. - * It should only return a `non-empty-string` of up to `n` bytes in size or throw an exception only if the buffer is empty. + * Reads a varint as the 64 bits it encodes: values of 2^63 and above come back as negative ints. + * Bits above the 64th, which only the 10th byte can carry, are dropped. + * + * @throws BufferUnderflow if the input or the current limit ends inside the varint + * @throws MalformedVarint if the varint does not end within 10 bytes + */ + public function readVarint(): int; + + /** + * Like readVarint(), but leaves the cursor where it is. * - * @param positive-int $n - * @return non-empty-string * @throws BufferUnderflow + * @throws MalformedVarint */ - public function peek(int $n): string; + public function peekVarint(): int; /** * Read must remove `n` bytes of data from the buffer and return a `non-empty-string`, or throw an exception diff --git a/src/Internal/Buffer/Reader.php b/src/Internal/Buffer/Reader.php index a61af6d..a97d2a5 100644 --- a/src/Internal/Buffer/Reader.php +++ b/src/Internal/Buffer/Reader.php @@ -5,6 +5,7 @@ namespace Thesis\Protobuf\Internal\Buffer; use Thesis\Protobuf\Exception\BufferUnderflow; +use Thesis\Protobuf\Exception\MalformedVarint; /** * Reads the input in place: a cursor moves over the original string and a nested record is read by narrowing @@ -27,14 +28,21 @@ public function __construct( } #[\Override] - public function peek(int $n): string + public function readVarint(): int { - if ($this->offset >= $this->limit) { - throw new BufferUnderflow('Buffer is empty.'); - } + $offset = $this->offset; + $value = $this->decodeVarint($offset); + $this->offset = $offset; + + return $value; + } - /** @var non-empty-string */ - return substr($this->bytes, $this->offset, min($n, $this->limit - $this->offset)); + #[\Override] + public function peekVarint(): int + { + $offset = $this->offset; + + return $this->decodeVarint($offset); } #[\Override] @@ -75,6 +83,32 @@ public function popLimit(int $limit): void $this->limit = $limit; } + /** + * @param non-negative-int $offset where the varint starts; on return, where it ends + * @throws BufferUnderflow + * @throws MalformedVarint + */ + private function decodeVarint(int &$offset): int + { + $value = 0; + + // Shifting a PHP int left drops the bits above the 64th, so the 10th byte contributes only its low bit. + for ($shift = 0; $shift < 70; $shift += 7) { + if ($offset >= $this->limit) { + throw new BufferUnderflow('Unexpected end of input: truncated varint.'); + } + + $byte = \ord($this->bytes[$offset++]); + $value |= ($byte & 0x7F) << $shift; + + if ($byte < 0x80) { + return $value; + } + } + + throw new MalformedVarint('Malformed varint: longer than 10 bytes.'); + } + #[\Override] public function count(): int { diff --git a/src/Internal/Buffer/buffer.php b/src/Internal/Buffer/buffer.php index c75e793..ec70f22 100644 --- a/src/Internal/Buffer/buffer.php +++ b/src/Internal/Buffer/buffer.php @@ -4,9 +4,7 @@ namespace Thesis\Protobuf\Internal\Buffer; -use BcMath\Number; use Thesis\Protobuf\Exception\MalformedMessage; -use Thesis\Protobuf\Internal\Serde\SerdeVarint; use Thesis\Protobuf\Internal\Wire; /** @@ -17,7 +15,7 @@ function copy(ByteBuffer $src, WriteBuffer $dst): void { $written = $src->flush(); - SerdeVarint::T->serialize($dst, new Number(\strlen($written))); + Wire\writeVarint($dst, \strlen($written)); if ($written !== '') { $dst->write($written); } @@ -32,5 +30,8 @@ function copy(ByteBuffer $src, WriteBuffer $dst): void */ function enterRecord(ReadBuffer $buffer): int { - return $buffer->pushLimit((int) Wire\readVarint($buffer)->value); + $length = $buffer->readVarint(); + + // A negative length is a varint of 2^63 or above: far more than any input holds. + return $buffer->pushLimit($length >= 0 ? $length : PHP_INT_MAX); } diff --git a/src/Internal/Serde/DeserializeEnum.php b/src/Internal/Serde/DeserializeEnum.php index 1dec734..e920230 100644 --- a/src/Internal/Serde/DeserializeEnum.php +++ b/src/Internal/Serde/DeserializeEnum.php @@ -4,8 +4,8 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -27,22 +27,14 @@ public function __construct( #[\Override] public function deserialize(ReadBuffer $buffer): \BackedEnum|UnknownEnumValue { - /** @var ?Number $p31 */ - static $p31; - $p31 ??= new Number(2)->pow(31); - - /** @var ?Number $p32 */ - static $p32; - $p32 ??= new Number(2)->pow(32); - - $raw = SerdeVarint::T->deserialize($buffer); + $raw = $buffer->readVarint(); // Enum numbers are int32: negative ones arrive sign-extended to 64 bits. - $num = $raw->mod($p32); - if ($num->compare($p31) >= 0) { - $num -= $p32; + $num = $raw & 0xFFFF_FFFF; + if ($num > 0x7FFF_FFFF) { + $num -= 0x1_0000_0000; } - return $this->enum::tryFrom((int) $num->value) ?? new UnknownEnumValue($raw); + return $this->enum::tryFrom($num) ?? new UnknownEnumValue(Wire\toUnsigned($raw)); } } diff --git a/src/Internal/Serde/SerdeBool.php b/src/Internal/Serde/SerdeBool.php index d4ac8fe..2c2e3ec 100644 --- a/src/Internal/Serde/SerdeBool.php +++ b/src/Internal/Serde/SerdeBool.php @@ -4,9 +4,9 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -22,14 +22,12 @@ enum SerdeBool implements #[\Override] public function serialize(WriteBuffer $buffer, mixed $value): void { - SerdeVarint::T->serialize($buffer, new Number((int) $value)); + Wire\writeVarint($buffer, $value ? 1 : 0); } #[\Override] public function deserialize(ReadBuffer $buffer): bool { - $num = SerdeVarint::T->deserialize($buffer); - - return (int) $num->value !== 0; + return $buffer->readVarint() !== 0; } } diff --git a/src/Internal/Serde/SerdeFixed64.php b/src/Internal/Serde/SerdeFixed64.php index a3947fb..2c86fb8 100644 --- a/src/Internal/Serde/SerdeFixed64.php +++ b/src/Internal/Serde/SerdeFixed64.php @@ -5,9 +5,9 @@ namespace Thesis\Protobuf\Internal\Serde; use BcMath\Number; -use Thesis\Endian; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -23,12 +23,17 @@ enum SerdeFixed64 implements #[\Override] public function serialize(WriteBuffer $buffer, mixed $value): void { - $buffer->write(Endian\Order::Little->packUint64(Range::uint64($value, 'fixed64'))); + /** @var non-empty-string $bytes */ + $bytes = pack('P', Wire\fromUnsigned(Range::uint64($value, 'fixed64'))); + $buffer->write($bytes); } #[\Override] public function deserialize(ReadBuffer $buffer): Number { - return Endian\Order::Little->unpackUint64($buffer->read(8)); + /** @var array{1: int} $value */ + $value = unpack('P', $buffer->read(8)); + + return Wire\toUnsigned($value[1]); } } diff --git a/src/Internal/Serde/SerdeInt32.php b/src/Internal/Serde/SerdeInt32.php index 1e80167..a65bb4e 100644 --- a/src/Internal/Serde/SerdeInt32.php +++ b/src/Internal/Serde/SerdeInt32.php @@ -4,9 +4,9 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -23,27 +23,15 @@ enum SerdeInt32 implements public function serialize(WriteBuffer $buffer, mixed $value): void { // A negative int32 is encoded as its 64-bit sign extension, exactly like an int64. - SerdeInt64::T->serialize($buffer, Range::int32($value, 'int32')); + Wire\writeVarint($buffer, Range::int32($value, 'int32')); } #[\Override] public function deserialize(ReadBuffer $buffer): int { - /** @var ?Number $p31 */ - static $p31; - $p31 ??= new Number(2)->pow(31); + // An int32 is the low 32 bits of the varint, sign-extended. + $value = $buffer->readVarint() & 0xFFFF_FFFF; - /** @var ?Number $p32 */ - static $p32; - $p32 ??= new Number(2)->pow(32); - - $num = SerdeVarint::T->deserialize($buffer); - $num = $num->mod($p32); - - if ($num->compare($p31) >= 0) { - $num -= $p32; - } - - return (int) $num->value; + return $value > 0x7FFF_FFFF ? $value - 0x1_0000_0000 : $value; } } diff --git a/src/Internal/Serde/SerdeInt64.php b/src/Internal/Serde/SerdeInt64.php index dab8836..06f6e29 100644 --- a/src/Internal/Serde/SerdeInt64.php +++ b/src/Internal/Serde/SerdeInt64.php @@ -4,9 +4,9 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -22,45 +22,13 @@ enum SerdeInt64 implements #[\Override] public function serialize(WriteBuffer $buffer, mixed $value): void { - /** @var ?Number $p64 */ - static $p64; - $p64 ??= new Number(2)->pow(64); - - /** @var ?Number $p63 */ - static $p63; - $p63 ??= new Number(2)->pow(63); - - $num = new Number($value)->mod($p64); - - if ($num->compare($p63) >= 0) { - $num -= $p64; - } - - if ($num->compare(0) < 0) { - $num += $p64; - } - - SerdeVarint::T->serialize($buffer, $num); + // A negative int64 is written as its 64 bits: a 10-byte varint. + Wire\writeVarint($buffer, $value); } #[\Override] public function deserialize(ReadBuffer $buffer): int { - /** @var ?Number $p63 */ - static $p63; - $p63 ??= new Number(2)->pow(63); - - /** @var ?Number $p64 */ - static $p64; - $p64 ??= new Number(2)->pow(64); - - $num = SerdeVarint::T->deserialize($buffer); - $num = $num->mod($p64); - - if ($num->compare($p63) >= 0) { - $num -= $p64; - } - - return (int) $num->value; + return $buffer->readVarint(); } } diff --git a/src/Internal/Serde/SerdeSFixed64.php b/src/Internal/Serde/SerdeSFixed64.php index 4f90719..38c3707 100644 --- a/src/Internal/Serde/SerdeSFixed64.php +++ b/src/Internal/Serde/SerdeSFixed64.php @@ -4,8 +4,6 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; -use Thesis\Endian; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; @@ -23,12 +21,17 @@ enum SerdeSFixed64 implements #[\Override] public function serialize(WriteBuffer $buffer, mixed $value): void { - $buffer->write(Endian\Order::Little->packInt64(new Number($value))); + /** @var non-empty-string $bytes */ + $bytes = pack('P', $value); + $buffer->write($bytes); } #[\Override] public function deserialize(ReadBuffer $buffer): int { - return (int) Endian\Order::Little->unpackInt64($buffer->read(8))->value; + /** @var array{1: int} $value */ + $value = unpack('P', $buffer->read(8)); + + return $value[1]; } } diff --git a/src/Internal/Serde/SerdeSInt32.php b/src/Internal/Serde/SerdeSInt32.php index 6744cd9..69bbdd1 100644 --- a/src/Internal/Serde/SerdeSInt32.php +++ b/src/Internal/Serde/SerdeSInt32.php @@ -4,7 +4,6 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; @@ -29,20 +28,9 @@ public function serialize(WriteBuffer $buffer, mixed $value): void #[\Override] public function deserialize(ReadBuffer $buffer): int { - /** @var ?Number $p31 */ - static $p31; - $p31 ??= new Number(2)->pow(31); + // A sint32 is ZigZag-32 of the low 32 bits of the varint. + $value = $buffer->readVarint() & 0xFFFF_FFFF; - /** @var ?Number $p32 */ - static $p32; - $p32 ??= new Number(2)->pow(32); - - $num = new Number(SerdeSInt64::T->deserialize($buffer))->mod($p32); - - if ($num->compare($p31) >= 0) { - $num -= $p32; - } - - return (int) $num->value; + return ($value >> 1) ^ -($value & 1); } } diff --git a/src/Internal/Serde/SerdeSInt64.php b/src/Internal/Serde/SerdeSInt64.php index 7f4c10f..e306804 100644 --- a/src/Internal/Serde/SerdeSInt64.php +++ b/src/Internal/Serde/SerdeSInt64.php @@ -4,10 +4,9 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; -use Thesis\Varint; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -23,12 +22,16 @@ enum SerdeSInt64 implements #[\Override] public function serialize(WriteBuffer $buffer, mixed $value): void { - SerdeVarint::T->serialize($buffer, Varint\BcMath::Codec->encodeZigZag(new Number($value))); + // ZigZag: 0, -1, 1, -2... become 0, 1, 2, 3... + Wire\writeVarint($buffer, ($value << 1) ^ ($value >> 63)); } #[\Override] public function deserialize(ReadBuffer $buffer): int { - return (int) Varint\BcMath::Codec->decodeZigZag(SerdeVarint::T->deserialize($buffer))->value; + $value = $buffer->readVarint(); + + // A logical shift right, then the sign restored from the low bit. + return (($value >> 1) & PHP_INT_MAX) ^ -($value & 1); } } diff --git a/src/Internal/Serde/SerdeString.php b/src/Internal/Serde/SerdeString.php index 4297b84..afc6e21 100644 --- a/src/Internal/Serde/SerdeString.php +++ b/src/Internal/Serde/SerdeString.php @@ -4,9 +4,9 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -22,7 +22,7 @@ enum SerdeString implements #[\Override] public function serialize(WriteBuffer $buffer, mixed $value): void { - SerdeVarint::T->serialize($buffer, new Number(\strlen($value))); + Wire\writeVarint($buffer, \strlen($value)); if ($value !== '') { $buffer->write($value); @@ -32,14 +32,13 @@ public function serialize(WriteBuffer $buffer, mixed $value): void #[\Override] public function deserialize(ReadBuffer $buffer): string { - // A varint is never negative; one above PHP_INT_MAX saturates and fails the read below. - /** @var non-negative-int $length */ - $length = (int) SerdeVarint::T->deserialize($buffer)->value; + $length = $buffer->readVarint(); if ($length === 0) { return ''; } - return $buffer->read($length); + // A negative length is a varint of 2^63 or above: far more than any input holds. + return $buffer->read($length > 0 ? $length : PHP_INT_MAX); } } diff --git a/src/Internal/Serde/SerdeUint32.php b/src/Internal/Serde/SerdeUint32.php index 8540095..f0411ae 100644 --- a/src/Internal/Serde/SerdeUint32.php +++ b/src/Internal/Serde/SerdeUint32.php @@ -4,9 +4,9 @@ namespace Thesis\Protobuf\Internal\Serde; -use BcMath\Number; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -22,19 +22,12 @@ enum SerdeUint32 implements #[\Override] public function serialize(WriteBuffer $buffer, mixed $value): void { - SerdeVarint::T->serialize($buffer, new Number(Range::uint32($value, 'uint32'))); + Wire\writeVarint($buffer, Range::uint32($value, 'uint32')); } #[\Override] public function deserialize(ReadBuffer $buffer): int { - /** @var ?Number $p32 */ - static $p32; - $p32 ??= new Number(2)->pow(32); - - $num = SerdeVarint::T->deserialize($buffer); - $num = $num->mod($p32); - - return (int) $num->value; + return $buffer->readVarint() & 0xFFFF_FFFF; } } diff --git a/src/Internal/Serde/SerdeVarint.php b/src/Internal/Serde/SerdeVarint.php index 4638efa..af49a83 100644 --- a/src/Internal/Serde/SerdeVarint.php +++ b/src/Internal/Serde/SerdeVarint.php @@ -8,7 +8,6 @@ use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; use Thesis\Protobuf\Internal\Wire; -use Thesis\Varint; /** * @internal @@ -24,12 +23,12 @@ enum SerdeVarint implements #[\Override] public function serialize(WriteBuffer $buffer, mixed $value): void { - $buffer->write(Varint\BcMath::Codec->encodeVarint($value)); + Wire\writeVarint($buffer, Wire\fromUnsigned($value)); } #[\Override] public function deserialize(ReadBuffer $buffer): Number { - return Wire\readVarint($buffer); + return Wire\toUnsigned($buffer->readVarint()); } } diff --git a/src/Internal/Serde/SerializeEnum.php b/src/Internal/Serde/SerializeEnum.php index 8f17fab..aadf779 100644 --- a/src/Internal/Serde/SerializeEnum.php +++ b/src/Internal/Serde/SerializeEnum.php @@ -5,6 +5,7 @@ namespace Thesis\Protobuf\Internal\Serde; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; +use Thesis\Protobuf\Internal\Wire; /** * @internal @@ -22,6 +23,6 @@ public function serialize(WriteBuffer $buffer, mixed $value): void $case = $value->value; // Enum numbers are int32: a negative one is sign-extended to 64 bits. - SerdeInt64::T->serialize($buffer, Range::int32($case, 'enum')); + Wire\writeVarint($buffer, Range::int32($case, 'enum')); } } diff --git a/src/Internal/Wire/wire.php b/src/Internal/Wire/wire.php index f4eead6..409c044 100644 --- a/src/Internal/Wire/wire.php +++ b/src/Internal/Wire/wire.php @@ -5,16 +5,12 @@ namespace Thesis\Protobuf\Internal\Wire; use BcMath\Number; -use Thesis\Protobuf\Exception\BufferUnderflow; use Thesis\Protobuf\Exception\MalformedMessage; -use Thesis\Protobuf\Exception\MalformedVarint; use Thesis\Protobuf\Internal\Buffer\ReadBuffer; use Thesis\Protobuf\Internal\Buffer\WriteBuffer; -use Thesis\Protobuf\Internal\Serde\SerdeVarint; use Thesis\Protobuf\Tag; use Thesis\Protobuf\UnknownFields\UnknownField; use Thesis\Protobuf\WireType; -use Thesis\Varint; /** * Return bytes from the buffer corresponding to the size of each type: @@ -31,11 +27,12 @@ function discardUnknown(ReadBuffer $buffer, Tag $tag): UnknownField return new UnknownField($tag, match ($tag->type) { WireType::FIXED32 => $buffer->read(4), WireType::FIXED64 => $buffer->read(8), - WireType::VARINT => readVarint($buffer), + WireType::VARINT => toUnsigned($buffer->readVarint()), WireType::BYTES => (static function () use ($buffer): string { - $length = (int) readVarint($buffer)->value; - if ($length > 0) { - return $buffer->read($length); + $length = $buffer->readVarint(); + if ($length !== 0) { + // A negative length is a varint of 2^63 or above: far more than any input holds. + return $buffer->read($length > 0 ? $length : PHP_INT_MAX); } return ''; @@ -44,32 +41,57 @@ function discardUnknown(ReadBuffer $buffer, Tag $tag): UnknownField } /** + * Writes the 64 bits of `value` as a varint: a negative int takes 10 bytes, as an int64 must. + * * @internal - * @throws MalformedMessage */ -function readVarint(ReadBuffer $buffer): Number +function writeVarint(WriteBuffer $buffer, int $value): void { - $number = decodeVarint($buffer->peek(10)); - $buffer->read($number->size); + if ($value >= 0 && $value < 0x80) { + $buffer->write(\chr($value)); + + return; + } + + $bytes = ''; - return $number->value; + while (($value & ~0x7F) !== 0) { + $bytes .= \chr(($value & 0x7F) | 0x80); + // A logical shift: the sign bit of a negative int must not be copied into the high bits. + $value = ($value >> 7) & 0x01FF_FFFF_FFFF_FFFF; + } + + $buffer->write($bytes . \chr($value)); } /** + * The value of 64 bits read as unsigned. + * * @internal - * @param non-empty-string $bytes up to 10 bytes from the start of a varint - * @throws MalformedMessage */ -function decodeVarint(string $bytes): Varint\Sized +function toUnsigned(int $value): Number { - try { - return Varint\BcMath::Codec->decodeVarintSized($bytes); - } catch (Varint\MalformedVarintNumber) { - // No terminating byte among the bytes given: either the input ends or the varint is too long. - throw \strlen($bytes) < 10 - ? new BufferUnderflow('Unexpected end of input: truncated varint.') - : new MalformedVarint('Malformed varint: longer than 10 bytes.'); + /** @var ?Number $p64 */ + static $p64; + + return $value >= 0 ? new Number($value) : new Number($value) + ($p64 ??= new Number('18446744073709551616')); +} + +/** + * The 64 bits of an unsigned value in [0, 2^64-1]: values of 2^63 and above become negative ints. + * + * @internal + */ +function fromUnsigned(Number $value): int +{ + /** @var ?Number $p64 */ + static $p64; + + if ($value->compare(PHP_INT_MAX) > 0) { + $value -= ($p64 ??= new Number('18446744073709551616')); } + + return (int) $value->value; } /** @@ -77,7 +99,7 @@ function decodeVarint(string $bytes): Varint\Sized */ function writeTag(WriteBuffer $buffer, Tag $tag): void { - SerdeVarint::T->serialize($buffer, $tag->number); + writeVarint($buffer, $tag->num << 3 | $tag->type->value); } /** @@ -86,7 +108,7 @@ function writeTag(WriteBuffer $buffer, Tag $tag): void */ function peekTag(ReadBuffer $buffer): Tag { - return Tag::from(decodeVarint($buffer->peek(10))->value); + return Tag::fromVarint($buffer->peekVarint()); } /** @@ -95,5 +117,5 @@ function peekTag(ReadBuffer $buffer): Tag */ function readTag(ReadBuffer $buffer): Tag { - return Tag::from(readVarint($buffer)); + return Tag::fromVarint($buffer->readVarint()); } diff --git a/src/Tag.php b/src/Tag.php index c953e9c..51bbe68 100644 --- a/src/Tag.php +++ b/src/Tag.php @@ -19,13 +19,22 @@ final class Tag */ public static function from(Number $number): self { - $tag = (int) $number->value; + return self::fromVarint((int) $number->value); + } - /** @var positive-int $num */ - $num = $tag >> 3; + /** + * @internal + * @throws Exception\UnsupportedWireType + */ + public static function fromVarint(int $tag): self + { + // A logical shift: a negative tag must not give a negative field number. + $num = ($tag >> 3) & 0x1FFF_FFFF_FFFF_FFFF; $type = $tag & 7; return new self( + // Field number 0 is invalid but still accepted here, as it always was. + /** @phpstan-ignore argument.type */ $num, WireType::tryFrom($type) ?? throw new Exception\UnsupportedWireType($type, $num), ); diff --git a/tests/BufferTest.php b/tests/BufferTest.php index 39913c7..45ccfb8 100644 --- a/tests/BufferTest.php +++ b/tests/BufferTest.php @@ -14,27 +14,18 @@ #[CoversClass(ByteBuffer::class)] final class BufferTest extends TestCase { - public function testPeekDoesNotMoveTheCursor(): void + public function testReadMovesTheCursorAndPeekDoesNot(): void { - $buffer = new Reader('test'); - self::assertCount(4, $buffer); - self::assertSame('te', $buffer->peek(2)); - self::assertCount(4, $buffer); - self::assertSame('test', $buffer->peek(5)); - self::assertCount(4, $buffer); - self::assertSame('test', $buffer->read(4)); + $buffer = new Reader("\x96\x01\x05"); + self::assertCount(3, $buffer); + self::assertSame(150, $buffer->peekVarint()); + self::assertCount(3, $buffer); + self::assertSame(150, $buffer->readVarint()); + self::assertCount(1, $buffer); + self::assertSame(5, $buffer->readVarint()); self::assertCount(0, $buffer); } - public function testPeekAtTheEndFails(): void - { - $buffer = new Reader('t'); - $buffer->read(1); - - $this->expectExceptionObject(new BufferUnderflow('Buffer is empty.')); - $buffer->peek(1); - } - public function testReadMovesTheCursor(): void { $buffer = new Reader('test'); @@ -52,7 +43,6 @@ public function testLimitNarrowsAndRestoresTheBuffer(): void $limit = $buffer->pushLimit(3); self::assertCount(3, $buffer); - self::assertSame('bcd', $buffer->peek(10), 'peek stops at the limit'); self::assertSame('bc', $buffer->read(2)); $inner = $buffer->pushLimit(1); diff --git a/tests/VarintTest.php b/tests/VarintTest.php new file mode 100644 index 0000000..60287f9 --- /dev/null +++ b/tests/VarintTest.php @@ -0,0 +1,118 @@ + $type + * @param non-empty-string $hex the field value, after the tag + */ + #[DataProvider('provideVarintsAreReadAsProtobufReadsThemCases')] + public function testVarintsAreReadAsProtobufReadsThem(Type $type, string $hex, mixed $expected): void + { + $message = new Serializer()->deserialize(messageT(fieldT(1, $type)), (string) hex2bin("08{$hex}")); + + self::assertEquals($expected, ($message->fields[1] ?? null)?->value->value); + } + + /** + * @return iterable, non-empty-string, mixed}> + */ + public static function provideVarintsAreReadAsProtobufReadsThemCases(): iterable + { + // The 10th byte carries only bit 63: higher bits are dropped, as official parsers do. + yield 'uint64 above 2^64-1 is truncated' => [uint64T, 'ffffffffffffffffff7f', new Number('18446744073709551615')]; + yield 'int64 above 2^64-1 is truncated' => [int64T, 'ffffffffffffffffff7f', -1]; + yield 'uint64 2^63' => [uint64T, '80808080808080808001', new Number('9223372036854775808')]; + // Non-canonical forms with redundant continuation bytes are valid. + yield 'overlong zero' => [int32T, '8080808000', 0]; + yield 'overlong one' => [uint64T, '818000', new Number(1)]; + // 32-bit types take the low 32 bits of whatever the varint holds. + yield 'int32 from a 5-byte -1' => [int32T, 'ffffffff0f', -1]; + yield 'uint32 from a 10-byte value' => [uint32T, 'ffffffffffffffffff01', 4294967295]; + yield 'sint32 from a 10-byte value' => [sint32T, 'feffffffffffffffff01', 2147483647]; + yield 'bool from a large value' => [boolT, '80808080808080808001', true]; + } + + public function testVarintEndsAtTheLimitOfTheNestedMessage(): void + { + // Field 1 is a 1-byte message holding only a tag; the value byte after it belongs to the outer message. + $type = messageT(fieldT(1, messageT(fieldT(1, int32T))), fieldT(2, int32T)); + + $this->expectException(BufferUnderflow::class); + $this->expectExceptionMessage('truncated varint'); + + new Serializer()->deserialize($type, "\x0a\x01\x08\x01"); + } + + /** + * @param non-empty-string $hex + */ + #[DataProvider('provideLengthsOf2To63AndAboveAreReportedAsTooLongCases')] + public function testLengthsOf2To63AndAboveAreReportedAsTooLong(string $hex): void + { + $type = messageT(fieldT(1, messageT()), fieldT(2, listT(int32T)), fieldT(3, stringT)); + + $this->expectException(BufferUnderflow::class); + $this->expectExceptionMessage(\sprintf("Expected '%d' bytes", PHP_INT_MAX)); + + new Serializer()->deserialize($type, (string) hex2bin($hex)); + } + + /** + * @return iterable + */ + public static function provideLengthsOf2To63AndAboveAreReportedAsTooLongCases(): iterable + { + yield 'nested message' => ['0a8080808080808080800100']; + yield 'packed list' => ['128080808080808080800100']; + yield 'string' => ['1a8080808080808080800100']; + yield 'unknown field' => ['228080808080808080800100']; + } + + public function testUnknownVarintsKeepTheirUnsignedValue(): void + { + $message = new Serializer()->deserialize(messageT(), (string) hex2bin('08fbffffffffffffffff01')); + + self::assertEquals(new Number('18446744073709551611'), $message->unknowns[0]->value ?? null); + } + + /** + * @param FieldDescriptor<*> $field + * @param non-empty-string $hex + */ + #[DataProvider('provideValuesAreWrittenAsCanonicalVarintsCases')] + public function testValuesAreWrittenAsCanonicalVarints(FieldDescriptor $field, string $hex): void + { + self::assertSame($hex, bin2hex(new Serializer()->serialize(message($field)))); + } + + /** + * @return iterable, non-empty-string}> + */ + public static function provideValuesAreWrittenAsCanonicalVarintsCases(): iterable + { + yield '127 takes one byte' => [fieldOf(1, int64Of(127)), '087f']; + yield '128 takes two bytes' => [fieldOf(1, int64Of(128)), '088001']; + yield 'int64 max' => [fieldOf(1, int64Of(PHP_INT_MAX)), '08ffffffffffffffff7f']; + yield 'int64 min' => [fieldOf(1, int64Of(PHP_INT_MIN)), '0880808080808080808001']; + yield 'uint64 2^63' => [fieldOf(1, uint64Of(new Number('9223372036854775808'))), '0880808080808080808001']; + yield 'sint64 min' => [fieldOf(1, sint64Of(PHP_INT_MIN)), '08ffffffffffffffffff01']; + yield 'field 2^29-1' => [fieldOf(536_870_911, boolOf(true)), 'f8ffffff0f01']; + } +} From eccdc0369f50661e1d6339a0005669a88b564fb3 Mon Sep 17 00:00:00 2001 From: kafkiansky Date: Tue, 17 Nov 2026 17:06:47 +0300 Subject: [PATCH 2/2] fix: narrow the last varint byte for chr() --- src/Internal/Wire/wire.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Internal/Wire/wire.php b/src/Internal/Wire/wire.php index 409c044..f14ad5b 100644 --- a/src/Internal/Wire/wire.php +++ b/src/Internal/Wire/wire.php @@ -61,7 +61,7 @@ function writeVarint(WriteBuffer $buffer, int $value): void $value = ($value >> 7) & 0x01FF_FFFF_FFFF_FFFF; } - $buffer->write($bytes . \chr($value)); + $buffer->write($bytes . \chr($value & 0x7F)); } /**