From d0cf698aef8e35f5819cb755224bcabc5d0831f8 Mon Sep 17 00:00:00 2001 From: kafkiansky Date: Thu, 12 Nov 2026 10:29:44 +0300 Subject: [PATCH] Fix ObjectStore: empty objects, integrity checks, cleanup of failed puts and get consumers --- README.md | 5 + .../ObjectStore/Internal/ChunkCollector.php | 105 ++++++ .../ObjectStore/Internal/DigestCalculator.php | 20 ++ src/JetStream/ObjectStore/Store.php | 196 +++++++---- src/JetStream/ObjectStore/StringReader.php | 5 +- .../JetStream/ObjectStore/ObjectStoreTest.php | 333 ++++++++++++++++++ 6 files changed, 595 insertions(+), 69 deletions(-) create mode 100644 src/JetStream/ObjectStore/Internal/ChunkCollector.php diff --git a/README.md b/README.md index 12f6acf..0819f4d 100644 --- a/README.md +++ b/README.md @@ -1445,6 +1445,11 @@ dump( $nc->drain(); ``` +- An empty object (`''` or an empty reader) is stored as meta without chunks. +- If the reader or a publish fails in the middle of `put()`, the chunks already published are purged and the error is rethrown; the previous version of the object stays as it was. +- `get()` reads the chunks through a temporary consumer, deleted after the object is read to the end. It throws `Nats\Exception\ObjectIsInvalid` at once if the stream holds fewer chunks than the meta tells. The size and the SHA-256 digest of the chunks are checked against the meta: on a mismatch, or when the chunks disappear while reading, the read of the stream throws `Nats\Exception\ObjectIsInvalid`. +- `info()->mtime` is the time the server stored the meta, with sub-second precision. + #### Watch Object Store ```php diff --git a/src/JetStream/ObjectStore/Internal/ChunkCollector.php b/src/JetStream/ObjectStore/Internal/ChunkCollector.php new file mode 100644 index 0000000..f8b5f53 --- /dev/null +++ b/src/JetStream/ObjectStore/Internal/ChunkCollector.php @@ -0,0 +1,105 @@ + $chunks + */ + public function __construct( + private readonly ObjectInfo $info, + private readonly Queue $chunks, + ) { + $this->digest = DigestCalculator::sha256(); + $this->exhausted = new DeferredCancellation(); + } + + /** + * Cancelled when the server has nothing more to deliver before the last chunk: the chunks are gone. + */ + public function exhausted(): Cancellation + { + return $this->exhausted->getCancellation(); + } + + public function idle(): void + { + if ($this->checksum === null) { + $this->exhausted->cancel(); + } + } + + public function __invoke(Delivery $delivery, Subscription $subscription): void + { + if ($this->checksum !== null) { + return; + } + + // The reader dropped the object before reading it to the end: nobody needs the rest. + if ($this->chunks->isDisposed()) { + $subscription->stop(); + + return; + } + + $payload = $delivery->message->payload ?? ''; + + if ($payload !== '') { + $this->digest->update($payload); + $this->size += \strlen($payload); + + $pushed = $this->chunks->pushAsync($payload); + $pushed->ignore(); + } + + if ($delivery->metadata?->pending === 0) { + $this->checksum = $this->digest->finish(); + $subscription->stop(); + } + } + + /** + * @throws ObjectIsInvalid + */ + public function verify(): void + { + if ($this->checksum === null) { + throw new ObjectIsInvalid("Object '{$this->info->name}' is missing chunks: received {$this->size} of {$this->info->size} bytes."); + } + + if ($this->size !== $this->info->size) { + throw new ObjectIsInvalid("Object '{$this->info->name}' size mismatch: expected {$this->info->size} bytes, received {$this->size}."); + } + + if (!hash_equals(DigestCalculator::decode($this->info->digest ?? ''), $this->checksum)) { + throw new ObjectIsInvalid("Object '{$this->info->name}' digest mismatch."); + } + } +} diff --git a/src/JetStream/ObjectStore/Internal/DigestCalculator.php b/src/JetStream/ObjectStore/Internal/DigestCalculator.php index c044538..8c138de 100644 --- a/src/JetStream/ObjectStore/Internal/DigestCalculator.php +++ b/src/JetStream/ObjectStore/Internal/DigestCalculator.php @@ -4,6 +4,8 @@ namespace Thesis\Nats\JetStream\ObjectStore\Internal; +use Thesis\Nats\Exception\ObjectIsInvalid; + /** * @internal */ @@ -27,6 +29,24 @@ public static function sha256(): self return new self(self::ALGO_SHA256); } + /** + * Decodes "SHA-256=" like nats.go DecodeObjectDigest. + * + * @throws ObjectIsInvalid + */ + public static function decode(string $digest): string + { + $parts = explode('=', $digest, 2); + + $decoded = \count($parts) === 2 ? base64_decode(strtr($parts[1], '-_', '+/'), true) : false; + + if ($decoded === false || $decoded === '') { + throw new ObjectIsInvalid("Invalid object digest format '{$digest}'."); + } + + return $decoded; + } + /** * @param non-empty-string $data */ diff --git a/src/JetStream/ObjectStore/Store.php b/src/JetStream/ObjectStore/Store.php index 958f025..6742bae 100644 --- a/src/JetStream/ObjectStore/Store.php +++ b/src/JetStream/ObjectStore/Store.php @@ -5,10 +5,11 @@ namespace Thesis\Nats\JetStream\ObjectStore; use Amp\ByteStream\ReadableIterableStream; -use Amp\ByteStream\WritableIterableStream; use Amp\Cancellation; +use Amp\CancelledException; +use Amp\Pipeline\Queue; +use Amp\TimeoutCancellation; use Thesis\Nats\Client; -use Thesis\Nats\Delivery as NatsDelivery; use Thesis\Nats\Exception\ObjectIsInvalid; use Thesis\Nats\Header\MsgRollup; use Thesis\Nats\Header\Timestamp; @@ -16,7 +17,9 @@ use Thesis\Nats\Internal\Id; use Thesis\Nats\Internal\Validation\Validate; use Thesis\Nats\JetStream; +use Thesis\Nats\JetStream\Api\AckPolicy; use Thesis\Nats\JetStream\Api\DeliverPolicy; +use Thesis\Nats\JetStream\Api\ReplayPolicy; use Thesis\Nats\JetStream\Delivery as JetStreamDelivery; use Thesis\Nats\JetStream\ObjectStore\Internal\DigestCalculator; use Thesis\Nats\Json\Encoder; @@ -24,6 +27,8 @@ use Thesis\Nats\NatsException; use Thesis\Nats\Serialization\Serializer; use Thesis\Nats\Subscription; +use Thesis\Time\TimeSpan; +use function Amp\async; /** * @api @@ -32,6 +37,9 @@ { /** @var int default chunk size is 128KB */ private const int DEFAULT_CHUNK_SIZE = 128 * 1024; + private const int GET_HEARTBEAT_SECONDS = 5; + private const int GET_INACTIVE_THRESHOLD_SECONDS = 300; + private const int CLEANUP_TIMEOUT_SECONDS = 5; public function __construct( public string $name, @@ -79,39 +87,69 @@ public function get(string $name): ?StoredObject return new StoredObject($info); } - $sink = new WritableIterableStream($info->size); - - $this->stream->createOrUpdateConsumer(new JetStream\Api\ConsumerConfig( - deliverSubject: $id = $this->nats->newInbox(), + /** @var Queue $chunks */ + $chunks = new Queue(max($info->chunks, 1)); + $collector = new Internal\ChunkCollector($info, $chunks); + + // An ordered consumer of nats.go: no acks, flow control with heartbeats, a replica in memory. + $consumer = $this->stream->createOrUpdateConsumer(new JetStream\Api\ConsumerConfig( + description: 'object store get consumer', + deliverPolicy: DeliverPolicy::All, + deliverSubject: $this->nats->newInbox(), + ackPolicy: AckPolicy::None, + maxDeliver: 1, filterSubject: "\$O.{$this->name}.C.{$info->nuid}", + flowControl: true, + idleHeartbeat: TimeSpan::fromSeconds(self::GET_HEARTBEAT_SECONDS), + replayPolicy: ReplayPolicy::Instant, + inactiveThreshold: TimeSpan::fromSeconds(self::GET_INACTIVE_THRESHOLD_SECONDS), + replicas: 1, + memoryStorage: true, )); - $this->nats->subscribe($id, static function ( - NatsDelivery $delivery, - Subscription $subscription, - ) use ($sink): void { - $metadata = $delivery->replyTo !== null ? JetStream\Metadata::parse($delivery->replyTo) : null; + // Each chunk is a message, so fewer messages than chunks means some of them are gone. + if ($consumer->info->numPending < $info->chunks) { + $completion = async($this->deleteConsumer(...), $consumer); + $completion->ignore(); - $payload = $delivery->message->payload; + throw new ObjectIsInvalid("Object '{$info->name}' is missing chunks: expected {$info->chunks}, stored {$consumer->info->numPending}."); + } - if ($metadata !== null && $payload !== null && $payload !== '') { - $sink->write($payload); + $subscription = $consumer->push($collector); + + // The reader gets the end of the object or the error first, the consumer is deleted afterwards. + $completion = async(function () use ( + $subscription, + $consumer, + $collector, + $chunks, + ): void { + try { + try { + $subscription->awaitCompletion($collector->exhausted()); + } catch (CancelledException) { + $subscription->stop(); + } + + $collector->verify(); + $chunks->complete(); + } catch (\Throwable $e) { + $chunks->error($e); } - if ($metadata?->pending === 0) { - $sink->close(); - $subscription->stop(); - } + $this->deleteConsumer($consumer); }); + $completion->ignore(); return new StoredObject( $info, - new ReadableIterableStream($sink->getIterator()), + new ReadableIterableStream($chunks->iterate()), ); } /** - * @param Reader|non-empty-string $object + * Chunks already published are purged if reading the object or publishing fails. + * * @throws NatsException */ public function put(ObjectMeta $meta, Reader|string $object): ObjectInfo @@ -130,52 +168,55 @@ public function put(ObjectMeta $meta, Reader|string $object): ObjectInfo $chunkSize = $meta->chunkSize ?? self::DEFAULT_CHUNK_SIZE; $digest = DigestCalculator::sha256(); - while (!$object->eof()) { - $data = $object->read($chunkSize); - - if ($data === null) { - break; - } + try { + while (!$object->eof()) { + $data = $object->read($chunkSize); - $digest->update($data); - $size += \strlen($data); - ++$chunks; + if ($data === null) { + break; + } - $this->js->publish($chunkSubject, new Message($data)); - } + $digest->update($data); + $size += \strlen($data); + ++$chunks; - if ($size === 0) { - throw new \LogicException('Empty object provided.'); - } + $this->js->publish($chunkSubject, new Message($data)); + } - $encodedDigest = self::base64encode($digest->finish()); - $subject = "\$O.{$this->name}.M." . self::base64encode($meta->name); + $encodedDigest = self::base64encode($digest->finish()); + + $objectInfo = new ObjectInfo( + name: $meta->name, + bucket: $this->name, + nuid: $nuid, + size: $size, + chunks: $chunks, + mtime: new \DateTimeImmutable(), + digest: "{$digest->name()}={$encodedDigest}", + deleted: false, + description: $meta->description, + headers: $meta->headers, + metadata: $meta->metadata, + options: new ObjectMetaOptions( + maxChunkSize: $meta->chunkSize, + ), + ); - $objectInfo = new ObjectInfo( - name: $meta->name, - bucket: $this->name, - nuid: $nuid, - size: $size, - chunks: $chunks, - mtime: new \DateTimeImmutable(), - digest: "{$digest->name()}={$encodedDigest}", - deleted: false, - description: $meta->description, - headers: $meta->headers, - metadata: $meta->metadata, - options: new ObjectMetaOptions( - maxChunkSize: $meta->chunkSize, - ), - ); + $this->js->publish( + subject: "\$O.{$this->name}.M." . self::base64encode($meta->name), + message: new Message( + payload: $this->json->encode($objectInfo), + headers: new Headers() + ->with(MsgRollup::header(), MsgRollup::ROLLUP_SUBJECT), + ), + ); + } catch (\Throwable $e) { + if ($chunks > 0) { + $this->purgeChunks($chunkSubject); + } - $this->js->publish( - subject: $subject, - message: new Message( - payload: $this->json->encode($objectInfo), - headers: new Headers() - ->with(MsgRollup::header(), MsgRollup::ROLLUP_SUBJECT), - ), - ); + throw $e; + } if ($info !== null) { $this->stream->purge( @@ -224,13 +265,13 @@ public function info(string $name): ?ObjectInfo $msg = $this->stream->getLastMessageForSubject("\$O.{$this->name}.M." . self::base64encode($name)); if ($msg !== null && $msg->payload !== null && $msg->payload !== '') { - $data = $this->json->decode($msg->payload); - $data['mtime'] = ($msg->headers?->get(Timestamp::Header) ?? new \DateTimeImmutable())->format('Y-m-d H:i:s'); - - return $this->serializer->deserialize( + $info = $this->serializer->deserialize( ObjectInfo::class, - $data, + $this->json->decode($msg->payload), ); + + // The time the server stored the meta, like nats.go: the mtime in the payload is set by the writer. + return $info->withTime($msg->headers?->get(Timestamp::Header) ?? $info->mtime); } return null; @@ -330,6 +371,31 @@ public function seal(): void $this->js->updateStream($info->config->seal()); } + /** + * A cleanup: bounded in time and not reported, the inactive threshold removes the consumer anyway. + */ + private function deleteConsumer(JetStream\Consumer $consumer): void + { + try { + $consumer->delete(new TimeoutCancellation(self::CLEANUP_TIMEOUT_SECONDS)); + } catch (\Throwable) { + } + } + + /** + * A cleanup: bounded in time and not reported, the error that made the put fail matters more. + */ + private function purgeChunks(string $subject): void + { + try { + $this->stream->purge( + subject: $subject, + cancellation: new TimeoutCancellation(self::CLEANUP_TIMEOUT_SECONDS), + ); + } catch (\Throwable) { + } + } + /** * Padding must be kept: nats.go encodes object names in meta subjects and object * digests with padded url-safe base64 (base64.URLEncoding) and rejects unpadded diff --git a/src/JetStream/ObjectStore/StringReader.php b/src/JetStream/ObjectStore/StringReader.php index 7025052..f1b73a9 100644 --- a/src/JetStream/ObjectStore/StringReader.php +++ b/src/JetStream/ObjectStore/StringReader.php @@ -12,12 +12,9 @@ final class StringReader implements Reader /** @var non-negative-int */ private int $cursor = 0; - /** @var positive-int */ + /** @var non-negative-int */ private readonly int $length; - /** - * @param non-empty-string $data - */ public function __construct( private readonly string $data, ) { diff --git a/tests/JetStream/ObjectStore/ObjectStoreTest.php b/tests/JetStream/ObjectStore/ObjectStoreTest.php index ac1023e..d46a5c8 100644 --- a/tests/JetStream/ObjectStore/ObjectStoreTest.php +++ b/tests/JetStream/ObjectStore/ObjectStoreTest.php @@ -4,14 +4,35 @@ namespace Thesis\Nats\JetStream\ObjectStore; +use Amp\TimeoutCancellation; use PHPUnit\Framework\Attributes\CoversClass; +use Thesis\Nats\Client; +use Thesis\Nats\Exception\ObjectIsInvalid; +use Thesis\Nats\Header\Timestamp; +use Thesis\Nats\JetStream; +use Thesis\Nats\Message; +use Thesis\Nats\NatsException; use Thesis\Nats\NatsTestCase; +use Thesis\Nats\PausableProxy; use Thesis\Nats\Subscription; +use function Amp\async; +use function Amp\delay; use function Thesis\Nats\Internal\Id\generateUniqueId; #[CoversClass(Store::class)] final class ObjectStoreTest extends NatsTestCase { + private const float TIMEOUT = 5; + + private ?PausableProxy $proxy = null; + + #[\Override] + protected function tearDown(): void + { + parent::tearDown(); + $this->proxy?->stop(); + } + public function testObjectStoreNames(): void { $js = $this->client()->jetStream(); @@ -168,4 +189,316 @@ public function testWatch(): void $js->deleteObjectStore($name); } + + public function testPutEmptyObject(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + $info = $store->put(new ObjectMeta(name: 'empty'), ''); + self::assertSame(0, $info->size); + self::assertSame(0, $info->chunks); + self::assertSame('SHA-256=' . strtr(base64_encode(hash('sha256', '', true)), '+/', '-_'), $info->digest); + + self::assertSame($info->nuid, $store->info('empty')?->nuid); + self::assertSame('', (string) $store->get('empty')); + + $handle = fopen('php://memory', 'r+'); + self::assertNotFalse($handle); + $store->put(new ObjectMeta(name: 'empty-resource'), new ResourceReader($handle)); + self::assertSame(0, $store->info('empty-resource')?->size); + self::assertSame('', (string) $store->get('empty-resource')); + + $js->deleteObjectStore($name); + } + + public function testGetReadsChunksAndDeletesItsConsumer(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $info = $store->put(new ObjectMeta(name: 'xfile', chunkSize: 4), $body = 'abcdefghijklmnopqrstuvwxyz'); + self::assertSame(7, $info->chunks); + + self::assertSame($body, (string) $store->get('xfile')); + self::assertSame($body, (string) $store->get('xfile')); + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testGetFailsOnDigestMismatch(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $info = $store->put(new ObjectMeta(name: 'xfile'), str_repeat('x', 10)); + + // Same size, other content. + $js->purgeStream("OBJ_{$name}", subject: "\$O.{$name}.C.{$info->nuid}"); + $js->publish("\$O.{$name}.C.{$info->nuid}", new Message(str_repeat('y', 10))); + + $object = $store->get('xfile'); + self::assertNotNull($object); + + try { + $body = $object->buffer(); + self::fail("A corrupted object must not be read, got '{$body}'."); + } catch (ObjectIsInvalid $e) { + self::assertSame("Object 'xfile' digest mismatch.", $e->getMessage()); + } + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testGetFailsOnSizeMismatch(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $info = $store->put(new ObjectMeta(name: 'xfile'), str_repeat('x', 10)); + + $js->publish("\$O.{$name}.C.{$info->nuid}", new Message('tail')); + + $this->expectException(ObjectIsInvalid::class); + $this->expectExceptionMessage("Object 'xfile' size mismatch: expected 10 bytes, received 14."); + + try { + (string) $store->get('xfile'); + } finally { + $js->deleteObjectStore($name); + } + } + + public function testPutPurgesChunksWhenReaderFails(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + try { + $store->put(new ObjectMeta(name: 'xfile', chunkSize: 8), self::failingReader()); + self::fail('The reader error must be rethrown.'); + } catch (\RuntimeException $e) { + self::assertSame('Disk is gone.', $e->getMessage()); + } + + self::assertNull($store->info('xfile')); + self::assertSame(0, $js->streamInfo("OBJ_{$name}")->state->messages); + + $js->deleteObjectStore($name); + } + + public function testPutPurgesChunksWhenPublishFails(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10), maxBytes: 2048)); + + try { + $store->put(new ObjectMeta(name: 'xfile', chunkSize: 256), str_repeat('x', 8192)); + self::fail('A put beyond the bucket limit must fail.'); + } catch (NatsException) { + } + + self::assertNull($store->info('xfile')); + self::assertSame(0, $js->streamInfo("OBJ_{$name}")->state->messages); + + $js->deleteObjectStore($name); + } + + public function testInfoReturnsTheServerTimeWithSubsecondPrecision(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $store->put(new ObjectMeta(name: 'xfile'), 'x'); + + $stored = $js->stream("OBJ_{$name}") + ?->getLastMessageForSubject("\$O.{$name}.M." . strtr(base64_encode('xfile'), '+/', '-_')) + ?->headers + ?->get(Timestamp::Header); + self::assertNotNull($stored); + + $info = $store->info('xfile'); + self::assertNotNull($info); + self::assertSame($stored->format('Y-m-d\TH:i:s.uP'), $info->mtime->format('Y-m-d\TH:i:s.uP')); + self::assertEqualsWithDelta(microtime(true), (float) $info->mtime->format('U.u'), 5); + + $js->deleteObjectStore($name); + } + + public function testGetFailsWhenAllChunksAreGone(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $info = $store->put(new ObjectMeta(name: 'xfile'), str_repeat('x', 10)); + + $js->purgeStream("OBJ_{$name}", subject: "\$O.{$name}.C.{$info->nuid}"); + + try { + $store->get('xfile'); + self::fail('An object without chunks must not be read.'); + } catch (ObjectIsInvalid $e) { + self::assertSame("Object 'xfile' is missing chunks: expected 1, stored 0.", $e->getMessage()); + } + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testGetFailsWhenSomeChunksAreGone(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $info = $store->put(new ObjectMeta(name: 'xfile', chunkSize: 4), 'abcdefghijklmnopqrstuvwxyz'); + + $js->purgeStream("OBJ_{$name}", keep: 3, subject: "\$O.{$name}.C.{$info->nuid}"); + + $this->expectException(ObjectIsInvalid::class); + $this->expectExceptionMessage("Object 'xfile' is missing chunks: expected 7, stored 3."); + + try { + $store->get('xfile'); + } finally { + $js->deleteObjectStore($name); + } + } + + public function testReadFailsWhenChunksAreGoneBeforeDelivery(): void + { + $this->proxy = PausableProxy::start($this->natsAddress()); + $js = $this->clientWithUrls([$this->proxy->address])->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $info = $store->put(new ObjectMeta(name: 'xfile'), str_repeat('x', 10)); + + // The consumer counts the chunk, but the server does not deliver it until the SUB of the reader arrives. + $held = $this->proxy->pauseBefore('SUB '); + $object = async(static fn(): ?StoredObject => $store->get('xfile')); + $held->await(new TimeoutCancellation(self::TIMEOUT)); + + $other = new Client($this->config()); + + try { + $other->jetStream()->purgeStream("OBJ_{$name}", subject: "\$O.{$name}.C.{$info->nuid}"); + } finally { + $other->stop(); + } + + $this->proxy->resume(); + + $read = async(static fn(): string => (string) $object->await()); + + // Nothing is delivered then, the idle heartbeat tells that the chunk is gone. + try { + $read->await(new TimeoutCancellation(self::TIMEOUT * 2)); + self::fail('An object without chunks must not be read.'); + } catch (ObjectIsInvalid $e) { + self::assertSame("Object 'xfile' is missing chunks: received 0 of 10 bytes.", $e->getMessage()); + } + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testReadDoesNotWaitForConsumerDeletion(): void + { + $this->proxy = PausableProxy::start($this->natsAddress()); + $js = $this->clientWithUrls([$this->proxy->address])->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $store->put(new ObjectMeta(name: 'xfile'), $body = str_repeat('x', 10)); + + $held = $this->proxy->pauseBefore('$JS.API.CONSUMER.DELETE.'); + + $object = $store->get('xfile'); + self::assertNotNull($object); + + $read = async($object->buffer(...)); + self::assertSame($body, $read->await(new TimeoutCancellation(self::TIMEOUT))); + + // The deletion was sent after the read ended and is still held back. + $held->await(new TimeoutCancellation(self::TIMEOUT)); + $this->proxy->resume(); + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testPutRethrowsWhenPurgeIsLost(): void + { + $this->proxy = PausableProxy::start($this->natsAddress()); + $js = $this->clientWithUrls([$this->proxy->address])->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + // The server never sees the purge, only the cleanup timeout ends the wait for it. + $held = $this->proxy->pauseBefore('$JS.API.STREAM.PURGE.'); + $put = async(static fn(): ObjectInfo => $store->put(new ObjectMeta(name: 'xfile', chunkSize: 8), self::failingReader())); + + try { + $put->await(new TimeoutCancellation(self::TIMEOUT * 2)); + self::fail('The reader error must be rethrown.'); + } catch (\RuntimeException $e) { + self::assertSame('Disk is gone.', $e->getMessage()); + } + + self::assertTrue($held->isComplete()); + $this->proxy->resume(); + + $js->deleteObjectStore($name); + } + + /** + * Gives two chunks, then fails. + */ + private static function failingReader(): Reader + { + return new class implements Reader { + private int $reads = 0; + + public function eof(): bool + { + return false; + } + + public function read(int $length): string + { + if (++$this->reads > 2) { + throw new \RuntimeException('Disk is gone.'); + } + + return str_repeat('x', $length); + } + }; + } + + /** + * The consumer of get() is deleted after the reader saw the end of the object. + * + * @param non-empty-string $store + */ + private function awaitNoConsumers(JetStream $js, string $store): void + { + $deadline = microtime(true) + self::TIMEOUT; + + $count = $js->streamInfo("OBJ_{$store}")->state->consumerCount; + + while ($count > 0 && microtime(true) < $deadline) { + delay(0.05); + $count = $js->streamInfo("OBJ_{$store}")->state->consumerCount; + } + + self::assertSame(0, $count); + } }