diff --git a/README.md b/README.md index b7c0497..0494111 100644 --- a/README.md +++ b/README.md @@ -858,6 +858,19 @@ $subscription = $consumer->push( Additionally, for push consumers, the `deliverSubject` parameter is mandatory, as it is this very parameter that distinguishes the push model from pull. +The deliveries the handler has not taken yet are bounded by the same pending limits as a core subscription (see [Slow consumers](#slow-consumers)), set through `PushConsumeConfig`: + +```php +$subscription = $consumer->push( + static function (Nats\JetStream\Delivery $delivery): void { + $delivery->ack(); + }, + new Nats\JetStream\PushConsumeConfig( + pendingLimits: new Nats\PendingLimits(messages: 10_000, bytes: null), + ), +); +``` + If you want to distribute messages from a single consumer across different subscriptions, you should use the `deliverGroup` parameter. This enables you to scale processing across multiple consumer instances. This parameter functions identically to `queueGroup` in NATS Core. @@ -1451,6 +1464,7 @@ $nc->drain(); - An empty object (`''` or an empty reader) is stored as meta without chunks. - If the reader or a publish fails in the middle of `put()`, the chunks already published are purged and the error is rethrown; the previous version of the object stays as it was. - `get()` reads the chunks through a temporary consumer, deleted after the object is read to the end. It throws `Nats\Exception\ObjectIsInvalid` at once if the stream holds fewer chunks than the meta tells. The size and the SHA-256 digest of the chunks are checked against the meta: on a mismatch, or when the chunks disappear while reading, the read of the stream throws `Nats\Exception\ObjectIsInvalid`. +- The chunks reach the reader of `get()` as fast as it reads them: a chunk is taken from the subscription only after the reader took the previous one, and the flow control reply waits behind the chunks before it, so the server stops sending. At most the flow control window of the server (32 MiB) is received ahead of a slow reader, not the whole object. Closing the stream before its end stops the reading and deletes the consumer. - `info()->mtime` is the time the server stored the meta, with sub-second precision. - `info()` returns null for a deleted object, `info($name, showDeleted: true)` returns its meta with `deleted: true`. diff --git a/src/JetStream/ObjectStore/Internal/ChunkCollector.php b/src/JetStream/ObjectStore/Internal/ChunkCollector.php index f8b5f53..d80b9ac 100644 --- a/src/JetStream/ObjectStore/Internal/ChunkCollector.php +++ b/src/JetStream/ObjectStore/Internal/ChunkCollector.php @@ -5,7 +5,9 @@ namespace Thesis\Nats\JetStream\ObjectStore\Internal; use Amp\Cancellation; +use Amp\CancelledException; use Amp\DeferredCancellation; +use Amp\Pipeline\DisposedException; use Amp\Pipeline\Queue; use Thesis\Nats\Exception\ObjectIsInvalid; use Thesis\Nats\JetStream\Delivery; @@ -15,6 +17,8 @@ /** * Passes the chunks of an object to its reader and verifies the size and the digest once the last chunk arrives. + * A chunk is handled only after the reader took the previous ones, so a slow reader holds back the flow control reply, + * and the server stops delivering. * * @internal */ @@ -24,6 +28,8 @@ final class ChunkCollector implements IdleListener private readonly DeferredCancellation $exhausted; + private readonly DeferredCancellation $released; + /** @var non-negative-int */ private int $size = 0; @@ -39,6 +45,7 @@ public function __construct( ) { $this->digest = DigestCalculator::sha256(); $this->exhausted = new DeferredCancellation(); + $this->released = new DeferredCancellation(); } /** @@ -49,6 +56,15 @@ public function exhausted(): Cancellation return $this->exhausted->getCancellation(); } + /** + * Called once the subscription completes: the chunks left are queued without waiting for the reader, + * so that the subscription is not held by a reader that does not read anymore. + */ + public function release(): void + { + $this->released->cancel(); + } + public function idle(): void { if ($this->checksum === null) { @@ -75,8 +91,17 @@ public function __invoke(Delivery $delivery, Subscription $subscription): void $this->digest->update($payload); $this->size += \strlen($payload); - $pushed = $this->chunks->pushAsync($payload); - $pushed->ignore(); + try { + $this->chunks + ->pushAsync($payload) + ->await($this->released->getCancellation()); + } catch (CancelledException) { + // The subscription is completed: the chunk stays queued for the reader, the rest is not coming. + } catch (DisposedException) { + $subscription->stop(); + + return; + } } if ($delivery->metadata?->pending === 0) { diff --git a/src/JetStream/ObjectStore/Store.php b/src/JetStream/ObjectStore/Store.php index fb54abf..794a5ad 100644 --- a/src/JetStream/ObjectStore/Store.php +++ b/src/JetStream/ObjectStore/Store.php @@ -34,6 +34,7 @@ use Thesis\Nats\Json\Encoder; use Thesis\Nats\Message; use Thesis\Nats\NatsException; +use Thesis\Nats\PendingLimits; use Thesis\Nats\Serialization\Serializer; use Thesis\Nats\Subscription; use Thesis\Time\TimeSpan; @@ -51,6 +52,9 @@ private const int INACTIVE_THRESHOLD_SECONDS = 300; private const int CLEANUP_TIMEOUT_SECONDS = 5; + /** @var int the chunks queued ahead of the reader of {@see get()}, the next one waits until the reader takes one */ + private const int READ_AHEAD_CHUNKS = 1; + /** * @param non-empty-string $name */ @@ -101,7 +105,7 @@ public function get(string $name): ?StoredObject } /** @var Queue $chunks */ - $chunks = new Queue(max($info->chunks, 1)); + $chunks = new Queue(self::READ_AHEAD_CHUNKS); $collector = new Internal\ChunkCollector($info, $chunks); // An ordered consumer of nats.go: no acks, flow control with heartbeats, a replica in memory. @@ -128,7 +132,11 @@ public function get(string $name): ?StoredObject throw new ObjectIsInvalid("Object '{$info->name}' is missing chunks: expected {$info->chunks}, stored {$consumer->info->numPending}."); } - $subscription = $consumer->push($collector); + // The collector waits for the reader, and the flow control reply goes out only after the chunks before it, + // so the server holds the rest of the object back: its window (32 MiB at most) bounds the chunks received ahead + // of the reader. Every dropped chunk would break the object, so the window is the only limit. + $subscription = $consumer->push($collector, new JetStream\PushConsumeConfig(pendingLimits: PendingLimits::unlimited())); + $subscription->onComplete($collector->release(...)); // The reader gets the end of the object or the error first, the consumer is deleted afterwards. $completion = async(function () use ( diff --git a/src/JetStream/PushConsumeConfig.php b/src/JetStream/PushConsumeConfig.php index 4023c4b..819ca46 100644 --- a/src/JetStream/PushConsumeConfig.php +++ b/src/JetStream/PushConsumeConfig.php @@ -4,6 +4,8 @@ namespace Thesis\Nats\JetStream; +use Thesis\Nats\PendingLimits; + /** * @api */ @@ -13,8 +15,10 @@ /** * @param positive-int $maxMissedHeartbeats + * @param PendingLimits $pendingLimits the deliveries the handler has not taken yet, the ones beyond them are dropped */ public function __construct( public int $maxMissedHeartbeats = self::DEFAULT_MAX_MISSED_HEARTBEATS, + public PendingLimits $pendingLimits = new PendingLimits(), ) {} } diff --git a/src/JetStream/PushConsumer.php b/src/JetStream/PushConsumer.php index d888cf7..f192a3b 100644 --- a/src/JetStream/PushConsumer.php +++ b/src/JetStream/PushConsumer.php @@ -57,6 +57,7 @@ public function consume( handler: $handler, queueGroup: $this->info->config->deliverGroup, cancellation: $cancellation, + pendingLimits: $config->pendingLimits, ); $watchdog->subscribe($subscription->error(...)); diff --git a/tests/JetStream/ObjectStore/ObjectStoreTest.php b/tests/JetStream/ObjectStore/ObjectStoreTest.php index 29294cd..850fcf5 100644 --- a/tests/JetStream/ObjectStore/ObjectStoreTest.php +++ b/tests/JetStream/ObjectStore/ObjectStoreTest.php @@ -28,6 +28,7 @@ use Thesis\Nats\Subscription; use Thesis\Time\TimeSpan; use function Amp\async; +use function Amp\ByteStream\buffer; use function Amp\delay; use function Thesis\Nats\Internal\Id\generateUniqueId; @@ -36,6 +37,9 @@ final class ObjectStoreTest extends NatsTestCase { private const float TIMEOUT = 5; + /** @var positive-int many times the initial flow control window of the server (2 MiB) */ + private const int LARGE_OBJECT_SIZE = 16 * 1024 * 1024; + private ?PausableProxy $proxy = null; #[\Override] @@ -449,6 +453,103 @@ public function testReadDoesNotWaitForConsumerDeletion(): void $js->deleteObjectStore($name); } + public function testSlowReaderHoldsTheRestOfTheObjectOnTheServer(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $info = $store->put(new ObjectMeta(name: 'xfile'), self::generatedObject(self::LARGE_OBJECT_SIZE)); + + memory_reset_peak_usage(); + $baseline = memory_get_usage(); + + $object = $store->get('xfile'); + self::assertNotNull($object); + + $digest = hash_init('sha256'); + $first = $object->stream->read(); + self::assertNotNull($first); + hash_update($digest, $first); + + // The reader stalls: the server stops at its flow control window instead of sending the whole object. + delay(1); + + $consumers = array_values([...$js->consumerNames("OBJ_{$name}")]); + self::assertCount(1, $consumers); + self::assertGreaterThan(0, $js->consumerInfo("OBJ_{$name}", $consumers[0])->numPending); + self::assertLessThan(self::LARGE_OBJECT_SIZE / 2, memory_get_peak_usage() - $baseline); + + $size = \strlen($first); + + while (($chunk = $object->stream->read()) !== null) { + hash_update($digest, $chunk); + $size += \strlen($chunk); + } + + self::assertSame($info->size, $size); + self::assertSame(hash_final($digest), hash('sha256', buffer(self::generatedObject(self::LARGE_OBJECT_SIZE)))); + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testReaderClosingStreamMidwayStopsTheGet(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $store->put(new ObjectMeta(name: 'xfile'), self::generatedObject(self::LARGE_OBJECT_SIZE)); + + $object = $store->get('xfile'); + self::assertNotNull($object); + self::assertNotNull($object->stream->read()); + + // The rest of the object waits for the reader, who gives up. + delay(0.5); + $object->stream->close(); + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testDrainDoesNotWaitForStalledReader(): void + { + $nc = $this->client(); + $js = $nc->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $store->put(new ObjectMeta(name: 'xfile'), self::generatedObject(self::LARGE_OBJECT_SIZE)); + + $object = $store->get('xfile'); + self::assertNotNull($object); + self::assertNotNull($object->stream->read()); + delay(0.5); + + // The drain does not wait for the reader to take the received chunks. + $started = microtime(true); + $nc->drain(); + self::assertLessThan(self::TIMEOUT, microtime(true) - $started); + + // The reader still gets the received chunks, but not the object. + try { + $read = $object->buffer(); + self::fail(\sprintf('A partly received object must not be read to the end, %d bytes were read.', \strlen($read))); + } catch (ObjectIsInvalid $e) { + self::assertStringContainsString("Object 'xfile' is missing chunks", $e->getMessage()); + } + + // The stopped client cannot delete the consumer, the server removes it after the inactive threshold. + $other = new Client($this->config()); + + try { + $other->jetStream()->deleteObjectStore($name); + } finally { + $other->stop(); + } + } + public function testPutRethrowsWhenPurgeIsLost(): void { $this->proxy = PausableProxy::start($this->natsAddress()); @@ -975,6 +1076,18 @@ public function read(int $length): string }; } + /** + * @param positive-int $size + */ + private static function generatedObject(int $size): ReadableIterableStream + { + return new ReadableIterableStream((static function () use ($size): \Generator { + for ($offset = 0; $offset < $size; $offset += 1024) { + yield str_repeat(\chr(intdiv($offset, 1024) % 256), min(1024, $size - $offset)); + } + })()); + } + /** * The consumer of get() is deleted after the reader saw the end of the object. *