diff --git a/README.md b/README.md index 0819f4d..b7c0497 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ Pure non-blocking (fiber based) strictly typed full-featured PHP driver for NATS - [Mirrors and sources](#mirrors-and-sources) - [NATS ObjectStore](https://docs.nats.io/nats-concepts/jetstream/obj_store) - [Store objects](#store-objects-in-the-buckets) + - [List and update objects](#list-and-update-objects) + - [Files and streams](#files-and-streams) + - [Object Store status](#object-store-status) - [Watch Object Store](#watch-object-store) - [NATS CRDT](#nats-crdt) - [Add Counter](#add-counter) @@ -1449,6 +1452,129 @@ $nc->drain(); - If the reader or a publish fails in the middle of `put()`, the chunks already published are purged and the error is rethrown; the previous version of the object stays as it was. - `get()` reads the chunks through a temporary consumer, deleted after the object is read to the end. It throws `Nats\Exception\ObjectIsInvalid` at once if the stream holds fewer chunks than the meta tells. The size and the SHA-256 digest of the chunks are checked against the meta: on a mismatch, or when the chunks disappear while reading, the read of the stream throws `Nats\Exception\ObjectIsInvalid`. - `info()->mtime` is the time the server stored the meta, with sub-second precision. +- `info()` returns null for a deleted object, `info($name, showDeleted: true)` returns its meta with `deleted: true`. + +#### List and update objects + +```php +jetStream(); + +$store = $js->createOrUpdateObjectStore(new StoreConfig( + store: 'code', +)); + +$store->put(new ObjectMeta('app.php'), 'put(new ObjectMeta('old.php'), 'delete('old.php'); + +foreach ($store->list() as $info) { + dump("{$info->name}: {$info->size} bytes, changed at {$info->mtime->format('c')}"); +} + +dump(count($store->list(showDeleted: true))); // 2 + +$store->updateMeta('app.php', new ObjectMeta( + name: 'main.php', + description: 'The entrypoint', + metadata: ['owner' => 'core'], +)); + +dump((string) $store->get('main.php')); + +$nc->drain(); +``` + +- `list()` returns the metas of the objects (links included) in the order they were last changed, an empty list for an empty store. It reads them through a temporary consumer, deleted afterwards. A part of the list is never returned: if the client is stopped or the connection is lost before every meta is received, `list()` throws `Nats\Exception\ObjectListIsIncomplete`. +- `updateMeta()` changes the name, the description, the headers and the metadata, the data is not copied. The chunk size of the new meta is ignored. It throws `Nats\Exception\ObjectNotFound` for a missing or deleted object and `Nats\Exception\ObjectAlreadyExists` when the new name belongs to another object; a deleted object does not hold its name. +- `addLink()` throws `Nats\Exception\ObjectAlreadyExists` when the name belongs to an object that is not a link. + +#### Files and streams + +```php +jetStream(); + +$store = $js->createOrUpdateObjectStore(new StoreConfig( + store: 'backups', +)); + +$store->putFile('/var/backups/db.sql', new ObjectMeta('db.sql')); +$store->getFile('db.sql', '/tmp/db.sql'); + +// Any amphp byte stream: a socket, a process output, another object. +$store->put(new ObjectMeta('stdin'), new ReadableResourceStream(STDIN)); +$object = $store->get('db.sql') ?? throw new \RuntimeException('db.sql is not found.'); +$store->put(new ObjectMeta('db-copy.sql'), $object->stream); + +$nc->drain(); +``` + +- `putFile()` stores the file under its path, like nats.go, unless a meta is given. +- `getFile()` writes the object to a temporary file next to the target and renames it over the target, so the file is replaced only by a complete, verified object. It returns null if there is no such object; if reading the object fails, e.g. on a digest mismatch, the temporary file is removed, an existing file is left untouched, and the error is rethrown. +- A file that cannot be opened or is not a regular file throws `Nats\Exception\FileIsNotAccessible`; so does a read error of a `ResourceReader`, and nothing is stored. A failing `ReadableStream` throws its own `Amp\ByteStream\StreamException`. +- `put()` accepts a string, a `Reader` (`StringReader`, `ResourceReader`, `StreamReader`) or an `Amp\ByteStream\ReadableStream`. The parts of a stream are glued, so that every chunk but the last one has the chunk size. + +#### Object Store status + +```php +jetStream(); + +$store = $js->createOrUpdateObjectStore(new StoreConfig( + store: 'code', + description: 'Source code', + ttl: TimeSpan::fromDays(7), +)); + +$status = $store->status(); + +dump( + $status->bucket, // code + $status->description, // Source code + $status->ttl, // 7 days, null if objects are kept forever + $status->storage, // StorageType::File + $status->replicas, // 1 + $status->sealed, // false + $status->size, // bytes of the stored chunks and metas + $status->compressed, // false + $status->metadata, + $status->info, // the stream info behind the store +); + +$nc->drain(); +``` #### Watch Object Store diff --git a/src/Exception/FileIsNotAccessible.php b/src/Exception/FileIsNotAccessible.php new file mode 100644 index 0000000..92ecd33 --- /dev/null +++ b/src/Exception/FileIsNotAccessible.php @@ -0,0 +1,12 @@ + + * An object even without limits: the server rejects an empty JSON array, e.g. when a stream config it returned is sent back. */ - public function jsonSerialize(): array + public function jsonSerialize(): \stdClass { - return array_filter( + return (object) array_filter( [ 'inactive_threshold' => $this->inactiveThreshold?->toNanoseconds(), 'max_ack_pending' => $this->maxAckPending, diff --git a/src/JetStream/KeyValue/Internal/InitialValues.php b/src/JetStream/Internal/InitialValues.php similarity index 97% rename from src/JetStream/KeyValue/Internal/InitialValues.php rename to src/JetStream/Internal/InitialValues.php index bfc4e73..c8ff34b 100644 --- a/src/JetStream/KeyValue/Internal/InitialValues.php +++ b/src/JetStream/Internal/InitialValues.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Thesis\Nats\JetStream\KeyValue\Internal; +namespace Thesis\Nats\JetStream\Internal; use Amp\DeferredFuture; use Amp\Future; diff --git a/src/JetStream/KeyValue/Bucket.php b/src/JetStream/KeyValue/Bucket.php index 1a51e6e..1fc028b 100644 --- a/src/JetStream/KeyValue/Bucket.php +++ b/src/JetStream/KeyValue/Bucket.php @@ -349,8 +349,8 @@ private function startWatch( $consumer = $this->stream->createOrUpdateConsumer($consumerConfig, $cancellation); $initialValues = $config->updatesOnly - ? Internal\InitialValues::none() - : new Internal\InitialValues($consumer->info->numPending + $consumer->info->delivered->consumerSeq); + ? JetStream\Internal\InitialValues::none() + : new JetStream\Internal\InitialValues($consumer->info->numPending + $consumer->info->delivered->consumerSeq); $subscription = $consumer->push( new Internal\WatchHandler( diff --git a/src/JetStream/KeyValue/Internal/WatchHandler.php b/src/JetStream/KeyValue/Internal/WatchHandler.php index ed9cc96..871e96e 100644 --- a/src/JetStream/KeyValue/Internal/WatchHandler.php +++ b/src/JetStream/KeyValue/Internal/WatchHandler.php @@ -7,6 +7,7 @@ use Thesis\Nats\Header; use Thesis\Nats\JetStream\Delivery; use Thesis\Nats\JetStream\Internal\IdleListener; +use Thesis\Nats\JetStream\Internal\InitialValues; use Thesis\Nats\JetStream\KeyValue\Entry; use Thesis\Nats\JetStream\KeyValue\EntryState; use Thesis\Nats\Subscription; diff --git a/src/JetStream/ObjectStore/Internal/InfoCollector.php b/src/JetStream/ObjectStore/Internal/InfoCollector.php new file mode 100644 index 0000000..0cb6d4c --- /dev/null +++ b/src/JetStream/ObjectStore/Internal/InfoCollector.php @@ -0,0 +1,96 @@ + */ + public private(set) array $objects = []; + + private int $consumerSequence = 0; + + /** @var DeferredFuture */ + private readonly DeferredFuture $lost; + + /** + * @param \Closure(Delivery): ?ObjectInfo $decode + */ + public function __construct( + private readonly \Closure $decode, + private readonly bool $showDeleted, + private readonly InitialValues $initialValues, + ) { + /** @var DeferredFuture $lost */ + $lost = new DeferredFuture(); + $this->lost = $lost; + $this->lost->getFuture()->ignore(); + } + + /** + * @throws ObjectListIsIncomplete + */ + public function __invoke(Delivery $delivery): void + { + $metadata = $delivery->metadata; + if ($metadata === null || $this->initialValues->delivered()) { + return; + } + + // Nothing is acknowledged, so the metas the server sent while the connection was lost are not sent again. + if ($metadata->consumerSequence !== ++$this->consumerSequence) { + throw new ObjectListIsIncomplete("Object metas were lost: expected delivery {$this->consumerSequence}, received {$metadata->consumerSequence}."); + } + + $info = ($this->decode)($delivery); + + if ($info !== null && ($this->showDeleted || $info->deleted !== true)) { + $this->objects[] = $info; + } + + $this->initialValues->receive(max($metadata->pending, 0)); + } + + /** + * The metas sent while the connection is lost are gone, and the server waits for the flow control reply sent to + * the lost connection, so the list would hang: it fails at once instead. + */ + public function disconnected(): void + { + if (!$this->lost->isComplete() && !$this->initialValues->delivered()) { + $this->lost->error(new ObjectListIsIncomplete('The connection was lost before every object was received.')); + } + } + + /** + * Errors once the connection is lost before every meta was received. + * + * @return Future + */ + public function lost(): Future + { + return $this->lost->getFuture(); + } + + /** + * The server has nothing more to deliver, e.g. the metas were purged after the list had started. + */ + public function idle(): void + { + $this->initialValues->complete(); + } +} diff --git a/src/JetStream/ObjectStore/ObjectInfo.php b/src/JetStream/ObjectStore/ObjectInfo.php index 2728ec0..e62a266 100644 --- a/src/JetStream/ObjectStore/ObjectInfo.php +++ b/src/JetStream/ObjectStore/ObjectInfo.php @@ -66,6 +66,29 @@ public function withTime(\DateTimeImmutable $mtime): self ); } + /** + * The options, i.e. the link and the chunk size, are kept: they describe the stored data, not the object. + * + * @internal + */ + public function withMeta(ObjectMeta $meta): self + { + return new self( + name: $meta->name, + bucket: $this->bucket, + nuid: $this->nuid, + size: $this->size, + chunks: $this->chunks, + mtime: $this->mtime, + digest: $this->digest, + deleted: $this->deleted, + description: $meta->description, + headers: $meta->headers, + metadata: $meta->metadata, + options: $this->options, + ); + } + /** * @internal */ diff --git a/src/JetStream/ObjectStore/ObjectStoreStatus.php b/src/JetStream/ObjectStore/ObjectStoreStatus.php new file mode 100644 index 0000000..243fd2e --- /dev/null +++ b/src/JetStream/ObjectStore/ObjectStoreStatus.php @@ -0,0 +1,64 @@ + */ + public array $metadata; + + /** + * @internal + * @param non-empty-string $bucket + */ + public function __construct( + public string $bucket, + public StreamInfo $info, + ) { + $this->description = $info->config->description; + $this->ttl = $info->config->maxAge !== null && !$info->config->maxAge->isZero() ? $info->config->maxAge : null; + $this->storage = $info->config->storageType; + $this->replicas = $info->config->replicas; + $this->sealed = $info->config->sealed === true; + $this->size = $info->state->bytes; + $this->backingStore = self::BACKING_STORE; + $this->compressed = $info->config->compression !== StoreCompression::None; + $this->metadata = $info->config->metadata ?? []; + } +} diff --git a/src/JetStream/ObjectStore/ResourceReader.php b/src/JetStream/ObjectStore/ResourceReader.php index 65463b0..95c4e9f 100644 --- a/src/JetStream/ObjectStore/ResourceReader.php +++ b/src/JetStream/ObjectStore/ResourceReader.php @@ -4,6 +4,8 @@ namespace Thesis\Nats\JetStream\ObjectStore; +use Thesis\Nats\Exception\FileIsNotAccessible; + /** * @api */ @@ -21,13 +23,18 @@ public function eof(): bool return feof($this->handle); } + /** + * @throws FileIsNotAccessible if reading fails: an object cut short must not be stored as a whole one + */ public function read(int $length): ?string { - $chunk = fread($this->handle, $length); - if ($chunk === false || $chunk === '') { - return null; + error_clear_last(); + $chunk = @fread($this->handle, $length); + + if ($chunk === false) { + throw new FileIsNotAccessible(\sprintf('Failed to read the resource: %s.', error_get_last()['message'] ?? 'unknown error')); } - return $chunk; + return $chunk !== '' ? $chunk : null; } } diff --git a/src/JetStream/ObjectStore/Store.php b/src/JetStream/ObjectStore/Store.php index 6742bae..fb54abf 100644 --- a/src/JetStream/ObjectStore/Store.php +++ b/src/JetStream/ObjectStore/Store.php @@ -5,12 +5,20 @@ namespace Thesis\Nats\JetStream\ObjectStore; use Amp\ByteStream\ReadableIterableStream; +use Amp\ByteStream\ReadableStream; +use Amp\ByteStream\StreamException; +use Amp\ByteStream\WritableResourceStream; use Amp\Cancellation; use Amp\CancelledException; +use Amp\Future; use Amp\Pipeline\Queue; use Amp\TimeoutCancellation; use Thesis\Nats\Client; +use Thesis\Nats\Exception\FileIsNotAccessible; +use Thesis\Nats\Exception\ObjectAlreadyExists; use Thesis\Nats\Exception\ObjectIsInvalid; +use Thesis\Nats\Exception\ObjectListIsIncomplete; +use Thesis\Nats\Exception\ObjectNotFound; use Thesis\Nats\Header\MsgRollup; use Thesis\Nats\Header\Timestamp; use Thesis\Nats\Headers; @@ -21,6 +29,7 @@ use Thesis\Nats\JetStream\Api\DeliverPolicy; use Thesis\Nats\JetStream\Api\ReplayPolicy; use Thesis\Nats\JetStream\Delivery as JetStreamDelivery; +use Thesis\Nats\JetStream\Internal\InitialValues; use Thesis\Nats\JetStream\ObjectStore\Internal\DigestCalculator; use Thesis\Nats\Json\Encoder; use Thesis\Nats\Message; @@ -29,6 +38,7 @@ use Thesis\Nats\Subscription; use Thesis\Time\TimeSpan; use function Amp\async; +use function Amp\ByteStream\pipe; /** * @api @@ -37,10 +47,13 @@ { /** @var int default chunk size is 128KB */ private const int DEFAULT_CHUNK_SIZE = 128 * 1024; - private const int GET_HEARTBEAT_SECONDS = 5; - private const int GET_INACTIVE_THRESHOLD_SECONDS = 300; + private const int HEARTBEAT_SECONDS = 5; + private const int INACTIVE_THRESHOLD_SECONDS = 300; private const int CLEANUP_TIMEOUT_SECONDS = 5; + /** + * @param non-empty-string $name + */ public function __construct( public string $name, private Client $nats, @@ -57,7 +70,7 @@ public function __construct( public function get(string $name): ?StoredObject { $info = $this->info($name); - if ($info === null || $info->deleted === true) { + if ($info === null) { return null; } @@ -100,9 +113,9 @@ public function get(string $name): ?StoredObject maxDeliver: 1, filterSubject: "\$O.{$this->name}.C.{$info->nuid}", flowControl: true, - idleHeartbeat: TimeSpan::fromSeconds(self::GET_HEARTBEAT_SECONDS), + idleHeartbeat: TimeSpan::fromSeconds(self::HEARTBEAT_SECONDS), replayPolicy: ReplayPolicy::Instant, - inactiveThreshold: TimeSpan::fromSeconds(self::GET_INACTIVE_THRESHOLD_SECONDS), + inactiveThreshold: TimeSpan::fromSeconds(self::INACTIVE_THRESHOLD_SECONDS), replicas: 1, memoryStorage: true, )); @@ -150,15 +163,18 @@ public function get(string $name): ?StoredObject /** * Chunks already published are purged if reading the object or publishing fails. * + * @throws StreamException if a {@see ReadableStream} fails * @throws NatsException */ - public function put(ObjectMeta $meta, Reader|string $object): ObjectInfo + public function put(ObjectMeta $meta, Reader|ReadableStream|string $object): ObjectInfo { - if (\is_string($object)) { - $object = new StringReader($object); - } + $object = match (true) { + \is_string($object) => new StringReader($object), + $object instanceof ReadableStream => new StreamReader($object), + default => $object, + }; - $info = $this->info($meta->name); + $info = $this->info($meta->name, showDeleted: true); $nuid = Id\generateUniqueId(); $chunkSubject = "\$O.{$this->name}.C.{$nuid}"; @@ -202,14 +218,7 @@ public function put(ObjectMeta $meta, Reader|string $object): ObjectInfo ), ); - $this->js->publish( - subject: "\$O.{$this->name}.M." . self::base64encode($meta->name), - message: new Message( - payload: $this->json->encode($objectInfo), - headers: new Headers() - ->with(MsgRollup::header(), MsgRollup::ROLLUP_SUBJECT), - ), - ); + $this->publishMeta($objectInfo); } catch (\Throwable $e) { if ($chunks > 0) { $this->purgeChunks($chunkSubject); @@ -232,49 +241,205 @@ public function put(ObjectMeta $meta, Reader|string $object): ObjectInfo */ public function delete(string $name): void { - $info = $this->info($name); + $info = $this->info($name, showDeleted: true); if ($info === null) { return; } - $info = $info - ->withoutTime() - ->asDeleted(); - - $this->js->publish( - subject: "\$O.{$info->bucket}.M." . self::base64encode($info->name), - message: new Message( - payload: $this->json->encode($info), - headers: new Headers() - ->with(MsgRollup::header(), MsgRollup::ROLLUP_SUBJECT), - ), - ); + $this->publishMeta($info->asDeleted()); $this->stream->purge( subject: "\$O.{$this->name}.C.{$info->nuid}", ); } + /** + * Changes the name, the description, the headers and the metadata of the object, the data stays as it is. + * {@see ObjectMeta::$chunkSize} is ignored: the chunks are already stored. + * + * @param non-empty-string $name + * @throws ObjectNotFound if there is no object with this name or it is deleted + * @throws ObjectAlreadyExists if the object is renamed and the new name is taken by another object + * @throws NatsException + */ + public function updateMeta(string $name, ObjectMeta $meta): ObjectInfo + { + $info = $this->info($name) ?? throw new ObjectNotFound("Object '{$name}' is not found."); + + if ($meta->name !== $name && $this->info($meta->name) !== null) { + throw new ObjectAlreadyExists("Object '{$meta->name}' already exists."); + } + + $info = $info->withMeta($meta); + $this->publishMeta($info); + + // The meta is stored under the new name, so the one under the old name is removed. + if ($meta->name !== $name) { + $this->stream->purge( + subject: "\$O.{$this->name}.M." . self::base64encode($name), + ); + } + + return $info->withTime(new \DateTimeImmutable()); + } + /** * @param non-empty-string $name + * @param bool $showDeleted return the meta of a deleted object instead of null */ - public function info(string $name): ?ObjectInfo + public function info(string $name, bool $showDeleted = false): ?ObjectInfo { Validate::objectName($name); $msg = $this->stream->getLastMessageForSubject("\$O.{$this->name}.M." . self::base64encode($name)); - if ($msg !== null && $msg->payload !== null && $msg->payload !== '') { - $info = $this->serializer->deserialize( - ObjectInfo::class, - $this->json->decode($msg->payload), - ); + if ($msg === null || $msg->payload === null || $msg->payload === '') { + return null; + } + + $info = $this->decodeInfo($msg->payload); + + if (!$showDeleted && $info->deleted === true) { + return null; + } + + // The time the server stored the meta, like nats.go: the mtime in the payload is set by the writer. + return $info->withTime($msg->headers?->get(Timestamp::Header) ?? $info->mtime); + } + + /** + * Returns the metas of the objects in the store, links included, in the order they were last changed. + * + * @param bool $showDeleted include the metas of deleted objects + * @return list + * @throws ObjectListIsIncomplete if the list stopped before every meta was received, a part is never returned + * @throws NatsException + */ + public function list(bool $showDeleted = false, ?Cancellation $cancellation = null): array + { + // An ordered consumer of nats.go: no acks, flow control with heartbeats, a replica in memory. + $consumer = $this->stream->createOrUpdateConsumer( + new JetStream\Api\ConsumerConfig( + description: 'object store list consumer', + deliverPolicy: DeliverPolicy::LastPerSubject, + deliverSubject: $this->nats->newInbox(), + ackPolicy: AckPolicy::None, + maxDeliver: 1, + filterSubject: "\$O.{$this->name}.M.>", + flowControl: true, + idleHeartbeat: TimeSpan::fromSeconds(self::HEARTBEAT_SECONDS), + replayPolicy: ReplayPolicy::Instant, + inactiveThreshold: TimeSpan::fromSeconds(self::INACTIVE_THRESHOLD_SECONDS), + replicas: 1, + memoryStorage: true, + ), + $cancellation, + ); + + $initialValues = new InitialValues($consumer->info->numPending); + $collector = new Internal\InfoCollector($this->decodeDelivered(...), $showDeleted, $initialValues); + + try { + if (!$initialValues->delivered()) { + $unwatch = $this->nats->watchConnection($collector->disconnected(...), static function (): void {}); + $subscription = null; + + try { + $subscription = $consumer->push($collector, cancellation: $cancellation); + + $completion = async($subscription->awaitCompletion(...)); + $completion->ignore(); + + Future\awaitFirst([$initialValues->future(), $completion, $collector->lost()], $cancellation); + } finally { + $unwatch(); + $subscription?->stop(); + } + + // The subscription completed first, e.g. the client was stopped. + if (!$initialValues->delivered()) { + throw new ObjectListIsIncomplete("The list of object store '{$this->name}' stopped before every object was received."); + } + } + } finally { + $this->deleteConsumer($consumer); + } + + return $collector->objects; + } + + /** + * @throws NatsException + */ + public function status(): ObjectStoreStatus + { + return new ObjectStoreStatus($this->name, $this->stream->actualInfo()); + } + + /** + * Puts the file into the store, under its path like nats.go unless the meta tells another name. + * + * @param non-empty-string $path + * @throws FileIsNotAccessible + * @throws NatsException + */ + public function putFile(string $path, ?ObjectMeta $meta = null): ObjectInfo + { + $meta ??= new ObjectMeta($path); + + // A directory opens on Linux, and reading it fails. + if (!is_file($path)) { + throw new FileIsNotAccessible("The file '{$path}' cannot be opened: not a regular file."); + } + + $handle = self::open($path, 'rb'); + + try { + return $this->put($meta, new ResourceReader($handle)); + } finally { + fclose($handle); + } + } + + /** + * Writes the object into the file, replacing it. The object is first written to a temporary file next to it, which is then + * renamed over the file, so an existing file is left untouched if there is no such object or reading the object fails. + * + * @param non-empty-string $name + * @param non-empty-string $path + * @throws FileIsNotAccessible + * @throws NatsException + */ + public function getFile(string $name, string $path, ?Cancellation $cancellation = null): ?ObjectInfo + { + $object = $this->get($name); + if ($object === null) { + return null; + } + + $temporary = \sprintf('%s.%s.part', $path, bin2hex(random_bytes(6))); + $file = new WritableResourceStream(self::open($temporary, 'xb', $path)); + + try { + pipe($object->stream, $file, $cancellation); + $file->end(); + + error_clear_last(); + if (!@rename($temporary, $path)) { + throw new FileIsNotAccessible(\sprintf( + "The file '%s' cannot be replaced: %s.", + $path, + error_get_last()['message'] ?? 'unknown error', + )); + } + } catch (\Throwable $e) { + $file->close(); + @unlink($temporary); - // The time the server stored the meta, like nats.go: the mtime in the payload is set by the writer. - return $info->withTime($msg->headers?->get(Timestamp::Header) ?? $info->mtime); + throw $e; } - return null; + return $object->info; } /** @@ -285,8 +450,7 @@ public function watch( WatchConfig $config = new WatchConfig(), ?Cancellation $cancellation = null, ): Subscription { - $serializer = $this->serializer; - $json = $this->json; + $decode = $this->decodeDelivered(...); return $this->stream ->createOrUpdateConsumer(new JetStream\Api\ConsumerConfig( @@ -299,17 +463,11 @@ public function watch( static function (JetStreamDelivery $delivery, Subscription $subscription) use ( $config, $handler, - $serializer, - $json, + $decode, ): void { - $payload = $delivery->message->payload ?? '{}'; - if ($payload === '') { - return; - } + $info = $decode($delivery); - $info = $serializer->deserialize(ObjectInfo::class, $json->decode($payload)); - - if ($config->ignoreDeletes && $info->deleted === true) { + if ($info === null || ($config->ignoreDeletes && $info->deleted === true)) { return; } @@ -333,9 +491,9 @@ public function addLink(string $name, ObjectInfo $object): ObjectInfo throw new ObjectIsInvalid('Not allowed to link to another link.'); } - $target = $this->info($name); + $target = $this->info($name, showDeleted: true); if ($target !== null && !$target->isLink()) { - throw new ObjectIsInvalid('Object is already exists.'); + throw new ObjectAlreadyExists("Object '{$name}' already exists."); } $info = new ObjectInfo( @@ -350,6 +508,28 @@ public function addLink(string $name, ObjectInfo $object): ObjectInfo ), ); + $this->publishMeta($info); + + return $info; + } + + /** + * @throws NatsException + */ + public function seal(): void + { + $info = $this->stream->actualInfo(); + $this->js->updateStream($info->config->seal()); + } + + /** + * The meta replaces the previous one of the object. The time is not stored, like in nats.go: + * the readers take the time the server stored the meta. + * + * @throws NatsException + */ + private function publishMeta(ObjectInfo $info): void + { $this->js->publish( subject: "\$O.{$info->bucket}.M." . self::base64encode($info->name), message: new Message( @@ -358,17 +538,51 @@ public function addLink(string $name, ObjectInfo $object): ObjectInfo ->with(MsgRollup::header(), MsgRollup::ROLLUP_SUBJECT), ), ); + } - return $info; + /** + * Decodes a meta delivered by a consumer, with the time the server stored it. + */ + private function decodeDelivered(JetStreamDelivery $delivery): ?ObjectInfo + { + $payload = $delivery->message->payload; + if ($payload === null || $payload === '') { + return null; + } + + $info = $this->decodeInfo($payload); + + return $delivery->metadata !== null ? $info->withTime($delivery->metadata->timestamp) : $info; } /** - * @throws NatsException + * @param non-empty-string $payload */ - public function seal(): void + private function decodeInfo(string $payload): ObjectInfo { - $info = $this->stream->actualInfo(); - $this->js->updateStream($info->config->seal()); + return $this->serializer->deserialize(ObjectInfo::class, $this->json->decode($payload)); + } + + /** + * @param non-empty-string $path + * @return resource + * @throws FileIsNotAccessible + */ + private static function open(string $path, string $mode, ?string $reportedPath = null): mixed + { + // The last error is cleared first, so that an earlier unrelated one is not reported as the reason. + error_clear_last(); + $handle = @fopen($path, $mode); + + if ($handle === false) { + throw new FileIsNotAccessible(\sprintf( + "The file '%s' cannot be opened: %s.", + $reportedPath ?? $path, + error_get_last()['message'] ?? 'unknown error', + )); + } + + return $handle; } /** diff --git a/src/JetStream/ObjectStore/StreamReader.php b/src/JetStream/ObjectStore/StreamReader.php new file mode 100644 index 0000000..2df52b9 --- /dev/null +++ b/src/JetStream/ObjectStore/StreamReader.php @@ -0,0 +1,69 @@ +buffer === '' && !$this->ended) { + $this->fill(1); + } + + return $this->buffer === ''; + } + + /** + * @throws StreamException + */ + public function read(int $length): ?string + { + $this->fill($length); + + $chunk = substr($this->buffer, 0, $length); + $this->buffer = substr($this->buffer, $length); + + return $chunk !== '' ? $chunk : null; + } + + /** + * @param positive-int $length + * @throws StreamException + */ + private function fill(int $length): void + { + while (!$this->ended && \strlen($this->buffer) < $length) { + $part = $this->stream->read($this->cancellation); + + if ($part === null) { + $this->ended = true; + } else { + $this->buffer .= $part; + } + } + } +} diff --git a/src/Serialization/Internal/CamelCaseKeys.php b/src/Serialization/Internal/CamelCaseKeys.php new file mode 100644 index 0000000..bca94b6 --- /dev/null +++ b/src/Serialization/Internal/CamelCaseKeys.php @@ -0,0 +1,72 @@ + + */ +final readonly class CamelCaseKeys implements \IteratorAggregate +{ + /** + * The properties of the mapped types that are maps: their keys are data, not property names. + */ + private const array MAPS = [ + 'metadata' => true, + 'headers' => true, + 'subjects' => true, + 'tiers' => true, + ]; + + /** @var array */ + private array $data; + + /** + * @param iterable $data + */ + public function __construct(iterable $data) + { + $this->data = self::convert($data, keepKeys: false); + } + + public function getIterator(): \Traversable + { + yield from $this->data; + } + + /** + * @param iterable $data + * @return array + */ + private static function convert(iterable $data, bool $keepKeys): array + { + $result = []; + + foreach ($data as $key => $value) { + \assert(\is_string($key) || \is_int($key)); + + if (is_iterable($value)) { + // The values of a map are converted again: e.g. a tier is an object with properties. + $value = self::convert($value, keepKeys: \is_string($key) && !$keepKeys && isset(self::MAPS[$key])); + } + + if (\is_string($key) && !$keepKeys) { + $key = lcfirst(str_replace([' ', '_', '-'], '', ucwords($key, ' _-'))); + + if (\array_key_exists($key, $result)) { + continue; + } + } + + $result[$key] = $value; + } + + return $result; + } +} diff --git a/src/Serialization/ValinorSerializer.php b/src/Serialization/ValinorSerializer.php index c07d912..5acf80c 100644 --- a/src/Serialization/ValinorSerializer.php +++ b/src/Serialization/ValinorSerializer.php @@ -47,7 +47,7 @@ public function deserialize(string $type, iterable $data): mixed { return $this->mapper->map( signature: $type, - source: Source::iterable($data)->camelCaseKeys(), + source: Source::iterable(new Internal\CamelCaseKeys($data)), ); } } diff --git a/tests/JetStream/ObjectStore/ObjectStoreTest.php b/tests/JetStream/ObjectStore/ObjectStoreTest.php index d46a5c8..29294cd 100644 --- a/tests/JetStream/ObjectStore/ObjectStoreTest.php +++ b/tests/JetStream/ObjectStore/ObjectStoreTest.php @@ -4,17 +4,29 @@ namespace Thesis\Nats\JetStream\ObjectStore; +use Amp\ByteStream\ReadableIterableStream; +use Amp\DeferredFuture; +use Amp\Future; use Amp\TimeoutCancellation; use PHPUnit\Framework\Attributes\CoversClass; use Thesis\Nats\Client; +use Thesis\Nats\Config; +use Thesis\Nats\Exception\FileIsNotAccessible; +use Thesis\Nats\Exception\ObjectAlreadyExists; use Thesis\Nats\Exception\ObjectIsInvalid; +use Thesis\Nats\Exception\ObjectListIsIncomplete; +use Thesis\Nats\Exception\ObjectNotFound; use Thesis\Nats\Header\Timestamp; use Thesis\Nats\JetStream; +use Thesis\Nats\JetStream\Api\StorageType; use Thesis\Nats\Message; use Thesis\Nats\NatsException; use Thesis\Nats\NatsTestCase; use Thesis\Nats\PausableProxy; +use Thesis\Nats\Serialization\Serializer; +use Thesis\Nats\Serialization\ValinorSerializer; use Thesis\Nats\Subscription; +use Thesis\Time\TimeSpan; use function Amp\async; use function Amp\delay; use function Thesis\Nats\Internal\Id\generateUniqueId; @@ -134,7 +146,9 @@ public function testDeleteObject(): void self::assertSame($info->nuid, $store->info('xfile')?->nuid); $store->delete('xfile'); - self::assertTrue($store->info('xfile')?->deleted); + self::assertNull($store->info('xfile')); + self::assertTrue($store->info('xfile', showDeleted: true)?->deleted); + self::assertNull($store->get('xfile')); $js->deleteObjectStore($name); } @@ -459,6 +473,484 @@ public function testPutRethrowsWhenPurgeIsLost(): void $js->deleteObjectStore($name); } + public function testListReturnsObjectsWithoutDeleted(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $a = $store->put(new ObjectMeta('a', description: 'first'), 'aaa'); + $store->put(new ObjectMeta('b'), 'bbb'); + $store->put(new ObjectMeta('c'), ''); + $store->addLink('link', $a); + $store->delete('b'); + $store->put(new ObjectMeta('a', description: 'second'), 'aaaa'); + + $objects = $store->list(cancellation: new TimeoutCancellation(self::TIMEOUT)); + + self::assertSame(['c', 'link', 'a'], array_map(static fn(ObjectInfo $info): string => $info->name, $objects)); + self::assertSame('second', $objects[2]->description ?? null); + self::assertSame(4, $objects[2]->size ?? null); + self::assertTrue(($objects[1] ?? null)?->isLink()); + self::assertEqualsWithDelta($store->info('a')?->mtime->getTimestamp(), ($objects[2] ?? null)?->mtime->getTimestamp(), 1); + + $all = $store->list(showDeleted: true); + + self::assertSame(['c', 'link', 'b', 'a'], array_map(static fn(ObjectInfo $info): string => $info->name, $all)); + self::assertTrue($all[2]->deleted ?? null); + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testListFailsWhenClientIsStoppedMidway(): void + { + $this->assertListFailsWhenInterrupted(static function (Client $client): void { + $stop = async($client->stop(...)); + $stop->ignore(); + + // The handler is held, so stop() waits for it: the subscription is stopped by now. + delay(0.1); + }); + } + + public function testListFailsWhenConnectionIsLost(): void + { + $this->assertListFailsWhenInterrupted(function (Client $client, Future $disconnected): void { + $this->kick('object-store-list'); + $disconnected->await(new TimeoutCancellation(self::TIMEOUT)); + }); + } + + public function testListOfEmptyStore(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + self::assertSame([], $store->list()); + + $store->put(new ObjectMeta('a'), 'aaa'); + $store->delete('a'); + + self::assertSame([], $store->list()); + self::assertCount(1, $store->list(showDeleted: true)); + + $this->awaitNoConsumers($js, $name); + + $js->deleteObjectStore($name); + } + + public function testListManyObjects(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + $names = []; + for ($i = 0; $i < 300; ++$i) { + $store->put(new ObjectMeta($names[] = "object-{$i}"), "{$i}"); + } + + self::assertSame( + $names, + array_map(static fn(ObjectInfo $info): string => $info->name, $store->list(cancellation: new TimeoutCancellation(self::TIMEOUT))), + ); + + $js->deleteObjectStore($name); + } + + public function testUpdateMetaKeepsData(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $put = $store->put(new ObjectMeta('xfile', chunkSize: 4), $body = 'abcdefghij'); + + $updated = $store->updateMeta('xfile', new ObjectMeta( + name: 'xfile', + description: 'updated', + headers: ['X-Kind' => ['text', 'plain']], + metadata: ['owner_team' => 'core', 'Build-Id' => '42'], + chunkSize: 1024, + )); + + self::assertSame('updated', $updated->description); + self::assertSame($put->nuid, $updated->nuid); + + $info = $store->info('xfile'); + self::assertNotNull($info); + self::assertSame('updated', $info->description); + self::assertSame(['X-Kind' => ['text', 'plain']], $info->headers); + self::assertSame(['owner_team' => 'core', 'Build-Id' => '42'], $info->metadata); + self::assertSame(['owner_team' => 'core', 'Build-Id' => '42'], ($store->list()[0] ?? null)?->metadata); + self::assertSame($put->digest, $info->digest); + self::assertSame(3, $info->chunks); + self::assertSame(4, $info->options?->maxChunkSize); + + self::assertSame($body, (string) $store->get('xfile')); + + $js->deleteObjectStore($name); + } + + public function testUpdateMetaRenamesObject(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $store->put(new ObjectMeta('old'), $body = 'abcdefghij'); + + // A deleted object does not hold its name. + $store->put(new ObjectMeta('new'), 'gone'); + $store->delete('new'); + + $store->updateMeta('old', new ObjectMeta('new')); + + self::assertNull($store->info('old')); + self::assertNull($store->info('old', showDeleted: true)); + self::assertSame($body, (string) $store->get('new')); + self::assertSame(['new'], array_map(static fn(ObjectInfo $info): string => $info->name, $store->list(showDeleted: true))); + + $js->deleteObjectStore($name); + } + + public function testUpdateMetaDoesNotRenameOntoExistingObject(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $store->put(new ObjectMeta('a'), 'aaa'); + $store->put(new ObjectMeta('b'), 'bbb'); + + try { + $store->updateMeta('a', new ObjectMeta('b')); + self::fail('The name of another object must not be taken.'); + } catch (ObjectAlreadyExists $e) { + self::assertSame("Object 'b' already exists.", $e->getMessage()); + } + + self::assertSame('aaa', (string) $store->get('a')); + self::assertSame('bbb', (string) $store->get('b')); + + $js->deleteObjectStore($name); + } + + public function testUpdateMetaOfMissingObject(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $store->put(new ObjectMeta('deleted'), 'aaa'); + $store->delete('deleted'); + + foreach (['missing', 'deleted'] as $object) { + try { + $store->updateMeta($object, new ObjectMeta($object, description: 'x')); + self::fail("Object '{$object}' must not be updated."); + } catch (ObjectNotFound $e) { + self::assertSame("Object '{$object}' is not found.", $e->getMessage()); + } + } + + self::assertNull($store->info('missing')); + + $js->deleteObjectStore($name); + } + + public function testAddLinkDoesNotReplaceObject(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $a = $store->put(new ObjectMeta('a'), 'aaa'); + $store->put(new ObjectMeta('b'), 'bbb'); + + $this->expectException(ObjectAlreadyExists::class); + + try { + $store->addLink('b', $a); + } finally { + $js->deleteObjectStore($name); + } + } + + public function testStatus(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig( + store: $name = generateUniqueId(10), + description: 'my store', + ttl: TimeSpan::fromHours(1), + storageType: StorageType::Memory, + compression: true, + metadata: ['owner_team' => 'core', 'Build-Id' => '42'], + )); + + $status = $store->status(); + self::assertSame($name, $status->bucket); + self::assertSame('my store', $status->description); + self::assertSame(3600, $status->ttl?->toSeconds()); + self::assertSame(StorageType::Memory, $status->storage); + self::assertSame(1, $status->replicas); + self::assertFalse($status->sealed); + self::assertSame(0, $status->size); + self::assertSame('JetStream', $status->backingStore); + self::assertTrue($status->compressed); + self::assertSame('core', $status->metadata['owner_team'] ?? null); + self::assertSame('42', $status->metadata['Build-Id'] ?? null); + self::assertArrayNotHasKey('ownerTeam', $status->metadata); + $metadata = $status->metadata; + + $store->put(new ObjectMeta('xfile'), str_repeat('x', 100)); + $store->seal(); + + $status = $store->status(); + self::assertTrue($status->sealed); + // The config the server returned is sent back by seal(), the metadata must come back unchanged. + self::assertSame($metadata, $status->metadata); + self::assertGreaterThan(100, $status->size); + + $js->deleteObjectStore($name); + } + + public function testStatusWithoutTtl(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + $status = $store->status(); + self::assertNull($status->ttl); + self::assertNull($status->description); + self::assertFalse($status->compressed); + self::assertSame(StorageType::File, $status->storage); + + $js->deleteObjectStore($name); + } + + public function testPutFileAndGetFile(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + $source = self::tempFile($body = random_bytes(300 * 1024)); + $target = self::tempFile('previous contents that are longer than nothing'); + + try { + $info = $store->putFile($source, new ObjectMeta('blob')); + self::assertSame(300 * 1024, $info->size); + self::assertSame(3, $info->chunks); + + self::assertSame($info->nuid, $store->getFile('blob', $target, new TimeoutCancellation(self::TIMEOUT))?->nuid); + self::assertSame($body, file_get_contents($target)); + + // Under its path by default, like nats.go. + self::assertSame($source, $store->putFile($source)->name); + self::assertSame($body, (string) $store->get($source)); + } finally { + @unlink($source); + @unlink($target); + $js->deleteObjectStore($name); + } + } + + public function testGetFileOfMissingObjectDoesNotCreateFile(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + $target = sys_get_temp_dir() . '/' . generateUniqueId(10); + + self::assertNull($store->getFile('missing', $target)); + self::assertFileDoesNotExist($target); + + $js->deleteObjectStore($name); + } + + public function testGetFileKeepsExistingFileWhenReadFails(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $info = $store->put(new ObjectMeta(name: 'xfile'), str_repeat('x', 10)); + + $js->purgeStream("OBJ_{$name}", subject: "\$O.{$name}.C.{$info->nuid}"); + $js->publish("\$O.{$name}.C.{$info->nuid}", new Message(str_repeat('y', 10))); + + $directory = sys_get_temp_dir() . '/' . generateUniqueId(10); + mkdir($directory); + $target = "{$directory}/xfile"; + file_put_contents($target, 'previous'); + + try { + $store->getFile('xfile', $target); + self::fail('A corrupted object must not be written.'); + } catch (ObjectIsInvalid $e) { + self::assertSame("Object 'xfile' digest mismatch.", $e->getMessage()); + } + + self::assertSame('previous', file_get_contents($target)); + self::assertSame(['xfile'], array_values(array_diff((array) scandir($directory), ['.', '..']))); + + unlink($target); + rmdir($directory); + + $js->deleteObjectStore($name); + } + + public function testFileIsNotAccessible(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + $store->put(new ObjectMeta('xfile'), 'x'); + + $directory = sys_get_temp_dir() . '/'; + + try { + $store->putFile($directory); + self::fail('A directory must not be put.'); + } catch (FileIsNotAccessible $e) { + self::assertSame("The file '{$directory}' cannot be opened: not a regular file.", $e->getMessage()); + } + + self::assertNull($store->info($directory, showDeleted: true)); + + $missing = sys_get_temp_dir() . '/' . generateUniqueId(10) . '/file'; + + try { + $store->putFile($missing); + self::fail('A missing file must not be put.'); + } catch (FileIsNotAccessible $e) { + self::assertStringStartsWith("The file '{$missing}' cannot be opened: ", $e->getMessage()); + } + + $this->expectException(FileIsNotAccessible::class); + + try { + $store->getFile('xfile', $missing); + } finally { + $js->deleteObjectStore($name); + } + } + + public function testPutReadableStream(): void + { + $js = $this->client()->jetStream(); + + $store = $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + $info = $store->put( + new ObjectMeta('stream', chunkSize: 4), + new ReadableIterableStream(['ab', 'cdefg', 'h', 'ij']), + ); + self::assertSame(10, $info->size); + self::assertSame(3, $info->chunks); + + $object = $store->get('stream'); + self::assertNotNull($object); + + // One object streamed into another. + $copy = $store->put(new ObjectMeta('copy'), $object->stream); + self::assertSame($info->digest, $copy->digest); + self::assertSame('abcdefghij', (string) $store->get('copy')); + + $js->deleteObjectStore($name); + } + + /** + * Holds the list on the 10th of 100 metas until the interruption is done: the list must fail, not return a part. + * + * @param \Closure(Client, Future): void $interrupt + */ + private function assertListFailsWhenInterrupted(\Closure $interrupt): void + { + $count = 100; + + $setup = $this->client(); + $js = $setup->jetStream(); + $js->createOrUpdateObjectStore(new StoreConfig($name = generateUniqueId(10))); + + for ($i = 0; $i < $count; ++$i) { + $js->objectStore($name)?->put(new ObjectMeta("object-{$i}"), ''); + } + + /** @var DeferredFuture $disconnected */ + $disconnected = new DeferredFuture(); + + $serializer = new class ($interrupt, $disconnected->getFuture()) implements Serializer { + private readonly ValinorSerializer $serializer; + + private int $metas = 0; + + /** + * @param \Closure(Client, Future): void $interrupt + * @param Future $disconnected + */ + public function __construct( + private readonly \Closure $interrupt, + private readonly Future $disconnected, + public ?Client $client = null, + ) { + $this->serializer = new ValinorSerializer(); + } + + public function deserialize(string $type, iterable $data): mixed + { + if ($type === ObjectInfo::class && ++$this->metas === 10 && $this->client !== null) { + // The handler waits here, so no more metas are taken until the interruption is done. + async($this->interrupt, $this->client, $this->disconnected)->await(); + } + + return $this->serializer->deserialize($type, $data); + } + }; + + $config = $this->config(); + $client = new Client( + new Config( + urls: $config->urls, + user: $config->user, + password: $config->password, + clientName: 'object-store-list', + reconnectWait: 0.1, + ), + serializer: $serializer, + ); + $serializer->client = $client; + $client->onDisconnect(static function () use ($disconnected): void { + if (!$disconnected->isComplete()) { + $disconnected->complete(); + } + }); + + $store = $client->jetStream()->objectStore($name); + self::assertNotNull($store); + + try { + $this->expectException(ObjectListIsIncomplete::class); + $store->list(cancellation: new TimeoutCancellation(self::TIMEOUT)); + } finally { + $client->stop(); + $js->deleteObjectStore($name); + } + } + + /** + * @return non-empty-string + */ + private static function tempFile(string $contents): string + { + $path = tempnam(sys_get_temp_dir(), 'nats'); + self::assertNotFalse($path); + file_put_contents($path, $contents); + + return $path; + } + /** * Gives two chunks, then fails. */ diff --git a/tests/JetStream/ObjectStore/ReaderTest.php b/tests/JetStream/ObjectStore/ReaderTest.php index 1c6358d..f96208a 100644 --- a/tests/JetStream/ObjectStore/ReaderTest.php +++ b/tests/JetStream/ObjectStore/ReaderTest.php @@ -4,12 +4,16 @@ namespace Thesis\Nats\JetStream\ObjectStore; +use Amp\ByteStream\ReadableBuffer; +use Amp\ByteStream\ReadableIterableStream; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; +use Thesis\Nats\Exception\FileIsNotAccessible; #[CoversClass(StringReader::class)] #[CoversClass(ResourceReader::class)] +#[CoversClass(StreamReader::class)] final class ReaderTest extends TestCase { /** @@ -96,4 +100,58 @@ public function testResourceReader(string $data, int $length, array $chunks): vo self::assertSame($chunks, $result); } + + /** + * @param list $parts + * @param positive-int $length + * @param list $chunks + */ + #[TestWith([['x', 'y', 'z'], 2, ['xy', 'z']])] + #[TestWith([['xyz'], 1, ['x', 'y', 'z']])] + #[TestWith([['ab', '', 'cdefg', 'h'], 3, ['abc', 'def', 'gh']])] + #[TestWith([['abc', 'def'], 3, ['abc', 'def']])] + #[TestWith([['abc'], 10, ['abc']])] + #[TestWith([[], 10, []])] + #[TestWith([[''], 10, []])] + public function testStreamReader(array $parts, int $length, array $chunks): void + { + $rdr = new StreamReader(new ReadableIterableStream($parts)); + + $result = []; + + while (!$rdr->eof()) { + $result[] = $rdr->read($length); + } + + self::assertSame($chunks, $result); + self::assertNull($rdr->read($length)); + } + + public function testStreamReaderReadsWithoutEofCheck(): void + { + $rdr = new StreamReader(new ReadableBuffer('abcde')); + + self::assertSame('abc', $rdr->read(3)); + self::assertSame('de', $rdr->read(3)); + self::assertNull($rdr->read(3)); + self::assertTrue($rdr->eof()); + } + + public function testResourceReaderFailsOnReadError(): void + { + // A directory opens on Linux, and reading it fails. + $handle = fopen(sys_get_temp_dir(), 'r'); + self::assertIsResource($handle); + + $rdr = new ResourceReader($handle); + + $this->expectException(FileIsNotAccessible::class); + $this->expectExceptionMessageMatches('/^Failed to read the resource: .+Is a directory.$/'); + + try { + $rdr->read(10); + } finally { + fclose($handle); + } + } } diff --git a/tests/Serialization/Internal/CamelCaseKeysTest.php b/tests/Serialization/Internal/CamelCaseKeysTest.php new file mode 100644 index 0000000..4dd0127 --- /dev/null +++ b/tests/Serialization/Internal/CamelCaseKeysTest.php @@ -0,0 +1,95 @@ + [ + 'max_msgs' => 10, + 'subjects' => ['orders.new_one'], + 'metadata' => ['team_name' => 'core', 'Build-Id' => '42', '_nats.level' => '1'], + ], + 'state' => [ + 'num_subjects' => 2, + 'subjects' => ['orders.new_one' => 1, 'orders.old-one' => 2], + ], + 'headers' => ['X-Kind' => ['text']], + 'tiers' => [ + 'R1_tier' => ['max_storage' => 1, 'metadata' => ['a_b' => 'c']], + ], + 'list' => [['first_key' => 1]], + ]); + + self::assertSame( + [ + 'config' => [ + 'maxMsgs' => 10, + 'subjects' => ['orders.new_one'], + 'metadata' => ['team_name' => 'core', 'Build-Id' => '42', '_nats.level' => '1'], + ], + 'state' => [ + 'numSubjects' => 2, + 'subjects' => ['orders.new_one' => 1, 'orders.old-one' => 2], + ], + 'headers' => ['X-Kind' => ['text']], + 'tiers' => [ + 'R1_tier' => ['maxStorage' => 1, 'metadata' => ['a_b' => 'c']], + ], + 'list' => [['firstKey' => 1]], + ], + iterator_to_array($data), + ); + } + + public function testFirstOfCollidingKeysWins(): void + { + self::assertSame(['maxMsgs' => 1], iterator_to_array(new CamelCaseKeys(['max_msgs' => 1, 'max-msgs' => 2]))); + } + + /** + * The map fields are known by name: a new one in the deserialized types must be added to the list. + */ + public function testKnowsEveryMapFieldOfDeserializedTypes(): void + { + $maps = new \ReflectionClassConstant(CamelCaseKeys::class, 'MAPS')->getValue(); + self::assertIsArray($maps); + + $api = glob(__DIR__ . '/../../../src/JetStream/Api/*.php'); + self::assertNotFalse($api); + + $files = [ + ...$api, + __DIR__ . '/../../../src/JetStream/ObjectStore/ObjectInfo.php', + ]; + + $fields = []; + + foreach ($files as $file) { + /** @var class-string $class */ + $class = 'Thesis\Nats\\' . str_replace('/', '\\', substr((string) realpath($file), \strlen((string) realpath(__DIR__ . '/../../../src/')) + 1, -4)); + + $doc = new \ReflectionClass($class)->getConstructor()?->getDocComment(); + + if (\is_string($doc) && preg_match_all('/@param\s+\??array<(?:non-empty-)?string,\s*.+>\s+\$(\w+)/', $doc, $matches) > 0) { + foreach ($matches[1] as $field) { + $fields[$field] = $class; + } + } + } + + self::assertNotEmpty($fields); + + foreach ($fields as $field => $class) { + self::assertArrayHasKey($field, $maps, "{$class}::\${$field} is a map: add it to CamelCaseKeys::MAPS."); + } + } +}