From f843837d1defcf5f23ad95280840c4656fa960bf Mon Sep 17 00:00:00 2001 From: kafkiansky Date: Wed, 11 Nov 2026 11:30:01 +0300 Subject: [PATCH] Reconnect automatically and restore the subscriptions --- README.md | 48 +- compose.yaml | 9 + phpunit.xml.dist | 1 + src/Client.php | 275 ++++++-- src/Config.php | 56 ++ src/Exception/ReconnectBufferExceeded.php | 24 + src/Internal/Connection/Connection.php | 5 +- src/Internal/Connection/ConnectionFactory.php | 8 +- src/Internal/Connection/ConnectionInfo.php | 6 + src/Internal/Connection/PingPongHandler.php | 4 +- src/Internal/Connection/Query.php | 41 ++ src/Internal/Connection/ReconnectState.php | 21 + .../Connection/ReconnectingConnection.php | 544 +++++++++++++++ src/Internal/Connection/Server.php | 46 ++ src/Internal/Connection/ServerPool.php | 154 +++++ src/Internal/Connection/SocketConnection.php | 10 +- .../Connection/SocketConnectionFactory.php | 38 +- src/Internal/Hooks/ConcurrentProvider.php | 55 +- src/Internal/Hooks/ConnectionClosed.php | 11 +- src/Internal/Hooks/Disconnected.php | 20 + src/Internal/Hooks/InfoReceived.php | 19 + src/Internal/Hooks/LameDuckModeEntered.php | 15 + src/Internal/Hooks/Provider.php | 32 +- src/Internal/Hooks/Reconnected.php | 15 + src/Internal/Hooks/ServersDiscovered.php | 20 + src/Internal/Protocol/Err.php | 19 + src/Internal/Subscription/Subscriber.php | 2 + .../Subscription/SubscriptionHandler.php | 22 +- src/Internal/Validation/Validate.php | 14 + src/JetStream/Internal/PullMessageHandler.php | 89 ++- src/JetStream/PullConsumer.php | 5 + tests/ClientTest.php | 4 +- tests/ConfigTest.php | 49 ++ tests/Internal/Connection/ServerPoolTest.php | 93 +++ tests/JetStream/PullConsumerTest.php | 204 ++++++ tests/NatsTestCase.php | 42 +- tests/PausableProxy.php | 31 +- tests/ReconnectTest.php | 650 ++++++++++++++++++ 38 files changed, 2587 insertions(+), 114 deletions(-) create mode 100644 src/Exception/ReconnectBufferExceeded.php create mode 100644 src/Internal/Connection/ReconnectState.php create mode 100644 src/Internal/Connection/ReconnectingConnection.php create mode 100644 src/Internal/Connection/Server.php create mode 100644 src/Internal/Connection/ServerPool.php create mode 100644 src/Internal/Hooks/Disconnected.php create mode 100644 src/Internal/Hooks/InfoReceived.php create mode 100644 src/Internal/Hooks/LameDuckModeEntered.php create mode 100644 src/Internal/Hooks/Reconnected.php create mode 100644 src/Internal/Hooks/ServersDiscovered.php create mode 100644 tests/Internal/Connection/ServerPoolTest.php create mode 100644 tests/ReconnectTest.php diff --git a/README.md b/README.md index 489e6c6..3c587b5 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Pure non-blocking (fiber based) strictly typed full-featured PHP driver for NATS - [Connection](#connection) - [Authentication](#authentication) - [Connect and ping](#connect-and-ping) + - [Reconnect](#reconnect) - [Publish-Subscribe](#pub-sub) - [Queues](#queues) - [Request-Reply](#request-reply) @@ -65,9 +66,9 @@ The library implements the full functionality of NATS Core, including pub-sub, q `Nats\Config::fromURI()` takes `[scheme://][user[:password]@]host[:port][,host[:port]...][?options]`. The scheme is `nats://` (`tcp://` is an alias, and it may be omitted), `tls://`, `ws://` and `wss://` are rejected as not supported yet. A host without a port connects to `4222`, an IPv6 host goes in brackets (`[::1]:4222`), -and the credentials are percent-encoded like in any url (`p%40ss` for `p@ss`). The servers of a list are tried in order until one accepts the connection; +and the credentials are percent-encoded like in any url (`p%40ss` for `p@ss`). The servers of a list are tried in a random order (in the given one with `noRandomize`, `no_randomize=true` in the uri) until one accepts the connection, see [Reconnect](#reconnect); each one may repeat the scheme and the credentials, as in nats.go (`nats://u:p@h1:4222,nats://u:p@h2:4222`): one user is sent to every server, so different credentials are rejected. The options are the snake_case names of the `Nats\Config` parameters, -e.g. `connection_timeout` (seconds, fractions allowed), `request_timeout`, `verbose`, `no_responders`, `ping`, `token`, `jwt`, `nkey`, `credentials_file`, `jwt_file`, `nkey_file`, `client_name`. +e.g. `connection_timeout` (seconds, fractions allowed), `request_timeout`, `verbose`, `no_responders`, `ping`, `token`, `jwt`, `nkey`, `credentials_file`, `jwt_file`, `nkey_file`, `client_name`, `allow_reconnect`, `max_reconnects` (negative for no limit), `reconnect_wait`, `reconnect_jitter`, `reconnect_buffer_size`, `no_randomize`. Userinfo without a colon is a token, as in nats.go (`nats://s3cr3t@host`), while `nats://alice:@host` is the user `alice` with an empty password, see [Authentication](#authentication): ```php @@ -150,6 +151,49 @@ $rtt = $nc->ping(); // The subscription is active on the server now. dump("Round trip: {$rtt}s"); ``` +#### Reconnect + +The client reconnects once the connection is lost, the way nats.go does it, and the subscriptions do not notice: +they are restored on the new connection with the same ids (an auto-unsubscribe keeps the rest of its limit), +the request inbox included, and a JetStream pull consumer pulls again. The publishes made while reconnecting are buffered, +up to `reconnectBufferSize` bytes (8 MiB by default), and sent after the subscriptions; once the buffer is full, `publish()` throws `Nats\Exception\ReconnectBufferExceeded`. +In verbose mode a publish waits for the reconnect instead, to get its acknowledgement, and so does `ping()`. + +The servers are shuffled (`noRandomize` keeps the given order) and tried one by one, pausing for `reconnectWait` plus a random +`reconnectJitter` after every round. A server that fails `maxReconnects` attempts in a row (60 by default, null for no limit) is given up. +Once no server is left, the client is closed: the subscriptions fail with `Nats\Exception\ConnectionWasClosed` and the next call +establishes a new connection. A server that rejects the credentials twice in a row closes the client at once, with `Nats\Exception\AuthorizationViolation`: +retrying would not help. The servers of a cluster announced by the server (`connect_urls`) +join the pool, and a server going to shut down (lame duck mode) makes the client move to another one right away. +Set `allowReconnect: false` to fail the subscriptions as soon as the connection is lost instead. + +```php +onDisconnect(static fn(?\Throwable $reason) => dump('Disconnected: ' . ($reason?->getMessage() ?? 'closed by the server'))); +$nc->onReconnect(static fn() => dump('Reconnected')); +$nc->onClose(static fn(?\Throwable $reason) => dump('Closed: ' . ($reason?->getMessage() ?? 'by the client'))); +$nc->onServersDiscovered(static fn(array $urls) => dump('Known servers: ' . implode(', ', $urls))); +$nc->onLameDuckMode(static fn() => dump('The server is going to shut down')); + +$nc->subscribe('foo.>', static function (Nats\Delivery $delivery): void { + dump($delivery->message->payload); // keeps receiving after a reconnect +}); + +trapSignal([\SIGTERM, \SIGINT]); + +$nc->drain(); +``` + #### Pub-Sub ```php diff --git a/compose.yaml b/compose.yaml index 183e4d3..d77aa40 100644 --- a/compose.yaml +++ b/compose.yaml @@ -30,5 +30,14 @@ services: image: nats command: --name nats-token -p 4222 --auth T0k3n + # A cluster of two servers: each one announces both in INFO (connect_urls), for the tests of the discovered servers. + nats-cluster-1: + image: nats + command: --name nats-cluster-1 -p 4222 --cluster_name test --cluster nats://0.0.0.0:6222 --routes nats://nats-cluster-2:6222 + + nats-cluster-2: + image: nats + command: --name nats-cluster-2 -p 4222 --cluster_name test --cluster nats://0.0.0.0:6222 --routes nats://nats-cluster-1:6222 + nats-cli: image: bitnamilegacy/natscli:latest diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 116b749..ed5a153 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -13,6 +13,7 @@ > + diff --git a/src/Client.php b/src/Client.php index fa39b48..96e6886 100644 --- a/src/Client.php +++ b/src/Client.php @@ -43,7 +43,7 @@ final class Client private readonly Connection\ConnectionFactory $connectionFactory; - /** @var ?Future */ + /** @var ?Future */ private ?Future $connection = null; /** @@ -61,6 +61,26 @@ final class Client /** @var list<\Closure(\Throwable): void> */ private array $errorCallbacks = []; + /** @var list<\Closure(?\Throwable): void> */ + private array $disconnectCallbacks = []; + + /** @var list<\Closure(): void> */ + private array $reconnectCallbacks = []; + + /** @var list<\Closure(?\Throwable): void> */ + private array $closeCallbacks = []; + + /** @var list<\Closure(non-empty-list): void> */ + private array $serversDiscoveredCallbacks = []; + + /** @var list<\Closure(): void> */ + private array $lameDuckModeCallbacks = []; + + /** @var array the internal listeners of the disconnects and reconnects */ + private array $connectionWatchers = []; + + private int $nextWatcherId = 0; + private readonly Id\SubscriptionIdGenerator $subscriptionIdGenerator; public function __construct( @@ -87,15 +107,16 @@ public static function default(): self } /** - * Connects to the server unless the client is already connected. Every call that needs the connection - * establishes it on demand, so calling this is optional: it makes network and authentication errors - * surface at startup rather than from the first publish or subscribe. + * Connects to the server unless the client is already connected, and waits for the reconnect in progress if any. + * Every call that needs the connection establishes it on demand, so calling this is optional: it makes network + * and authentication errors surface at startup rather than from the first publish or subscribe. * * @throws NatsException */ public function connect(?Cancellation $cancellation = null): void { - $this->connection($cancellation); + // The connection object outlives a lost connection: while reconnecting, wait for the reconnect. + $this->connection($cancellation)->awaitConnected($cancellation); } /** @@ -409,6 +430,88 @@ public function onError(callable $callback): void $this->errorCallbacks[] = $callback(...); } + /** + * Registers a callback for a lost connection: the client is reconnecting, the subscriptions are kept and the publishes + * are buffered meanwhile, see {@see Config::$allowReconnect}. The callback gets the server error or the failure + * that broke the connection, null if the server just closed it. + * + * @param callable(?\Throwable): void $callback + */ + public function onDisconnect(callable $callback): void + { + $this->disconnectCallbacks[] = $callback(...); + } + + /** + * Registers a callback for a restored connection: the subscriptions are active again and the buffered publishes are sent. + * + * @param callable(): void $callback + */ + public function onReconnect(callable $callback): void + { + $this->reconnectCallbacks[] = $callback(...); + } + + /** + * Registers a callback for the connection closed for good: by {@see stop()} or {@see drain()}, by a fatal server error, + * or once no server is left to reconnect to. The callback gets the reason, null for a regular close. + * The next call establishes a new connection. + * + * @param callable(?\Throwable): void $callback + */ + public function onClose(callable $callback): void + { + $this->closeCallbacks[] = $callback(...); + } + + /** + * Registers a callback for the servers of the cluster announced after the client has connected, + * the client reconnects to them as well. The callback gets the `host:port` of every known server. + * + * @param callable(non-empty-list): void $callback + */ + public function onServersDiscovered(callable $callback): void + { + $this->serversDiscoveredCallbacks[] = $callback(...); + } + + /** + * Registers a callback for the server going to shut down (lame duck mode). If the client knows another server, + * it moves there right away, otherwise it reconnects once the server closes the connection. + * + * @param callable(): void $callback + */ + public function onLameDuckMode(callable $callback): void + { + $this->lameDuckModeCallbacks[] = $callback(...); + } + + /** + * Listens to the disconnects and reconnects, e.g. for a JetStream consumer to pull again once reconnected. + * + * @internal + * @param \Closure(): void $onDisconnect + * @param \Closure(): void $onReconnect + * @return \Closure(): void stops listening + */ + public function watchConnection(\Closure $onDisconnect, \Closure $onReconnect): \Closure + { + $id = $this->nextWatcherId++; + $this->connectionWatchers[$id] = [$onDisconnect, $onReconnect]; + + $client = \WeakReference::create($this); + + return static function () use ( + $client, + $id, + ): void { + $self = $client->get(); + if ($self !== null) { + unset($self->connectionWatchers[$id]); + } + }; + } + public function createService(Micro\ServiceConfig $config): Micro\Service { $identity = new Micro\ServiceIdentity( @@ -446,6 +549,11 @@ public function stop(?Cancellation $cancellation = null): void $this->disconnect(static fn(Subscription $subscription) => $subscription->stop($cancellation), $cancellation); } + /** + * Unsubscribes, lets the handlers finish the messages already received and closes the connection. While reconnecting, + * unlike nats.go that fails here, it waits for the reconnect to drain over the new connection: pass a cancellation + * to bound the wait. + */ public function drain(?Cancellation $cancellation = null): void { $this->disconnect(static fn(Subscription $subscription) => $subscription->drain($cancellation), $cancellation); @@ -506,6 +614,7 @@ private function register( queueGroup: $queueGroup, push: $handler->push(...), subscription: $subscription = $handler->subscription, + remaining: $handler->remaining(...), ); try { @@ -634,28 +743,67 @@ private function invokeSubscriber(Hooks\MessageReceived $event): void } /** - * Routes a server error that was not an acknowledgement of a command, the same way nats.go does: - * a subscription permission violation fails that subscription only, a publish permission violation - * is reported to the error callbacks, anything else is fatal and the connection is closed. - * A fatal error is reported once the connection is torn down, see onClose in connection(). - * - * @return ?\Throwable the error the connection is closed with + * Routes a permission violation, the only server error the connection passes on, the same way nats.go does: + * a subscription permission violation fails that subscription only, a publish one is reported to the error callbacks. + * The connection deals with the others itself: it reconnects or closes, see {@see closed()}. */ - private function handleError(Hooks\ErrorReceived $event, Connection\Connection $connection): ?\Throwable + private function handleError(Hooks\ErrorReceived $event): void { $e = $event->error->toException(); - if ($e instanceof Exception\PermissionViolated) { - if (!$this->failSubscriptions($event->error, $e)) { - $this->reportError($e); + if (!$e instanceof Exception\PermissionViolated || !$this->failSubscriptions($event->error, $e)) { + $this->reportError($e); + } + } + + /** + * @return list the commands that restore the subscriptions on a new connection, with the same sids, + * and with the rest of the limit of an auto-unsubscribe + */ + private function resubscriptions(): array + { + $frames = []; + + foreach ($this->subscribers as $sid => $subscriber) { + // A numeric sid becomes an integer key of the array, which the type of the property does not tell. + $sid = (string) $sid; // @phpstan-ignore cast.useless + $remaining = $subscriber->remaining !== null ? ($subscriber->remaining)() : null; + + if ($remaining === 0) { + // The server has already removed it, the handler is finishing the received messages. + continue; } - return null; + $frames[] = Internal\Command::sub($subscriber->subject, $sid, $subscriber->queueGroup); + + if ($remaining !== null) { + $frames[] = Internal\Command::unsub($sid, $remaining); + } } - $connection->close(); + return $frames; + } - return $e; + private function disconnected(?\Throwable $reason): void + { + foreach ($this->disconnectCallbacks as $callback) { + EventLoop::queue($callback, $reason); + } + + foreach ($this->connectionWatchers as [$onDisconnect]) { + EventLoop::queue($onDisconnect); + } + } + + private function reconnected(): void + { + foreach ($this->reconnectCallbacks as $callback) { + EventLoop::queue($callback); + } + + foreach ($this->connectionWatchers as [, $onReconnect]) { + EventLoop::queue($onReconnect); + } } /** @@ -728,8 +876,8 @@ private function disconnect(\Closure $do, ?Cancellation $cancellation = null): v } /** - * Tears the client down once the connection has been closed from the outside: by the server (e.g. a kick - * or a restart) or because of a fatal server error. Nothing is sent over the connection, it is gone and so + * Tears the client down once the connection has been closed from the outside: the client has given up reconnecting + * (or reconnecting is off and the server has closed the connection), or a fatal server error has closed it. Nothing is sent over the connection, it is gone and so * are the server-side subscriptions: the subscriptions and the pending requests are only failed locally. * * @param ?\Throwable $error the fatal server error that closed the connection, if any @@ -772,9 +920,10 @@ private function closed(Connection\Connection $connection, ?\Throwable $error): // and nothing may escape to the event loop, which has no one to catch it. Future\awaitAll($completes); } finally { - if ($error !== null) { - // Reported after the teardown, so that the callback observes the subscriptions already failed - // and the next call establishing a new connection. + // Reported after the teardown, so that the callback observes the subscriptions already failed + // and the next call establishing a new connection. Running out of servers is not a server error, + // it is reported to the close callbacks only. + if ($error !== null && !$error instanceof ConnectionWasClosed) { $this->reportError($error); } } @@ -833,49 +982,77 @@ private static function awaitOrNull(Future $future, ?Cancellation $cancellation } } - private function connection(?Cancellation $cancellation = null): Connection\Connection + private function connection(?Cancellation $cancellation = null): Connection\ReconnectingConnection { - $this->connection ??= async(weakClosure(function (): Connection\Connection { - try { - $connection = $this->connectionFactory->connect(); - } catch (\Throwable $e) { - // Forget the failed attempt so that the next call tries to connect again. - $this->connection = null; - - throw $e; - } - - $this->activeConnection = $connection; - - /** @var ?\Throwable $reason the fatal server error that closed the connection */ - $reason = null; - + $this->connection ??= async(weakClosure(function (): Connection\ReconnectingConnection { // The connection's reader runs in the event loop: strong references from its hooks would keep the client // alive for as long as the socket is open, so the destructor that closes the socket would never run. // Once the client is gone, its hooks do nothing: e.g. the close event of the connection it closed itself. $client = \WeakReference::create($this); + $connection = new Connection\ReconnectingConnection( + config: $this->config, + factory: $this->connectionFactory, + subscriptions: static fn(): array => $client->get()?->resubscriptions() ?? [], + ); + $connection->hooks->onMessage(static function (Hooks\MessageReceived $event) use ($client): void { $client->get()?->invokeSubscriber($event); }); - $connection->hooks->onError(static function (Hooks\ErrorReceived $event) use ( - $client, - $connection, - &$reason, - ): void { + $connection->hooks->onError(static function (Hooks\ErrorReceived $event) use ($client): void { + $client->get()?->handleError($event); + }); + $connection->hooks->onDisconnect(static function (Hooks\Disconnected $event) use ($client): void { + $client->get()?->disconnected($event->reason); + }); + $connection->hooks->onReconnect(static function () use ($client): void { + $client->get()?->reconnected(); + }); + $connection->hooks->onServersDiscovered(static function (Hooks\ServersDiscovered $event) use ($client): void { $self = $client->get(); - if ($self !== null) { - $reason ??= $self->handleError($event, $connection); + if ($self === null) { + return; + } + + foreach ($self->serversDiscoveredCallbacks as $callback) { + EventLoop::queue($callback, $event->urls); + } + }); + $connection->hooks->onLameDuckMode(static function () use ($client): void { + $self = $client->get(); + if ($self === null) { + return; + } + + foreach ($self->lameDuckModeCallbacks as $callback) { + EventLoop::queue($callback); } }); - $connection->hooks->onClose(static function () use ( + $connection->hooks->onClose(static function (Hooks\ConnectionClosed $event) use ( $client, $connection, - &$reason, ): void { - $client->get()?->closed($connection, $reason); + $self = $client->get(); + if ($self !== null) { + foreach ($self->closeCallbacks as $callback) { + EventLoop::queue($callback, $event->reason); + } + + $self->closed($connection, $event->reason); + } }); + try { + $connection->start(); + } catch (\Throwable $e) { + // Forget the failed attempt so that the next call tries to connect again. + $this->connection = null; + + throw $e; + } + + $this->activeConnection = $connection; + return $connection; })); diff --git a/src/Config.php b/src/Config.php index e51ed37..cca4e9b 100644 --- a/src/Config.php +++ b/src/Config.php @@ -17,6 +17,10 @@ private const int DEFAULT_PING_INTERVAL = 10000; private const int DEFAULT_MAX_PINGS = 5; private const string DEFAULT_CLIENT_NAME = 'thesis/nats'; + private const int DEFAULT_MAX_RECONNECTS = 60; + private const float DEFAULT_RECONNECT_WAIT = 2.0; + private const float DEFAULT_RECONNECT_JITTER = 0.1; + private const int DEFAULT_RECONNECT_BUFFER_SIZE = 8 * 1024 * 1024; /** * @var non-empty-string @@ -41,6 +45,14 @@ * read on every connection attempt; exclusive with the other JWT and NKey options * @param ?non-empty-string $jwtFile the path to a file holding the user JWT, bare or decorated, read on every connection attempt; exclusive with $jwt * @param ?non-empty-string $nkeyFile the path to a file holding the NKey user seed, bare or decorated, read on every connection attempt; exclusive with $nkey + * @param bool $allowReconnect whether the client reconnects once the connection is lost, restoring the subscriptions; + * otherwise the subscriptions fail and the next call establishes a new connection + * @param ?non-negative-int $maxReconnects the failed reconnects in a row after which a server is given up, null for no limit; + * once every server is given up, the client is closed + * @param float $reconnectWait in seconds, the pause after every server has been tried + * @param float $reconnectJitter in seconds, the upper bound of a random delay added to the pause, so that the clients do not reconnect at once + * @param non-negative-int $reconnectBufferSize in bytes, the publishes buffered while reconnecting, 0 to fail them instead + * @param bool $noRandomize whether to try the servers in the given order instead of a random one * @throws \InvalidArgumentException */ public function __construct( @@ -68,9 +80,31 @@ public function __construct( public ?string $credentialsFile = null, public ?string $jwtFile = null, public ?string $nkeyFile = null, + public bool $allowReconnect = true, + public ?int $maxReconnects = self::DEFAULT_MAX_RECONNECTS, + public float $reconnectWait = self::DEFAULT_RECONNECT_WAIT, + public float $reconnectJitter = self::DEFAULT_RECONNECT_JITTER, + public int $reconnectBufferSize = self::DEFAULT_RECONNECT_BUFFER_SIZE, + public bool $noRandomize = false, ) { Validate::duration($connectionTimeout, 'connection timeout'); + // A negative limit would give up every server at once, while the uri means no limit by it: refused as a mistake, + // null is the way to say no limit here. + if ($maxReconnects !== null) { + Validate::nonNegative($maxReconnects, 'max reconnects'); + } + + Validate::nonNegative($reconnectBufferSize, 'reconnect buffer size'); + + if (!is_finite($reconnectWait) || $reconnectWait < 0) { + throw new \InvalidArgumentException('The reconnect wait must be a finite non-negative number of seconds.'); + } + + if (!is_finite($reconnectJitter) || $reconnectJitter < 0) { + throw new \InvalidArgumentException('The reconnect jitter must be a finite non-negative number of seconds.'); + } + // nats-server itself refuses to have a token next to users, so both of them here are a mistake. if ($token !== null && ($user !== null || $password !== null)) { throw new \InvalidArgumentException('A token cannot be combined with a user and a password.'); @@ -134,6 +168,16 @@ public static function fromURI(#[\SensitiveParameter] string $uri): self credentialsFile: $query->credentialsFile, jwtFile: $query->jwtFile, nkeyFile: $query->nkeyFile, + allowReconnect: $query->allowReconnect ?? true, + maxReconnects: match (true) { + $query->maxReconnects === null => self::DEFAULT_MAX_RECONNECTS, + $query->maxReconnects < 0 => null, + default => $query->maxReconnects, + }, + reconnectWait: $query->reconnectWait ?? self::DEFAULT_RECONNECT_WAIT, + reconnectJitter: $query->reconnectJitter ?? self::DEFAULT_RECONNECT_JITTER, + reconnectBufferSize: $query->reconnectBufferSize ?? self::DEFAULT_RECONNECT_BUFFER_SIZE, + noRandomize: $query->noRandomize ?? false, ); } @@ -158,6 +202,12 @@ public static function fromURI(#[\SensitiveParameter] string $uri): self * credentials_file?: non-empty-string, * jwt_file?: non-empty-string, * nkey_file?: non-empty-string, + * allow_reconnect?: bool, + * max_reconnects?: ?non-negative-int, + * reconnect_wait?: float, + * reconnect_jitter?: float, + * reconnect_buffer_size?: non-negative-int, + * no_randomize?: bool, * } $options * @throws \InvalidArgumentException */ @@ -183,6 +233,12 @@ public static function fromArray(#[\SensitiveParameter] array $options): self credentialsFile: $options['credentials_file'] ?? null, jwtFile: $options['jwt_file'] ?? null, nkeyFile: $options['nkey_file'] ?? null, + allowReconnect: $options['allow_reconnect'] ?? true, + maxReconnects: \array_key_exists('max_reconnects', $options) ? $options['max_reconnects'] : self::DEFAULT_MAX_RECONNECTS, + reconnectWait: $options['reconnect_wait'] ?? self::DEFAULT_RECONNECT_WAIT, + reconnectJitter: $options['reconnect_jitter'] ?? self::DEFAULT_RECONNECT_JITTER, + reconnectBufferSize: $options['reconnect_buffer_size'] ?? self::DEFAULT_RECONNECT_BUFFER_SIZE, + noRandomize: $options['no_randomize'] ?? false, ); } } diff --git a/src/Exception/ReconnectBufferExceeded.php b/src/Exception/ReconnectBufferExceeded.php new file mode 100644 index 0000000..dfc356d --- /dev/null +++ b/src/Exception/ReconnectBufferExceeded.php @@ -0,0 +1,24 @@ +headers, supportJetstream: $info->jetstream ?? false, maxPayload: $info->maxPayload, + connectUrls: $info->connectUrls ?? [], + serverId: $info->serverId, ); } @@ -25,11 +27,15 @@ public static function fromServerInfo(ServerInfo $info): self * @param non-empty-string $serverVersion * @param ?positive-int $maxPayload the largest message (headers and payload) the server accepts, in bytes, * null if the server does not announce it + * @param list $connectUrls the `host:port` of the servers of the cluster the server announces + * @param ?non-empty-string $serverId */ public function __construct( public string $serverVersion, public bool $allowHeaders = false, public bool $supportJetstream = false, public ?int $maxPayload = null, + public array $connectUrls = [], + public ?string $serverId = null, ) {} } diff --git a/src/Internal/Connection/PingPongHandler.php b/src/Internal/Connection/PingPongHandler.php index 166ba6e..24d1324 100644 --- a/src/Internal/Connection/PingPongHandler.php +++ b/src/Internal/Connection/PingPongHandler.php @@ -4,6 +4,7 @@ namespace Thesis\Nats\Internal\Connection; +use Thesis\Nats\Exception\StaleConnection; use Thesis\Nats\Internal\Protocol; use Thesis\Nats\Internal\Timer; @@ -58,6 +59,7 @@ public function stop(): void private function forceStop(): void { $this->stop(); - $this->connection->close(); + // The server has stopped answering: the reason the client reconnects, the same nats.go reports. + $this->connection->close(new StaleConnection()); } } diff --git a/src/Internal/Connection/Query.php b/src/Internal/Connection/Query.php index 88b0e81..a3c9991 100644 --- a/src/Internal/Connection/Query.php +++ b/src/Internal/Connection/Query.php @@ -25,6 +25,10 @@ * @param ?non-empty-string $credentialsFile * @param ?non-empty-string $jwtFile * @param ?non-empty-string $nkeyFile + * @param ?int $maxReconnects any integer, the config decides what a negative one means + * @param ?float $reconnectWait non-negative and finite + * @param ?float $reconnectJitter non-negative and finite + * @param ?non-negative-int $reconnectBufferSize */ private function __construct( public ?bool $verbose = null, @@ -46,6 +50,12 @@ private function __construct( public ?string $credentialsFile = null, public ?string $jwtFile = null, public ?string $nkeyFile = null, + public ?bool $allowReconnect = null, + public ?int $maxReconnects = null, + public ?float $reconnectWait = null, + public ?float $reconnectJitter = null, + public ?int $reconnectBufferSize = null, + public ?bool $noRandomize = null, ) {} public static function parse(#[\SensitiveParameter] string $query): self @@ -69,6 +79,12 @@ public static function parse(#[\SensitiveParameter] string $query): self credentialsFile: self::string($options, 'credentials_file'), jwtFile: self::string($options, 'jwt_file'), nkeyFile: self::string($options, 'nkey_file'), + allowReconnect: self::bool($options, 'allow_reconnect'), + maxReconnects: self::int($options, 'max_reconnects'), + reconnectWait: self::nonNegativeFloat($options, 'reconnect_wait'), + reconnectJitter: self::nonNegativeFloat($options, 'reconnect_jitter'), + reconnectBufferSize: self::nonNegativeInt($options, 'reconnect_buffer_size'), + noRandomize: self::bool($options, 'no_randomize'), ); } @@ -109,6 +125,31 @@ private static function positiveInt(array $options, string $name): ?int return $value !== null && $value > 0 ? $value : null; } + /** + * @param array $options + * @return ?non-negative-int + */ + private static function nonNegativeInt(array $options, string $name): ?int + { + $value = self::int($options, $name); + + return $value !== null && $value >= 0 ? $value : null; + } + + /** + * @param array $options + */ + private static function nonNegativeFloat(array $options, string $name): ?float + { + if (!isset($options[$name])) { + return null; + } + + $value = filter_var($options[$name], FILTER_VALIDATE_FLOAT); + + return $value !== false && is_finite($value) && $value >= 0 ? $value : null; + } + /** * @param array $options */ diff --git a/src/Internal/Connection/ReconnectState.php b/src/Internal/Connection/ReconnectState.php new file mode 100644 index 0000000..64498da --- /dev/null +++ b/src/Internal/Connection/ReconnectState.php @@ -0,0 +1,21 @@ + the commands sent while reconnecting */ + private array $pending = []; + + /** @var non-negative-int the size of the publishes among the pending commands */ + private int $pendingBytes = 0; + + /** + * Pending while reconnecting, for the calls that cannot be buffered: a ping and a publish in verbose mode. + * + * @var ?DeferredFuture + */ + private ?DeferredFuture $reconnected = null; + + private readonly DeferredCancellation $closing; + + /** + * @param \Closure(): list $subscriptions the commands that restore the subscriptions on a new connection + */ + public function __construct( + private readonly Config $config, + private readonly ConnectionFactory $factory, + private readonly \Closure $subscriptions, + ) { + $this->hooks = new Hooks\ConcurrentProvider(); + $this->pool = new ServerPool($config->urls, randomize: !$config->noRandomize, maxReconnects: $config->maxReconnects); + $this->closing = new DeferredCancellation(); + } + + /** + * Establishes the first connection: every server is tried once, in the order of the pool. A server that cannot + * be reached is skipped, any other failure (e.g. an authentication error) is thrown at once. + * + * @throws \Throwable + */ + public function start(): void + { + /** @var array $errors */ + $errors = []; + + foreach ($this->pool->servers() as $server) { + try { + $connection = $this->factory->connect($server->url); + } catch (ServerIsNotAvailable $e) { + $errors[$server->url] = $e; + + continue; + } + + if ($this->state === ReconnectState::Closed) { + $connection->close(); + + throw new ConnectionWasClosed(); + } + + $this->pool->use($server); + $this->attach($connection); + $this->pool->discover($connection->info()->connectUrls); + $this->state = ReconnectState::Connected; + + return; + } + + throw new ServerIsNotAvailable( + \sprintf('No available nats host: %s.', implode('; ', array_map( + static fn(string $url, ServerIsNotAvailable $e): string => "{$url}: {$e->getMessage()}", + array_keys($errors), + array_values($errors), + ))), + previous: $errors !== [] ? $errors[array_key_last($errors)] : null, + ); + } + + public function execute(Protocol\Frame|array $frames, ?Cancellation $cancellation = null): void + { + if ($this->state === ReconnectState::Reconnecting) { + $frames = $frames instanceof Protocol\Frame ? [$frames] : $frames; + + // In verbose mode a publish is acknowledged, and a buffered one would be acknowledged by nobody. + if (!$this->config->verbose || !self::publishes($frames)) { + $this->buffer($frames); + + return; + } + + $this->awaitReconnect($cancellation); + } + + $this->connected()->execute($frames, $cancellation); + } + + /** + * Returns once connected: at once, or after the reconnect in progress. + * + * @throws ConnectionWasClosed if the client gives up reconnecting meanwhile or is closed + * @throws CancelledException + */ + public function awaitConnected(?Cancellation $cancellation = null): void + { + if ($this->state === ReconnectState::Reconnecting) { + $this->awaitReconnect($cancellation); + } + + $this->connected(); + } + + public function ping(?Cancellation $cancellation = null): float + { + if ($this->state === ReconnectState::Reconnecting) { + $this->awaitReconnect($cancellation); + } + + return $this->connected()->ping($cancellation); + } + + public function info(): ConnectionInfo + { + return $this->info ?? throw new ConnectionIsNotAvailable(); + } + + public function close(?\Throwable $reason = null): void + { + $this->finish($reason); + } + + private function connected(): Connection + { + if ($this->state === ReconnectState::Closed) { + throw new ConnectionWasClosed(); + } + + return $this->current ?? throw new ConnectionIsNotAvailable(); + } + + /** + * @throws ConnectionWasClosed if the client gives up reconnecting meanwhile + * @throws CancelledException + */ + private function awaitReconnect(?Cancellation $cancellation): void + { + $this->reconnected?->getFuture()->await($cancellation); + } + + /** + * @param non-empty-list $frames + * @throws ReconnectBufferExceeded + */ + private function buffer(array $frames): void + { + // Only the publishes count: the subscription commands are tiny, and a subscription must not fail on a full buffer. + $bytes = 0; + foreach ($frames as $frame) { + if ($frame instanceof Protocol\Pub) { + $bytes += \strlen($frame->encode()); + } + } + + if ($bytes > 0 && $this->pendingBytes + $bytes > $this->config->reconnectBufferSize) { + throw ReconnectBufferExceeded::forSize($this->config->reconnectBufferSize); + } + + // Appended in place: copying the whole buffer on every publish would make a reconnect quadratic. + array_push($this->pending, ...$frames); + $this->pendingBytes += $bytes; + } + + private function attach(Connection $connection): void + { + $this->current = $connection; + $this->info = $connection->info(); + + // The reader of the connection runs in the event loop: a strong reference from its hooks would keep this + // connection, and the client behind it, alive for as long as the socket is open. + $self = \WeakReference::create($this); + + $connection->hooks->onMessage(static function (Hooks\MessageReceived $event) use ($self): void { + $self->get()?->hooks->dispatch($event); + }); + $connection->hooks->onError(static function (Hooks\ErrorReceived $event) use ( + $self, + $connection, + ): void { + $self->get()?->onServerError($connection, $event); + }); + $connection->hooks->onInfo(static function (Hooks\InfoReceived $event) use ( + $self, + $connection, + ): void { + $self->get()?->onServerInfo($connection, $event->info); + }); + $connection->hooks->onClose(static function (Hooks\ConnectionClosed $event) use ( + $self, + $connection, + ): void { + $self->get()?->onClosed($connection, $event->reason); + }); + } + + /** + * A permission violation is passed on, the client attributes it to a subscription. The errors another connection + * may not have close the current one and the client reconnects, any other one closes the client. + */ + private function onServerError(Connection $connection, Hooks\ErrorReceived $event): void + { + if ($connection !== $this->current) { + return; + } + + $e = $event->error->toException(); + + if ($e instanceof PermissionViolated) { + $this->hooks->dispatch($event); + + return; + } + + if ($this->config->allowReconnect && $event->error->isReconnectable()) { + $connection->close($e); + + return; + } + + $this->finish($e); + } + + private function onServerInfo(Connection $connection, Protocol\ServerInfo $info): void + { + if ($connection !== $this->current) { + return; + } + + if ($this->pool->discover($info->connectUrls ?? [])) { + $this->discovered(); + } + + if ($info->ldm !== true) { + return; + } + + $this->hooks->dispatch(Hooks\LameDuckModeEntered::Event); + + // The server is going to close the connections: move to another one now, while the server still serves. + // The next server is tried at once, the pause comes only after a whole round of the pool. + // With nowhere to move to, the connection stays until the server closes it. + if ($this->config->allowReconnect && $this->state === ReconnectState::Connected && $this->pool->count() > 1) { + $connection->close(); + } + } + + private function discovered(): void + { + $urls = $this->pool->urls(); + + if ($urls !== []) { + $this->hooks->dispatch(new Hooks\ServersDiscovered($urls)); + } + } + + private function onClosed(Connection $connection, ?\Throwable $reason): void + { + if ($connection !== $this->current || $this->state !== ReconnectState::Connected) { + return; + } + + $this->current = null; + + if (!$this->config->allowReconnect) { + $this->finish($reason); + + return; + } + + $this->state = ReconnectState::Reconnecting; + + /** @var DeferredFuture $reconnected */ + $reconnected = new DeferredFuture(); + // Nobody may be waiting when the client gives up. + $reconnected->getFuture()->ignore(); + $this->reconnected = $reconnected; + + $this->hooks->dispatch(new Hooks\Disconnected($reason)); + + $future = async($this->reconnect(...)); + $future->ignore(); + } + + /** + * Tries the servers of the pool one by one, pausing once all of them have been tried, until one of them accepts + * the connection or every one of them has used up its reconnects, see {@see ServerPool::next()}. + */ + private function reconnect(): void + { + $cancellation = $this->closing->getCancellation(); + $tried = 0; + $error = null; + + while (($server = $this->pool->next()) !== null) { + // A round of the pool is tried at once, the pause comes after it, as in nats.go. + if ($tried + 1 >= $this->pool->count()) { + $tried = 0; + + try { + delay($this->pause(), cancellation: $cancellation); + } catch (CancelledException) { + return; + } + } else { + ++$tried; + } + + if (!$this->reconnecting()) { + return; + } + + try { + $connection = $this->factory->connect($server->url); + } catch (\Throwable $e) { + $error = $e; + + // Credentials rejected twice in a row by the same server are not going to be accepted, as in nats.go. + if ($server->failed($e) && $e instanceof AuthorizationViolation) { + $this->finish($e); + + return; + } + + continue; + } + + if (!$this->reconnecting()) { + $connection->close(); + + return; + } + + try { + $this->restore($connection); + } catch (NatsException $e) { + $error = $e; + $server->failed($e); + $this->current = null; + $connection->close(); + + continue; + } catch (\Throwable $e) { + // Not a connection failure: retrying would fail the same way, and the error would never surface. + $this->current = null; + $connection->close(); + $this->finish($e); + + return; + } + + if (!$this->reconnecting()) { + // Closed while the subscriptions were being restored: finish() has not seen the new connection yet. + $connection->close(); + + return; + } + + $server->connected(); + + if ($this->pool->discover($connection->info()->connectUrls)) { + $this->discovered(); + } + + $this->state = ReconnectState::Connected; + [$reconnected, $this->reconnected] = [$this->reconnected, null]; + $reconnected?->complete(); + $this->hooks->dispatch(Hooks\Reconnected::Event); + + return; + } + + $this->finish(new ConnectionWasClosed('The connection was lost and no server is left to reconnect to.', previous: $error)); + } + + /** + * Sends the subscriptions, then the commands buffered meanwhile, including the ones buffered while sending, + * and waits for the PONG of a PING after them: SUB is not acknowledged, and the reconnect callbacks promise + * the subscriptions active. A server that does not answer in time is treated as a failed attempt. + * The subscriptions sent while reconnecting are buffered too and may be sent twice: the server ignores + * a subscription with a sid it already has. + * + * @throws \Throwable + */ + private function restore(Connection $connection): void + { + $this->attach($connection); + + $frames = ($this->subscriptions)(); + + do { + [$pending, $this->pending, $this->pendingBytes] = [$this->pending, [], 0]; + $frames = [...$frames, ...$pending]; + + try { + if ($frames !== []) { + try { + $connection->execute($frames); + } catch (PermissionViolated $e) { + // Verbose mode: a subscription the user is no longer allowed to have fails alone, as it would + // on its own. The acknowledgements of the commands after it are not awaited, so the PING below + // is what proves the server has the rest of them. + $this->hooks->dispatch(new Hooks\ErrorReceived(new Protocol\Err($e->getMessage() !== '' ? $e->getMessage() : 'Permissions Violation'))); + } + } + + $connection->ping(new TimeoutCancellation($this->config->connectionTimeout)); + } catch (CancelledException $e) { + $this->requeue($pending); + + throw new ServerIsNotAvailable( + \sprintf('The nats server did not answer PING within %s seconds after the reconnect.', $this->config->connectionTimeout), + previous: $e, + ); + } catch (\Throwable $e) { + // Kept for the next server, together with the ones buffered while sending. + $this->requeue($pending); + + throw $e; + } + + $frames = []; + } while ($this->hasPending()); + } + + /** + * The state changes while the reconnect awaits, which PHPStan does not see in a property comparison. + */ + private function reconnecting(): bool + { + return $this->state === ReconnectState::Reconnecting; + } + + /** + * Commands may be buffered while the restore awaits, which PHPStan does not see in a property comparison. + */ + private function hasPending(): bool + { + return $this->pending !== []; + } + + /** + * @param list $frames sent before the ones pending now + */ + private function requeue(array $frames): void + { + $this->pending = [...$frames, ...$this->pending]; + $this->pendingBytes = 0; + + foreach ($this->pending as $frame) { + if ($frame instanceof Protocol\Pub) { + $this->pendingBytes += \strlen($frame->encode()); + } + } + } + + /** + * @return float in seconds + */ + private function pause(): float + { + $jitter = $this->config->reconnectJitter > 0 + ? random_int(0, (int) ($this->config->reconnectJitter * 1_000_000)) / 1_000_000 + : 0; + + return $this->config->reconnectWait + $jitter; + } + + /** + * Closes the client for good: the reconnects stop, the buffered commands are dropped, and the calls waiting + * for the reconnect fail. + */ + private function finish(?\Throwable $reason = null): void + { + if ($this->state === ReconnectState::Closed) { + return; + } + + $this->state = ReconnectState::Closed; + $this->closing->cancel(); + + [$current, $this->current] = [$this->current, null]; + $this->pending = []; + $this->pendingBytes = 0; + + [$reconnected, $this->reconnected] = [$this->reconnected, null]; + $reconnected?->error($reason instanceof ConnectionWasClosed ? $reason : new ConnectionWasClosed(previous: $reason)); + + $current?->close(); + $this->hooks->dispatch(new Hooks\ConnectionClosed($reason)); + } + + /** + * @param non-empty-list $frames + */ + private static function publishes(array $frames): bool + { + return array_any($frames, static fn(Protocol\Frame $frame): bool => $frame instanceof Protocol\Pub); + } +} diff --git a/src/Internal/Connection/Server.php b/src/Internal/Connection/Server.php new file mode 100644 index 0000000..69f7e08 --- /dev/null +++ b/src/Internal/Connection/Server.php @@ -0,0 +1,46 @@ + the error the last attempt failed with */ + public private(set) ?string $lastError = null; + + /** + * @param non-empty-string $url `host:port` + * @param bool $implicit whether the server was announced by the cluster rather than configured + */ + public function __construct( + public readonly string $url, + public readonly bool $implicit, + ) {} + + /** + * @return bool whether the attempt failed with the same kind of error as the previous one + */ + public function failed(\Throwable $e): bool + { + ++$this->reconnects; + $repeated = $this->lastError === $e::class; + $this->lastError = $e::class; + + return $repeated; + } + + public function connected(): void + { + $this->reconnects = 0; + $this->lastError = null; + } +} diff --git a/src/Internal/Connection/ServerPool.php b/src/Internal/Connection/ServerPool.php new file mode 100644 index 0000000..7f9d252 --- /dev/null +++ b/src/Internal/Connection/ServerPool.php @@ -0,0 +1,154 @@ + */ + private array $servers = []; + + /** + * Every url ever seen, to tell a new server from one that has merely restarted. + * + * @var array + */ + private array $known = []; + + /** + * @param non-empty-list $urls + * @param ?non-negative-int $maxReconnects the failed reconnects in a row after which a server is dropped, null for no limit + */ + public function __construct( + array $urls, + private readonly bool $randomize, + private readonly ?int $maxReconnects, + ) { + foreach ($urls as $url) { + $this->add($url, implicit: false); + } + + if ($this->randomize) { + $this->shuffle(0); + } + } + + /** + * @return list in the order to try them + */ + public function servers(): array + { + return $this->servers; + } + + public function count(): int + { + return \count($this->servers); + } + + /** + * Makes the server the current one, e.g. the one the initial connection was established with. + */ + public function use(Server $server): void + { + $index = array_search($server, $this->servers, true); + + if ($index !== false && $index !== 0) { + array_splice($this->servers, $index, 1); + array_unshift($this->servers, $server); + } + } + + /** + * Moves the current server to the end of the pool, or drops it once it has used up its reconnects. + * + * @return ?Server the server to try next, null if no server is left + */ + public function next(): ?Server + { + $current = array_shift($this->servers); + + if ($current !== null && ($this->maxReconnects === null || $current->reconnects < $this->maxReconnects)) { + $this->servers[] = $current; + } + + return $this->servers[0] ?? null; + } + + /** + * Applies the servers a server of the cluster announces: adds the new ones, drops the announced ones it no longer + * lists, keeps the configured ones and the current one. + * + * @param list $urls the `host:port` from connect_urls, an empty list means the server does not announce them + * @return bool whether a server never seen before was added + */ + public function discover(array $urls): bool + { + if ($urls === []) { + return false; + } + + $announced = array_fill_keys($urls, true); + $current = $this->servers[0] ?? null; + + $this->servers = array_values(array_filter( + $this->servers, + static fn(Server $server): bool => !$server->implicit || $server === $current || isset($announced[$server->url]), + )); + + foreach ($this->servers as $server) { + unset($announced[$server->url]); + } + + $new = false; + + foreach (array_keys($announced) as $url) { + $new = $new || !isset($this->known[$url]); + $this->add($url, implicit: true); + } + + if ($new && $this->randomize) { + // The current server keeps its place. + $this->shuffle(1); + } + + return $new; + } + + /** + * @return list + */ + public function urls(): array + { + return array_map(static fn(Server $server): string => $server->url, $this->servers); + } + + /** + * @param non-empty-string $url + */ + private function add(string $url, bool $implicit): void + { + $this->servers[] = new Server($url, $implicit); + $this->known[$url] = true; + } + + /** + * @param non-negative-int $offset + */ + private function shuffle(int $offset): void + { + $head = \array_slice($this->servers, 0, $offset); + $tail = \array_slice($this->servers, $offset); + shuffle($tail); + + $this->servers = [...$head, ...$tail]; + } +} diff --git a/src/Internal/Connection/SocketConnection.php b/src/Internal/Connection/SocketConnection.php index b7bef55..083656e 100644 --- a/src/Internal/Connection/SocketConnection.php +++ b/src/Internal/Connection/SocketConnection.php @@ -230,6 +230,8 @@ private function handshake(Cancellation $cancellation): void tlsRequired: false, name: $this->config->clientName, version: version(self::PACKAGE_NAME), + // Asks the server for the INFO updates: the servers of the cluster and the lame duck mode notification. + protocol: Protocol\Proto::Cluster, authToken: $this->config->token, user: $this->config->user, pass: $this->config->password, @@ -275,7 +277,7 @@ public function info(): ConnectionInfo return $this->info ?? throw new ConnectionIsNotAvailable(); } - public function close(): void + public function close(?\Throwable $reason = null): void { if ($this->state === ConnectionState::Closed || $this->state === ConnectionState::GracefulClosed) { return; @@ -285,7 +287,7 @@ public function close(): void $this->socket->close(); self::failPending($this->queue, new ConnectionWasClosed()); self::failPongs($this->pongs, new ConnectionWasClosed()); - $this->hooks->dispatch(Hooks\ConnectionClosed::Event); + $this->hooks->dispatch(new Hooks\ConnectionClosed($reason)); } private function run(): void @@ -347,6 +349,8 @@ private function run(): void $event = match (true) { $frame instanceof Protocol\Ping => Hooks\PingReceived::Event, $frame instanceof Protocol\Pong => Hooks\PongReceived::Event, + // The INFO of the handshake is read before the reader starts, so any INFO here is an update. + $frame instanceof Protocol\ServerInfo => new Hooks\InfoReceived($frame), $frame instanceof Protocol\Msg => new Hooks\MessageReceived( subject: $frame->subject, sid: $frame->sid, @@ -374,7 +378,7 @@ private function run(): void if ($state !== ConnectionState::GracefulClosed) { // @phpstan-ignore notIdentical.alwaysTrue $state = ConnectionState::Closed; $socket->close(); - $hooks->dispatch(Hooks\ConnectionClosed::Event); + $hooks->dispatch(new Hooks\ConnectionClosed($error)); } // The server closed the connection without answering the handshake. diff --git a/src/Internal/Connection/SocketConnectionFactory.php b/src/Internal/Connection/SocketConnectionFactory.php index 87b0e12..3f5c14b 100644 --- a/src/Internal/Connection/SocketConnectionFactory.php +++ b/src/Internal/Connection/SocketConnectionFactory.php @@ -6,7 +6,6 @@ use Amp\Socket\ConnectContext; use Amp\Socket\DnsSocketConnector; -use Amp\Socket\Socket; use Amp\Socket\SocketConnector; use Thesis\Nats\Config; use Thesis\Nats\Exception\ServerIsNotAvailable; @@ -32,21 +31,23 @@ public static function fromConfig(Config $config): self } // The default amp connector retries each url with an exponential backoff, which takes seconds - // before an unavailable host is reported. The hosts are iterated here instead, so connect only once. + // before an unavailable host is reported. The servers are iterated by the server pool instead, so connect only once. return new self($config, $context, new DnsSocketConnector()); } - public function connect(): Connection + public function connect(string $url): Connection { // The credentials are read and a malformed seed is rejected before the socket is opened: // otherwise the server would be left waiting for CONNECT. $credentials = Credentials::fromConfig($this->config); - $connection = new SocketConnection( - $this->config, - $this->createSocket(), - $credentials, - ); + try { + $socket = $this->connector->connect($url, $this->context); + } catch (\Throwable $e) { + throw new ServerIsNotAvailable($e->getMessage(), previous: $e); + } + + $connection = new SocketConnection($this->config, $socket, $credentials); try { $connection->startup(); @@ -59,25 +60,4 @@ public function connect(): Connection return $connection; } - - private function createSocket(): Socket - { - $exceptions = []; - - foreach ($this->config->urls as $url) { - try { - return $this->connector->connect($url, $this->context); - } catch (\Throwable $e) { - $exceptions[$url] = $e->getMessage(); - } - } - - throw new ServerIsNotAvailable(vsprintf('No available nats host: %s.', [ - implode('; ', array_map( - static fn(string $url, string $exception): string => "{$url}: {$exception}", - array_keys($exceptions), - array_values($exceptions), - )), - ])); - } } diff --git a/src/Internal/Hooks/ConcurrentProvider.php b/src/Internal/Hooks/ConcurrentProvider.php index 451a24e..bca820d 100644 --- a/src/Internal/Hooks/ConcurrentProvider.php +++ b/src/Internal/Hooks/ConcurrentProvider.php @@ -13,6 +13,11 @@ * @phpstan-import-type OnPongCallback from Provider * @phpstan-import-type OnErrorCallback from Provider * @phpstan-import-type OnCloseCallback from Provider + * @phpstan-import-type OnInfoCallback from Provider + * @phpstan-import-type OnDisconnectCallback from Provider + * @phpstan-import-type OnReconnectCallback from Provider + * @phpstan-import-type OnServersDiscoveredCallback from Provider + * @phpstan-import-type OnLameDuckModeCallback from Provider */ final class ConcurrentProvider implements Provider { @@ -31,6 +36,21 @@ final class ConcurrentProvider implements Provider /** @var list */ private array $closeCallbacks = []; + /** @var list */ + private array $infoCallbacks = []; + + /** @var list */ + private array $disconnectCallbacks = []; + + /** @var list */ + private array $reconnectCallbacks = []; + + /** @var list */ + private array $serversDiscoveredCallbacks = []; + + /** @var list */ + private array $lameDuckModeCallbacks = []; + public function onMessage(\Closure $callback): void { $this->messageCallbacks[] = $callback; @@ -56,14 +76,45 @@ public function onError(\Closure $callback): void $this->errorCallbacks[] = $callback; } - public function dispatch(MessageReceived|PingReceived|PongReceived|ErrorReceived|ConnectionClosed $event): void + public function onInfo(\Closure $callback): void { + $this->infoCallbacks[] = $callback; + } + + public function onDisconnect(\Closure $callback): void + { + $this->disconnectCallbacks[] = $callback; + } + + public function onReconnect(\Closure $callback): void + { + $this->reconnectCallbacks[] = $callback; + } + + public function onServersDiscovered(\Closure $callback): void + { + $this->serversDiscoveredCallbacks[] = $callback; + } + + public function onLameDuckMode(\Closure $callback): void + { + $this->lameDuckModeCallbacks[] = $callback; + } + + public function dispatch( + MessageReceived|PingReceived|PongReceived|ErrorReceived|ConnectionClosed|InfoReceived|Disconnected|Reconnected|ServersDiscovered|LameDuckModeEntered $event, + ): void { $callbacks = match (true) { $event instanceof MessageReceived => $this->messageCallbacks, $event instanceof PingReceived => $this->pingCallbacks, $event instanceof PongReceived => $this->pongCallbacks, $event instanceof ErrorReceived => $this->errorCallbacks, - $event instanceof ConnectionClosed => $this->closeCallbacks, // @phpstan-ignore instanceof.alwaysTrue + $event instanceof ConnectionClosed => $this->closeCallbacks, + $event instanceof InfoReceived => $this->infoCallbacks, + $event instanceof Disconnected => $this->disconnectCallbacks, + $event instanceof Reconnected => $this->reconnectCallbacks, + $event instanceof ServersDiscovered => $this->serversDiscoveredCallbacks, + $event instanceof LameDuckModeEntered => $this->lameDuckModeCallbacks, // @phpstan-ignore instanceof.alwaysTrue }; foreach ($callbacks as $callback) { diff --git a/src/Internal/Hooks/ConnectionClosed.php b/src/Internal/Hooks/ConnectionClosed.php index 46a21ca..5571b49 100644 --- a/src/Internal/Hooks/ConnectionClosed.php +++ b/src/Internal/Hooks/ConnectionClosed.php @@ -5,9 +5,16 @@ namespace Thesis\Nats\Internal\Hooks; /** + * The connection is closed for good: nothing will be received or sent over it anymore. + * * @internal */ -enum ConnectionClosed +final readonly class ConnectionClosed { - case Event; + /** + * @param ?\Throwable $reason the server error or the failure that closed the connection, null for a regular close + */ + public function __construct( + public ?\Throwable $reason = null, + ) {} } diff --git a/src/Internal/Hooks/Disconnected.php b/src/Internal/Hooks/Disconnected.php new file mode 100644 index 0000000..affd44a --- /dev/null +++ b/src/Internal/Hooks/Disconnected.php @@ -0,0 +1,20 @@ + $urls the `host:port` of every known server + */ + public function __construct( + public array $urls, + ) {} +} diff --git a/src/Internal/Protocol/Err.php b/src/Internal/Protocol/Err.php index c400728..d9e4ddf 100644 --- a/src/Internal/Protocol/Err.php +++ b/src/Internal/Protocol/Err.php @@ -11,6 +11,16 @@ */ final readonly class Err implements Frame { + /** @var list lowercase prefixes, the server is not consistent in the case of its errors */ + private const array RECONNECTABLE = [ + 'stale connection', + 'maximum connections exceeded', + 'maximum account active connections exceeded', + 'authorization violation', + 'user authentication expired', + 'user authentication revoked', + 'account authentication expired', + ]; private const string SUBSCRIPTION_VIOLATION = '/^Permissions Violation for Subscription to "([^"]+)"(?: using queue "([^"]+)")?/'; /** @var non-empty-string */ @@ -45,6 +55,15 @@ public function violatedSubscription(): ?array return [$matches[1], $matches[2] ?? null]; } + /** + * Whether the server closes the connection for a reason another attempt may not have, the same errors nats.go + * reconnects after: a stale connection, a connection limit, credentials that expired or were revoked. + */ + public function isReconnectable(): bool + { + return array_any(self::RECONNECTABLE, fn(string $prefix): bool => str_starts_with(strtolower($this->message), $prefix)); + } + public function toException(): \Exception { return match (true) { diff --git a/src/Internal/Subscription/Subscriber.php b/src/Internal/Subscription/Subscriber.php index 0d1bc06..2e46287 100644 --- a/src/Internal/Subscription/Subscriber.php +++ b/src/Internal/Subscription/Subscriber.php @@ -17,11 +17,13 @@ * @param ?non-empty-string $queueGroup * @param \Closure(Delivery): bool $push returns false if the subscriber does not accept deliveries anymore * @param ?Subscription $subscription null for the internal subscriptions which are not exposed to the user + * @param ?\Closure(): ?non-negative-int $remaining the messages the server is still to deliver, see {@see SubscriptionHandler::remaining()} */ public function __construct( public string $subject, public ?string $queueGroup, public \Closure $push, public ?Subscription $subscription = null, + public ?\Closure $remaining = null, ) {} } diff --git a/src/Internal/Subscription/SubscriptionHandler.php b/src/Internal/Subscription/SubscriptionHandler.php index 368ebbc..c1203b3 100644 --- a/src/Internal/Subscription/SubscriptionHandler.php +++ b/src/Internal/Subscription/SubscriptionHandler.php @@ -34,7 +34,7 @@ final class SubscriptionHandler * @param \Closure(): void $unsubscribe * @param callable(Delivery, Subscription): void $handler * @param \Closure(SlowConsumer): void $onSlowConsumer - * @param ?positive-int $remaining the number of messages after which the server removes the subscription + * @param ?non-negative-int $remaining the number of messages after which the server removes the subscription * @param ?\Closure(): void $onReceive called once a message arrives, before it waits for the handler */ public function __construct( @@ -213,23 +213,25 @@ private function reportSlowConsumer(): void ($this->onSlowConsumer)(SlowConsumer::forSubscription($this->subscription)); } + /** + * @return ?non-negative-int the messages the server is still to deliver before it removes the subscription, null for no limit + */ + public function remaining(): ?int + { + return $this->remaining; + } + /** * @return bool whether it is the last message the server delivers to the subscription */ private function countDown(): bool { - if ($this->remaining === null) { - return false; - } - - if ($this->remaining > 1) { - --$this->remaining; - + if ($this->remaining === null || $this->remaining === 0) { return false; } - $this->remaining = null; + --$this->remaining; - return true; + return $this->remaining === 0; } } diff --git a/src/Internal/Validation/Validate.php b/src/Internal/Validation/Validate.php index fb0abe8..43d6e0b 100644 --- a/src/Internal/Validation/Validate.php +++ b/src/Internal/Validation/Validate.php @@ -61,6 +61,20 @@ public static function count(int $count, string $what): void } } + /** + * A limit or a size, which the phpdoc types promise to be non-negative: checked anyway, since a negative one + * would not fail but silently misbehave, e.g. give up every server at once. + * + * @param non-empty-string $what + * @throws \InvalidArgumentException + */ + public static function nonNegative(int $value, string $what): void + { + if ($value < 0) { + throw new \InvalidArgumentException("The {$what} must be non-negative, {$value} given."); + } + } + /** * A literal subject, e.g. of publish, request or reply-to: wildcards are not allowed. * diff --git a/src/JetStream/Internal/PullMessageHandler.php b/src/JetStream/Internal/PullMessageHandler.php index 2942900..52a558d 100644 --- a/src/JetStream/Internal/PullMessageHandler.php +++ b/src/JetStream/Internal/PullMessageHandler.php @@ -10,11 +10,15 @@ use Thesis\Nats\Delivery as NatsDelivery; use Thesis\Nats\Description; use Thesis\Nats\Exception\BadRequestSent; +use Thesis\Nats\Exception\ConnectionIsNotAvailable; +use Thesis\Nats\Exception\ConnectionWasClosed; use Thesis\Nats\Exception\ConsumerDeleted; use Thesis\Nats\Exception\ConsumerIsPushBased; use Thesis\Nats\Exception\MaxRequestBatchExceeded; use Thesis\Nats\Exception\MaxRequestExpiresExceeded; use Thesis\Nats\Exception\MaxRequestMaxBytesExceeded; +use Thesis\Nats\Exception\ReconnectBufferExceeded; +use Thesis\Nats\Exception\ServerIsNotAvailable; use Thesis\Nats\Header; use Thesis\Nats\JetStream\Api\PullRequest; use Thesis\Nats\JetStream\Delivery as JetStreamDelivery; @@ -24,6 +28,7 @@ use Thesis\Nats\NatsException; use Thesis\Nats\Status; use Thesis\Nats\Subscription; +use function Amp\async; /** * @internal @@ -42,6 +47,14 @@ final class PullMessageHandler private readonly Heartbeat\Watchdog $watchdog; + /** Nothing is pulled while reconnecting: the pull requests cannot reach the server, the reconnect pulls again. */ + private bool $connected = true; + + private ?Subscription $subscription = null; + + /** The failure of a pull request that came before the subscription was bound, see bind(). */ + private ?\Throwable $failure = null; + /** @var Pipeline\Queue */ private readonly Pipeline\Queue $pulls; @@ -87,12 +100,63 @@ public function __invoke(NatsDelivery $delivery, Subscription $subscription): vo // The handler may stop the subscription itself: stop() then runs before the subscription is marked completed, // and rescheduling the watchdog would pull into the completed queue later. - if (!$subscription->completed() && !$this->pulls->isComplete()) { + if (!$subscription->completed() && !$this->pulls->isComplete() && $this->connected) { $this->watchdog->reset(); $this->replenish(); } } + /** + * The subscription the pulled messages are delivered to: it fails if a pull request cannot be sent. + */ + public function bind(Subscription $subscription): void + { + $this->subscription = $subscription; + + if ($this->failure !== null) { + $this->fail($this->failure); + } + } + + /** + * The heartbeats cannot come while reconnecting: the watchdog would pull into the void. + */ + public function disconnected(): void + { + $this->connected = false; + $this->watchdog->stop(); + } + + /** + * The pull requests were bound to the lost connection, the server forgets them: pull a full batch again, as nats.go does. + */ + public function reconnected(): void + { + $this->connected = true; + + if ($this->pulls->isComplete()) { + return; + } + + $this->reset(); + $this->pull(); + $this->watchdog->reset(); + } + + private function fail(\Throwable $e): void + { + $this->stop(); + + if ($this->subscription === null) { + $this->failure = $e; + + return; + } + + $future = async($this->subscription->error(...), $e); + $future->ignore(); + } + public function stop(): void { $this->watchdog->stop(); @@ -221,19 +285,32 @@ private function tick( Encoder $json, Client $nc, ): void { + $fail = $this->fail(...); + EventLoop::queue(static function () use ( $queue, $subject, $reply, $json, $nc, + $fail, ): void { foreach ($queue->iterate() as $pull) { - $nc->publish( - subject: $subject, - message: new Message($json->encode($pull)), - replyTo: $reply, - ); + try { + $nc->publish( + subject: $subject, + message: new Message($json->encode($pull)), + replyTo: $reply, + ); + } catch (ConnectionWasClosed|ConnectionIsNotAvailable|ServerIsNotAvailable|ReconnectBufferExceeded) { + // The connection is lost: the reconnect pulls again, and a client that gives up fails the subscription itself. + continue; + } catch (\Throwable $e) { + // Nothing in the event loop would catch it: the subscription is the one to know. + $fail($e); + + return; + } } }); diff --git a/src/JetStream/PullConsumer.php b/src/JetStream/PullConsumer.php index ff2d9c6..d47a9bc 100644 --- a/src/JetStream/PullConsumer.php +++ b/src/JetStream/PullConsumer.php @@ -141,8 +141,13 @@ public function consume( cancellation: $cancellation, ); + $messageHandler->bind($subscription); + + $unwatch = $this->nats->watchConnection($messageHandler->disconnected(...), $messageHandler->reconnected(...)); + $this->subscriptions[] = $subscription->onComplete( $messageHandler->stop(...), + $unwatch, ); return $subscription; diff --git a/tests/ClientTest.php b/tests/ClientTest.php index 59209a1..0801416 100644 --- a/tests/ClientTest.php +++ b/tests/ClientTest.php @@ -201,10 +201,10 @@ public function testSuspendSubscriptionOnClientDisconnect(): void $subscription->awaitCompletion(); } - public function testSubscriptionsAndRequestsFailWhenServerClosesConnection(): void + public function testSubscriptionsAndRequestsFailWhenServerClosesConnectionWithoutReconnect(): void { $name = Id\generateUniqueId(); - $client = $this->clientNamed($name); + $client = $this->clientNamed($name, allowReconnect: false); // The teardown runs in the event loop, where an escaped exception has no one to catch it. $loopErrors = []; diff --git a/tests/ConfigTest.php b/tests/ConfigTest.php index 27517fe..4932163 100644 --- a/tests/ConfigTest.php +++ b/tests/ConfigTest.php @@ -313,6 +313,18 @@ final class ConfigTest extends TestCase new Config(urls: ['h1:4222', 'h2:4222'], user: 'alice'), ], )] + #[TestWith( + [ + 'nats://127.0.0.1:4222?allow_reconnect=false&max_reconnects=5&reconnect_wait=0.5&reconnect_jitter=0&reconnect_buffer_size=1024&no_randomize=true', + new Config(allowReconnect: false, maxReconnects: 5, reconnectWait: 0.5, reconnectJitter: 0.0, reconnectBufferSize: 1024, noRandomize: true), + ], + )] + #[TestWith( + [ + 'nats://127.0.0.1:4222?max_reconnects=-1&reconnect_wait=-1&reconnect_buffer_size=-5', + new Config(maxReconnects: null), + ], + )] public function testFromURI(string $uri, Config $config): void { self::assertEquals($config, Config::fromURI($uri)); @@ -402,6 +414,43 @@ public function testFromArrayWithRequestTimeout(): void self::assertEquals(new Config(requestTimeout: 0.5), Config::fromArray(['request_timeout' => 0.5])); } + public function testFromArrayWithReconnectOptions(): void + { + self::assertEquals( + new Config(allowReconnect: false, maxReconnects: null, reconnectWait: 0.0, reconnectJitter: 0.5, reconnectBufferSize: 0, noRandomize: true), + Config::fromArray([ + 'allow_reconnect' => false, + 'max_reconnects' => null, + 'reconnect_wait' => 0.0, + 'reconnect_jitter' => 0.5, + 'reconnect_buffer_size' => 0, + 'no_randomize' => true, + ]), + ); + } + + public function testMaxReconnectsMustBeNonNegative(): void + { + $this->expectException(\InvalidArgumentException::class); + Config::fromArray(['max_reconnects' => -1]); // @phpstan-ignore argument.type + } + + public function testReconnectBufferSizeMustBeNonNegative(): void + { + $this->expectException(\InvalidArgumentException::class); + new Config(reconnectBufferSize: -1); // @phpstan-ignore argument.type + } + + #[TestWith([-1.0, 0.1])] + #[TestWith([INF, 0.1])] + #[TestWith([1.0, -0.1])] + #[TestWith([1.0, NAN])] + public function testReconnectWaitAndJitterMustBeNonNegative(float $wait, float $jitter): void + { + $this->expectException(\InvalidArgumentException::class); + new Config(reconnectWait: $wait, reconnectJitter: $jitter); + } + public function testFromArrayWithAuthenticationFiles(): void { self::assertEquals( diff --git a/tests/Internal/Connection/ServerPoolTest.php b/tests/Internal/Connection/ServerPoolTest.php new file mode 100644 index 0000000..7022aa9 --- /dev/null +++ b/tests/Internal/Connection/ServerPoolTest.php @@ -0,0 +1,93 @@ +next()?->url); + self::assertSame('c:4222', $pool->next()?->url); + self::assertSame('a:4222', $pool->next()?->url); + self::assertSame(['a:4222', 'b:4222', 'c:4222'], $pool->urls()); + } + + public function testServerIsDroppedOnceItUsesUpItsReconnects(): void + { + $pool = new ServerPool(['a:4222', 'b:4222'], randomize: false, maxReconnects: 1); + + $b = $pool->next(); + self::assertSame('b:4222', $b?->url); + $b->failed(new \RuntimeException()); + + // "a" has never failed, "b" has used up its only reconnect. + self::assertSame('a:4222', $pool->next()?->url); + self::assertSame(['a:4222'], $pool->urls()); + + ($pool->servers()[0] ?? self::fail('The pool is empty.'))->failed(new \RuntimeException()); + self::assertNull($pool->next()); + } + + public function testSuccessfulConnectResetsReconnects(): void + { + $server = new Server('a:4222', implicit: false); + + self::assertFalse($server->failed(new \RuntimeException())); + self::assertTrue($server->failed(new \RuntimeException())); + self::assertFalse($server->failed(new \LogicException())); + + $server->connected(); + self::assertSame(0, $server->reconnects); + self::assertNull($server->lastError); + } + + public function testDiscoverAddsAnnouncedAndDropsGoneImplicitServers(): void + { + $pool = new ServerPool(['a:4222'], randomize: false, maxReconnects: null); + + self::assertFalse($pool->discover([])); + self::assertTrue($pool->discover(['a:4222', 'b:4222', 'c:4222'])); + self::assertSame(['a:4222', 'b:4222', 'c:4222'], $pool->urls()); + + // "c" has left the cluster, the configured "a" stays even if not announced. + self::assertFalse($pool->discover(['b:4222'])); + self::assertSame(['a:4222', 'b:4222'], $pool->urls()); + + // "c" is back: it was seen before, so it is not a new server. + self::assertFalse($pool->discover(['b:4222', 'c:4222'])); + self::assertSame(['a:4222', 'b:4222', 'c:4222'], $pool->urls()); + } + + public function testDiscoverKeepsCurrentImplicitServer(): void + { + $pool = new ServerPool(['a:4222'], randomize: false, maxReconnects: null); + $pool->discover(['b:4222']); + $pool->use($pool->servers()[1] ?? self::fail('The server was not discovered.')); + + $pool->discover(['c:4222']); + + self::assertSame(['b:4222', 'a:4222', 'c:4222'], $pool->urls()); + } + + public function testRandomizedPoolKeepsAllServers(): void + { + $urls = array_map(static fn(int $i): string => "h{$i}:4222", range(1, 20)); + $pool = new ServerPool($urls, randomize: true, maxReconnects: null); + + $shuffled = $pool->urls(); + sort($shuffled); + $expected = $urls; + sort($expected); + + self::assertSame($expected, $shuffled); + } +} diff --git a/tests/JetStream/PullConsumerTest.php b/tests/JetStream/PullConsumerTest.php index 14e8d49..4e04876 100644 --- a/tests/JetStream/PullConsumerTest.php +++ b/tests/JetStream/PullConsumerTest.php @@ -8,6 +8,10 @@ use Amp\TimeoutCancellation; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; +use Revolt\EventLoop; +use Thesis\Nats\Client; +use Thesis\Nats\Config; use Thesis\Nats\Delivery; use Thesis\Nats\Description; use Thesis\Nats\Exception\MaxRequestBatchExceeded; @@ -23,6 +27,7 @@ use Thesis\Nats\Message; use Thesis\Nats\NatsException; use Thesis\Nats\NatsTestCase; +use Thesis\Nats\PausableProxy; use Thesis\Nats\Status; use Thesis\Nats\Subscription; use Thesis\Time\TimeSpan; @@ -66,6 +71,205 @@ public function testAckDelivery(): void $stream->delete(); } + public function testConsumeContinuesAfterReconnect(): void + { + $proxy = PausableProxy::start($this->natsAddress()); + $config = $this->config(); + // Stopped at the end: the client of the base test case is the publisher's. + $nc = new Client(new Config( + urls: [$proxy->address], + user: $config->user, + password: $config->password, + reconnectWait: 0.1, + )); + $js = $nc->jetStream(); + $publisher = $this->client()->jetStream(); + + $disconnected = new DeferredFuture(); + $nc->onDisconnect(static function () use ($disconnected): void { + if (!$disconnected->isComplete()) { + $disconnected->complete(); + } + }); + + /** @var DeferredFuture $reconnected */ + $reconnected = new DeferredFuture(); + $nc->onReconnect(static function () use ($reconnected): void { + if (!$reconnected->isComplete()) { + $reconnected->complete(); + } + }); + + $subject = generateUniqueId(10); + + $stream = $js->createStream(new StreamConfig( + name: generateUniqueId(10), + subjects: ["{$subject}.*"], + )); + + $consumer = $stream->createConsumer( + new ConsumerConfig(durableName: generateUniqueId(10), ackPolicy: AckPolicy::Explicit), + ); + + /** @var \ArrayObject $received */ + $received = new \ArrayObject(); + $subscription = $consumer->pull(static function (JetStreamDelivery $delivery) use ($received): void { + $received[] = (string) $delivery->message->payload; + $delivery->ack(); + }); + + try { + $publisher->publish("{$subject}.x", new Message('before')); + self::waitUntil(static fn(): bool => \count($received) === 1); + + $proxy->kill(); + $disconnected->getFuture()->await(new TimeoutCancellation(5)); + + // Nobody listens to the reply subject of the pull request now: the server drops the request. + // Without pulling again, the message would wait for the missed heartbeats (2 x 15 seconds by default). + $publisher->publish("{$subject}.x", new Message('while disconnected')); + + $restarted = PausableProxy::start($this->natsAddress(), $proxy->address); + $reconnected->getFuture()->await(new TimeoutCancellation(5)); + self::waitUntil(static fn(): bool => \count($received) === 2); + + self::assertSame(['before', 'while disconnected'], $received->getArrayCopy()); + + $subscription->stop(); + $stream->delete(); + $restarted->kill(); + } finally { + $nc->stop(); + $proxy->kill(); + } + } + + /** + * @param non-negative-int $reconnectBufferSize without a buffer, a pull request sent while reconnecting fails at once, + * with one it would be sent after the reconnect, next to the reconnect's own + */ + #[TestWith([0])] + #[TestWith([1024 * 1024])] + public function testHandlerFinishingWhileDisconnectedDoesNotPull(int $reconnectBufferSize): void + { + $proxy = PausableProxy::start($this->natsAddress()); + $config = $this->config(); + $nc = new Client(new Config( + urls: [$proxy->address], + user: $config->user, + password: $config->password, + reconnectWait: 0.1, + reconnectBufferSize: $reconnectBufferSize, + )); + $js = $nc->jetStream(); + $publisher = $this->client()->jetStream(); + + /** @var DeferredFuture $disconnected */ + $disconnected = new DeferredFuture(); + $nc->onDisconnect(static function () use ($disconnected): void { + if (!$disconnected->isComplete()) { + $disconnected->complete(); + } + }); + + /** @var DeferredFuture $reconnected */ + $reconnected = new DeferredFuture(); + $nc->onReconnect(static function () use ($reconnected): void { + if (!$reconnected->isComplete()) { + $reconnected->complete(); + } + }); + + /** @var \ArrayObject $loopErrors */ + $loopErrors = new \ArrayObject(); + $errorHandler = EventLoop::getErrorHandler(); + EventLoop::setErrorHandler(static fn(\Throwable $e) => $loopErrors->append($e)); + + $subject = generateUniqueId(10); + $streamName = generateUniqueId(10); + $durable = generateUniqueId(10); + $stream = $js->createStream(new StreamConfig(name: $streamName, subjects: ["{$subject}.*"])); + $consumer = $stream->createConsumer(new ConsumerConfig(durableName: $durable, ackPolicy: AckPolicy::None)); + + // The pull requests reach the subscribers of the API subject as well. + $observer = new Client($this->config()); + $pulls = new \ArrayObject(); + $observer->subscribe("\$JS.API.CONSUMER.MSG.NEXT.{$streamName}.{$durable}", static fn() => $pulls->append(true)); + $observer->ping(); + + /** @var DeferredFuture $started */ + $started = new DeferredFuture(); + + /** @var DeferredFuture $gate */ + $gate = new DeferredFuture(); + + /** @var \ArrayObject $received */ + $received = new \ArrayObject(); + $subscription = $consumer->pull(static function (JetStreamDelivery $delivery) use ( + $received, + $started, + $gate, + ): void { + if (!$started->isComplete()) { + $started->complete(); + $gate->getFuture()->await(); + } + + $received[] = (string) $delivery->message->payload; + }, new PullConsumeConfig(maxMessages: 1)); + + try { + $publisher->publish("{$subject}.x", new Message('first')); + $started->getFuture()->await(new TimeoutCancellation(5)); + + $proxy->kill(); + $disconnected->getFuture()->await(new TimeoutCancellation(5)); + + // The handler finishes while there is no connection: nothing must be pulled until the reconnect. + $gate->complete(); + delay(0.3); + self::assertCount(0, $loopErrors, implode("\n", array_map(static fn(\Throwable $e): string => (string) $e, $loopErrors->getArrayCopy()))); + + $restarted = PausableProxy::start($this->natsAddress(), $proxy->address); + $reconnected->getFuture()->await(new TimeoutCancellation(5)); + + // The first pull and the full batch the reconnect pulls, nothing pulled while disconnected. + $observer->ping(); + delay(0.3); + self::assertCount(2, $pulls, 'A pull request was sent while disconnected.'); + + $publisher->publish("{$subject}.x", new Message('second')); + self::waitUntil(static fn(): bool => \count($received) === 2); + self::assertSame(['first', 'second'], $received->getArrayCopy()); + self::assertFalse($subscription->completed()); + + $subscription->stop(); + $stream->delete(); + $restarted->kill(); + } finally { + EventLoop::setErrorHandler($errorHandler); + $observer->stop(); + $nc->stop(); + $proxy->kill(); + } + } + + /** + * @param \Closure(): bool $condition + */ + private static function waitUntil(\Closure $condition): void + { + $deadline = microtime(true) + 5; + + while (!$condition()) { + if (microtime(true) > $deadline) { + self::fail('The condition was not met in time.'); + } + + delay(0.01); + } + } + public function testStopFromHandlerDoesNotPullAfterwards(): void { $nc = $this->client(); diff --git a/tests/NatsTestCase.php b/tests/NatsTestCase.php index 465cc4e..daedeaf 100644 --- a/tests/NatsTestCase.php +++ b/tests/NatsTestCase.php @@ -90,16 +90,23 @@ final protected function clientWithUrls(array $urls): Client * * @param non-empty-string $name */ - final protected function clientNamed(string $name): Client - { + final protected function clientNamed( + string $name, + bool $allowReconnect = true, + float $reconnectWait = 0.1, + bool $verbose = false, + ): Client { $config = $this->config(); return $this->clientWithConfig(new Config( urls: $config->urls, + verbose: $verbose, user: $config->user, password: $config->password, noResponders: $config->noResponders, clientName: $name, + allowReconnect: $allowReconnect, + reconnectWait: $reconnectWait, )); } @@ -110,12 +117,41 @@ final protected function clientNamed(string $name): Client * @param non-empty-string $name */ final protected function kick(string $name): void + { + $this->requestServer('KICK', $name); + } + + /** + * Sends the client connected under the given name the lame duck mode notification, as if the server were going + * to shut down; the server itself keeps running. + * + * @param non-empty-string $name + */ + final protected function lameDuck(string $name): void + { + $this->requestServer('LDM', $name); + } + + /** + * @param non-empty-string $name + * @return ?int the server-side id of the connection of the client connected under the given name + */ + final protected function connectionId(string $name): ?int + { + return $this->connectionNamed($name)['cid'] ?? null; + } + + /** + * @param non-empty-string $request + * @param non-empty-string $name + */ + private function requestServer(string $request, string $name): void { $connection = $this->connectionNamed($name) ?? self::fail("The client '{$name}' is not connected."); self::decodeServerResponse( $this->sysClient()->request( - "\$SYS.REQ.SERVER.{$connection['server']}.KICK", + "\$SYS.REQ.SERVER.{$connection['server']}.{$request}", new Message(json_encode(['cid' => $connection['cid']], JSON_THROW_ON_ERROR)), new TimeoutCancellation(5), ), diff --git a/tests/PausableProxy.php b/tests/PausableProxy.php index a03f17e..cc3271c 100644 --- a/tests/PausableProxy.php +++ b/tests/PausableProxy.php @@ -26,6 +26,9 @@ final class PausableProxy /** @var ?array{non-empty-string, DeferredFuture} */ private ?array $pauseBefore = null; + /** @var list */ + private array $sockets = []; + /** * @param non-empty-string $address */ @@ -36,10 +39,11 @@ private function __construct( /** * @param non-empty-string $upstream + * @param non-empty-string $listen */ - public static function start(string $upstream): self + public static function start(string $upstream, string $listen = '127.0.0.1:0'): self { - $server = listen('127.0.0.1:0'); + $server = listen($listen); $address = $server->getAddress()->toString(); \assert($address !== ''); @@ -52,6 +56,8 @@ public static function start(string $upstream): self ): void { while (($client = $server->accept()) !== null) { $nats = connect($upstream); + $proxy->sockets[] = $client; + $proxy->sockets[] = $nats; $future = async(static fn() => $proxy->forward($client, $nats, pausable: true)); $future->ignore(); @@ -95,6 +101,27 @@ public function resume(): void $paused?->complete(); } + /** + * Closes the connections passing through the proxy, as if the server went away; new ones are still accepted. + */ + public function disconnect(): void + { + [$sockets, $this->sockets] = [$this->sockets, []]; + + foreach ($sockets as $socket) { + $socket->close(); + } + } + + /** + * Stops accepting and closes the connections: the server is down until a proxy is started on the same address again. + */ + public function kill(): void + { + $this->server->close(); + $this->disconnect(); + } + public function stop(): void { $this->resume(); diff --git a/tests/ReconnectTest.php b/tests/ReconnectTest.php new file mode 100644 index 0000000..0be90d3 --- /dev/null +++ b/tests/ReconnectTest.php @@ -0,0 +1,650 @@ + */ + private array $proxies = []; + + /** @var list */ + private array $clients = []; + + /** @var list */ + private array $files = []; + + #[\Override] + protected function tearDown(): void + { + foreach ($this->clients as $client) { + $client->stop(); + } + + foreach ($this->proxies as $proxy) { + $proxy->kill(); + } + + foreach ($this->files as $file) { + @unlink($file); + } + + parent::tearDown(); + } + + public function testSubscriptionsAndRequestsSurviveServerSideDisconnect(): void + { + $name = Id\generateUniqueId(); + $client = $this->clientNamed($name); + /** @var \ArrayObject $events */ + $events = new \ArrayObject(); + $reconnected = $this->watch($client, $events); + $subject = Id\generateUniqueId(); + + /** @var \ArrayObject $received */ + $received = new \ArrayObject(); + $subscription = $client->subscribe("{$subject}.*", static function (Delivery $delivery) use ($received): void { + $received[] = (string) $delivery->message->payload; + }); + + $responder = $this->otherClient(); + $responder->subscribe("{$subject}.rpc", static function (Delivery $delivery): void { + $delivery->reply(new Message('pong')); + }); + $responder->ping(); + + // The request inbox is subscribed before the disconnect. + self::assertSame('pong', $client->request("{$subject}.rpc", new Message('ping'), timeout: 5)->message->payload); + + $cid = $this->connectionId($name); + $this->kick($name); + $reconnected->await(new TimeoutCancellation(5)); + + self::assertNotSame($cid, $this->connectionId($name)); + self::assertSame(['disconnect', 'reconnect'], $events->getArrayCopy()); + self::assertFalse($subscription->completed()); + + // The same subscription and the same request inbox, restored on the new connection. + $client->ping(); + $responder->publish("{$subject}.event", new Message('after')); + self::assertSame('pong', $client->request("{$subject}.rpc", new Message('ping'), timeout: 5)->message->payload); + self::assertContains('after', $received->getArrayCopy()); + } + + public function testPublishesAreBufferedWhileReconnecting(): void + { + $proxy = $this->proxy(); + $client = $this->clientVia($proxy, reconnectWait: 0.1); + $disconnected = new DeferredFuture(); + $client->onDisconnect(static fn() => $disconnected->complete()); + $reconnected = $this->watch($client); + $subject = Id\generateUniqueId(); + + $observer = $this->otherClient(); + $deliveries = $observer->subscribeIterator($subject); + $observer->ping(); + $client->connect(); + + $proxy->kill(); + $disconnected->getFuture()->await(new TimeoutCancellation(5)); + + // Nothing to send them to: they wait for the reconnect, in order. + foreach (['one', 'two', 'three'] as $payload) { + $client->publish($subject, new Message($payload)); + } + + $this->restart($proxy); + $reconnected->await(new TimeoutCancellation(5)); + + $received = []; + foreach ($deliveries as $delivery) { + $received[] = (string) $delivery->message->payload; + + if (\count($received) === 3) { + $deliveries->stop(); + } + } + + self::assertSame(['one', 'two', 'three'], $received); + } + + public function testMissedPongsReportStaleConnection(): void + { + $proxy = $this->proxy(); + $config = $this->config(); + $client = $this->track(new Client(new Config( + urls: [$proxy->address], + connectionTimeout: 1, + user: $config->user, + password: $config->password, + ping: 100, + maxPings: 1, + reconnectWait: 0.1, + ))); + + /** @var DeferredFuture $disconnected */ + $disconnected = new DeferredFuture(); + $client->onDisconnect(static function (?\Throwable $reason) use ($disconnected): void { + if (!$disconnected->isComplete()) { + $disconnected->complete($reason); + } + }); + $reconnected = $this->watch($client); + $client->connect(); + + // The PINGs no longer reach the server, so no PONG comes back. + $proxy->pause(); + + self::assertInstanceOf(StaleConnection::class, $disconnected->getFuture()->await(new TimeoutCancellation(5))); + + $proxy->resume(); + $reconnected->await(new TimeoutCancellation(5)); + } + + public function testReconnectWaitsUntilServerHasRestoredSubscriptions(): void + { + $proxy = $this->proxy(); + $client = $this->clientVia($proxy, reconnectWait: 0.1); + $disconnected = new DeferredFuture(); + $client->onDisconnect(static fn() => $disconnected->complete()); + $reconnected = $this->watch($client); + $subject = Id\generateUniqueId(); + $deliveries = $client->subscribeIterator($subject); + $client->ping(); + + $proxy->kill(); + $disconnected->getFuture()->await(new TimeoutCancellation(5)); + + // The new connection is established, but the restored SUB (and the PING after it) is held back. + $restarted = $this->restart($proxy); + $held = $restarted->pauseBefore('SUB '); + $held->await(new TimeoutCancellation(5)); + delay(0.3); + self::assertFalse($reconnected->isComplete(), 'The server has not seen the subscriptions yet.'); + + $restarted->resume(); + $reconnected->await(new TimeoutCancellation(5)); + + // The reconnect callbacks promise active subscriptions: a message sent right away is not lost. + $this->otherClient()->publish($subject, new Message('right after the reconnect')); + + foreach ($deliveries as $delivery) { + self::assertSame('right after the reconnect', $delivery->message->payload); + $deliveries->stop(); + } + } + + public function testReconnectConfirmsAllowedSubscriptionsAfterForbiddenOne(): void + { + $proxy = $this->proxy(); + + // May subscribe only under "allowed.>"; verbose, so the forbidden SUB fails the restore's execute at once. + $client = $this->track(new Client(new Config( + urls: [$proxy->address], + verbose: true, + user: 'restricted', + password: 'Pswd1', + reconnectWait: 0.1, + ))); + $disconnected = new DeferredFuture(); + $client->onDisconnect(static fn() => $disconnected->complete()); + $reconnected = $this->watch($client); + $client->connect(); + + $proxy->kill(); + $disconnected->getFuture()->await(new TimeoutCancellation(5)); + + // Both are buffered while reconnecting and sent in this order by the restore. + $id = Id\generateUniqueId(); + $forbidden = $client->subscribe("forbidden.{$id}", static fn() => null); + $allowed = $client->subscribeIterator("allowed.{$id}"); + + // The forbidden SUB reaches the server and is rejected, the allowed one (and the PING after it) is held back. + $restarted = $this->restart($proxy); + $restarted->pauseBefore('SUB allowed.')->await(new TimeoutCancellation(5)); + + delay(0.3); + self::assertFalse($reconnected->isComplete(), 'The server has not seen the allowed subscription yet.'); + + $restarted->resume(); + $reconnected->await(new TimeoutCancellation(5)); + + // The forbidden one fails alone. + try { + $forbidden->awaitCompletion(new TimeoutCancellation(5)); + self::fail('The subscription is forbidden, it must fail.'); + } catch (PermissionViolated) { + } + + $this->otherClient()->publish("allowed.{$id}", new Message('right after the reconnect')); + + foreach ($allowed as $delivery) { + self::assertSame('right after the reconnect', $delivery->message->payload); + $allowed->stop(); + } + } + + public function testConnectWaitsForReconnect(): void + { + $proxy = $this->proxy(); + $client = $this->clientVia($proxy, reconnectWait: 0.1); + $disconnected = new DeferredFuture(); + $client->onDisconnect(static fn() => $disconnected->complete()); + $client->connect(); + + $proxy->kill(); + $disconnected->getFuture()->await(new TimeoutCancellation(5)); + + try { + $client->connect(new TimeoutCancellation(0.2)); + self::fail('The server is down, connect must not return.'); + } catch (CancelledException) { + } + + $this->restart($proxy); + $client->connect(new TimeoutCancellation(5)); + self::assertGreaterThan(0, $client->ping(new TimeoutCancellation(5))); + } + + public function testPublishFailsOnceReconnectBufferIsFull(): void + { + $proxy = $this->proxy(); + $client = $this->clientVia($proxy, reconnectBufferSize: 64); + $disconnected = new DeferredFuture(); + $client->onDisconnect(static fn() => $disconnected->complete()); + $client->connect(); + + $proxy->kill(); + $disconnected->getFuture()->await(new TimeoutCancellation(5)); + + $client->publish('events.small', new Message('fits')); + + self::expectException(ReconnectBufferExceeded::class); + $client->publish('events.large', new Message(str_repeat('x', 64))); + } + + public function testVerbosePublishAndPingWaitForReconnect(): void + { + $proxy = $this->proxy(); + $client = $this->clientVia($proxy, reconnectWait: 0.1, verbose: true); + $disconnected = new DeferredFuture(); + $client->onDisconnect(static fn() => $disconnected->complete()); + $subject = Id\generateUniqueId(); + + $observer = $this->otherClient(); + $deliveries = $observer->subscribeIterator($subject); + $observer->ping(); + $client->connect(); + + $proxy->kill(); + $disconnected->getFuture()->await(new TimeoutCancellation(5)); + + // A verbose publish is acknowledged by the server, so it cannot be buffered. + $publish = async($client->publish(...), $subject, new Message('acknowledged')); + $ping = async($client->ping(...)); + delay(0.3); + self::assertFalse($publish->isComplete()); + self::assertFalse($ping->isComplete()); + + $this->restart($proxy); + $publish->await(new TimeoutCancellation(5)); + $ping->await(new TimeoutCancellation(5)); + + foreach ($deliveries as $delivery) { + self::assertSame('acknowledged', $delivery->message->payload); + $deliveries->stop(); + } + } + + public function testAutoUnsubscribeLimitSurvivesReconnect(): void + { + $name = Id\generateUniqueId(); + $client = $this->clientNamed($name); + $reconnected = $this->watch($client); + $subject = Id\generateUniqueId(); + $publisher = $this->otherClient(); + + /** @var \ArrayObject $received */ + $received = new \ArrayObject(); + $subscription = $client->subscribe($subject, static function (Delivery $delivery) use ($received): void { + $received[] = (string) $delivery->message->payload; + }, max: 3); + $client->ping(); + + $publisher->publish($subject, new Message('1')); + $this->waitFor(static fn(): bool => \count($received) === 1); + + $this->kick($name); + $reconnected->await(new TimeoutCancellation(5)); + $client->ping(); + + // Two messages are left of the limit: the restored subscription is removed by the server after them. + foreach (['2', '3', '4', '5'] as $payload) { + $publisher->publish($subject, new Message($payload)); + } + + $subscription->awaitCompletion(new TimeoutCancellation(5)); + self::assertSame(['1', '2', '3'], $received->getArrayCopy()); + } + + public function testClientGivesUpOnceServersUseUpTheirReconnects(): void + { + $proxy = $this->proxy(); + $client = $this->clientVia($proxy, reconnectWait: 0.05, maxReconnects: 2); + + /** @var DeferredFuture $closed */ + $closed = new DeferredFuture(); + $client->onClose(static fn(?\Throwable $reason) => $closed->complete($reason)); + $errors = new \ArrayObject(); + $client->onError(static fn(\Throwable $e) => $errors->append($e)); + $subscription = $client->subscribe(Id\generateUniqueId(), static fn() => null); + + $proxy->kill(); + + $reason = $closed->getFuture()->await(new TimeoutCancellation(5)); + self::assertInstanceOf(ConnectionWasClosed::class, $reason); + self::assertStringContainsString('no server is left', $reason->getMessage()); + + try { + $subscription->awaitCompletion(new TimeoutCancellation(1)); + self::fail('The client has given up reconnecting, the subscription must fail.'); + } catch (ConnectionWasClosed $e) { + self::assertSame($reason, $e); + } + + // Running out of servers is not a server error. + self::assertCount(0, $errors); + } + + #[RequiresPhpExtension('sodium')] + public function testRepeatedAuthorizationErrorStopsReconnecting(): void + { + $name = Id\generateUniqueId(); + $file = $this->file(self::SEED); + $client = $this->track(new Client(new Config( + urls: [$this->natsAddress()], + clientName: $name, + nkeyFile: $file, + reconnectWait: 0.05, + maxReconnects: null, + ))); + + /** @var DeferredFuture $closed */ + $closed = new DeferredFuture(); + $client->onClose(static fn(?\Throwable $reason) => $closed->complete($reason)); + $client->connect(); + + // The credentials are revoked: every attempt is rejected the same way, retrying forever makes no sense. + file_put_contents($file, self::UNKNOWN_SEED); + $this->kick($name); + + self::assertInstanceOf(AuthorizationViolation::class, $closed->getFuture()->await(new TimeoutCancellation(5))); + } + + public function testLameDuckModeMovesToAnotherServer(): void + { + $name = Id\generateUniqueId(); + $proxy = $this->proxy(); + $config = $this->config(); + + // Both urls lead to the same server: the client starts with the first one and has the second one to move to. + $client = $this->track(new Client(new Config( + urls: [$this->natsAddress(), $proxy->address], + user: $config->user, + password: $config->password, + clientName: $name, + reconnectWait: 10, + noRandomize: true, + ))); + $lameDuck = new DeferredFuture(); + $client->onLameDuckMode(static fn() => $lameDuck->complete()); + $reconnected = $this->watch($client); + $subject = Id\generateUniqueId(); + $deliveries = $client->subscribeIterator($subject); + $client->ping(); + $cid = $this->connectionId($name); + + $this->lameDuck($name); + + $lameDuck->getFuture()->await(new TimeoutCancellation(5)); + // Right away, without the 10 seconds pause between the rounds of the reconnects. + $reconnected->await(new TimeoutCancellation(2)); + self::assertNotSame($cid, $this->connectionId($name)); + + $client->publish($subject, new Message('moved')); + + foreach ($deliveries as $delivery) { + self::assertSame('moved', $delivery->message->payload); + $deliveries->stop(); + } + } + + public function testLameDuckModeWithNowhereToMoveOnlyNotifies(): void + { + $name = Id\generateUniqueId(); + $client = $this->clientNamed($name); + $lameDuck = new DeferredFuture(); + $client->onLameDuckMode(static fn() => $lameDuck->complete()); + /** @var \ArrayObject $events */ + $events = new \ArrayObject(); + $this->watch($client, $events); + $client->connect(); + $cid = $this->connectionId($name); + + $this->lameDuck($name); + + $lameDuck->getFuture()->await(new TimeoutCancellation(5)); + delay(0.3); + self::assertSame($cid, $this->connectionId($name)); + self::assertCount(0, $events); + } + + public function testReconnectsToServerDiscoveredFromCluster(): void + { + $cluster = $this->clusterUrls(); + // Only the first server is configured, behind a proxy: the other one is known from its INFO only. + $proxy = $this->proxy($cluster[0]); + $client = $this->track(new Client(new Config(urls: [$proxy->address], reconnectWait: 0.1, noRandomize: true))); + $reconnected = $this->watch($client); + $subject = Id\generateUniqueId(); + $deliveries = $client->subscribeIterator($subject); + $client->ping(); + + $proxy->kill(); + $reconnected->await(new TimeoutCancellation(5)); + + // The publisher is connected to the second server of the cluster: the message is routed to the restored subscription. + $publisher = $this->track(new Client(new Config(urls: [$cluster[1] ?? self::fail('The cluster has a single server.')]))); + $publisher->publish($subject, new Message('routed')); + + foreach ($deliveries as $delivery) { + self::assertSame('routed', $delivery->message->payload); + $deliveries->stop(); + } + } + + public function testServersAnnouncedAfterConnectAreReported(): void + { + $server = listen('127.0.0.1:0'); + $address = $server->getAddress()->toString(); + \assert($address !== ''); + + $peer = async(static function () use ($server): void { + $peer = $server->accept() ?? self::fail('The client has not connected.'); + $peer->write(new Protocol\ServerInfo(version: '2.12.0', serverId: 'fake', headers: true)->encode()); + + $buffer = ''; + + while (!str_contains($buffer, "PING\r\n")) { + $buffer .= $peer->read() ?? self::fail('The client has closed the connection.'); + } + + // A server joined the cluster after the client had connected. + $peer->write("PONG\r\n"); + $peer->write(new Protocol\ServerInfo(version: '2.12.0', serverId: 'fake', headers: true, connectUrls: ['127.0.0.1:4333'])->encode()); + $peer->read(); + }); + $peer->ignore(); + + $client = $this->track(new Client(new Config(urls: [$address], connectionTimeout: 5))); + + /** @var DeferredFuture> $discovered */ + $discovered = new DeferredFuture(); + $client->onServersDiscovered(static fn(array $urls) => $discovered->complete($urls)); + $client->connect(); + + $urls = $discovered->getFuture()->await(new TimeoutCancellation(5)); + $expected = [$address, '127.0.0.1:4333']; + sort($urls); + sort($expected); + self::assertSame($expected, $urls); + + $server->close(); + } + + /** + * @param ?\ArrayObject $events + * @return Future completed on the first reconnect + */ + private function watch(Client $client, ?\ArrayObject $events = null): Future + { + /** @var DeferredFuture $reconnected */ + $reconnected = new DeferredFuture(); + + $client->onDisconnect(static fn() => $events?->append('disconnect')); + $client->onReconnect(static function () use ( + $reconnected, + $events, + ): void { + $events?->append('reconnect'); + + if (!$reconnected->isComplete()) { + $reconnected->complete(); + } + }); + + return $reconnected->getFuture(); + } + + /** + * @param ?non-empty-string $upstream + */ + private function proxy(?string $upstream = null): PausableProxy + { + return $this->proxies[] = PausableProxy::start($upstream ?? $this->natsAddress()); + } + + /** + * Brings the server behind a killed proxy back on the same address. + */ + private function restart(PausableProxy $proxy): PausableProxy + { + return $this->proxies[] = PausableProxy::start($this->natsAddress(), $proxy->address); + } + + /** + * @param ?non-negative-int $maxReconnects + * @param non-negative-int $reconnectBufferSize + */ + private function clientVia( + PausableProxy $proxy, + float $reconnectWait = 2.0, + ?int $maxReconnects = 60, + int $reconnectBufferSize = 8 * 1024 * 1024, + bool $verbose = false, + ): Client { + $config = $this->config(); + + return $this->track(new Client(new Config( + urls: [$proxy->address], + verbose: $verbose, + user: $config->user, + password: $config->password, + noResponders: true, + maxReconnects: $maxReconnects, + reconnectWait: $reconnectWait, + reconnectBufferSize: $reconnectBufferSize, + ))); + } + + private function otherClient(): Client + { + return $this->track(new Client($this->config())); + } + + private function track(Client $client): Client + { + return $this->clients[] = $client; + } + + /** + * @param \Closure(): bool $condition + */ + private function waitFor(\Closure $condition): void + { + $deadline = microtime(true) + 5; + + while (!$condition()) { + if (microtime(true) > $deadline) { + self::fail('The condition was not met in time.'); + } + + delay(0.01); + } + } + + /** + * @return non-empty-string + */ + private function file(string $contents): string + { + $file = tempnam(sys_get_temp_dir(), 'nats'); + \assert($file !== false); + + file_put_contents($file, $contents); + + return $this->files[] = $file; + } + + /** + * @return non-empty-list the `host:port` of the servers of the cluster + */ + private function clusterUrls(): array + { + $dsn = getenv('THESIS_NATS_CLUSTER_DSN'); + if (!\is_string($dsn) || $dsn === '') { + self::markTestSkipped('THESIS_NATS_CLUSTER_DSN must be set.'); + } + + return Config::fromURI($dsn)->urls; + } +}