diff --git a/README.md b/README.md
index 489e6c6..3c587b5 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@ Pure non-blocking (fiber based) strictly typed full-featured PHP driver for NATS
- [Connection](#connection)
- [Authentication](#authentication)
- [Connect and ping](#connect-and-ping)
+ - [Reconnect](#reconnect)
- [Publish-Subscribe](#pub-sub)
- [Queues](#queues)
- [Request-Reply](#request-reply)
@@ -65,9 +66,9 @@ The library implements the full functionality of NATS Core, including pub-sub, q
`Nats\Config::fromURI()` takes `[scheme://][user[:password]@]host[:port][,host[:port]...][?options]`. The scheme is `nats://` (`tcp://` is an alias, and it may be omitted),
`tls://`, `ws://` and `wss://` are rejected as not supported yet. A host without a port connects to `4222`, an IPv6 host goes in brackets (`[::1]:4222`),
-and the credentials are percent-encoded like in any url (`p%40ss` for `p@ss`). The servers of a list are tried in order until one accepts the connection;
+and the credentials are percent-encoded like in any url (`p%40ss` for `p@ss`). The servers of a list are tried in a random order (in the given one with `noRandomize`, `no_randomize=true` in the uri) until one accepts the connection, see [Reconnect](#reconnect);
each one may repeat the scheme and the credentials, as in nats.go (`nats://u:p@h1:4222,nats://u:p@h2:4222`): one user is sent to every server, so different credentials are rejected. The options are the snake_case names of the `Nats\Config` parameters,
-e.g. `connection_timeout` (seconds, fractions allowed), `request_timeout`, `verbose`, `no_responders`, `ping`, `token`, `jwt`, `nkey`, `credentials_file`, `jwt_file`, `nkey_file`, `client_name`.
+e.g. `connection_timeout` (seconds, fractions allowed), `request_timeout`, `verbose`, `no_responders`, `ping`, `token`, `jwt`, `nkey`, `credentials_file`, `jwt_file`, `nkey_file`, `client_name`, `allow_reconnect`, `max_reconnects` (negative for no limit), `reconnect_wait`, `reconnect_jitter`, `reconnect_buffer_size`, `no_randomize`.
Userinfo without a colon is a token, as in nats.go (`nats://s3cr3t@host`), while `nats://alice:@host` is the user `alice` with an empty password, see [Authentication](#authentication):
```php
@@ -150,6 +151,49 @@ $rtt = $nc->ping(); // The subscription is active on the server now.
dump("Round trip: {$rtt}s");
```
+#### Reconnect
+
+The client reconnects once the connection is lost, the way nats.go does it, and the subscriptions do not notice:
+they are restored on the new connection with the same ids (an auto-unsubscribe keeps the rest of its limit),
+the request inbox included, and a JetStream pull consumer pulls again. The publishes made while reconnecting are buffered,
+up to `reconnectBufferSize` bytes (8 MiB by default), and sent after the subscriptions; once the buffer is full, `publish()` throws `Nats\Exception\ReconnectBufferExceeded`.
+In verbose mode a publish waits for the reconnect instead, to get its acknowledgement, and so does `ping()`.
+
+The servers are shuffled (`noRandomize` keeps the given order) and tried one by one, pausing for `reconnectWait` plus a random
+`reconnectJitter` after every round. A server that fails `maxReconnects` attempts in a row (60 by default, null for no limit) is given up.
+Once no server is left, the client is closed: the subscriptions fail with `Nats\Exception\ConnectionWasClosed` and the next call
+establishes a new connection. A server that rejects the credentials twice in a row closes the client at once, with `Nats\Exception\AuthorizationViolation`:
+retrying would not help. The servers of a cluster announced by the server (`connect_urls`)
+join the pool, and a server going to shut down (lame duck mode) makes the client move to another one right away.
+Set `allowReconnect: false` to fail the subscriptions as soon as the connection is lost instead.
+
+```php
+onDisconnect(static fn(?\Throwable $reason) => dump('Disconnected: ' . ($reason?->getMessage() ?? 'closed by the server')));
+$nc->onReconnect(static fn() => dump('Reconnected'));
+$nc->onClose(static fn(?\Throwable $reason) => dump('Closed: ' . ($reason?->getMessage() ?? 'by the client')));
+$nc->onServersDiscovered(static fn(array $urls) => dump('Known servers: ' . implode(', ', $urls)));
+$nc->onLameDuckMode(static fn() => dump('The server is going to shut down'));
+
+$nc->subscribe('foo.>', static function (Nats\Delivery $delivery): void {
+ dump($delivery->message->payload); // keeps receiving after a reconnect
+});
+
+trapSignal([\SIGTERM, \SIGINT]);
+
+$nc->drain();
+```
+
#### Pub-Sub
```php
diff --git a/compose.yaml b/compose.yaml
index 183e4d3..d77aa40 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -30,5 +30,14 @@ services:
image: nats
command: --name nats-token -p 4222 --auth T0k3n
+ # A cluster of two servers: each one announces both in INFO (connect_urls), for the tests of the discovered servers.
+ nats-cluster-1:
+ image: nats
+ command: --name nats-cluster-1 -p 4222 --cluster_name test --cluster nats://0.0.0.0:6222 --routes nats://nats-cluster-2:6222
+
+ nats-cluster-2:
+ image: nats
+ command: --name nats-cluster-2 -p 4222 --cluster_name test --cluster nats://0.0.0.0:6222 --routes nats://nats-cluster-1:6222
+
nats-cli:
image: bitnamilegacy/natscli:latest
diff --git a/phpunit.xml.dist b/phpunit.xml.dist
index 116b749..ed5a153 100644
--- a/phpunit.xml.dist
+++ b/phpunit.xml.dist
@@ -13,6 +13,7 @@
>
+
diff --git a/src/Client.php b/src/Client.php
index fa39b48..96e6886 100644
--- a/src/Client.php
+++ b/src/Client.php
@@ -43,7 +43,7 @@ final class Client
private readonly Connection\ConnectionFactory $connectionFactory;
- /** @var ?Future */
+ /** @var ?Future */
private ?Future $connection = null;
/**
@@ -61,6 +61,26 @@ final class Client
/** @var list<\Closure(\Throwable): void> */
private array $errorCallbacks = [];
+ /** @var list<\Closure(?\Throwable): void> */
+ private array $disconnectCallbacks = [];
+
+ /** @var list<\Closure(): void> */
+ private array $reconnectCallbacks = [];
+
+ /** @var list<\Closure(?\Throwable): void> */
+ private array $closeCallbacks = [];
+
+ /** @var list<\Closure(non-empty-list): void> */
+ private array $serversDiscoveredCallbacks = [];
+
+ /** @var list<\Closure(): void> */
+ private array $lameDuckModeCallbacks = [];
+
+ /** @var array the internal listeners of the disconnects and reconnects */
+ private array $connectionWatchers = [];
+
+ private int $nextWatcherId = 0;
+
private readonly Id\SubscriptionIdGenerator $subscriptionIdGenerator;
public function __construct(
@@ -87,15 +107,16 @@ public static function default(): self
}
/**
- * Connects to the server unless the client is already connected. Every call that needs the connection
- * establishes it on demand, so calling this is optional: it makes network and authentication errors
- * surface at startup rather than from the first publish or subscribe.
+ * Connects to the server unless the client is already connected, and waits for the reconnect in progress if any.
+ * Every call that needs the connection establishes it on demand, so calling this is optional: it makes network
+ * and authentication errors surface at startup rather than from the first publish or subscribe.
*
* @throws NatsException
*/
public function connect(?Cancellation $cancellation = null): void
{
- $this->connection($cancellation);
+ // The connection object outlives a lost connection: while reconnecting, wait for the reconnect.
+ $this->connection($cancellation)->awaitConnected($cancellation);
}
/**
@@ -409,6 +430,88 @@ public function onError(callable $callback): void
$this->errorCallbacks[] = $callback(...);
}
+ /**
+ * Registers a callback for a lost connection: the client is reconnecting, the subscriptions are kept and the publishes
+ * are buffered meanwhile, see {@see Config::$allowReconnect}. The callback gets the server error or the failure
+ * that broke the connection, null if the server just closed it.
+ *
+ * @param callable(?\Throwable): void $callback
+ */
+ public function onDisconnect(callable $callback): void
+ {
+ $this->disconnectCallbacks[] = $callback(...);
+ }
+
+ /**
+ * Registers a callback for a restored connection: the subscriptions are active again and the buffered publishes are sent.
+ *
+ * @param callable(): void $callback
+ */
+ public function onReconnect(callable $callback): void
+ {
+ $this->reconnectCallbacks[] = $callback(...);
+ }
+
+ /**
+ * Registers a callback for the connection closed for good: by {@see stop()} or {@see drain()}, by a fatal server error,
+ * or once no server is left to reconnect to. The callback gets the reason, null for a regular close.
+ * The next call establishes a new connection.
+ *
+ * @param callable(?\Throwable): void $callback
+ */
+ public function onClose(callable $callback): void
+ {
+ $this->closeCallbacks[] = $callback(...);
+ }
+
+ /**
+ * Registers a callback for the servers of the cluster announced after the client has connected,
+ * the client reconnects to them as well. The callback gets the `host:port` of every known server.
+ *
+ * @param callable(non-empty-list): void $callback
+ */
+ public function onServersDiscovered(callable $callback): void
+ {
+ $this->serversDiscoveredCallbacks[] = $callback(...);
+ }
+
+ /**
+ * Registers a callback for the server going to shut down (lame duck mode). If the client knows another server,
+ * it moves there right away, otherwise it reconnects once the server closes the connection.
+ *
+ * @param callable(): void $callback
+ */
+ public function onLameDuckMode(callable $callback): void
+ {
+ $this->lameDuckModeCallbacks[] = $callback(...);
+ }
+
+ /**
+ * Listens to the disconnects and reconnects, e.g. for a JetStream consumer to pull again once reconnected.
+ *
+ * @internal
+ * @param \Closure(): void $onDisconnect
+ * @param \Closure(): void $onReconnect
+ * @return \Closure(): void stops listening
+ */
+ public function watchConnection(\Closure $onDisconnect, \Closure $onReconnect): \Closure
+ {
+ $id = $this->nextWatcherId++;
+ $this->connectionWatchers[$id] = [$onDisconnect, $onReconnect];
+
+ $client = \WeakReference::create($this);
+
+ return static function () use (
+ $client,
+ $id,
+ ): void {
+ $self = $client->get();
+ if ($self !== null) {
+ unset($self->connectionWatchers[$id]);
+ }
+ };
+ }
+
public function createService(Micro\ServiceConfig $config): Micro\Service
{
$identity = new Micro\ServiceIdentity(
@@ -446,6 +549,11 @@ public function stop(?Cancellation $cancellation = null): void
$this->disconnect(static fn(Subscription $subscription) => $subscription->stop($cancellation), $cancellation);
}
+ /**
+ * Unsubscribes, lets the handlers finish the messages already received and closes the connection. While reconnecting,
+ * unlike nats.go that fails here, it waits for the reconnect to drain over the new connection: pass a cancellation
+ * to bound the wait.
+ */
public function drain(?Cancellation $cancellation = null): void
{
$this->disconnect(static fn(Subscription $subscription) => $subscription->drain($cancellation), $cancellation);
@@ -506,6 +614,7 @@ private function register(
queueGroup: $queueGroup,
push: $handler->push(...),
subscription: $subscription = $handler->subscription,
+ remaining: $handler->remaining(...),
);
try {
@@ -634,28 +743,67 @@ private function invokeSubscriber(Hooks\MessageReceived $event): void
}
/**
- * Routes a server error that was not an acknowledgement of a command, the same way nats.go does:
- * a subscription permission violation fails that subscription only, a publish permission violation
- * is reported to the error callbacks, anything else is fatal and the connection is closed.
- * A fatal error is reported once the connection is torn down, see onClose in connection().
- *
- * @return ?\Throwable the error the connection is closed with
+ * Routes a permission violation, the only server error the connection passes on, the same way nats.go does:
+ * a subscription permission violation fails that subscription only, a publish one is reported to the error callbacks.
+ * The connection deals with the others itself: it reconnects or closes, see {@see closed()}.
*/
- private function handleError(Hooks\ErrorReceived $event, Connection\Connection $connection): ?\Throwable
+ private function handleError(Hooks\ErrorReceived $event): void
{
$e = $event->error->toException();
- if ($e instanceof Exception\PermissionViolated) {
- if (!$this->failSubscriptions($event->error, $e)) {
- $this->reportError($e);
+ if (!$e instanceof Exception\PermissionViolated || !$this->failSubscriptions($event->error, $e)) {
+ $this->reportError($e);
+ }
+ }
+
+ /**
+ * @return list the commands that restore the subscriptions on a new connection, with the same sids,
+ * and with the rest of the limit of an auto-unsubscribe
+ */
+ private function resubscriptions(): array
+ {
+ $frames = [];
+
+ foreach ($this->subscribers as $sid => $subscriber) {
+ // A numeric sid becomes an integer key of the array, which the type of the property does not tell.
+ $sid = (string) $sid; // @phpstan-ignore cast.useless
+ $remaining = $subscriber->remaining !== null ? ($subscriber->remaining)() : null;
+
+ if ($remaining === 0) {
+ // The server has already removed it, the handler is finishing the received messages.
+ continue;
}
- return null;
+ $frames[] = Internal\Command::sub($subscriber->subject, $sid, $subscriber->queueGroup);
+
+ if ($remaining !== null) {
+ $frames[] = Internal\Command::unsub($sid, $remaining);
+ }
}
- $connection->close();
+ return $frames;
+ }
- return $e;
+ private function disconnected(?\Throwable $reason): void
+ {
+ foreach ($this->disconnectCallbacks as $callback) {
+ EventLoop::queue($callback, $reason);
+ }
+
+ foreach ($this->connectionWatchers as [$onDisconnect]) {
+ EventLoop::queue($onDisconnect);
+ }
+ }
+
+ private function reconnected(): void
+ {
+ foreach ($this->reconnectCallbacks as $callback) {
+ EventLoop::queue($callback);
+ }
+
+ foreach ($this->connectionWatchers as [, $onReconnect]) {
+ EventLoop::queue($onReconnect);
+ }
}
/**
@@ -728,8 +876,8 @@ private function disconnect(\Closure $do, ?Cancellation $cancellation = null): v
}
/**
- * Tears the client down once the connection has been closed from the outside: by the server (e.g. a kick
- * or a restart) or because of a fatal server error. Nothing is sent over the connection, it is gone and so
+ * Tears the client down once the connection has been closed from the outside: the client has given up reconnecting
+ * (or reconnecting is off and the server has closed the connection), or a fatal server error has closed it. Nothing is sent over the connection, it is gone and so
* are the server-side subscriptions: the subscriptions and the pending requests are only failed locally.
*
* @param ?\Throwable $error the fatal server error that closed the connection, if any
@@ -772,9 +920,10 @@ private function closed(Connection\Connection $connection, ?\Throwable $error):
// and nothing may escape to the event loop, which has no one to catch it.
Future\awaitAll($completes);
} finally {
- if ($error !== null) {
- // Reported after the teardown, so that the callback observes the subscriptions already failed
- // and the next call establishing a new connection.
+ // Reported after the teardown, so that the callback observes the subscriptions already failed
+ // and the next call establishing a new connection. Running out of servers is not a server error,
+ // it is reported to the close callbacks only.
+ if ($error !== null && !$error instanceof ConnectionWasClosed) {
$this->reportError($error);
}
}
@@ -833,49 +982,77 @@ private static function awaitOrNull(Future $future, ?Cancellation $cancellation
}
}
- private function connection(?Cancellation $cancellation = null): Connection\Connection
+ private function connection(?Cancellation $cancellation = null): Connection\ReconnectingConnection
{
- $this->connection ??= async(weakClosure(function (): Connection\Connection {
- try {
- $connection = $this->connectionFactory->connect();
- } catch (\Throwable $e) {
- // Forget the failed attempt so that the next call tries to connect again.
- $this->connection = null;
-
- throw $e;
- }
-
- $this->activeConnection = $connection;
-
- /** @var ?\Throwable $reason the fatal server error that closed the connection */
- $reason = null;
-
+ $this->connection ??= async(weakClosure(function (): Connection\ReconnectingConnection {
// The connection's reader runs in the event loop: strong references from its hooks would keep the client
// alive for as long as the socket is open, so the destructor that closes the socket would never run.
// Once the client is gone, its hooks do nothing: e.g. the close event of the connection it closed itself.
$client = \WeakReference::create($this);
+ $connection = new Connection\ReconnectingConnection(
+ config: $this->config,
+ factory: $this->connectionFactory,
+ subscriptions: static fn(): array => $client->get()?->resubscriptions() ?? [],
+ );
+
$connection->hooks->onMessage(static function (Hooks\MessageReceived $event) use ($client): void {
$client->get()?->invokeSubscriber($event);
});
- $connection->hooks->onError(static function (Hooks\ErrorReceived $event) use (
- $client,
- $connection,
- &$reason,
- ): void {
+ $connection->hooks->onError(static function (Hooks\ErrorReceived $event) use ($client): void {
+ $client->get()?->handleError($event);
+ });
+ $connection->hooks->onDisconnect(static function (Hooks\Disconnected $event) use ($client): void {
+ $client->get()?->disconnected($event->reason);
+ });
+ $connection->hooks->onReconnect(static function () use ($client): void {
+ $client->get()?->reconnected();
+ });
+ $connection->hooks->onServersDiscovered(static function (Hooks\ServersDiscovered $event) use ($client): void {
$self = $client->get();
- if ($self !== null) {
- $reason ??= $self->handleError($event, $connection);
+ if ($self === null) {
+ return;
+ }
+
+ foreach ($self->serversDiscoveredCallbacks as $callback) {
+ EventLoop::queue($callback, $event->urls);
+ }
+ });
+ $connection->hooks->onLameDuckMode(static function () use ($client): void {
+ $self = $client->get();
+ if ($self === null) {
+ return;
+ }
+
+ foreach ($self->lameDuckModeCallbacks as $callback) {
+ EventLoop::queue($callback);
}
});
- $connection->hooks->onClose(static function () use (
+ $connection->hooks->onClose(static function (Hooks\ConnectionClosed $event) use (
$client,
$connection,
- &$reason,
): void {
- $client->get()?->closed($connection, $reason);
+ $self = $client->get();
+ if ($self !== null) {
+ foreach ($self->closeCallbacks as $callback) {
+ EventLoop::queue($callback, $event->reason);
+ }
+
+ $self->closed($connection, $event->reason);
+ }
});
+ try {
+ $connection->start();
+ } catch (\Throwable $e) {
+ // Forget the failed attempt so that the next call tries to connect again.
+ $this->connection = null;
+
+ throw $e;
+ }
+
+ $this->activeConnection = $connection;
+
return $connection;
}));
diff --git a/src/Config.php b/src/Config.php
index e51ed37..cca4e9b 100644
--- a/src/Config.php
+++ b/src/Config.php
@@ -17,6 +17,10 @@
private const int DEFAULT_PING_INTERVAL = 10000;
private const int DEFAULT_MAX_PINGS = 5;
private const string DEFAULT_CLIENT_NAME = 'thesis/nats';
+ private const int DEFAULT_MAX_RECONNECTS = 60;
+ private const float DEFAULT_RECONNECT_WAIT = 2.0;
+ private const float DEFAULT_RECONNECT_JITTER = 0.1;
+ private const int DEFAULT_RECONNECT_BUFFER_SIZE = 8 * 1024 * 1024;
/**
* @var non-empty-string
@@ -41,6 +45,14 @@
* read on every connection attempt; exclusive with the other JWT and NKey options
* @param ?non-empty-string $jwtFile the path to a file holding the user JWT, bare or decorated, read on every connection attempt; exclusive with $jwt
* @param ?non-empty-string $nkeyFile the path to a file holding the NKey user seed, bare or decorated, read on every connection attempt; exclusive with $nkey
+ * @param bool $allowReconnect whether the client reconnects once the connection is lost, restoring the subscriptions;
+ * otherwise the subscriptions fail and the next call establishes a new connection
+ * @param ?non-negative-int $maxReconnects the failed reconnects in a row after which a server is given up, null for no limit;
+ * once every server is given up, the client is closed
+ * @param float $reconnectWait in seconds, the pause after every server has been tried
+ * @param float $reconnectJitter in seconds, the upper bound of a random delay added to the pause, so that the clients do not reconnect at once
+ * @param non-negative-int $reconnectBufferSize in bytes, the publishes buffered while reconnecting, 0 to fail them instead
+ * @param bool $noRandomize whether to try the servers in the given order instead of a random one
* @throws \InvalidArgumentException
*/
public function __construct(
@@ -68,9 +80,31 @@ public function __construct(
public ?string $credentialsFile = null,
public ?string $jwtFile = null,
public ?string $nkeyFile = null,
+ public bool $allowReconnect = true,
+ public ?int $maxReconnects = self::DEFAULT_MAX_RECONNECTS,
+ public float $reconnectWait = self::DEFAULT_RECONNECT_WAIT,
+ public float $reconnectJitter = self::DEFAULT_RECONNECT_JITTER,
+ public int $reconnectBufferSize = self::DEFAULT_RECONNECT_BUFFER_SIZE,
+ public bool $noRandomize = false,
) {
Validate::duration($connectionTimeout, 'connection timeout');
+ // A negative limit would give up every server at once, while the uri means no limit by it: refused as a mistake,
+ // null is the way to say no limit here.
+ if ($maxReconnects !== null) {
+ Validate::nonNegative($maxReconnects, 'max reconnects');
+ }
+
+ Validate::nonNegative($reconnectBufferSize, 'reconnect buffer size');
+
+ if (!is_finite($reconnectWait) || $reconnectWait < 0) {
+ throw new \InvalidArgumentException('The reconnect wait must be a finite non-negative number of seconds.');
+ }
+
+ if (!is_finite($reconnectJitter) || $reconnectJitter < 0) {
+ throw new \InvalidArgumentException('The reconnect jitter must be a finite non-negative number of seconds.');
+ }
+
// nats-server itself refuses to have a token next to users, so both of them here are a mistake.
if ($token !== null && ($user !== null || $password !== null)) {
throw new \InvalidArgumentException('A token cannot be combined with a user and a password.');
@@ -134,6 +168,16 @@ public static function fromURI(#[\SensitiveParameter] string $uri): self
credentialsFile: $query->credentialsFile,
jwtFile: $query->jwtFile,
nkeyFile: $query->nkeyFile,
+ allowReconnect: $query->allowReconnect ?? true,
+ maxReconnects: match (true) {
+ $query->maxReconnects === null => self::DEFAULT_MAX_RECONNECTS,
+ $query->maxReconnects < 0 => null,
+ default => $query->maxReconnects,
+ },
+ reconnectWait: $query->reconnectWait ?? self::DEFAULT_RECONNECT_WAIT,
+ reconnectJitter: $query->reconnectJitter ?? self::DEFAULT_RECONNECT_JITTER,
+ reconnectBufferSize: $query->reconnectBufferSize ?? self::DEFAULT_RECONNECT_BUFFER_SIZE,
+ noRandomize: $query->noRandomize ?? false,
);
}
@@ -158,6 +202,12 @@ public static function fromURI(#[\SensitiveParameter] string $uri): self
* credentials_file?: non-empty-string,
* jwt_file?: non-empty-string,
* nkey_file?: non-empty-string,
+ * allow_reconnect?: bool,
+ * max_reconnects?: ?non-negative-int,
+ * reconnect_wait?: float,
+ * reconnect_jitter?: float,
+ * reconnect_buffer_size?: non-negative-int,
+ * no_randomize?: bool,
* } $options
* @throws \InvalidArgumentException
*/
@@ -183,6 +233,12 @@ public static function fromArray(#[\SensitiveParameter] array $options): self
credentialsFile: $options['credentials_file'] ?? null,
jwtFile: $options['jwt_file'] ?? null,
nkeyFile: $options['nkey_file'] ?? null,
+ allowReconnect: $options['allow_reconnect'] ?? true,
+ maxReconnects: \array_key_exists('max_reconnects', $options) ? $options['max_reconnects'] : self::DEFAULT_MAX_RECONNECTS,
+ reconnectWait: $options['reconnect_wait'] ?? self::DEFAULT_RECONNECT_WAIT,
+ reconnectJitter: $options['reconnect_jitter'] ?? self::DEFAULT_RECONNECT_JITTER,
+ reconnectBufferSize: $options['reconnect_buffer_size'] ?? self::DEFAULT_RECONNECT_BUFFER_SIZE,
+ noRandomize: $options['no_randomize'] ?? false,
);
}
}
diff --git a/src/Exception/ReconnectBufferExceeded.php b/src/Exception/ReconnectBufferExceeded.php
new file mode 100644
index 0000000..dfc356d
--- /dev/null
+++ b/src/Exception/ReconnectBufferExceeded.php
@@ -0,0 +1,24 @@
+headers,
supportJetstream: $info->jetstream ?? false,
maxPayload: $info->maxPayload,
+ connectUrls: $info->connectUrls ?? [],
+ serverId: $info->serverId,
);
}
@@ -25,11 +27,15 @@ public static function fromServerInfo(ServerInfo $info): self
* @param non-empty-string $serverVersion
* @param ?positive-int $maxPayload the largest message (headers and payload) the server accepts, in bytes,
* null if the server does not announce it
+ * @param list $connectUrls the `host:port` of the servers of the cluster the server announces
+ * @param ?non-empty-string $serverId
*/
public function __construct(
public string $serverVersion,
public bool $allowHeaders = false,
public bool $supportJetstream = false,
public ?int $maxPayload = null,
+ public array $connectUrls = [],
+ public ?string $serverId = null,
) {}
}
diff --git a/src/Internal/Connection/PingPongHandler.php b/src/Internal/Connection/PingPongHandler.php
index 166ba6e..24d1324 100644
--- a/src/Internal/Connection/PingPongHandler.php
+++ b/src/Internal/Connection/PingPongHandler.php
@@ -4,6 +4,7 @@
namespace Thesis\Nats\Internal\Connection;
+use Thesis\Nats\Exception\StaleConnection;
use Thesis\Nats\Internal\Protocol;
use Thesis\Nats\Internal\Timer;
@@ -58,6 +59,7 @@ public function stop(): void
private function forceStop(): void
{
$this->stop();
- $this->connection->close();
+ // The server has stopped answering: the reason the client reconnects, the same nats.go reports.
+ $this->connection->close(new StaleConnection());
}
}
diff --git a/src/Internal/Connection/Query.php b/src/Internal/Connection/Query.php
index 88b0e81..a3c9991 100644
--- a/src/Internal/Connection/Query.php
+++ b/src/Internal/Connection/Query.php
@@ -25,6 +25,10 @@
* @param ?non-empty-string $credentialsFile
* @param ?non-empty-string $jwtFile
* @param ?non-empty-string $nkeyFile
+ * @param ?int $maxReconnects any integer, the config decides what a negative one means
+ * @param ?float $reconnectWait non-negative and finite
+ * @param ?float $reconnectJitter non-negative and finite
+ * @param ?non-negative-int $reconnectBufferSize
*/
private function __construct(
public ?bool $verbose = null,
@@ -46,6 +50,12 @@ private function __construct(
public ?string $credentialsFile = null,
public ?string $jwtFile = null,
public ?string $nkeyFile = null,
+ public ?bool $allowReconnect = null,
+ public ?int $maxReconnects = null,
+ public ?float $reconnectWait = null,
+ public ?float $reconnectJitter = null,
+ public ?int $reconnectBufferSize = null,
+ public ?bool $noRandomize = null,
) {}
public static function parse(#[\SensitiveParameter] string $query): self
@@ -69,6 +79,12 @@ public static function parse(#[\SensitiveParameter] string $query): self
credentialsFile: self::string($options, 'credentials_file'),
jwtFile: self::string($options, 'jwt_file'),
nkeyFile: self::string($options, 'nkey_file'),
+ allowReconnect: self::bool($options, 'allow_reconnect'),
+ maxReconnects: self::int($options, 'max_reconnects'),
+ reconnectWait: self::nonNegativeFloat($options, 'reconnect_wait'),
+ reconnectJitter: self::nonNegativeFloat($options, 'reconnect_jitter'),
+ reconnectBufferSize: self::nonNegativeInt($options, 'reconnect_buffer_size'),
+ noRandomize: self::bool($options, 'no_randomize'),
);
}
@@ -109,6 +125,31 @@ private static function positiveInt(array $options, string $name): ?int
return $value !== null && $value > 0 ? $value : null;
}
+ /**
+ * @param array $options
+ * @return ?non-negative-int
+ */
+ private static function nonNegativeInt(array $options, string $name): ?int
+ {
+ $value = self::int($options, $name);
+
+ return $value !== null && $value >= 0 ? $value : null;
+ }
+
+ /**
+ * @param array $options
+ */
+ private static function nonNegativeFloat(array $options, string $name): ?float
+ {
+ if (!isset($options[$name])) {
+ return null;
+ }
+
+ $value = filter_var($options[$name], FILTER_VALIDATE_FLOAT);
+
+ return $value !== false && is_finite($value) && $value >= 0 ? $value : null;
+ }
+
/**
* @param array $options
*/
diff --git a/src/Internal/Connection/ReconnectState.php b/src/Internal/Connection/ReconnectState.php
new file mode 100644
index 0000000..64498da
--- /dev/null
+++ b/src/Internal/Connection/ReconnectState.php
@@ -0,0 +1,21 @@
+ the commands sent while reconnecting */
+ private array $pending = [];
+
+ /** @var non-negative-int the size of the publishes among the pending commands */
+ private int $pendingBytes = 0;
+
+ /**
+ * Pending while reconnecting, for the calls that cannot be buffered: a ping and a publish in verbose mode.
+ *
+ * @var ?DeferredFuture
+ */
+ private ?DeferredFuture $reconnected = null;
+
+ private readonly DeferredCancellation $closing;
+
+ /**
+ * @param \Closure(): list $subscriptions the commands that restore the subscriptions on a new connection
+ */
+ public function __construct(
+ private readonly Config $config,
+ private readonly ConnectionFactory $factory,
+ private readonly \Closure $subscriptions,
+ ) {
+ $this->hooks = new Hooks\ConcurrentProvider();
+ $this->pool = new ServerPool($config->urls, randomize: !$config->noRandomize, maxReconnects: $config->maxReconnects);
+ $this->closing = new DeferredCancellation();
+ }
+
+ /**
+ * Establishes the first connection: every server is tried once, in the order of the pool. A server that cannot
+ * be reached is skipped, any other failure (e.g. an authentication error) is thrown at once.
+ *
+ * @throws \Throwable
+ */
+ public function start(): void
+ {
+ /** @var array $errors */
+ $errors = [];
+
+ foreach ($this->pool->servers() as $server) {
+ try {
+ $connection = $this->factory->connect($server->url);
+ } catch (ServerIsNotAvailable $e) {
+ $errors[$server->url] = $e;
+
+ continue;
+ }
+
+ if ($this->state === ReconnectState::Closed) {
+ $connection->close();
+
+ throw new ConnectionWasClosed();
+ }
+
+ $this->pool->use($server);
+ $this->attach($connection);
+ $this->pool->discover($connection->info()->connectUrls);
+ $this->state = ReconnectState::Connected;
+
+ return;
+ }
+
+ throw new ServerIsNotAvailable(
+ \sprintf('No available nats host: %s.', implode('; ', array_map(
+ static fn(string $url, ServerIsNotAvailable $e): string => "{$url}: {$e->getMessage()}",
+ array_keys($errors),
+ array_values($errors),
+ ))),
+ previous: $errors !== [] ? $errors[array_key_last($errors)] : null,
+ );
+ }
+
+ public function execute(Protocol\Frame|array $frames, ?Cancellation $cancellation = null): void
+ {
+ if ($this->state === ReconnectState::Reconnecting) {
+ $frames = $frames instanceof Protocol\Frame ? [$frames] : $frames;
+
+ // In verbose mode a publish is acknowledged, and a buffered one would be acknowledged by nobody.
+ if (!$this->config->verbose || !self::publishes($frames)) {
+ $this->buffer($frames);
+
+ return;
+ }
+
+ $this->awaitReconnect($cancellation);
+ }
+
+ $this->connected()->execute($frames, $cancellation);
+ }
+
+ /**
+ * Returns once connected: at once, or after the reconnect in progress.
+ *
+ * @throws ConnectionWasClosed if the client gives up reconnecting meanwhile or is closed
+ * @throws CancelledException
+ */
+ public function awaitConnected(?Cancellation $cancellation = null): void
+ {
+ if ($this->state === ReconnectState::Reconnecting) {
+ $this->awaitReconnect($cancellation);
+ }
+
+ $this->connected();
+ }
+
+ public function ping(?Cancellation $cancellation = null): float
+ {
+ if ($this->state === ReconnectState::Reconnecting) {
+ $this->awaitReconnect($cancellation);
+ }
+
+ return $this->connected()->ping($cancellation);
+ }
+
+ public function info(): ConnectionInfo
+ {
+ return $this->info ?? throw new ConnectionIsNotAvailable();
+ }
+
+ public function close(?\Throwable $reason = null): void
+ {
+ $this->finish($reason);
+ }
+
+ private function connected(): Connection
+ {
+ if ($this->state === ReconnectState::Closed) {
+ throw new ConnectionWasClosed();
+ }
+
+ return $this->current ?? throw new ConnectionIsNotAvailable();
+ }
+
+ /**
+ * @throws ConnectionWasClosed if the client gives up reconnecting meanwhile
+ * @throws CancelledException
+ */
+ private function awaitReconnect(?Cancellation $cancellation): void
+ {
+ $this->reconnected?->getFuture()->await($cancellation);
+ }
+
+ /**
+ * @param non-empty-list $frames
+ * @throws ReconnectBufferExceeded
+ */
+ private function buffer(array $frames): void
+ {
+ // Only the publishes count: the subscription commands are tiny, and a subscription must not fail on a full buffer.
+ $bytes = 0;
+ foreach ($frames as $frame) {
+ if ($frame instanceof Protocol\Pub) {
+ $bytes += \strlen($frame->encode());
+ }
+ }
+
+ if ($bytes > 0 && $this->pendingBytes + $bytes > $this->config->reconnectBufferSize) {
+ throw ReconnectBufferExceeded::forSize($this->config->reconnectBufferSize);
+ }
+
+ // Appended in place: copying the whole buffer on every publish would make a reconnect quadratic.
+ array_push($this->pending, ...$frames);
+ $this->pendingBytes += $bytes;
+ }
+
+ private function attach(Connection $connection): void
+ {
+ $this->current = $connection;
+ $this->info = $connection->info();
+
+ // The reader of the connection runs in the event loop: a strong reference from its hooks would keep this
+ // connection, and the client behind it, alive for as long as the socket is open.
+ $self = \WeakReference::create($this);
+
+ $connection->hooks->onMessage(static function (Hooks\MessageReceived $event) use ($self): void {
+ $self->get()?->hooks->dispatch($event);
+ });
+ $connection->hooks->onError(static function (Hooks\ErrorReceived $event) use (
+ $self,
+ $connection,
+ ): void {
+ $self->get()?->onServerError($connection, $event);
+ });
+ $connection->hooks->onInfo(static function (Hooks\InfoReceived $event) use (
+ $self,
+ $connection,
+ ): void {
+ $self->get()?->onServerInfo($connection, $event->info);
+ });
+ $connection->hooks->onClose(static function (Hooks\ConnectionClosed $event) use (
+ $self,
+ $connection,
+ ): void {
+ $self->get()?->onClosed($connection, $event->reason);
+ });
+ }
+
+ /**
+ * A permission violation is passed on, the client attributes it to a subscription. The errors another connection
+ * may not have close the current one and the client reconnects, any other one closes the client.
+ */
+ private function onServerError(Connection $connection, Hooks\ErrorReceived $event): void
+ {
+ if ($connection !== $this->current) {
+ return;
+ }
+
+ $e = $event->error->toException();
+
+ if ($e instanceof PermissionViolated) {
+ $this->hooks->dispatch($event);
+
+ return;
+ }
+
+ if ($this->config->allowReconnect && $event->error->isReconnectable()) {
+ $connection->close($e);
+
+ return;
+ }
+
+ $this->finish($e);
+ }
+
+ private function onServerInfo(Connection $connection, Protocol\ServerInfo $info): void
+ {
+ if ($connection !== $this->current) {
+ return;
+ }
+
+ if ($this->pool->discover($info->connectUrls ?? [])) {
+ $this->discovered();
+ }
+
+ if ($info->ldm !== true) {
+ return;
+ }
+
+ $this->hooks->dispatch(Hooks\LameDuckModeEntered::Event);
+
+ // The server is going to close the connections: move to another one now, while the server still serves.
+ // The next server is tried at once, the pause comes only after a whole round of the pool.
+ // With nowhere to move to, the connection stays until the server closes it.
+ if ($this->config->allowReconnect && $this->state === ReconnectState::Connected && $this->pool->count() > 1) {
+ $connection->close();
+ }
+ }
+
+ private function discovered(): void
+ {
+ $urls = $this->pool->urls();
+
+ if ($urls !== []) {
+ $this->hooks->dispatch(new Hooks\ServersDiscovered($urls));
+ }
+ }
+
+ private function onClosed(Connection $connection, ?\Throwable $reason): void
+ {
+ if ($connection !== $this->current || $this->state !== ReconnectState::Connected) {
+ return;
+ }
+
+ $this->current = null;
+
+ if (!$this->config->allowReconnect) {
+ $this->finish($reason);
+
+ return;
+ }
+
+ $this->state = ReconnectState::Reconnecting;
+
+ /** @var DeferredFuture $reconnected */
+ $reconnected = new DeferredFuture();
+ // Nobody may be waiting when the client gives up.
+ $reconnected->getFuture()->ignore();
+ $this->reconnected = $reconnected;
+
+ $this->hooks->dispatch(new Hooks\Disconnected($reason));
+
+ $future = async($this->reconnect(...));
+ $future->ignore();
+ }
+
+ /**
+ * Tries the servers of the pool one by one, pausing once all of them have been tried, until one of them accepts
+ * the connection or every one of them has used up its reconnects, see {@see ServerPool::next()}.
+ */
+ private function reconnect(): void
+ {
+ $cancellation = $this->closing->getCancellation();
+ $tried = 0;
+ $error = null;
+
+ while (($server = $this->pool->next()) !== null) {
+ // A round of the pool is tried at once, the pause comes after it, as in nats.go.
+ if ($tried + 1 >= $this->pool->count()) {
+ $tried = 0;
+
+ try {
+ delay($this->pause(), cancellation: $cancellation);
+ } catch (CancelledException) {
+ return;
+ }
+ } else {
+ ++$tried;
+ }
+
+ if (!$this->reconnecting()) {
+ return;
+ }
+
+ try {
+ $connection = $this->factory->connect($server->url);
+ } catch (\Throwable $e) {
+ $error = $e;
+
+ // Credentials rejected twice in a row by the same server are not going to be accepted, as in nats.go.
+ if ($server->failed($e) && $e instanceof AuthorizationViolation) {
+ $this->finish($e);
+
+ return;
+ }
+
+ continue;
+ }
+
+ if (!$this->reconnecting()) {
+ $connection->close();
+
+ return;
+ }
+
+ try {
+ $this->restore($connection);
+ } catch (NatsException $e) {
+ $error = $e;
+ $server->failed($e);
+ $this->current = null;
+ $connection->close();
+
+ continue;
+ } catch (\Throwable $e) {
+ // Not a connection failure: retrying would fail the same way, and the error would never surface.
+ $this->current = null;
+ $connection->close();
+ $this->finish($e);
+
+ return;
+ }
+
+ if (!$this->reconnecting()) {
+ // Closed while the subscriptions were being restored: finish() has not seen the new connection yet.
+ $connection->close();
+
+ return;
+ }
+
+ $server->connected();
+
+ if ($this->pool->discover($connection->info()->connectUrls)) {
+ $this->discovered();
+ }
+
+ $this->state = ReconnectState::Connected;
+ [$reconnected, $this->reconnected] = [$this->reconnected, null];
+ $reconnected?->complete();
+ $this->hooks->dispatch(Hooks\Reconnected::Event);
+
+ return;
+ }
+
+ $this->finish(new ConnectionWasClosed('The connection was lost and no server is left to reconnect to.', previous: $error));
+ }
+
+ /**
+ * Sends the subscriptions, then the commands buffered meanwhile, including the ones buffered while sending,
+ * and waits for the PONG of a PING after them: SUB is not acknowledged, and the reconnect callbacks promise
+ * the subscriptions active. A server that does not answer in time is treated as a failed attempt.
+ * The subscriptions sent while reconnecting are buffered too and may be sent twice: the server ignores
+ * a subscription with a sid it already has.
+ *
+ * @throws \Throwable
+ */
+ private function restore(Connection $connection): void
+ {
+ $this->attach($connection);
+
+ $frames = ($this->subscriptions)();
+
+ do {
+ [$pending, $this->pending, $this->pendingBytes] = [$this->pending, [], 0];
+ $frames = [...$frames, ...$pending];
+
+ try {
+ if ($frames !== []) {
+ try {
+ $connection->execute($frames);
+ } catch (PermissionViolated $e) {
+ // Verbose mode: a subscription the user is no longer allowed to have fails alone, as it would
+ // on its own. The acknowledgements of the commands after it are not awaited, so the PING below
+ // is what proves the server has the rest of them.
+ $this->hooks->dispatch(new Hooks\ErrorReceived(new Protocol\Err($e->getMessage() !== '' ? $e->getMessage() : 'Permissions Violation')));
+ }
+ }
+
+ $connection->ping(new TimeoutCancellation($this->config->connectionTimeout));
+ } catch (CancelledException $e) {
+ $this->requeue($pending);
+
+ throw new ServerIsNotAvailable(
+ \sprintf('The nats server did not answer PING within %s seconds after the reconnect.', $this->config->connectionTimeout),
+ previous: $e,
+ );
+ } catch (\Throwable $e) {
+ // Kept for the next server, together with the ones buffered while sending.
+ $this->requeue($pending);
+
+ throw $e;
+ }
+
+ $frames = [];
+ } while ($this->hasPending());
+ }
+
+ /**
+ * The state changes while the reconnect awaits, which PHPStan does not see in a property comparison.
+ */
+ private function reconnecting(): bool
+ {
+ return $this->state === ReconnectState::Reconnecting;
+ }
+
+ /**
+ * Commands may be buffered while the restore awaits, which PHPStan does not see in a property comparison.
+ */
+ private function hasPending(): bool
+ {
+ return $this->pending !== [];
+ }
+
+ /**
+ * @param list $frames sent before the ones pending now
+ */
+ private function requeue(array $frames): void
+ {
+ $this->pending = [...$frames, ...$this->pending];
+ $this->pendingBytes = 0;
+
+ foreach ($this->pending as $frame) {
+ if ($frame instanceof Protocol\Pub) {
+ $this->pendingBytes += \strlen($frame->encode());
+ }
+ }
+ }
+
+ /**
+ * @return float in seconds
+ */
+ private function pause(): float
+ {
+ $jitter = $this->config->reconnectJitter > 0
+ ? random_int(0, (int) ($this->config->reconnectJitter * 1_000_000)) / 1_000_000
+ : 0;
+
+ return $this->config->reconnectWait + $jitter;
+ }
+
+ /**
+ * Closes the client for good: the reconnects stop, the buffered commands are dropped, and the calls waiting
+ * for the reconnect fail.
+ */
+ private function finish(?\Throwable $reason = null): void
+ {
+ if ($this->state === ReconnectState::Closed) {
+ return;
+ }
+
+ $this->state = ReconnectState::Closed;
+ $this->closing->cancel();
+
+ [$current, $this->current] = [$this->current, null];
+ $this->pending = [];
+ $this->pendingBytes = 0;
+
+ [$reconnected, $this->reconnected] = [$this->reconnected, null];
+ $reconnected?->error($reason instanceof ConnectionWasClosed ? $reason : new ConnectionWasClosed(previous: $reason));
+
+ $current?->close();
+ $this->hooks->dispatch(new Hooks\ConnectionClosed($reason));
+ }
+
+ /**
+ * @param non-empty-list $frames
+ */
+ private static function publishes(array $frames): bool
+ {
+ return array_any($frames, static fn(Protocol\Frame $frame): bool => $frame instanceof Protocol\Pub);
+ }
+}
diff --git a/src/Internal/Connection/Server.php b/src/Internal/Connection/Server.php
new file mode 100644
index 0000000..69f7e08
--- /dev/null
+++ b/src/Internal/Connection/Server.php
@@ -0,0 +1,46 @@
+ the error the last attempt failed with */
+ public private(set) ?string $lastError = null;
+
+ /**
+ * @param non-empty-string $url `host:port`
+ * @param bool $implicit whether the server was announced by the cluster rather than configured
+ */
+ public function __construct(
+ public readonly string $url,
+ public readonly bool $implicit,
+ ) {}
+
+ /**
+ * @return bool whether the attempt failed with the same kind of error as the previous one
+ */
+ public function failed(\Throwable $e): bool
+ {
+ ++$this->reconnects;
+ $repeated = $this->lastError === $e::class;
+ $this->lastError = $e::class;
+
+ return $repeated;
+ }
+
+ public function connected(): void
+ {
+ $this->reconnects = 0;
+ $this->lastError = null;
+ }
+}
diff --git a/src/Internal/Connection/ServerPool.php b/src/Internal/Connection/ServerPool.php
new file mode 100644
index 0000000..7f9d252
--- /dev/null
+++ b/src/Internal/Connection/ServerPool.php
@@ -0,0 +1,154 @@
+ */
+ private array $servers = [];
+
+ /**
+ * Every url ever seen, to tell a new server from one that has merely restarted.
+ *
+ * @var array
+ */
+ private array $known = [];
+
+ /**
+ * @param non-empty-list $urls
+ * @param ?non-negative-int $maxReconnects the failed reconnects in a row after which a server is dropped, null for no limit
+ */
+ public function __construct(
+ array $urls,
+ private readonly bool $randomize,
+ private readonly ?int $maxReconnects,
+ ) {
+ foreach ($urls as $url) {
+ $this->add($url, implicit: false);
+ }
+
+ if ($this->randomize) {
+ $this->shuffle(0);
+ }
+ }
+
+ /**
+ * @return list in the order to try them
+ */
+ public function servers(): array
+ {
+ return $this->servers;
+ }
+
+ public function count(): int
+ {
+ return \count($this->servers);
+ }
+
+ /**
+ * Makes the server the current one, e.g. the one the initial connection was established with.
+ */
+ public function use(Server $server): void
+ {
+ $index = array_search($server, $this->servers, true);
+
+ if ($index !== false && $index !== 0) {
+ array_splice($this->servers, $index, 1);
+ array_unshift($this->servers, $server);
+ }
+ }
+
+ /**
+ * Moves the current server to the end of the pool, or drops it once it has used up its reconnects.
+ *
+ * @return ?Server the server to try next, null if no server is left
+ */
+ public function next(): ?Server
+ {
+ $current = array_shift($this->servers);
+
+ if ($current !== null && ($this->maxReconnects === null || $current->reconnects < $this->maxReconnects)) {
+ $this->servers[] = $current;
+ }
+
+ return $this->servers[0] ?? null;
+ }
+
+ /**
+ * Applies the servers a server of the cluster announces: adds the new ones, drops the announced ones it no longer
+ * lists, keeps the configured ones and the current one.
+ *
+ * @param list $urls the `host:port` from connect_urls, an empty list means the server does not announce them
+ * @return bool whether a server never seen before was added
+ */
+ public function discover(array $urls): bool
+ {
+ if ($urls === []) {
+ return false;
+ }
+
+ $announced = array_fill_keys($urls, true);
+ $current = $this->servers[0] ?? null;
+
+ $this->servers = array_values(array_filter(
+ $this->servers,
+ static fn(Server $server): bool => !$server->implicit || $server === $current || isset($announced[$server->url]),
+ ));
+
+ foreach ($this->servers as $server) {
+ unset($announced[$server->url]);
+ }
+
+ $new = false;
+
+ foreach (array_keys($announced) as $url) {
+ $new = $new || !isset($this->known[$url]);
+ $this->add($url, implicit: true);
+ }
+
+ if ($new && $this->randomize) {
+ // The current server keeps its place.
+ $this->shuffle(1);
+ }
+
+ return $new;
+ }
+
+ /**
+ * @return list
+ */
+ public function urls(): array
+ {
+ return array_map(static fn(Server $server): string => $server->url, $this->servers);
+ }
+
+ /**
+ * @param non-empty-string $url
+ */
+ private function add(string $url, bool $implicit): void
+ {
+ $this->servers[] = new Server($url, $implicit);
+ $this->known[$url] = true;
+ }
+
+ /**
+ * @param non-negative-int $offset
+ */
+ private function shuffle(int $offset): void
+ {
+ $head = \array_slice($this->servers, 0, $offset);
+ $tail = \array_slice($this->servers, $offset);
+ shuffle($tail);
+
+ $this->servers = [...$head, ...$tail];
+ }
+}
diff --git a/src/Internal/Connection/SocketConnection.php b/src/Internal/Connection/SocketConnection.php
index b7bef55..083656e 100644
--- a/src/Internal/Connection/SocketConnection.php
+++ b/src/Internal/Connection/SocketConnection.php
@@ -230,6 +230,8 @@ private function handshake(Cancellation $cancellation): void
tlsRequired: false,
name: $this->config->clientName,
version: version(self::PACKAGE_NAME),
+ // Asks the server for the INFO updates: the servers of the cluster and the lame duck mode notification.
+ protocol: Protocol\Proto::Cluster,
authToken: $this->config->token,
user: $this->config->user,
pass: $this->config->password,
@@ -275,7 +277,7 @@ public function info(): ConnectionInfo
return $this->info ?? throw new ConnectionIsNotAvailable();
}
- public function close(): void
+ public function close(?\Throwable $reason = null): void
{
if ($this->state === ConnectionState::Closed || $this->state === ConnectionState::GracefulClosed) {
return;
@@ -285,7 +287,7 @@ public function close(): void
$this->socket->close();
self::failPending($this->queue, new ConnectionWasClosed());
self::failPongs($this->pongs, new ConnectionWasClosed());
- $this->hooks->dispatch(Hooks\ConnectionClosed::Event);
+ $this->hooks->dispatch(new Hooks\ConnectionClosed($reason));
}
private function run(): void
@@ -347,6 +349,8 @@ private function run(): void
$event = match (true) {
$frame instanceof Protocol\Ping => Hooks\PingReceived::Event,
$frame instanceof Protocol\Pong => Hooks\PongReceived::Event,
+ // The INFO of the handshake is read before the reader starts, so any INFO here is an update.
+ $frame instanceof Protocol\ServerInfo => new Hooks\InfoReceived($frame),
$frame instanceof Protocol\Msg => new Hooks\MessageReceived(
subject: $frame->subject,
sid: $frame->sid,
@@ -374,7 +378,7 @@ private function run(): void
if ($state !== ConnectionState::GracefulClosed) { // @phpstan-ignore notIdentical.alwaysTrue
$state = ConnectionState::Closed;
$socket->close();
- $hooks->dispatch(Hooks\ConnectionClosed::Event);
+ $hooks->dispatch(new Hooks\ConnectionClosed($error));
}
// The server closed the connection without answering the handshake.
diff --git a/src/Internal/Connection/SocketConnectionFactory.php b/src/Internal/Connection/SocketConnectionFactory.php
index 87b0e12..3f5c14b 100644
--- a/src/Internal/Connection/SocketConnectionFactory.php
+++ b/src/Internal/Connection/SocketConnectionFactory.php
@@ -6,7 +6,6 @@
use Amp\Socket\ConnectContext;
use Amp\Socket\DnsSocketConnector;
-use Amp\Socket\Socket;
use Amp\Socket\SocketConnector;
use Thesis\Nats\Config;
use Thesis\Nats\Exception\ServerIsNotAvailable;
@@ -32,21 +31,23 @@ public static function fromConfig(Config $config): self
}
// The default amp connector retries each url with an exponential backoff, which takes seconds
- // before an unavailable host is reported. The hosts are iterated here instead, so connect only once.
+ // before an unavailable host is reported. The servers are iterated by the server pool instead, so connect only once.
return new self($config, $context, new DnsSocketConnector());
}
- public function connect(): Connection
+ public function connect(string $url): Connection
{
// The credentials are read and a malformed seed is rejected before the socket is opened:
// otherwise the server would be left waiting for CONNECT.
$credentials = Credentials::fromConfig($this->config);
- $connection = new SocketConnection(
- $this->config,
- $this->createSocket(),
- $credentials,
- );
+ try {
+ $socket = $this->connector->connect($url, $this->context);
+ } catch (\Throwable $e) {
+ throw new ServerIsNotAvailable($e->getMessage(), previous: $e);
+ }
+
+ $connection = new SocketConnection($this->config, $socket, $credentials);
try {
$connection->startup();
@@ -59,25 +60,4 @@ public function connect(): Connection
return $connection;
}
-
- private function createSocket(): Socket
- {
- $exceptions = [];
-
- foreach ($this->config->urls as $url) {
- try {
- return $this->connector->connect($url, $this->context);
- } catch (\Throwable $e) {
- $exceptions[$url] = $e->getMessage();
- }
- }
-
- throw new ServerIsNotAvailable(vsprintf('No available nats host: %s.', [
- implode('; ', array_map(
- static fn(string $url, string $exception): string => "{$url}: {$exception}",
- array_keys($exceptions),
- array_values($exceptions),
- )),
- ]));
- }
}
diff --git a/src/Internal/Hooks/ConcurrentProvider.php b/src/Internal/Hooks/ConcurrentProvider.php
index 451a24e..bca820d 100644
--- a/src/Internal/Hooks/ConcurrentProvider.php
+++ b/src/Internal/Hooks/ConcurrentProvider.php
@@ -13,6 +13,11 @@
* @phpstan-import-type OnPongCallback from Provider
* @phpstan-import-type OnErrorCallback from Provider
* @phpstan-import-type OnCloseCallback from Provider
+ * @phpstan-import-type OnInfoCallback from Provider
+ * @phpstan-import-type OnDisconnectCallback from Provider
+ * @phpstan-import-type OnReconnectCallback from Provider
+ * @phpstan-import-type OnServersDiscoveredCallback from Provider
+ * @phpstan-import-type OnLameDuckModeCallback from Provider
*/
final class ConcurrentProvider implements Provider
{
@@ -31,6 +36,21 @@ final class ConcurrentProvider implements Provider
/** @var list */
private array $closeCallbacks = [];
+ /** @var list */
+ private array $infoCallbacks = [];
+
+ /** @var list */
+ private array $disconnectCallbacks = [];
+
+ /** @var list */
+ private array $reconnectCallbacks = [];
+
+ /** @var list */
+ private array $serversDiscoveredCallbacks = [];
+
+ /** @var list */
+ private array $lameDuckModeCallbacks = [];
+
public function onMessage(\Closure $callback): void
{
$this->messageCallbacks[] = $callback;
@@ -56,14 +76,45 @@ public function onError(\Closure $callback): void
$this->errorCallbacks[] = $callback;
}
- public function dispatch(MessageReceived|PingReceived|PongReceived|ErrorReceived|ConnectionClosed $event): void
+ public function onInfo(\Closure $callback): void
{
+ $this->infoCallbacks[] = $callback;
+ }
+
+ public function onDisconnect(\Closure $callback): void
+ {
+ $this->disconnectCallbacks[] = $callback;
+ }
+
+ public function onReconnect(\Closure $callback): void
+ {
+ $this->reconnectCallbacks[] = $callback;
+ }
+
+ public function onServersDiscovered(\Closure $callback): void
+ {
+ $this->serversDiscoveredCallbacks[] = $callback;
+ }
+
+ public function onLameDuckMode(\Closure $callback): void
+ {
+ $this->lameDuckModeCallbacks[] = $callback;
+ }
+
+ public function dispatch(
+ MessageReceived|PingReceived|PongReceived|ErrorReceived|ConnectionClosed|InfoReceived|Disconnected|Reconnected|ServersDiscovered|LameDuckModeEntered $event,
+ ): void {
$callbacks = match (true) {
$event instanceof MessageReceived => $this->messageCallbacks,
$event instanceof PingReceived => $this->pingCallbacks,
$event instanceof PongReceived => $this->pongCallbacks,
$event instanceof ErrorReceived => $this->errorCallbacks,
- $event instanceof ConnectionClosed => $this->closeCallbacks, // @phpstan-ignore instanceof.alwaysTrue
+ $event instanceof ConnectionClosed => $this->closeCallbacks,
+ $event instanceof InfoReceived => $this->infoCallbacks,
+ $event instanceof Disconnected => $this->disconnectCallbacks,
+ $event instanceof Reconnected => $this->reconnectCallbacks,
+ $event instanceof ServersDiscovered => $this->serversDiscoveredCallbacks,
+ $event instanceof LameDuckModeEntered => $this->lameDuckModeCallbacks, // @phpstan-ignore instanceof.alwaysTrue
};
foreach ($callbacks as $callback) {
diff --git a/src/Internal/Hooks/ConnectionClosed.php b/src/Internal/Hooks/ConnectionClosed.php
index 46a21ca..5571b49 100644
--- a/src/Internal/Hooks/ConnectionClosed.php
+++ b/src/Internal/Hooks/ConnectionClosed.php
@@ -5,9 +5,16 @@
namespace Thesis\Nats\Internal\Hooks;
/**
+ * The connection is closed for good: nothing will be received or sent over it anymore.
+ *
* @internal
*/
-enum ConnectionClosed
+final readonly class ConnectionClosed
{
- case Event;
+ /**
+ * @param ?\Throwable $reason the server error or the failure that closed the connection, null for a regular close
+ */
+ public function __construct(
+ public ?\Throwable $reason = null,
+ ) {}
}
diff --git a/src/Internal/Hooks/Disconnected.php b/src/Internal/Hooks/Disconnected.php
new file mode 100644
index 0000000..affd44a
--- /dev/null
+++ b/src/Internal/Hooks/Disconnected.php
@@ -0,0 +1,20 @@
+ $urls the `host:port` of every known server
+ */
+ public function __construct(
+ public array $urls,
+ ) {}
+}
diff --git a/src/Internal/Protocol/Err.php b/src/Internal/Protocol/Err.php
index c400728..d9e4ddf 100644
--- a/src/Internal/Protocol/Err.php
+++ b/src/Internal/Protocol/Err.php
@@ -11,6 +11,16 @@
*/
final readonly class Err implements Frame
{
+ /** @var list lowercase prefixes, the server is not consistent in the case of its errors */
+ private const array RECONNECTABLE = [
+ 'stale connection',
+ 'maximum connections exceeded',
+ 'maximum account active connections exceeded',
+ 'authorization violation',
+ 'user authentication expired',
+ 'user authentication revoked',
+ 'account authentication expired',
+ ];
private const string SUBSCRIPTION_VIOLATION = '/^Permissions Violation for Subscription to "([^"]+)"(?: using queue "([^"]+)")?/';
/** @var non-empty-string */
@@ -45,6 +55,15 @@ public function violatedSubscription(): ?array
return [$matches[1], $matches[2] ?? null];
}
+ /**
+ * Whether the server closes the connection for a reason another attempt may not have, the same errors nats.go
+ * reconnects after: a stale connection, a connection limit, credentials that expired or were revoked.
+ */
+ public function isReconnectable(): bool
+ {
+ return array_any(self::RECONNECTABLE, fn(string $prefix): bool => str_starts_with(strtolower($this->message), $prefix));
+ }
+
public function toException(): \Exception
{
return match (true) {
diff --git a/src/Internal/Subscription/Subscriber.php b/src/Internal/Subscription/Subscriber.php
index 0d1bc06..2e46287 100644
--- a/src/Internal/Subscription/Subscriber.php
+++ b/src/Internal/Subscription/Subscriber.php
@@ -17,11 +17,13 @@
* @param ?non-empty-string $queueGroup
* @param \Closure(Delivery): bool $push returns false if the subscriber does not accept deliveries anymore
* @param ?Subscription $subscription null for the internal subscriptions which are not exposed to the user
+ * @param ?\Closure(): ?non-negative-int $remaining the messages the server is still to deliver, see {@see SubscriptionHandler::remaining()}
*/
public function __construct(
public string $subject,
public ?string $queueGroup,
public \Closure $push,
public ?Subscription $subscription = null,
+ public ?\Closure $remaining = null,
) {}
}
diff --git a/src/Internal/Subscription/SubscriptionHandler.php b/src/Internal/Subscription/SubscriptionHandler.php
index 368ebbc..c1203b3 100644
--- a/src/Internal/Subscription/SubscriptionHandler.php
+++ b/src/Internal/Subscription/SubscriptionHandler.php
@@ -34,7 +34,7 @@ final class SubscriptionHandler
* @param \Closure(): void $unsubscribe
* @param callable(Delivery, Subscription): void $handler
* @param \Closure(SlowConsumer): void $onSlowConsumer
- * @param ?positive-int $remaining the number of messages after which the server removes the subscription
+ * @param ?non-negative-int $remaining the number of messages after which the server removes the subscription
* @param ?\Closure(): void $onReceive called once a message arrives, before it waits for the handler
*/
public function __construct(
@@ -213,23 +213,25 @@ private function reportSlowConsumer(): void
($this->onSlowConsumer)(SlowConsumer::forSubscription($this->subscription));
}
+ /**
+ * @return ?non-negative-int the messages the server is still to deliver before it removes the subscription, null for no limit
+ */
+ public function remaining(): ?int
+ {
+ return $this->remaining;
+ }
+
/**
* @return bool whether it is the last message the server delivers to the subscription
*/
private function countDown(): bool
{
- if ($this->remaining === null) {
- return false;
- }
-
- if ($this->remaining > 1) {
- --$this->remaining;
-
+ if ($this->remaining === null || $this->remaining === 0) {
return false;
}
- $this->remaining = null;
+ --$this->remaining;
- return true;
+ return $this->remaining === 0;
}
}
diff --git a/src/Internal/Validation/Validate.php b/src/Internal/Validation/Validate.php
index fb0abe8..43d6e0b 100644
--- a/src/Internal/Validation/Validate.php
+++ b/src/Internal/Validation/Validate.php
@@ -61,6 +61,20 @@ public static function count(int $count, string $what): void
}
}
+ /**
+ * A limit or a size, which the phpdoc types promise to be non-negative: checked anyway, since a negative one
+ * would not fail but silently misbehave, e.g. give up every server at once.
+ *
+ * @param non-empty-string $what
+ * @throws \InvalidArgumentException
+ */
+ public static function nonNegative(int $value, string $what): void
+ {
+ if ($value < 0) {
+ throw new \InvalidArgumentException("The {$what} must be non-negative, {$value} given.");
+ }
+ }
+
/**
* A literal subject, e.g. of publish, request or reply-to: wildcards are not allowed.
*
diff --git a/src/JetStream/Internal/PullMessageHandler.php b/src/JetStream/Internal/PullMessageHandler.php
index 2942900..52a558d 100644
--- a/src/JetStream/Internal/PullMessageHandler.php
+++ b/src/JetStream/Internal/PullMessageHandler.php
@@ -10,11 +10,15 @@
use Thesis\Nats\Delivery as NatsDelivery;
use Thesis\Nats\Description;
use Thesis\Nats\Exception\BadRequestSent;
+use Thesis\Nats\Exception\ConnectionIsNotAvailable;
+use Thesis\Nats\Exception\ConnectionWasClosed;
use Thesis\Nats\Exception\ConsumerDeleted;
use Thesis\Nats\Exception\ConsumerIsPushBased;
use Thesis\Nats\Exception\MaxRequestBatchExceeded;
use Thesis\Nats\Exception\MaxRequestExpiresExceeded;
use Thesis\Nats\Exception\MaxRequestMaxBytesExceeded;
+use Thesis\Nats\Exception\ReconnectBufferExceeded;
+use Thesis\Nats\Exception\ServerIsNotAvailable;
use Thesis\Nats\Header;
use Thesis\Nats\JetStream\Api\PullRequest;
use Thesis\Nats\JetStream\Delivery as JetStreamDelivery;
@@ -24,6 +28,7 @@
use Thesis\Nats\NatsException;
use Thesis\Nats\Status;
use Thesis\Nats\Subscription;
+use function Amp\async;
/**
* @internal
@@ -42,6 +47,14 @@ final class PullMessageHandler
private readonly Heartbeat\Watchdog $watchdog;
+ /** Nothing is pulled while reconnecting: the pull requests cannot reach the server, the reconnect pulls again. */
+ private bool $connected = true;
+
+ private ?Subscription $subscription = null;
+
+ /** The failure of a pull request that came before the subscription was bound, see bind(). */
+ private ?\Throwable $failure = null;
+
/** @var Pipeline\Queue */
private readonly Pipeline\Queue $pulls;
@@ -87,12 +100,63 @@ public function __invoke(NatsDelivery $delivery, Subscription $subscription): vo
// The handler may stop the subscription itself: stop() then runs before the subscription is marked completed,
// and rescheduling the watchdog would pull into the completed queue later.
- if (!$subscription->completed() && !$this->pulls->isComplete()) {
+ if (!$subscription->completed() && !$this->pulls->isComplete() && $this->connected) {
$this->watchdog->reset();
$this->replenish();
}
}
+ /**
+ * The subscription the pulled messages are delivered to: it fails if a pull request cannot be sent.
+ */
+ public function bind(Subscription $subscription): void
+ {
+ $this->subscription = $subscription;
+
+ if ($this->failure !== null) {
+ $this->fail($this->failure);
+ }
+ }
+
+ /**
+ * The heartbeats cannot come while reconnecting: the watchdog would pull into the void.
+ */
+ public function disconnected(): void
+ {
+ $this->connected = false;
+ $this->watchdog->stop();
+ }
+
+ /**
+ * The pull requests were bound to the lost connection, the server forgets them: pull a full batch again, as nats.go does.
+ */
+ public function reconnected(): void
+ {
+ $this->connected = true;
+
+ if ($this->pulls->isComplete()) {
+ return;
+ }
+
+ $this->reset();
+ $this->pull();
+ $this->watchdog->reset();
+ }
+
+ private function fail(\Throwable $e): void
+ {
+ $this->stop();
+
+ if ($this->subscription === null) {
+ $this->failure = $e;
+
+ return;
+ }
+
+ $future = async($this->subscription->error(...), $e);
+ $future->ignore();
+ }
+
public function stop(): void
{
$this->watchdog->stop();
@@ -221,19 +285,32 @@ private function tick(
Encoder $json,
Client $nc,
): void {
+ $fail = $this->fail(...);
+
EventLoop::queue(static function () use (
$queue,
$subject,
$reply,
$json,
$nc,
+ $fail,
): void {
foreach ($queue->iterate() as $pull) {
- $nc->publish(
- subject: $subject,
- message: new Message($json->encode($pull)),
- replyTo: $reply,
- );
+ try {
+ $nc->publish(
+ subject: $subject,
+ message: new Message($json->encode($pull)),
+ replyTo: $reply,
+ );
+ } catch (ConnectionWasClosed|ConnectionIsNotAvailable|ServerIsNotAvailable|ReconnectBufferExceeded) {
+ // The connection is lost: the reconnect pulls again, and a client that gives up fails the subscription itself.
+ continue;
+ } catch (\Throwable $e) {
+ // Nothing in the event loop would catch it: the subscription is the one to know.
+ $fail($e);
+
+ return;
+ }
}
});
diff --git a/src/JetStream/PullConsumer.php b/src/JetStream/PullConsumer.php
index ff2d9c6..d47a9bc 100644
--- a/src/JetStream/PullConsumer.php
+++ b/src/JetStream/PullConsumer.php
@@ -141,8 +141,13 @@ public function consume(
cancellation: $cancellation,
);
+ $messageHandler->bind($subscription);
+
+ $unwatch = $this->nats->watchConnection($messageHandler->disconnected(...), $messageHandler->reconnected(...));
+
$this->subscriptions[] = $subscription->onComplete(
$messageHandler->stop(...),
+ $unwatch,
);
return $subscription;
diff --git a/tests/ClientTest.php b/tests/ClientTest.php
index 59209a1..0801416 100644
--- a/tests/ClientTest.php
+++ b/tests/ClientTest.php
@@ -201,10 +201,10 @@ public function testSuspendSubscriptionOnClientDisconnect(): void
$subscription->awaitCompletion();
}
- public function testSubscriptionsAndRequestsFailWhenServerClosesConnection(): void
+ public function testSubscriptionsAndRequestsFailWhenServerClosesConnectionWithoutReconnect(): void
{
$name = Id\generateUniqueId();
- $client = $this->clientNamed($name);
+ $client = $this->clientNamed($name, allowReconnect: false);
// The teardown runs in the event loop, where an escaped exception has no one to catch it.
$loopErrors = [];
diff --git a/tests/ConfigTest.php b/tests/ConfigTest.php
index 27517fe..4932163 100644
--- a/tests/ConfigTest.php
+++ b/tests/ConfigTest.php
@@ -313,6 +313,18 @@ final class ConfigTest extends TestCase
new Config(urls: ['h1:4222', 'h2:4222'], user: 'alice'),
],
)]
+ #[TestWith(
+ [
+ 'nats://127.0.0.1:4222?allow_reconnect=false&max_reconnects=5&reconnect_wait=0.5&reconnect_jitter=0&reconnect_buffer_size=1024&no_randomize=true',
+ new Config(allowReconnect: false, maxReconnects: 5, reconnectWait: 0.5, reconnectJitter: 0.0, reconnectBufferSize: 1024, noRandomize: true),
+ ],
+ )]
+ #[TestWith(
+ [
+ 'nats://127.0.0.1:4222?max_reconnects=-1&reconnect_wait=-1&reconnect_buffer_size=-5',
+ new Config(maxReconnects: null),
+ ],
+ )]
public function testFromURI(string $uri, Config $config): void
{
self::assertEquals($config, Config::fromURI($uri));
@@ -402,6 +414,43 @@ public function testFromArrayWithRequestTimeout(): void
self::assertEquals(new Config(requestTimeout: 0.5), Config::fromArray(['request_timeout' => 0.5]));
}
+ public function testFromArrayWithReconnectOptions(): void
+ {
+ self::assertEquals(
+ new Config(allowReconnect: false, maxReconnects: null, reconnectWait: 0.0, reconnectJitter: 0.5, reconnectBufferSize: 0, noRandomize: true),
+ Config::fromArray([
+ 'allow_reconnect' => false,
+ 'max_reconnects' => null,
+ 'reconnect_wait' => 0.0,
+ 'reconnect_jitter' => 0.5,
+ 'reconnect_buffer_size' => 0,
+ 'no_randomize' => true,
+ ]),
+ );
+ }
+
+ public function testMaxReconnectsMustBeNonNegative(): void
+ {
+ $this->expectException(\InvalidArgumentException::class);
+ Config::fromArray(['max_reconnects' => -1]); // @phpstan-ignore argument.type
+ }
+
+ public function testReconnectBufferSizeMustBeNonNegative(): void
+ {
+ $this->expectException(\InvalidArgumentException::class);
+ new Config(reconnectBufferSize: -1); // @phpstan-ignore argument.type
+ }
+
+ #[TestWith([-1.0, 0.1])]
+ #[TestWith([INF, 0.1])]
+ #[TestWith([1.0, -0.1])]
+ #[TestWith([1.0, NAN])]
+ public function testReconnectWaitAndJitterMustBeNonNegative(float $wait, float $jitter): void
+ {
+ $this->expectException(\InvalidArgumentException::class);
+ new Config(reconnectWait: $wait, reconnectJitter: $jitter);
+ }
+
public function testFromArrayWithAuthenticationFiles(): void
{
self::assertEquals(
diff --git a/tests/Internal/Connection/ServerPoolTest.php b/tests/Internal/Connection/ServerPoolTest.php
new file mode 100644
index 0000000..7022aa9
--- /dev/null
+++ b/tests/Internal/Connection/ServerPoolTest.php
@@ -0,0 +1,93 @@
+next()?->url);
+ self::assertSame('c:4222', $pool->next()?->url);
+ self::assertSame('a:4222', $pool->next()?->url);
+ self::assertSame(['a:4222', 'b:4222', 'c:4222'], $pool->urls());
+ }
+
+ public function testServerIsDroppedOnceItUsesUpItsReconnects(): void
+ {
+ $pool = new ServerPool(['a:4222', 'b:4222'], randomize: false, maxReconnects: 1);
+
+ $b = $pool->next();
+ self::assertSame('b:4222', $b?->url);
+ $b->failed(new \RuntimeException());
+
+ // "a" has never failed, "b" has used up its only reconnect.
+ self::assertSame('a:4222', $pool->next()?->url);
+ self::assertSame(['a:4222'], $pool->urls());
+
+ ($pool->servers()[0] ?? self::fail('The pool is empty.'))->failed(new \RuntimeException());
+ self::assertNull($pool->next());
+ }
+
+ public function testSuccessfulConnectResetsReconnects(): void
+ {
+ $server = new Server('a:4222', implicit: false);
+
+ self::assertFalse($server->failed(new \RuntimeException()));
+ self::assertTrue($server->failed(new \RuntimeException()));
+ self::assertFalse($server->failed(new \LogicException()));
+
+ $server->connected();
+ self::assertSame(0, $server->reconnects);
+ self::assertNull($server->lastError);
+ }
+
+ public function testDiscoverAddsAnnouncedAndDropsGoneImplicitServers(): void
+ {
+ $pool = new ServerPool(['a:4222'], randomize: false, maxReconnects: null);
+
+ self::assertFalse($pool->discover([]));
+ self::assertTrue($pool->discover(['a:4222', 'b:4222', 'c:4222']));
+ self::assertSame(['a:4222', 'b:4222', 'c:4222'], $pool->urls());
+
+ // "c" has left the cluster, the configured "a" stays even if not announced.
+ self::assertFalse($pool->discover(['b:4222']));
+ self::assertSame(['a:4222', 'b:4222'], $pool->urls());
+
+ // "c" is back: it was seen before, so it is not a new server.
+ self::assertFalse($pool->discover(['b:4222', 'c:4222']));
+ self::assertSame(['a:4222', 'b:4222', 'c:4222'], $pool->urls());
+ }
+
+ public function testDiscoverKeepsCurrentImplicitServer(): void
+ {
+ $pool = new ServerPool(['a:4222'], randomize: false, maxReconnects: null);
+ $pool->discover(['b:4222']);
+ $pool->use($pool->servers()[1] ?? self::fail('The server was not discovered.'));
+
+ $pool->discover(['c:4222']);
+
+ self::assertSame(['b:4222', 'a:4222', 'c:4222'], $pool->urls());
+ }
+
+ public function testRandomizedPoolKeepsAllServers(): void
+ {
+ $urls = array_map(static fn(int $i): string => "h{$i}:4222", range(1, 20));
+ $pool = new ServerPool($urls, randomize: true, maxReconnects: null);
+
+ $shuffled = $pool->urls();
+ sort($shuffled);
+ $expected = $urls;
+ sort($expected);
+
+ self::assertSame($expected, $shuffled);
+ }
+}
diff --git a/tests/JetStream/PullConsumerTest.php b/tests/JetStream/PullConsumerTest.php
index 14e8d49..4e04876 100644
--- a/tests/JetStream/PullConsumerTest.php
+++ b/tests/JetStream/PullConsumerTest.php
@@ -8,6 +8,10 @@
use Amp\TimeoutCancellation;
use PHPUnit\Framework\Attributes\CoversClass;
use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\Attributes\TestWith;
+use Revolt\EventLoop;
+use Thesis\Nats\Client;
+use Thesis\Nats\Config;
use Thesis\Nats\Delivery;
use Thesis\Nats\Description;
use Thesis\Nats\Exception\MaxRequestBatchExceeded;
@@ -23,6 +27,7 @@
use Thesis\Nats\Message;
use Thesis\Nats\NatsException;
use Thesis\Nats\NatsTestCase;
+use Thesis\Nats\PausableProxy;
use Thesis\Nats\Status;
use Thesis\Nats\Subscription;
use Thesis\Time\TimeSpan;
@@ -66,6 +71,205 @@ public function testAckDelivery(): void
$stream->delete();
}
+ public function testConsumeContinuesAfterReconnect(): void
+ {
+ $proxy = PausableProxy::start($this->natsAddress());
+ $config = $this->config();
+ // Stopped at the end: the client of the base test case is the publisher's.
+ $nc = new Client(new Config(
+ urls: [$proxy->address],
+ user: $config->user,
+ password: $config->password,
+ reconnectWait: 0.1,
+ ));
+ $js = $nc->jetStream();
+ $publisher = $this->client()->jetStream();
+
+ $disconnected = new DeferredFuture();
+ $nc->onDisconnect(static function () use ($disconnected): void {
+ if (!$disconnected->isComplete()) {
+ $disconnected->complete();
+ }
+ });
+
+ /** @var DeferredFuture $reconnected */
+ $reconnected = new DeferredFuture();
+ $nc->onReconnect(static function () use ($reconnected): void {
+ if (!$reconnected->isComplete()) {
+ $reconnected->complete();
+ }
+ });
+
+ $subject = generateUniqueId(10);
+
+ $stream = $js->createStream(new StreamConfig(
+ name: generateUniqueId(10),
+ subjects: ["{$subject}.*"],
+ ));
+
+ $consumer = $stream->createConsumer(
+ new ConsumerConfig(durableName: generateUniqueId(10), ackPolicy: AckPolicy::Explicit),
+ );
+
+ /** @var \ArrayObject $received */
+ $received = new \ArrayObject();
+ $subscription = $consumer->pull(static function (JetStreamDelivery $delivery) use ($received): void {
+ $received[] = (string) $delivery->message->payload;
+ $delivery->ack();
+ });
+
+ try {
+ $publisher->publish("{$subject}.x", new Message('before'));
+ self::waitUntil(static fn(): bool => \count($received) === 1);
+
+ $proxy->kill();
+ $disconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ // Nobody listens to the reply subject of the pull request now: the server drops the request.
+ // Without pulling again, the message would wait for the missed heartbeats (2 x 15 seconds by default).
+ $publisher->publish("{$subject}.x", new Message('while disconnected'));
+
+ $restarted = PausableProxy::start($this->natsAddress(), $proxy->address);
+ $reconnected->getFuture()->await(new TimeoutCancellation(5));
+ self::waitUntil(static fn(): bool => \count($received) === 2);
+
+ self::assertSame(['before', 'while disconnected'], $received->getArrayCopy());
+
+ $subscription->stop();
+ $stream->delete();
+ $restarted->kill();
+ } finally {
+ $nc->stop();
+ $proxy->kill();
+ }
+ }
+
+ /**
+ * @param non-negative-int $reconnectBufferSize without a buffer, a pull request sent while reconnecting fails at once,
+ * with one it would be sent after the reconnect, next to the reconnect's own
+ */
+ #[TestWith([0])]
+ #[TestWith([1024 * 1024])]
+ public function testHandlerFinishingWhileDisconnectedDoesNotPull(int $reconnectBufferSize): void
+ {
+ $proxy = PausableProxy::start($this->natsAddress());
+ $config = $this->config();
+ $nc = new Client(new Config(
+ urls: [$proxy->address],
+ user: $config->user,
+ password: $config->password,
+ reconnectWait: 0.1,
+ reconnectBufferSize: $reconnectBufferSize,
+ ));
+ $js = $nc->jetStream();
+ $publisher = $this->client()->jetStream();
+
+ /** @var DeferredFuture $disconnected */
+ $disconnected = new DeferredFuture();
+ $nc->onDisconnect(static function () use ($disconnected): void {
+ if (!$disconnected->isComplete()) {
+ $disconnected->complete();
+ }
+ });
+
+ /** @var DeferredFuture $reconnected */
+ $reconnected = new DeferredFuture();
+ $nc->onReconnect(static function () use ($reconnected): void {
+ if (!$reconnected->isComplete()) {
+ $reconnected->complete();
+ }
+ });
+
+ /** @var \ArrayObject $loopErrors */
+ $loopErrors = new \ArrayObject();
+ $errorHandler = EventLoop::getErrorHandler();
+ EventLoop::setErrorHandler(static fn(\Throwable $e) => $loopErrors->append($e));
+
+ $subject = generateUniqueId(10);
+ $streamName = generateUniqueId(10);
+ $durable = generateUniqueId(10);
+ $stream = $js->createStream(new StreamConfig(name: $streamName, subjects: ["{$subject}.*"]));
+ $consumer = $stream->createConsumer(new ConsumerConfig(durableName: $durable, ackPolicy: AckPolicy::None));
+
+ // The pull requests reach the subscribers of the API subject as well.
+ $observer = new Client($this->config());
+ $pulls = new \ArrayObject();
+ $observer->subscribe("\$JS.API.CONSUMER.MSG.NEXT.{$streamName}.{$durable}", static fn() => $pulls->append(true));
+ $observer->ping();
+
+ /** @var DeferredFuture $started */
+ $started = new DeferredFuture();
+
+ /** @var DeferredFuture $gate */
+ $gate = new DeferredFuture();
+
+ /** @var \ArrayObject $received */
+ $received = new \ArrayObject();
+ $subscription = $consumer->pull(static function (JetStreamDelivery $delivery) use (
+ $received,
+ $started,
+ $gate,
+ ): void {
+ if (!$started->isComplete()) {
+ $started->complete();
+ $gate->getFuture()->await();
+ }
+
+ $received[] = (string) $delivery->message->payload;
+ }, new PullConsumeConfig(maxMessages: 1));
+
+ try {
+ $publisher->publish("{$subject}.x", new Message('first'));
+ $started->getFuture()->await(new TimeoutCancellation(5));
+
+ $proxy->kill();
+ $disconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ // The handler finishes while there is no connection: nothing must be pulled until the reconnect.
+ $gate->complete();
+ delay(0.3);
+ self::assertCount(0, $loopErrors, implode("\n", array_map(static fn(\Throwable $e): string => (string) $e, $loopErrors->getArrayCopy())));
+
+ $restarted = PausableProxy::start($this->natsAddress(), $proxy->address);
+ $reconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ // The first pull and the full batch the reconnect pulls, nothing pulled while disconnected.
+ $observer->ping();
+ delay(0.3);
+ self::assertCount(2, $pulls, 'A pull request was sent while disconnected.');
+
+ $publisher->publish("{$subject}.x", new Message('second'));
+ self::waitUntil(static fn(): bool => \count($received) === 2);
+ self::assertSame(['first', 'second'], $received->getArrayCopy());
+ self::assertFalse($subscription->completed());
+
+ $subscription->stop();
+ $stream->delete();
+ $restarted->kill();
+ } finally {
+ EventLoop::setErrorHandler($errorHandler);
+ $observer->stop();
+ $nc->stop();
+ $proxy->kill();
+ }
+ }
+
+ /**
+ * @param \Closure(): bool $condition
+ */
+ private static function waitUntil(\Closure $condition): void
+ {
+ $deadline = microtime(true) + 5;
+
+ while (!$condition()) {
+ if (microtime(true) > $deadline) {
+ self::fail('The condition was not met in time.');
+ }
+
+ delay(0.01);
+ }
+ }
+
public function testStopFromHandlerDoesNotPullAfterwards(): void
{
$nc = $this->client();
diff --git a/tests/NatsTestCase.php b/tests/NatsTestCase.php
index 465cc4e..daedeaf 100644
--- a/tests/NatsTestCase.php
+++ b/tests/NatsTestCase.php
@@ -90,16 +90,23 @@ final protected function clientWithUrls(array $urls): Client
*
* @param non-empty-string $name
*/
- final protected function clientNamed(string $name): Client
- {
+ final protected function clientNamed(
+ string $name,
+ bool $allowReconnect = true,
+ float $reconnectWait = 0.1,
+ bool $verbose = false,
+ ): Client {
$config = $this->config();
return $this->clientWithConfig(new Config(
urls: $config->urls,
+ verbose: $verbose,
user: $config->user,
password: $config->password,
noResponders: $config->noResponders,
clientName: $name,
+ allowReconnect: $allowReconnect,
+ reconnectWait: $reconnectWait,
));
}
@@ -110,12 +117,41 @@ final protected function clientNamed(string $name): Client
* @param non-empty-string $name
*/
final protected function kick(string $name): void
+ {
+ $this->requestServer('KICK', $name);
+ }
+
+ /**
+ * Sends the client connected under the given name the lame duck mode notification, as if the server were going
+ * to shut down; the server itself keeps running.
+ *
+ * @param non-empty-string $name
+ */
+ final protected function lameDuck(string $name): void
+ {
+ $this->requestServer('LDM', $name);
+ }
+
+ /**
+ * @param non-empty-string $name
+ * @return ?int the server-side id of the connection of the client connected under the given name
+ */
+ final protected function connectionId(string $name): ?int
+ {
+ return $this->connectionNamed($name)['cid'] ?? null;
+ }
+
+ /**
+ * @param non-empty-string $request
+ * @param non-empty-string $name
+ */
+ private function requestServer(string $request, string $name): void
{
$connection = $this->connectionNamed($name) ?? self::fail("The client '{$name}' is not connected.");
self::decodeServerResponse(
$this->sysClient()->request(
- "\$SYS.REQ.SERVER.{$connection['server']}.KICK",
+ "\$SYS.REQ.SERVER.{$connection['server']}.{$request}",
new Message(json_encode(['cid' => $connection['cid']], JSON_THROW_ON_ERROR)),
new TimeoutCancellation(5),
),
diff --git a/tests/PausableProxy.php b/tests/PausableProxy.php
index a03f17e..cc3271c 100644
--- a/tests/PausableProxy.php
+++ b/tests/PausableProxy.php
@@ -26,6 +26,9 @@ final class PausableProxy
/** @var ?array{non-empty-string, DeferredFuture} */
private ?array $pauseBefore = null;
+ /** @var list */
+ private array $sockets = [];
+
/**
* @param non-empty-string $address
*/
@@ -36,10 +39,11 @@ private function __construct(
/**
* @param non-empty-string $upstream
+ * @param non-empty-string $listen
*/
- public static function start(string $upstream): self
+ public static function start(string $upstream, string $listen = '127.0.0.1:0'): self
{
- $server = listen('127.0.0.1:0');
+ $server = listen($listen);
$address = $server->getAddress()->toString();
\assert($address !== '');
@@ -52,6 +56,8 @@ public static function start(string $upstream): self
): void {
while (($client = $server->accept()) !== null) {
$nats = connect($upstream);
+ $proxy->sockets[] = $client;
+ $proxy->sockets[] = $nats;
$future = async(static fn() => $proxy->forward($client, $nats, pausable: true));
$future->ignore();
@@ -95,6 +101,27 @@ public function resume(): void
$paused?->complete();
}
+ /**
+ * Closes the connections passing through the proxy, as if the server went away; new ones are still accepted.
+ */
+ public function disconnect(): void
+ {
+ [$sockets, $this->sockets] = [$this->sockets, []];
+
+ foreach ($sockets as $socket) {
+ $socket->close();
+ }
+ }
+
+ /**
+ * Stops accepting and closes the connections: the server is down until a proxy is started on the same address again.
+ */
+ public function kill(): void
+ {
+ $this->server->close();
+ $this->disconnect();
+ }
+
public function stop(): void
{
$this->resume();
diff --git a/tests/ReconnectTest.php b/tests/ReconnectTest.php
new file mode 100644
index 0000000..0be90d3
--- /dev/null
+++ b/tests/ReconnectTest.php
@@ -0,0 +1,650 @@
+ */
+ private array $proxies = [];
+
+ /** @var list */
+ private array $clients = [];
+
+ /** @var list */
+ private array $files = [];
+
+ #[\Override]
+ protected function tearDown(): void
+ {
+ foreach ($this->clients as $client) {
+ $client->stop();
+ }
+
+ foreach ($this->proxies as $proxy) {
+ $proxy->kill();
+ }
+
+ foreach ($this->files as $file) {
+ @unlink($file);
+ }
+
+ parent::tearDown();
+ }
+
+ public function testSubscriptionsAndRequestsSurviveServerSideDisconnect(): void
+ {
+ $name = Id\generateUniqueId();
+ $client = $this->clientNamed($name);
+ /** @var \ArrayObject $events */
+ $events = new \ArrayObject();
+ $reconnected = $this->watch($client, $events);
+ $subject = Id\generateUniqueId();
+
+ /** @var \ArrayObject $received */
+ $received = new \ArrayObject();
+ $subscription = $client->subscribe("{$subject}.*", static function (Delivery $delivery) use ($received): void {
+ $received[] = (string) $delivery->message->payload;
+ });
+
+ $responder = $this->otherClient();
+ $responder->subscribe("{$subject}.rpc", static function (Delivery $delivery): void {
+ $delivery->reply(new Message('pong'));
+ });
+ $responder->ping();
+
+ // The request inbox is subscribed before the disconnect.
+ self::assertSame('pong', $client->request("{$subject}.rpc", new Message('ping'), timeout: 5)->message->payload);
+
+ $cid = $this->connectionId($name);
+ $this->kick($name);
+ $reconnected->await(new TimeoutCancellation(5));
+
+ self::assertNotSame($cid, $this->connectionId($name));
+ self::assertSame(['disconnect', 'reconnect'], $events->getArrayCopy());
+ self::assertFalse($subscription->completed());
+
+ // The same subscription and the same request inbox, restored on the new connection.
+ $client->ping();
+ $responder->publish("{$subject}.event", new Message('after'));
+ self::assertSame('pong', $client->request("{$subject}.rpc", new Message('ping'), timeout: 5)->message->payload);
+ self::assertContains('after', $received->getArrayCopy());
+ }
+
+ public function testPublishesAreBufferedWhileReconnecting(): void
+ {
+ $proxy = $this->proxy();
+ $client = $this->clientVia($proxy, reconnectWait: 0.1);
+ $disconnected = new DeferredFuture();
+ $client->onDisconnect(static fn() => $disconnected->complete());
+ $reconnected = $this->watch($client);
+ $subject = Id\generateUniqueId();
+
+ $observer = $this->otherClient();
+ $deliveries = $observer->subscribeIterator($subject);
+ $observer->ping();
+ $client->connect();
+
+ $proxy->kill();
+ $disconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ // Nothing to send them to: they wait for the reconnect, in order.
+ foreach (['one', 'two', 'three'] as $payload) {
+ $client->publish($subject, new Message($payload));
+ }
+
+ $this->restart($proxy);
+ $reconnected->await(new TimeoutCancellation(5));
+
+ $received = [];
+ foreach ($deliveries as $delivery) {
+ $received[] = (string) $delivery->message->payload;
+
+ if (\count($received) === 3) {
+ $deliveries->stop();
+ }
+ }
+
+ self::assertSame(['one', 'two', 'three'], $received);
+ }
+
+ public function testMissedPongsReportStaleConnection(): void
+ {
+ $proxy = $this->proxy();
+ $config = $this->config();
+ $client = $this->track(new Client(new Config(
+ urls: [$proxy->address],
+ connectionTimeout: 1,
+ user: $config->user,
+ password: $config->password,
+ ping: 100,
+ maxPings: 1,
+ reconnectWait: 0.1,
+ )));
+
+ /** @var DeferredFuture\Throwable> $disconnected */
+ $disconnected = new DeferredFuture();
+ $client->onDisconnect(static function (?\Throwable $reason) use ($disconnected): void {
+ if (!$disconnected->isComplete()) {
+ $disconnected->complete($reason);
+ }
+ });
+ $reconnected = $this->watch($client);
+ $client->connect();
+
+ // The PINGs no longer reach the server, so no PONG comes back.
+ $proxy->pause();
+
+ self::assertInstanceOf(StaleConnection::class, $disconnected->getFuture()->await(new TimeoutCancellation(5)));
+
+ $proxy->resume();
+ $reconnected->await(new TimeoutCancellation(5));
+ }
+
+ public function testReconnectWaitsUntilServerHasRestoredSubscriptions(): void
+ {
+ $proxy = $this->proxy();
+ $client = $this->clientVia($proxy, reconnectWait: 0.1);
+ $disconnected = new DeferredFuture();
+ $client->onDisconnect(static fn() => $disconnected->complete());
+ $reconnected = $this->watch($client);
+ $subject = Id\generateUniqueId();
+ $deliveries = $client->subscribeIterator($subject);
+ $client->ping();
+
+ $proxy->kill();
+ $disconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ // The new connection is established, but the restored SUB (and the PING after it) is held back.
+ $restarted = $this->restart($proxy);
+ $held = $restarted->pauseBefore('SUB ');
+ $held->await(new TimeoutCancellation(5));
+ delay(0.3);
+ self::assertFalse($reconnected->isComplete(), 'The server has not seen the subscriptions yet.');
+
+ $restarted->resume();
+ $reconnected->await(new TimeoutCancellation(5));
+
+ // The reconnect callbacks promise active subscriptions: a message sent right away is not lost.
+ $this->otherClient()->publish($subject, new Message('right after the reconnect'));
+
+ foreach ($deliveries as $delivery) {
+ self::assertSame('right after the reconnect', $delivery->message->payload);
+ $deliveries->stop();
+ }
+ }
+
+ public function testReconnectConfirmsAllowedSubscriptionsAfterForbiddenOne(): void
+ {
+ $proxy = $this->proxy();
+
+ // May subscribe only under "allowed.>"; verbose, so the forbidden SUB fails the restore's execute at once.
+ $client = $this->track(new Client(new Config(
+ urls: [$proxy->address],
+ verbose: true,
+ user: 'restricted',
+ password: 'Pswd1',
+ reconnectWait: 0.1,
+ )));
+ $disconnected = new DeferredFuture();
+ $client->onDisconnect(static fn() => $disconnected->complete());
+ $reconnected = $this->watch($client);
+ $client->connect();
+
+ $proxy->kill();
+ $disconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ // Both are buffered while reconnecting and sent in this order by the restore.
+ $id = Id\generateUniqueId();
+ $forbidden = $client->subscribe("forbidden.{$id}", static fn() => null);
+ $allowed = $client->subscribeIterator("allowed.{$id}");
+
+ // The forbidden SUB reaches the server and is rejected, the allowed one (and the PING after it) is held back.
+ $restarted = $this->restart($proxy);
+ $restarted->pauseBefore('SUB allowed.')->await(new TimeoutCancellation(5));
+
+ delay(0.3);
+ self::assertFalse($reconnected->isComplete(), 'The server has not seen the allowed subscription yet.');
+
+ $restarted->resume();
+ $reconnected->await(new TimeoutCancellation(5));
+
+ // The forbidden one fails alone.
+ try {
+ $forbidden->awaitCompletion(new TimeoutCancellation(5));
+ self::fail('The subscription is forbidden, it must fail.');
+ } catch (PermissionViolated) {
+ }
+
+ $this->otherClient()->publish("allowed.{$id}", new Message('right after the reconnect'));
+
+ foreach ($allowed as $delivery) {
+ self::assertSame('right after the reconnect', $delivery->message->payload);
+ $allowed->stop();
+ }
+ }
+
+ public function testConnectWaitsForReconnect(): void
+ {
+ $proxy = $this->proxy();
+ $client = $this->clientVia($proxy, reconnectWait: 0.1);
+ $disconnected = new DeferredFuture();
+ $client->onDisconnect(static fn() => $disconnected->complete());
+ $client->connect();
+
+ $proxy->kill();
+ $disconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ try {
+ $client->connect(new TimeoutCancellation(0.2));
+ self::fail('The server is down, connect must not return.');
+ } catch (CancelledException) {
+ }
+
+ $this->restart($proxy);
+ $client->connect(new TimeoutCancellation(5));
+ self::assertGreaterThan(0, $client->ping(new TimeoutCancellation(5)));
+ }
+
+ public function testPublishFailsOnceReconnectBufferIsFull(): void
+ {
+ $proxy = $this->proxy();
+ $client = $this->clientVia($proxy, reconnectBufferSize: 64);
+ $disconnected = new DeferredFuture();
+ $client->onDisconnect(static fn() => $disconnected->complete());
+ $client->connect();
+
+ $proxy->kill();
+ $disconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ $client->publish('events.small', new Message('fits'));
+
+ self::expectException(ReconnectBufferExceeded::class);
+ $client->publish('events.large', new Message(str_repeat('x', 64)));
+ }
+
+ public function testVerbosePublishAndPingWaitForReconnect(): void
+ {
+ $proxy = $this->proxy();
+ $client = $this->clientVia($proxy, reconnectWait: 0.1, verbose: true);
+ $disconnected = new DeferredFuture();
+ $client->onDisconnect(static fn() => $disconnected->complete());
+ $subject = Id\generateUniqueId();
+
+ $observer = $this->otherClient();
+ $deliveries = $observer->subscribeIterator($subject);
+ $observer->ping();
+ $client->connect();
+
+ $proxy->kill();
+ $disconnected->getFuture()->await(new TimeoutCancellation(5));
+
+ // A verbose publish is acknowledged by the server, so it cannot be buffered.
+ $publish = async($client->publish(...), $subject, new Message('acknowledged'));
+ $ping = async($client->ping(...));
+ delay(0.3);
+ self::assertFalse($publish->isComplete());
+ self::assertFalse($ping->isComplete());
+
+ $this->restart($proxy);
+ $publish->await(new TimeoutCancellation(5));
+ $ping->await(new TimeoutCancellation(5));
+
+ foreach ($deliveries as $delivery) {
+ self::assertSame('acknowledged', $delivery->message->payload);
+ $deliveries->stop();
+ }
+ }
+
+ public function testAutoUnsubscribeLimitSurvivesReconnect(): void
+ {
+ $name = Id\generateUniqueId();
+ $client = $this->clientNamed($name);
+ $reconnected = $this->watch($client);
+ $subject = Id\generateUniqueId();
+ $publisher = $this->otherClient();
+
+ /** @var \ArrayObject $received */
+ $received = new \ArrayObject();
+ $subscription = $client->subscribe($subject, static function (Delivery $delivery) use ($received): void {
+ $received[] = (string) $delivery->message->payload;
+ }, max: 3);
+ $client->ping();
+
+ $publisher->publish($subject, new Message('1'));
+ $this->waitFor(static fn(): bool => \count($received) === 1);
+
+ $this->kick($name);
+ $reconnected->await(new TimeoutCancellation(5));
+ $client->ping();
+
+ // Two messages are left of the limit: the restored subscription is removed by the server after them.
+ foreach (['2', '3', '4', '5'] as $payload) {
+ $publisher->publish($subject, new Message($payload));
+ }
+
+ $subscription->awaitCompletion(new TimeoutCancellation(5));
+ self::assertSame(['1', '2', '3'], $received->getArrayCopy());
+ }
+
+ public function testClientGivesUpOnceServersUseUpTheirReconnects(): void
+ {
+ $proxy = $this->proxy();
+ $client = $this->clientVia($proxy, reconnectWait: 0.05, maxReconnects: 2);
+
+ /** @var DeferredFuture\Throwable> $closed */
+ $closed = new DeferredFuture();
+ $client->onClose(static fn(?\Throwable $reason) => $closed->complete($reason));
+ $errors = new \ArrayObject();
+ $client->onError(static fn(\Throwable $e) => $errors->append($e));
+ $subscription = $client->subscribe(Id\generateUniqueId(), static fn() => null);
+
+ $proxy->kill();
+
+ $reason = $closed->getFuture()->await(new TimeoutCancellation(5));
+ self::assertInstanceOf(ConnectionWasClosed::class, $reason);
+ self::assertStringContainsString('no server is left', $reason->getMessage());
+
+ try {
+ $subscription->awaitCompletion(new TimeoutCancellation(1));
+ self::fail('The client has given up reconnecting, the subscription must fail.');
+ } catch (ConnectionWasClosed $e) {
+ self::assertSame($reason, $e);
+ }
+
+ // Running out of servers is not a server error.
+ self::assertCount(0, $errors);
+ }
+
+ #[RequiresPhpExtension('sodium')]
+ public function testRepeatedAuthorizationErrorStopsReconnecting(): void
+ {
+ $name = Id\generateUniqueId();
+ $file = $this->file(self::SEED);
+ $client = $this->track(new Client(new Config(
+ urls: [$this->natsAddress()],
+ clientName: $name,
+ nkeyFile: $file,
+ reconnectWait: 0.05,
+ maxReconnects: null,
+ )));
+
+ /** @var DeferredFuture\Throwable> $closed */
+ $closed = new DeferredFuture();
+ $client->onClose(static fn(?\Throwable $reason) => $closed->complete($reason));
+ $client->connect();
+
+ // The credentials are revoked: every attempt is rejected the same way, retrying forever makes no sense.
+ file_put_contents($file, self::UNKNOWN_SEED);
+ $this->kick($name);
+
+ self::assertInstanceOf(AuthorizationViolation::class, $closed->getFuture()->await(new TimeoutCancellation(5)));
+ }
+
+ public function testLameDuckModeMovesToAnotherServer(): void
+ {
+ $name = Id\generateUniqueId();
+ $proxy = $this->proxy();
+ $config = $this->config();
+
+ // Both urls lead to the same server: the client starts with the first one and has the second one to move to.
+ $client = $this->track(new Client(new Config(
+ urls: [$this->natsAddress(), $proxy->address],
+ user: $config->user,
+ password: $config->password,
+ clientName: $name,
+ reconnectWait: 10,
+ noRandomize: true,
+ )));
+ $lameDuck = new DeferredFuture();
+ $client->onLameDuckMode(static fn() => $lameDuck->complete());
+ $reconnected = $this->watch($client);
+ $subject = Id\generateUniqueId();
+ $deliveries = $client->subscribeIterator($subject);
+ $client->ping();
+ $cid = $this->connectionId($name);
+
+ $this->lameDuck($name);
+
+ $lameDuck->getFuture()->await(new TimeoutCancellation(5));
+ // Right away, without the 10 seconds pause between the rounds of the reconnects.
+ $reconnected->await(new TimeoutCancellation(2));
+ self::assertNotSame($cid, $this->connectionId($name));
+
+ $client->publish($subject, new Message('moved'));
+
+ foreach ($deliveries as $delivery) {
+ self::assertSame('moved', $delivery->message->payload);
+ $deliveries->stop();
+ }
+ }
+
+ public function testLameDuckModeWithNowhereToMoveOnlyNotifies(): void
+ {
+ $name = Id\generateUniqueId();
+ $client = $this->clientNamed($name);
+ $lameDuck = new DeferredFuture();
+ $client->onLameDuckMode(static fn() => $lameDuck->complete());
+ /** @var \ArrayObject $events */
+ $events = new \ArrayObject();
+ $this->watch($client, $events);
+ $client->connect();
+ $cid = $this->connectionId($name);
+
+ $this->lameDuck($name);
+
+ $lameDuck->getFuture()->await(new TimeoutCancellation(5));
+ delay(0.3);
+ self::assertSame($cid, $this->connectionId($name));
+ self::assertCount(0, $events);
+ }
+
+ public function testReconnectsToServerDiscoveredFromCluster(): void
+ {
+ $cluster = $this->clusterUrls();
+ // Only the first server is configured, behind a proxy: the other one is known from its INFO only.
+ $proxy = $this->proxy($cluster[0]);
+ $client = $this->track(new Client(new Config(urls: [$proxy->address], reconnectWait: 0.1, noRandomize: true)));
+ $reconnected = $this->watch($client);
+ $subject = Id\generateUniqueId();
+ $deliveries = $client->subscribeIterator($subject);
+ $client->ping();
+
+ $proxy->kill();
+ $reconnected->await(new TimeoutCancellation(5));
+
+ // The publisher is connected to the second server of the cluster: the message is routed to the restored subscription.
+ $publisher = $this->track(new Client(new Config(urls: [$cluster[1] ?? self::fail('The cluster has a single server.')])));
+ $publisher->publish($subject, new Message('routed'));
+
+ foreach ($deliveries as $delivery) {
+ self::assertSame('routed', $delivery->message->payload);
+ $deliveries->stop();
+ }
+ }
+
+ public function testServersAnnouncedAfterConnectAreReported(): void
+ {
+ $server = listen('127.0.0.1:0');
+ $address = $server->getAddress()->toString();
+ \assert($address !== '');
+
+ $peer = async(static function () use ($server): void {
+ $peer = $server->accept() ?? self::fail('The client has not connected.');
+ $peer->write(new Protocol\ServerInfo(version: '2.12.0', serverId: 'fake', headers: true)->encode());
+
+ $buffer = '';
+
+ while (!str_contains($buffer, "PING\r\n")) {
+ $buffer .= $peer->read() ?? self::fail('The client has closed the connection.');
+ }
+
+ // A server joined the cluster after the client had connected.
+ $peer->write("PONG\r\n");
+ $peer->write(new Protocol\ServerInfo(version: '2.12.0', serverId: 'fake', headers: true, connectUrls: ['127.0.0.1:4333'])->encode());
+ $peer->read();
+ });
+ $peer->ignore();
+
+ $client = $this->track(new Client(new Config(urls: [$address], connectionTimeout: 5)));
+
+ /** @var DeferredFuture> $discovered */
+ $discovered = new DeferredFuture();
+ $client->onServersDiscovered(static fn(array $urls) => $discovered->complete($urls));
+ $client->connect();
+
+ $urls = $discovered->getFuture()->await(new TimeoutCancellation(5));
+ $expected = [$address, '127.0.0.1:4333'];
+ sort($urls);
+ sort($expected);
+ self::assertSame($expected, $urls);
+
+ $server->close();
+ }
+
+ /**
+ * @param ?\ArrayObject $events
+ * @return Future completed on the first reconnect
+ */
+ private function watch(Client $client, ?\ArrayObject $events = null): Future
+ {
+ /** @var DeferredFuture $reconnected */
+ $reconnected = new DeferredFuture();
+
+ $client->onDisconnect(static fn() => $events?->append('disconnect'));
+ $client->onReconnect(static function () use (
+ $reconnected,
+ $events,
+ ): void {
+ $events?->append('reconnect');
+
+ if (!$reconnected->isComplete()) {
+ $reconnected->complete();
+ }
+ });
+
+ return $reconnected->getFuture();
+ }
+
+ /**
+ * @param ?non-empty-string $upstream
+ */
+ private function proxy(?string $upstream = null): PausableProxy
+ {
+ return $this->proxies[] = PausableProxy::start($upstream ?? $this->natsAddress());
+ }
+
+ /**
+ * Brings the server behind a killed proxy back on the same address.
+ */
+ private function restart(PausableProxy $proxy): PausableProxy
+ {
+ return $this->proxies[] = PausableProxy::start($this->natsAddress(), $proxy->address);
+ }
+
+ /**
+ * @param ?non-negative-int $maxReconnects
+ * @param non-negative-int $reconnectBufferSize
+ */
+ private function clientVia(
+ PausableProxy $proxy,
+ float $reconnectWait = 2.0,
+ ?int $maxReconnects = 60,
+ int $reconnectBufferSize = 8 * 1024 * 1024,
+ bool $verbose = false,
+ ): Client {
+ $config = $this->config();
+
+ return $this->track(new Client(new Config(
+ urls: [$proxy->address],
+ verbose: $verbose,
+ user: $config->user,
+ password: $config->password,
+ noResponders: true,
+ maxReconnects: $maxReconnects,
+ reconnectWait: $reconnectWait,
+ reconnectBufferSize: $reconnectBufferSize,
+ )));
+ }
+
+ private function otherClient(): Client
+ {
+ return $this->track(new Client($this->config()));
+ }
+
+ private function track(Client $client): Client
+ {
+ return $this->clients[] = $client;
+ }
+
+ /**
+ * @param \Closure(): bool $condition
+ */
+ private function waitFor(\Closure $condition): void
+ {
+ $deadline = microtime(true) + 5;
+
+ while (!$condition()) {
+ if (microtime(true) > $deadline) {
+ self::fail('The condition was not met in time.');
+ }
+
+ delay(0.01);
+ }
+ }
+
+ /**
+ * @return non-empty-string
+ */
+ private function file(string $contents): string
+ {
+ $file = tempnam(sys_get_temp_dir(), 'nats');
+ \assert($file !== false);
+
+ file_put_contents($file, $contents);
+
+ return $this->files[] = $file;
+ }
+
+ /**
+ * @return non-empty-list the `host:port` of the servers of the cluster
+ */
+ private function clusterUrls(): array
+ {
+ $dsn = getenv('THESIS_NATS_CLUSTER_DSN');
+ if (!\is_string($dsn) || $dsn === '') {
+ self::markTestSkipped('THESIS_NATS_CLUSTER_DSN must be set.');
+ }
+
+ return Config::fromURI($dsn)->urls;
+ }
+}