diff --git a/compose.yaml b/compose.yaml index 56e58e7..e8d8f18 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,8 +6,22 @@ services: environment: IN_CONTAINER: true HISTFILE: /workspace/var/.docker_history + THESIS_PGMQ_DSN: host=pgmq user=postgres password=postgres volumes: - .:/workspace - ~/.composer/cache:/composer/cache tty: true init: true + depends_on: + pgmq: + condition: service_healthy + + pgmq: + image: ghcr.io/pgmq/pg18-pgmq:v1.11.1 + environment: + POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 2s + timeout: 5s + retries: 20 diff --git a/src/MessageBus/Handling/Handlers.php b/src/MessageBus/Handling/Handlers.php index b55ba5f..9aacd48 100644 --- a/src/MessageBus/Handling/Handlers.php +++ b/src/MessageBus/Handling/Handlers.php @@ -54,7 +54,6 @@ public function with(string $messageClass, callable $handler, string $qualifier } $copy = clone $this; - /** @phpstan-ignore assign.propertyType */ $copy->handlers[$messageClass][$qualifier] = $handler; return $copy; diff --git a/src/MessageBus/Pgmq/PgmqTransport.php b/src/MessageBus/Pgmq/PgmqTransport.php index 66bdca6..4489662 100644 --- a/src/MessageBus/Pgmq/PgmqTransport.php +++ b/src/MessageBus/Pgmq/PgmqTransport.php @@ -60,7 +60,61 @@ public function createQueue(string $name): void Pgmq\createExtension($this->postgres); Pgmq\createQueue($this->postgres, $name); + // Consumers are woken up by pgmq_dispatch_v1() itself (see below), not by pgmq's insert trigger. The trigger + // updates the queue's row in pgmq.notify_insert_throttle on every insert, and that row lock is held until the + // sender commits: concurrent senders to one queue wait for each other, and two transactions sending to two + // queues in opposite order deadlock. Undo an earlier setup, without stalling a live queue: + // + // * the throttle row goes first: without it the trigger updates nothing and locks nothing. Deleting it waits + // only for senders holding that row, not for the whole table; + // * the trigger itself is dropped with a short lock_timeout: DROP TRIGGER takes an ACCESS EXCLUSIVE lock that + // waits for every open transaction on the queue and blocks its readers and senders meanwhile. If the queue is + // busy, it is skipped with a warning; the trigger is harmless without its row, and the next setup retries. + // + // Both steps only read the catalog when there is nothing to undo, so repeated setups run no DDL. + $this->postgres->query(\sprintf( + <<<'SQL' + DO $undo$ + DECLARE + v_queue text := %s; + v_lock_timeout text := current_setting('lock_timeout'); + BEGIN + PERFORM set_config('lock_timeout', '2s', true); + + BEGIN + DELETE FROM pgmq.notify_insert_throttle WHERE queue_name = v_queue; + EXCEPTION WHEN lock_not_available THEN + RAISE WARNING 'pgmq queue "%%" is busy: its insert-notify throttle row is left for the next setup', v_queue; + END; + + IF EXISTS ( + SELECT 1 + FROM pg_trigger tg + JOIN pg_class c ON c.oid = tg.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'pgmq' + AND c.relname = pgmq.format_table_name(v_queue, 'q') + AND tg.tgname = 'trigger_notify_queue_insert_listeners' + ) THEN + BEGIN + PERFORM pgmq.disable_notify_insert(v_queue); + EXCEPTION WHEN lock_not_available THEN + RAISE WARNING 'pgmq queue "%%" is busy: its insert-notify trigger is left for the next setup', v_queue; + END; + END IF; + + PERFORM set_config('lock_timeout', v_lock_timeout, true); + END + $undo$ + SQL, + $this->postgres->quoteLiteral($name), + )); + $this->postgres->query('create schema if not exists thesis_message_bus;'); + // Wakes consumers with pg_notify() on the channel pgmq's insert trigger used, so listeners need no change. + // No row is locked: a notification is delivered on commit (never for a rolled-back send), and identical + // notifications of one transaction are folded into one — a natural throttle per queue and transaction. + // Delayed messages are not announced: they are invisible until the delay passes, and the poll picks them up. $this->postgres->query( <<<'SQL' create or replace function thesis_message_bus.pgmq_dispatch_v1( @@ -75,12 +129,28 @@ public function createQueue(string $name): void as $$ declare index integer; + target text; begin for index in 1..coalesce(array_length(operations, 1), 0) loop if operations[index] = 'send' then return query select id from pgmq.send(addresses[index], payloads[index], headers[index], delays[index]) as id; + + if delays[index] <= 0 then + perform pg_notify('pgmq.' || pgmq.format_table_name(addresses[index], 'q') || '.INSERT', null); + end if; elsif operations[index] = 'publish' then return next pgmq.send_topic(addresses[index], payloads[index], headers[index], delays[index])::bigint; + + if delays[index] <= 0 then + -- The same queues pgmq.send_topic() has just sent to. + for target in + select distinct tb.queue_name + from pgmq.topic_bindings tb + where addresses[index] ~ tb.compiled_regex + loop + perform pg_notify('pgmq.' || pgmq.format_table_name(target, 'q') || '.INSERT', null); + end loop; + end if; else raise exception 'unknown message bus operation: %', operations[index]; end if; @@ -189,14 +259,16 @@ public function startConsumer(string $queue, ConsumerHandler $handler): PgmqCons $timeoutWatcher, new ChannelWatcher( $polls, - $this->postgres->listen($queue->enableNotifyInsert()), + // pgmq_dispatch_v1() notifies this channel; see createQueue() for why the insert trigger is not used. Lower case: + // the SQL side names the channel after the queue table, and pgmq lower-cases table names. + $this->postgres->listen(Pgmq\channelName(strtolower($queue->name))), $timeoutWatcher, ), ]); $iterator = $polls->iterate(); - EventLoop::queue(function () use ($queue, $handler, $completion, $watcher, $iterator): void { + EventLoop::queue(function () use ($queue, $handler, $completion, $watcher, $iterator, $polls): void { $watcher->watch(); try { @@ -238,6 +310,12 @@ public function startConsumer(string $queue, ConsumerHandler $handler): PgmqCons } finally { EventLoop::cancel($updateVisibilityId); } + + // A full batch means the queue may hold more messages. Inserts that already happened will not + // notify again, so waiting for the next signal would leave the backlog idle for up to $pollInterval. + if (\count($messages) === $this->batchSize && !$polls->isComplete()) { + $polls->pushAsync(null)->ignore(); + } } $completion->complete(); diff --git a/tests/MessageBus/Pgmq/PgmqDispatchNotifyTest.php b/tests/MessageBus/Pgmq/PgmqDispatchNotifyTest.php new file mode 100644 index 0000000..ed73eaa --- /dev/null +++ b/tests/MessageBus/Pgmq/PgmqDispatchNotifyTest.php @@ -0,0 +1,235 @@ +startConsumer($queue, new SignallingHandler()); + + // A sender keeps its transaction open after sending, as a handler does while it finishes its unit of work. + $first = $postgres->beginTransaction(); + $transport->dispatchInTransaction($first, [self::send($queue)]); + + try { + /** @var Future $second */ + $second = async(static function () use ($transport, $queue): void { + $transport->dispatch([self::send($queue)]); + }); + $second->await(new TimeoutCancellation(1)); + } finally { + $first->rollback(); + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + } + + public function crosswiseSendsToTwoQueuesDoNotDeadlock(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres); + $one = self::createQueue($transport); + $two = self::createQueue($transport); + $consumers = [ + $transport->startConsumer($one, new SignallingHandler()), + $transport->startConsumer($two, new SignallingHandler()), + ]; + + $a = $postgres->beginTransaction(); + $b = $postgres->beginTransaction(); + $failures = []; + + try { + $transport->dispatchInTransaction($a, [self::send($one)]); + $transport->dispatchInTransaction($b, [self::send($two)]); + + $aToTwo = async(static fn() => $transport->dispatchInTransaction($a, [self::send($two)])); + delay(0.1); + $bToOne = async(static fn() => $transport->dispatchInTransaction($b, [self::send($one)])); + + foreach ([$aToTwo, $bToOne] as $future) { + try { + $future->await(new TimeoutCancellation(3)); + } catch (\Throwable $e) { + $failures[] = strtok($e->getMessage(), "\n"); + } + } + + foreach ([$a, $b] as $transaction) { + if ($transaction->isActive()) { + $transaction->commit(); + } + } + } finally { + foreach ($consumers as $consumer) { + $consumer->stop(); + $consumer->awaitCompletion(); + } + Pgmq\dropQueue($postgres, $one); + Pgmq\dropQueue($postgres, $two); + } + + Assert::same($failures, []); + } + + public function publishWakesSubscribedConsumerOnCommitOnly(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10)); + $queue = self::createQueue($transport); + $topic = 'test.' . $queue; + $transport->subscribe($queue, [$topic]); + + $handler = new SignallingHandler(); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // Let the consumer make its initial (empty) read first, so only a notification can deliver the message. + delay(0.2); + $tx = $postgres->beginTransaction(); + $transport->dispatchInTransaction($tx, [self::publish($topic)]); + + // Not committed yet: nothing is announced. + delay(0.5); + Assert::false($handler->received->isComplete()); + + $tx->commit(); + $handler->received->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($handler->received->isComplete()); + } + + public function createQueueDropsTheInsertTriggerOfAnEarlierSetup(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres); + $queue = self::createQueue($transport); + $postgres->execute('SELECT pgmq.enable_notify_insert(?, 30)', [$queue]); + + try { + $transport->createQueue($queue); + + /** @var array{triggers: int, rows: int} $row */ + $row = $postgres->execute( + <<<'SQL' + SELECT + (SELECT count(*)::int FROM pg_trigger tg JOIN pg_class c ON c.oid = tg.tgrelid + WHERE c.relname = pgmq.format_table_name(:queue, 'q') + AND tg.tgname = 'trigger_notify_queue_insert_listeners') AS triggers, + (SELECT count(*)::int FROM pgmq.notify_insert_throttle WHERE queue_name = :queue) AS rows + SQL, + ['queue' => $queue], + )->fetchRow(); + } finally { + Pgmq\dropQueue($postgres, $queue); + } + + Assert::same($row, ['triggers' => 0, 'rows' => 0]); + } + + public function createQueueDoesNotStallABusyQueueWhileUndoingAnEarlierSetup(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres); + $queue = self::createQueue($transport); + $postgres->execute('SELECT pgmq.enable_notify_insert(?, 30)', [$queue]); + + // A long transaction has read the queue and is still open. Its lock does not stop pgmq.create() of the existing + // queue, but DROP TRIGGER (ACCESS EXCLUSIVE) would wait for it, and every reader and sender behind that. + $sender = $postgres->beginTransaction(); + $sender->query(\sprintf('SELECT count(*) FROM pgmq.%s', 'q_' . $queue)); + + try { + $startedAt = microtime(true); + $transport->createQueue($queue); + $took = microtime(true) - $startedAt; + + // The throttle row is gone even though the trigger could not be dropped: a send no longer waits for anyone. + /** @var array{rows: int} $row */ + $row = $postgres->execute('SELECT count(*)::int AS rows FROM pgmq.notify_insert_throttle WHERE queue_name = ?', [$queue])->fetchRow(); + $other = $postgres->beginTransaction(); + $transport->dispatchInTransaction($other, [self::send($queue)]); + $third = async(static fn() => $transport->dispatch([self::send($queue)])); + $third->await(new TimeoutCancellation(1)); + $other->rollback(); + } finally { + $sender->rollback(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($took < 4.0); + Assert::same($row, ['rows' => 0]); + } + + private static function postgres(): PostgresConnection + { + $dsn = getenv('THESIS_PGMQ_DSN'); + + if (!\is_string($dsn) || $dsn === '') { + throw new \LogicException('Set the THESIS_PGMQ_DSN environment variable (see compose.yaml).'); + } + + return new PostgresConnectionPool(PostgresConfig::fromString($dsn)); + } + + /** + * @return non-empty-string + */ + private static function createQueue(PgmqTransport $transport): string + { + $queue = 'q' . bin2hex(random_bytes(8)); + $transport->createQueue($queue); + + return $queue; + } + + /** + * @param non-empty-string $queue + */ + private static function send(string $queue): OutboundEnvelope + { + return new OutboundEnvelope(operation: Operation::Send, address: $queue, payload: '{}', headers: new Headers()); + } + + /** + * @param non-empty-string $topic + */ + private static function publish(string $topic): OutboundEnvelope + { + return new OutboundEnvelope(operation: Operation::Publish, address: $topic, payload: '{}', headers: new Headers()); + } +} diff --git a/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php new file mode 100644 index 0000000..bcca256 --- /dev/null +++ b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php @@ -0,0 +1,198 @@ +beginTransaction(); + $transport->dispatchInTransaction($producer, [self::envelope($queue)]); + + // A worker (re)starts. Recreating the notify trigger takes a table lock that waits for every open producer + // transaction and blocks new inserts meanwhile, so the consumer must not touch it. + /** @var Future $start */ + $start = async(static fn(): Consumer => $transport->startConsumer($queue, new SignallingHandler())); + + try { + $start->await(new TimeoutCancellation(1)); + } finally { + $producer->rollback(); + + $consumer = $start->await(); + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + } + + public function createdQueueWakesConsumerOnInsert(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10)); + $queue = self::createQueue($transport); + + $handler = new SignallingHandler(); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // Let the consumer make its initial (empty) read first, so only a notification can deliver the message. + delay(0.2); + $transport->dispatch([self::envelope($queue)]); + + // Notifications are set up by createQueue(): the insert wakes the consumer long before the poll interval. + $handler->received->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($handler->received->isComplete()); + } + + public function consumerStartRestoresNotificationsLostInCrashRecovery(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10)); + $queue = self::createQueue($transport); + + // pgmq.notify_insert_throttle is UNLOGGED: crash recovery (or a promoted replica) leaves it empty, and the + // trigger notifies only when it finds the queue's row. A consumer (re)start must bring notifications back. + $postgres->execute('DELETE FROM pgmq.notify_insert_throttle WHERE queue_name = ?', [$queue]); + + $handler = new SignallingHandler(); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // Let the consumer make its initial (empty) read first, so only a notification can deliver the message. + delay(0.2); + $transport->dispatch([self::envelope($queue)]); + + $handler->received->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($handler->received->isComplete()); + } + + public function consumerStartRestoresDisabledTrigger(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10)); + $queue = self::createQueue($transport); + $postgres->execute('SELECT pgmq.enable_notify_insert(?, 30)', [$queue]); + $postgres->query(\sprintf('ALTER TABLE pgmq.q_%s DISABLE TRIGGER trigger_notify_queue_insert_listeners', $queue)); + + $handler = new SignallingHandler(); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // Let the consumer make its initial (empty) read first, so only a notification can deliver the message. + delay(0.2); + $transport->dispatch([self::envelope($queue)]); + + $handler->received->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($handler->received->isComplete()); + } + + private static function postgres(): PostgresConnection + { + $dsn = getenv('THESIS_PGMQ_DSN'); + + if (!\is_string($dsn) || $dsn === '') { + throw new \LogicException('Set the THESIS_PGMQ_DSN environment variable (see compose.yaml).'); + } + + return new PostgresConnectionPool(PostgresConfig::fromString($dsn)); + } + + /** + * @return non-empty-string + */ + private static function createQueue(PgmqTransport $transport): string + { + $queue = 'q' . bin2hex(random_bytes(8)); + $transport->createQueue($queue); + + return $queue; + } + + /** + * @param non-empty-string $queue + */ + private static function envelope(string $queue): OutboundEnvelope + { + return new OutboundEnvelope( + operation: Operation::Send, + address: $queue, + payload: '{}', + headers: new Headers(), + ); + } +} + +/** + * @internal + */ +final readonly class SignallingHandler implements ConsumerHandler +{ + /** @var DeferredFuture */ + public DeferredFuture $received; + + public function __construct() + { + $this->received = new DeferredFuture(); + } + + #[\Override] + public function handle(InboundEnvelope $envelope): Disposition + { + if (!$this->received->isComplete()) { + $this->received->complete(); + } + + return Disposition::Ack; + } +} diff --git a/tests/MessageBus/Pgmq/PgmqTransportTest.php b/tests/MessageBus/Pgmq/PgmqTransportTest.php new file mode 100644 index 0000000..84923b3 --- /dev/null +++ b/tests/MessageBus/Pgmq/PgmqTransportTest.php @@ -0,0 +1,122 @@ +dispatch(self::envelopes($queue, 25)); + + $handler = new CountingHandler(25); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // The backlog is already there: no further inserts will wake the consumer up, so a full batch must + // trigger the next read immediately instead of waiting for the poll interval (2 x 10 s for 25 messages). + $handler->done->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::same($handler->handled, 25); + } + + private static function postgres(): PostgresConnection + { + $dsn = getenv('THESIS_PGMQ_DSN'); + + if (!\is_string($dsn) || $dsn === '') { + throw new \LogicException('Set the THESIS_PGMQ_DSN environment variable (see compose.yaml).'); + } + + return new PostgresConnectionPool(PostgresConfig::fromString($dsn)); + } + + /** + * @return non-empty-string + */ + private static function createQueue(PostgresConnection $postgres): string + { + $queue = 'q' . bin2hex(random_bytes(8)); + new PgmqTransport($postgres)->createQueue($queue); + + return $queue; + } + + /** + * @param non-empty-string $queue + * @param positive-int $count + * @return non-empty-list + */ + private static function envelopes(string $queue, int $count): array + { + return array_map( + static fn(int $index): OutboundEnvelope => new OutboundEnvelope( + operation: Operation::Send, + address: $queue, + payload: json_encode(['index' => $index], JSON_THROW_ON_ERROR), + headers: new Headers(), + ), + range(1, $count), + ); + } +} + +/** + * @internal + */ +final class CountingHandler implements ConsumerHandler +{ + public int $handled = 0; + + /** @var DeferredFuture */ + public readonly DeferredFuture $done; + + /** + * @param positive-int $expected + */ + public function __construct( + private readonly int $expected, + ) { + $this->done = new DeferredFuture(); + } + + #[\Override] + public function handle(InboundEnvelope $envelope): Disposition + { + ++$this->handled; + + if ($this->handled === $this->expected) { + $this->done->complete(); + } + + return Disposition::Ack; + } +}