From 3e33964bbca3db6be085c003c7d35d87ba3ab400 Mon Sep 17 00:00:00 2001 From: max Date: Wed, 7 Oct 2026 12:38:02 +0300 Subject: [PATCH 01/10] Pgmq: poll again right after a full batch instead of waiting for the next signal A backlog that is already in the queue produces no further insert notifications, so after reading one batch the consumer slept until the poll interval elapsed: a queue of N messages took ~N/batch * pollInterval (5 s by default) to drain. --- src/MessageBus/Pgmq/PgmqTransport.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/MessageBus/Pgmq/PgmqTransport.php b/src/MessageBus/Pgmq/PgmqTransport.php index 66bdca6..d627fb4 100644 --- a/src/MessageBus/Pgmq/PgmqTransport.php +++ b/src/MessageBus/Pgmq/PgmqTransport.php @@ -196,7 +196,7 @@ public function startConsumer(string $queue, ConsumerHandler $handler): PgmqCons $iterator = $polls->iterate(); - EventLoop::queue(function () use ($queue, $handler, $completion, $watcher, $iterator): void { + EventLoop::queue(function () use ($queue, $handler, $completion, $watcher, $iterator, $polls): void { $watcher->watch(); try { @@ -238,6 +238,12 @@ public function startConsumer(string $queue, ConsumerHandler $handler): PgmqCons } finally { EventLoop::cancel($updateVisibilityId); } + + // A full batch means the queue may hold more messages. Inserts that already happened will not + // notify again, so waiting for the next signal would leave the backlog idle for up to $pollInterval. + if (\count($messages) === $this->batchSize && !$polls->isComplete()) { + $polls->pushAsync(null)->ignore(); + } } $completion->complete(); From 273b5006aa334fbe0f9ac1209022a12201ffe943 Mon Sep 17 00:00:00 2001 From: max Date: Wed, 7 Oct 2026 12:38:11 +0300 Subject: [PATCH 02/10] Add integration tests for PgmqTransport against a real pgmq Adds a pgmq service to compose.yaml (THESIS_PGMQ_DSN) and the first PgmqTransport tests: a backlog must drain without waiting for the poll interval, a consumer start must not wait for open producer transactions, and createQueue() must set up insert notifications. --- compose.yaml | 14 ++ .../MessageBus/Pgmq/PgmqNotifyTriggerTest.php | 144 ++++++++++++++++++ tests/MessageBus/Pgmq/PgmqTransportTest.php | 122 +++++++++++++++ 3 files changed, 280 insertions(+) create mode 100644 tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php create mode 100644 tests/MessageBus/Pgmq/PgmqTransportTest.php diff --git a/compose.yaml b/compose.yaml index 56e58e7..e8d8f18 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,8 +6,22 @@ services: environment: IN_CONTAINER: true HISTFILE: /workspace/var/.docker_history + THESIS_PGMQ_DSN: host=pgmq user=postgres password=postgres volumes: - .:/workspace - ~/.composer/cache:/composer/cache tty: true init: true + depends_on: + pgmq: + condition: service_healthy + + pgmq: + image: ghcr.io/pgmq/pg18-pgmq:v1.11.1 + environment: + POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 2s + timeout: 5s + retries: 20 diff --git a/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php new file mode 100644 index 0000000..8e39528 --- /dev/null +++ b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php @@ -0,0 +1,144 @@ +beginTransaction(); + $transport->dispatchInTransaction($producer, [self::envelope($queue)]); + + // A worker (re)starts. Recreating the notify trigger takes a table lock that waits for every open producer + // transaction and blocks new inserts meanwhile, so the consumer must not touch it. + /** @var Future $start */ + $start = async(static fn(): Consumer => $transport->startConsumer($queue, new SignallingHandler())); + + try { + $start->await(new TimeoutCancellation(1)); + } finally { + $producer->rollback(); + + $consumer = $start->await(); + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + } + + public function createdQueueWakesConsumerOnInsert(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10)); + $queue = self::createQueue($transport); + + $handler = new SignallingHandler(); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // Let the consumer make its initial (empty) read first, so only a notification can deliver the message. + delay(0.2); + $transport->dispatch([self::envelope($queue)]); + + // Notifications are set up by createQueue(): the insert wakes the consumer long before the poll interval. + $handler->received->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($handler->received->isComplete()); + } + + private static function postgres(): PostgresConnection + { + $dsn = getenv('THESIS_PGMQ_DSN'); + + if (!\is_string($dsn) || $dsn === '') { + throw new \LogicException('Set the THESIS_PGMQ_DSN environment variable (see compose.yaml).'); + } + + return new PostgresConnectionPool(PostgresConfig::fromString($dsn)); + } + + /** + * @return non-empty-string + */ + private static function createQueue(PgmqTransport $transport): string + { + $queue = 'q' . bin2hex(random_bytes(8)); + $transport->createQueue($queue); + + return $queue; + } + + /** + * @param non-empty-string $queue + */ + private static function envelope(string $queue): OutboundEnvelope + { + return new OutboundEnvelope( + operation: Operation::Send, + address: $queue, + payload: '{}', + headers: new Headers(), + ); + } +} + +/** + * @internal + */ +final readonly class SignallingHandler implements ConsumerHandler +{ + /** @var DeferredFuture */ + public DeferredFuture $received; + + public function __construct() + { + $this->received = new DeferredFuture(); + } + + #[\Override] + public function handle(InboundEnvelope $envelope): Disposition + { + if (!$this->received->isComplete()) { + $this->received->complete(); + } + + return Disposition::Ack; + } +} diff --git a/tests/MessageBus/Pgmq/PgmqTransportTest.php b/tests/MessageBus/Pgmq/PgmqTransportTest.php new file mode 100644 index 0000000..84923b3 --- /dev/null +++ b/tests/MessageBus/Pgmq/PgmqTransportTest.php @@ -0,0 +1,122 @@ +dispatch(self::envelopes($queue, 25)); + + $handler = new CountingHandler(25); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // The backlog is already there: no further inserts will wake the consumer up, so a full batch must + // trigger the next read immediately instead of waiting for the poll interval (2 x 10 s for 25 messages). + $handler->done->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::same($handler->handled, 25); + } + + private static function postgres(): PostgresConnection + { + $dsn = getenv('THESIS_PGMQ_DSN'); + + if (!\is_string($dsn) || $dsn === '') { + throw new \LogicException('Set the THESIS_PGMQ_DSN environment variable (see compose.yaml).'); + } + + return new PostgresConnectionPool(PostgresConfig::fromString($dsn)); + } + + /** + * @return non-empty-string + */ + private static function createQueue(PostgresConnection $postgres): string + { + $queue = 'q' . bin2hex(random_bytes(8)); + new PgmqTransport($postgres)->createQueue($queue); + + return $queue; + } + + /** + * @param non-empty-string $queue + * @param positive-int $count + * @return non-empty-list + */ + private static function envelopes(string $queue, int $count): array + { + return array_map( + static fn(int $index): OutboundEnvelope => new OutboundEnvelope( + operation: Operation::Send, + address: $queue, + payload: json_encode(['index' => $index], JSON_THROW_ON_ERROR), + headers: new Headers(), + ), + range(1, $count), + ); + } +} + +/** + * @internal + */ +final class CountingHandler implements ConsumerHandler +{ + public int $handled = 0; + + /** @var DeferredFuture */ + public readonly DeferredFuture $done; + + /** + * @param positive-int $expected + */ + public function __construct( + private readonly int $expected, + ) { + $this->done = new DeferredFuture(); + } + + #[\Override] + public function handle(InboundEnvelope $envelope): Disposition + { + ++$this->handled; + + if ($this->handled === $this->expected) { + $this->done->complete(); + } + + return Disposition::Ack; + } +} From 482a848a0fd7e98ecbe03dd0ea13f53c4a162f93 Mon Sep 17 00:00:00 2001 From: max Date: Wed, 7 Oct 2026 13:46:15 +0300 Subject: [PATCH 03/10] Test that a consumer start restores notifications lost in crash recovery pgmq.notify_insert_throttle is UNLOGGED: crash recovery (or a promoted replica) empties it, and the insert trigger then never notifies. The test removes the queue's row and expects a consumer (re)start to bring notifications back. --- .../MessageBus/Pgmq/PgmqNotifyTriggerTest.php | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php index 8e39528..aa898a9 100644 --- a/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php +++ b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php @@ -83,6 +83,34 @@ public function createdQueueWakesConsumerOnInsert(): void Assert::true($handler->received->isComplete()); } + public function consumerStartRestoresNotificationsLostInCrashRecovery(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10)); + $queue = self::createQueue($transport); + + // pgmq.notify_insert_throttle is UNLOGGED: crash recovery (or a promoted replica) leaves it empty, and the + // trigger notifies only when it finds the queue's row. A consumer (re)start must bring notifications back. + $postgres->execute('DELETE FROM pgmq.notify_insert_throttle WHERE queue_name = ?', [$queue]); + + $handler = new SignallingHandler(); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // Let the consumer make its initial (empty) read first, so only a notification can deliver the message. + delay(0.2); + $transport->dispatch([self::envelope($queue)]); + + $handler->received->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($handler->received->isComplete()); + } + private static function postgres(): PostgresConnection { $dsn = getenv('THESIS_PGMQ_DSN'); From b09e52a0c39e8d623a37f6d78465f430dc01981e Mon Sep 17 00:00:00 2001 From: max Date: Wed, 7 Oct 2026 12:42:21 +0300 Subject: [PATCH 04/10] Remove the @phpstan-ignore that no longer matches Recent PHPStan 2.2.x infers the nested property assignment correctly, so the ignore is unmatched and phpstan fails on every cell of the CI matrix (8.4/8.5 x lowest/highest). (cherry picked from commit ec4557352169cfe1a978542690ccd9bfbedfba04) --- src/MessageBus/Handling/Handlers.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/MessageBus/Handling/Handlers.php b/src/MessageBus/Handling/Handlers.php index b55ba5f..9aacd48 100644 --- a/src/MessageBus/Handling/Handlers.php +++ b/src/MessageBus/Handling/Handlers.php @@ -54,7 +54,6 @@ public function with(string $messageClass, callable $handler, string $qualifier } $copy = clone $this; - /** @phpstan-ignore assign.propertyType */ $copy->handlers[$messageClass][$qualifier] = $handler; return $copy; From 020c15231905f8f4978bb742b468eb9a25433788 Mon Sep 17 00:00:00 2001 From: max Date: Wed, 7 Oct 2026 12:42:21 +0300 Subject: [PATCH 05/10] Remove the @phpstan-ignore that no longer matches Recent PHPStan 2.2.x infers the nested property assignment correctly, so the ignore is unmatched and phpstan fails on every cell of the CI matrix (8.4/8.5 x lowest/highest). (cherry picked from commit ec4557352169cfe1a978542690ccd9bfbedfba04) --- src/MessageBus/Handling/Handlers.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/MessageBus/Handling/Handlers.php b/src/MessageBus/Handling/Handlers.php index b55ba5f..9aacd48 100644 --- a/src/MessageBus/Handling/Handlers.php +++ b/src/MessageBus/Handling/Handlers.php @@ -54,7 +54,6 @@ public function with(string $messageClass, callable $handler, string $qualifier } $copy = clone $this; - /** @phpstan-ignore assign.propertyType */ $copy->handlers[$messageClass][$qualifier] = $handler; return $copy; From f470d6764d9096ac2441b46077cf4963cc2bb557 Mon Sep 17 00:00:00 2001 From: max Date: Wed, 7 Oct 2026 14:33:29 +0300 Subject: [PATCH 06/10] Test concurrent consumer starts and a disabled notify trigger - 4 consumers (separate pools, like worker processes) starting at once on a queue without notifications create the trigger exactly once (counted with an event trigger); - a consumer start restores a disabled trigger. --- .../MessageBus/Pgmq/PgmqNotifyTriggerTest.php | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php index aa898a9..663403e 100644 --- a/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php +++ b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php @@ -111,6 +111,111 @@ public function consumerStartRestoresNotificationsLostInCrashRecovery(): void Assert::true($handler->received->isComplete()); } + public function concurrentConsumerStartsCreateTheTriggerOnce(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres); + $queue = self::createQueue($transport); + // Simulates a queue whose notifications are not set up (created before, or lost). + $postgres->execute('SELECT pgmq.disable_notify_insert(?)', [$queue]); + + // A producer keeps an insert into the queue open, so the consumer starts are guaranteed to overlap. + $producer = $postgres->beginTransaction(); + $transport->dispatchInTransaction($producer, [self::envelope($queue)]); + + $failures = []; + $creations = self::countTriggerCreations($postgres, $queue, static function () use ($producer, $queue, &$failures): void { + $starts = []; + + for ($i = 0; $i < 4; ++$i) { + // Each consumer has its own pool, as separate worker processes do. + $own = new PgmqTransport(self::postgres()); + $starts[] = async(static fn(): Consumer => $own->startConsumer($queue, new SignallingHandler())); + } + + delay(0.5); + $producer->rollback(); + + foreach ($starts as $start) { + try { + /** @var Consumer $consumer */ + $consumer = $start->await(new TimeoutCancellation(5)); + $consumer->stop(); + $consumer->awaitCompletion(); + } catch (\Throwable $e) { + $failures[] = $e->getMessage(); + } + } + }); + + Pgmq\dropQueue($postgres, $queue); + + Assert::same($failures, []); + Assert::same($creations, 1); + } + + public function consumerStartRestoresDisabledTrigger(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10)); + $queue = self::createQueue($transport); + $postgres->execute('SELECT pgmq.enable_notify_insert(?, 30)', [$queue]); + $postgres->query(\sprintf('ALTER TABLE pgmq.q_%s DISABLE TRIGGER trigger_notify_queue_insert_listeners', $queue)); + + $handler = new SignallingHandler(); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // Let the consumer make its initial (empty) read first, so only a notification can deliver the message. + delay(0.2); + $transport->dispatch([self::envelope($queue)]); + + $handler->received->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($handler->received->isComplete()); + } + + /** + * Runs $action and returns how many times the notify trigger of the queue was created meanwhile (event trigger). + * + * @param non-empty-string $queue + * @param callable(): void $action + */ + private static function countTriggerCreations(PostgresConnection $postgres, string $queue, callable $action): int + { + $postgres->query('CREATE TABLE IF NOT EXISTS public.thesis_test_ddl (object_identity text)'); + $postgres->query('TRUNCATE public.thesis_test_ddl'); + $postgres->query(<<<'SQL' + CREATE OR REPLACE FUNCTION public.thesis_test_log_ddl() RETURNS event_trigger LANGUAGE plpgsql AS $$ + DECLARE r record; + BEGIN + FOR r IN SELECT * FROM pg_event_trigger_ddl_commands() LOOP + INSERT INTO public.thesis_test_ddl (object_identity) VALUES (r.object_identity); + END LOOP; + END $$ + SQL); + $postgres->query('DROP EVENT TRIGGER IF EXISTS thesis_test_ddl'); + $postgres->query("CREATE EVENT TRIGGER thesis_test_ddl ON ddl_command_end WHEN TAG IN ('CREATE TRIGGER') EXECUTE FUNCTION public.thesis_test_log_ddl()"); + + try { + $action(); + } finally { + $postgres->query('DROP EVENT TRIGGER IF EXISTS thesis_test_ddl'); + } + + /** @var array{n: int} $row */ + $row = $postgres + ->execute('SELECT count(*)::int AS n FROM public.thesis_test_ddl WHERE object_identity LIKE ?', ['% on pgmq.q_' . $queue]) + ->fetchRow(); + + return $row['n']; + } + private static function postgres(): PostgresConnection { $dsn = getenv('THESIS_PGMQ_DSN'); From 4234fc6a837d98aadc3176e4863d1f6c9f41029f Mon Sep 17 00:00:00 2001 From: max Date: Mon, 9 Nov 2026 13:39:11 +0200 Subject: [PATCH 07/10] Pgmq: notify consumers from pgmq_dispatch_v1() instead of the insert trigger --- src/MessageBus/Pgmq/PgmqTransport.php | 53 ++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/MessageBus/Pgmq/PgmqTransport.php b/src/MessageBus/Pgmq/PgmqTransport.php index 66bdca6..717c963 100644 --- a/src/MessageBus/Pgmq/PgmqTransport.php +++ b/src/MessageBus/Pgmq/PgmqTransport.php @@ -60,7 +60,41 @@ public function createQueue(string $name): void Pgmq\createExtension($this->postgres); Pgmq\createQueue($this->postgres, $name); + // Consumers are woken up by pgmq_dispatch_v1() itself (see below), not by pgmq's insert trigger. The trigger + // updates the queue's row in pgmq.notify_insert_throttle on every insert, and that row lock is held until the + // sender commits: concurrent senders to one queue wait for each other, and two transactions sending to two + // queues in opposite order deadlock. Drop it if an earlier version set it up. The check reads the catalog only; + // the DDL (and its table lock) runs once per queue, under the lock pgmq takes to create and drop queues. + $this->postgres->query(\sprintf( + <<<'SQL' + DO $disable$ + BEGIN + PERFORM pgmq.acquire_queue_lock(%1$s); + + IF EXISTS ( + SELECT 1 + FROM pg_trigger tg + JOIN pg_class c ON c.oid = tg.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'pgmq' + AND c.relname = pgmq.format_table_name(%1$s, 'q') + AND tg.tgname = 'trigger_notify_queue_insert_listeners' + ) OR EXISTS ( + SELECT 1 FROM pgmq.notify_insert_throttle t WHERE t.queue_name = %1$s + ) THEN + PERFORM pgmq.disable_notify_insert(%1$s); + END IF; + END + $disable$ + SQL, + $this->postgres->quoteLiteral($name), + )); + $this->postgres->query('create schema if not exists thesis_message_bus;'); + // Wakes consumers with pg_notify() on the channel pgmq's insert trigger used, so listeners need no change. + // No row is locked: a notification is delivered on commit (never for a rolled-back send), and identical + // notifications of one transaction are folded into one — a natural throttle per queue and transaction. + // Delayed messages are not announced: they are invisible until the delay passes, and the poll picks them up. $this->postgres->query( <<<'SQL' create or replace function thesis_message_bus.pgmq_dispatch_v1( @@ -75,12 +109,28 @@ public function createQueue(string $name): void as $$ declare index integer; + target text; begin for index in 1..coalesce(array_length(operations, 1), 0) loop if operations[index] = 'send' then return query select id from pgmq.send(addresses[index], payloads[index], headers[index], delays[index]) as id; + + if delays[index] <= 0 then + perform pg_notify('pgmq.' || pgmq.format_table_name(addresses[index], 'q') || '.INSERT', null); + end if; elsif operations[index] = 'publish' then return next pgmq.send_topic(addresses[index], payloads[index], headers[index], delays[index])::bigint; + + if delays[index] <= 0 then + -- The same queues pgmq.send_topic() has just sent to. + for target in + select distinct tb.queue_name + from pgmq.topic_bindings tb + where addresses[index] ~ tb.compiled_regex + loop + perform pg_notify('pgmq.' || pgmq.format_table_name(target, 'q') || '.INSERT', null); + end loop; + end if; else raise exception 'unknown message bus operation: %', operations[index]; end if; @@ -189,7 +239,8 @@ public function startConsumer(string $queue, ConsumerHandler $handler): PgmqCons $timeoutWatcher, new ChannelWatcher( $polls, - $this->postgres->listen($queue->enableNotifyInsert()), + // pgmq_dispatch_v1() notifies this channel; see createQueue() for why the insert trigger is not used. + $this->postgres->listen(Pgmq\channelName($queue->name)), $timeoutWatcher, ), ]); From beedc4e2b6a913fc63e17436b3f674f05c231194 Mon Sep 17 00:00:00 2001 From: max Date: Mon, 9 Nov 2026 14:09:12 +0200 Subject: [PATCH 08/10] Pgmq: undo an earlier notify-trigger setup without stalling a busy queue; listen on the lower-case channel --- src/MessageBus/Pgmq/PgmqTransport.php | 43 ++++++++++++++++++++------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/src/MessageBus/Pgmq/PgmqTransport.php b/src/MessageBus/Pgmq/PgmqTransport.php index 717c963..49a6fdd 100644 --- a/src/MessageBus/Pgmq/PgmqTransport.php +++ b/src/MessageBus/Pgmq/PgmqTransport.php @@ -63,13 +63,29 @@ public function createQueue(string $name): void // Consumers are woken up by pgmq_dispatch_v1() itself (see below), not by pgmq's insert trigger. The trigger // updates the queue's row in pgmq.notify_insert_throttle on every insert, and that row lock is held until the // sender commits: concurrent senders to one queue wait for each other, and two transactions sending to two - // queues in opposite order deadlock. Drop it if an earlier version set it up. The check reads the catalog only; - // the DDL (and its table lock) runs once per queue, under the lock pgmq takes to create and drop queues. + // queues in opposite order deadlock. Undo an earlier setup, without stalling a live queue: + // + // * the throttle row goes first: without it the trigger updates nothing and locks nothing. Deleting it waits + // only for senders holding that row, not for the whole table; + // * the trigger itself is dropped with a short lock_timeout: DROP TRIGGER takes an ACCESS EXCLUSIVE lock that + // waits for every open transaction on the queue and blocks its readers and senders meanwhile. If the queue is + // busy, it is skipped with a warning; the trigger is harmless without its row, and the next setup retries. + // + // Both steps only read the catalog when there is nothing to undo, so repeated setups run no DDL. $this->postgres->query(\sprintf( <<<'SQL' - DO $disable$ + DO $undo$ + DECLARE + v_queue text := %s; + v_lock_timeout text := current_setting('lock_timeout'); BEGIN - PERFORM pgmq.acquire_queue_lock(%1$s); + PERFORM set_config('lock_timeout', '2s', true); + + BEGIN + DELETE FROM pgmq.notify_insert_throttle WHERE queue_name = v_queue; + EXCEPTION WHEN lock_not_available THEN + RAISE WARNING 'pgmq queue "%%" is busy: its insert-notify throttle row is left for the next setup', v_queue; + END; IF EXISTS ( SELECT 1 @@ -77,15 +93,19 @@ public function createQueue(string $name): void JOIN pg_class c ON c.oid = tg.tgrelid JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'pgmq' - AND c.relname = pgmq.format_table_name(%1$s, 'q') + AND c.relname = pgmq.format_table_name(v_queue, 'q') AND tg.tgname = 'trigger_notify_queue_insert_listeners' - ) OR EXISTS ( - SELECT 1 FROM pgmq.notify_insert_throttle t WHERE t.queue_name = %1$s ) THEN - PERFORM pgmq.disable_notify_insert(%1$s); + BEGIN + PERFORM pgmq.disable_notify_insert(v_queue); + EXCEPTION WHEN lock_not_available THEN + RAISE WARNING 'pgmq queue "%%" is busy: its insert-notify trigger is left for the next setup', v_queue; + END; END IF; + + PERFORM set_config('lock_timeout', v_lock_timeout, true); END - $disable$ + $undo$ SQL, $this->postgres->quoteLiteral($name), )); @@ -239,8 +259,9 @@ public function startConsumer(string $queue, ConsumerHandler $handler): PgmqCons $timeoutWatcher, new ChannelWatcher( $polls, - // pgmq_dispatch_v1() notifies this channel; see createQueue() for why the insert trigger is not used. - $this->postgres->listen(Pgmq\channelName($queue->name)), + // pgmq_dispatch_v1() notifies this channel; see createQueue() for why the insert trigger is not used. Lower case: + // the SQL side names the channel after the queue table, and pgmq lower-cases table names. + $this->postgres->listen(Pgmq\channelName(strtolower($queue->name))), $timeoutWatcher, ), ]); From bc5930ed700db92d66dc479433b5c8037eed93ee Mon Sep 17 00:00:00 2001 From: max Date: Mon, 9 Nov 2026 14:09:19 +0200 Subject: [PATCH 09/10] Test that senders do not serialize or deadlock on insert notifications --- .../Pgmq/PgmqDispatchNotifyTest.php | 235 ++++++++++++++++++ .../MessageBus/Pgmq/PgmqNotifyTriggerTest.php | 79 ------ 2 files changed, 235 insertions(+), 79 deletions(-) create mode 100644 tests/MessageBus/Pgmq/PgmqDispatchNotifyTest.php diff --git a/tests/MessageBus/Pgmq/PgmqDispatchNotifyTest.php b/tests/MessageBus/Pgmq/PgmqDispatchNotifyTest.php new file mode 100644 index 0000000..ed73eaa --- /dev/null +++ b/tests/MessageBus/Pgmq/PgmqDispatchNotifyTest.php @@ -0,0 +1,235 @@ +startConsumer($queue, new SignallingHandler()); + + // A sender keeps its transaction open after sending, as a handler does while it finishes its unit of work. + $first = $postgres->beginTransaction(); + $transport->dispatchInTransaction($first, [self::send($queue)]); + + try { + /** @var Future $second */ + $second = async(static function () use ($transport, $queue): void { + $transport->dispatch([self::send($queue)]); + }); + $second->await(new TimeoutCancellation(1)); + } finally { + $first->rollback(); + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + } + + public function crosswiseSendsToTwoQueuesDoNotDeadlock(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres); + $one = self::createQueue($transport); + $two = self::createQueue($transport); + $consumers = [ + $transport->startConsumer($one, new SignallingHandler()), + $transport->startConsumer($two, new SignallingHandler()), + ]; + + $a = $postgres->beginTransaction(); + $b = $postgres->beginTransaction(); + $failures = []; + + try { + $transport->dispatchInTransaction($a, [self::send($one)]); + $transport->dispatchInTransaction($b, [self::send($two)]); + + $aToTwo = async(static fn() => $transport->dispatchInTransaction($a, [self::send($two)])); + delay(0.1); + $bToOne = async(static fn() => $transport->dispatchInTransaction($b, [self::send($one)])); + + foreach ([$aToTwo, $bToOne] as $future) { + try { + $future->await(new TimeoutCancellation(3)); + } catch (\Throwable $e) { + $failures[] = strtok($e->getMessage(), "\n"); + } + } + + foreach ([$a, $b] as $transaction) { + if ($transaction->isActive()) { + $transaction->commit(); + } + } + } finally { + foreach ($consumers as $consumer) { + $consumer->stop(); + $consumer->awaitCompletion(); + } + Pgmq\dropQueue($postgres, $one); + Pgmq\dropQueue($postgres, $two); + } + + Assert::same($failures, []); + } + + public function publishWakesSubscribedConsumerOnCommitOnly(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10)); + $queue = self::createQueue($transport); + $topic = 'test.' . $queue; + $transport->subscribe($queue, [$topic]); + + $handler = new SignallingHandler(); + $consumer = $transport->startConsumer($queue, $handler); + + try { + // Let the consumer make its initial (empty) read first, so only a notification can deliver the message. + delay(0.2); + $tx = $postgres->beginTransaction(); + $transport->dispatchInTransaction($tx, [self::publish($topic)]); + + // Not committed yet: nothing is announced. + delay(0.5); + Assert::false($handler->received->isComplete()); + + $tx->commit(); + $handler->received->getFuture()->await(new TimeoutCancellation(2)); + } finally { + $consumer->stop(); + $consumer->awaitCompletion(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($handler->received->isComplete()); + } + + public function createQueueDropsTheInsertTriggerOfAnEarlierSetup(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres); + $queue = self::createQueue($transport); + $postgres->execute('SELECT pgmq.enable_notify_insert(?, 30)', [$queue]); + + try { + $transport->createQueue($queue); + + /** @var array{triggers: int, rows: int} $row */ + $row = $postgres->execute( + <<<'SQL' + SELECT + (SELECT count(*)::int FROM pg_trigger tg JOIN pg_class c ON c.oid = tg.tgrelid + WHERE c.relname = pgmq.format_table_name(:queue, 'q') + AND tg.tgname = 'trigger_notify_queue_insert_listeners') AS triggers, + (SELECT count(*)::int FROM pgmq.notify_insert_throttle WHERE queue_name = :queue) AS rows + SQL, + ['queue' => $queue], + )->fetchRow(); + } finally { + Pgmq\dropQueue($postgres, $queue); + } + + Assert::same($row, ['triggers' => 0, 'rows' => 0]); + } + + public function createQueueDoesNotStallABusyQueueWhileUndoingAnEarlierSetup(): void + { + $postgres = self::postgres(); + $transport = new PgmqTransport($postgres); + $queue = self::createQueue($transport); + $postgres->execute('SELECT pgmq.enable_notify_insert(?, 30)', [$queue]); + + // A long transaction has read the queue and is still open. Its lock does not stop pgmq.create() of the existing + // queue, but DROP TRIGGER (ACCESS EXCLUSIVE) would wait for it, and every reader and sender behind that. + $sender = $postgres->beginTransaction(); + $sender->query(\sprintf('SELECT count(*) FROM pgmq.%s', 'q_' . $queue)); + + try { + $startedAt = microtime(true); + $transport->createQueue($queue); + $took = microtime(true) - $startedAt; + + // The throttle row is gone even though the trigger could not be dropped: a send no longer waits for anyone. + /** @var array{rows: int} $row */ + $row = $postgres->execute('SELECT count(*)::int AS rows FROM pgmq.notify_insert_throttle WHERE queue_name = ?', [$queue])->fetchRow(); + $other = $postgres->beginTransaction(); + $transport->dispatchInTransaction($other, [self::send($queue)]); + $third = async(static fn() => $transport->dispatch([self::send($queue)])); + $third->await(new TimeoutCancellation(1)); + $other->rollback(); + } finally { + $sender->rollback(); + Pgmq\dropQueue($postgres, $queue); + } + + Assert::true($took < 4.0); + Assert::same($row, ['rows' => 0]); + } + + private static function postgres(): PostgresConnection + { + $dsn = getenv('THESIS_PGMQ_DSN'); + + if (!\is_string($dsn) || $dsn === '') { + throw new \LogicException('Set the THESIS_PGMQ_DSN environment variable (see compose.yaml).'); + } + + return new PostgresConnectionPool(PostgresConfig::fromString($dsn)); + } + + /** + * @return non-empty-string + */ + private static function createQueue(PgmqTransport $transport): string + { + $queue = 'q' . bin2hex(random_bytes(8)); + $transport->createQueue($queue); + + return $queue; + } + + /** + * @param non-empty-string $queue + */ + private static function send(string $queue): OutboundEnvelope + { + return new OutboundEnvelope(operation: Operation::Send, address: $queue, payload: '{}', headers: new Headers()); + } + + /** + * @param non-empty-string $topic + */ + private static function publish(string $topic): OutboundEnvelope + { + return new OutboundEnvelope(operation: Operation::Publish, address: $topic, payload: '{}', headers: new Headers()); + } +} diff --git a/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php index 663403e..bcca256 100644 --- a/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php +++ b/tests/MessageBus/Pgmq/PgmqNotifyTriggerTest.php @@ -111,49 +111,6 @@ public function consumerStartRestoresNotificationsLostInCrashRecovery(): void Assert::true($handler->received->isComplete()); } - public function concurrentConsumerStartsCreateTheTriggerOnce(): void - { - $postgres = self::postgres(); - $transport = new PgmqTransport($postgres); - $queue = self::createQueue($transport); - // Simulates a queue whose notifications are not set up (created before, or lost). - $postgres->execute('SELECT pgmq.disable_notify_insert(?)', [$queue]); - - // A producer keeps an insert into the queue open, so the consumer starts are guaranteed to overlap. - $producer = $postgres->beginTransaction(); - $transport->dispatchInTransaction($producer, [self::envelope($queue)]); - - $failures = []; - $creations = self::countTriggerCreations($postgres, $queue, static function () use ($producer, $queue, &$failures): void { - $starts = []; - - for ($i = 0; $i < 4; ++$i) { - // Each consumer has its own pool, as separate worker processes do. - $own = new PgmqTransport(self::postgres()); - $starts[] = async(static fn(): Consumer => $own->startConsumer($queue, new SignallingHandler())); - } - - delay(0.5); - $producer->rollback(); - - foreach ($starts as $start) { - try { - /** @var Consumer $consumer */ - $consumer = $start->await(new TimeoutCancellation(5)); - $consumer->stop(); - $consumer->awaitCompletion(); - } catch (\Throwable $e) { - $failures[] = $e->getMessage(); - } - } - }); - - Pgmq\dropQueue($postgres, $queue); - - Assert::same($failures, []); - Assert::same($creations, 1); - } - public function consumerStartRestoresDisabledTrigger(): void { $postgres = self::postgres(); @@ -180,42 +137,6 @@ public function consumerStartRestoresDisabledTrigger(): void Assert::true($handler->received->isComplete()); } - /** - * Runs $action and returns how many times the notify trigger of the queue was created meanwhile (event trigger). - * - * @param non-empty-string $queue - * @param callable(): void $action - */ - private static function countTriggerCreations(PostgresConnection $postgres, string $queue, callable $action): int - { - $postgres->query('CREATE TABLE IF NOT EXISTS public.thesis_test_ddl (object_identity text)'); - $postgres->query('TRUNCATE public.thesis_test_ddl'); - $postgres->query(<<<'SQL' - CREATE OR REPLACE FUNCTION public.thesis_test_log_ddl() RETURNS event_trigger LANGUAGE plpgsql AS $$ - DECLARE r record; - BEGIN - FOR r IN SELECT * FROM pg_event_trigger_ddl_commands() LOOP - INSERT INTO public.thesis_test_ddl (object_identity) VALUES (r.object_identity); - END LOOP; - END $$ - SQL); - $postgres->query('DROP EVENT TRIGGER IF EXISTS thesis_test_ddl'); - $postgres->query("CREATE EVENT TRIGGER thesis_test_ddl ON ddl_command_end WHEN TAG IN ('CREATE TRIGGER') EXECUTE FUNCTION public.thesis_test_log_ddl()"); - - try { - $action(); - } finally { - $postgres->query('DROP EVENT TRIGGER IF EXISTS thesis_test_ddl'); - } - - /** @var array{n: int} $row */ - $row = $postgres - ->execute('SELECT count(*)::int AS n FROM public.thesis_test_ddl WHERE object_identity LIKE ?', ['% on pgmq.q_' . $queue]) - ->fetchRow(); - - return $row['n']; - } - private static function postgres(): PostgresConnection { $dsn = getenv('THESIS_PGMQ_DSN'); From 0763081fb19ad0976813947f492a879e14bb6a53 Mon Sep 17 00:00:00 2001 From: max Date: Wed, 7 Oct 2026 12:42:21 +0300 Subject: [PATCH 10/10] Remove the @phpstan-ignore that no longer matches Recent PHPStan 2.2.x infers the nested property assignment correctly, so the ignore is unmatched and phpstan fails on every cell of the CI matrix (8.4/8.5 x lowest/highest). --- src/MessageBus/Handling/Handlers.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/MessageBus/Handling/Handlers.php b/src/MessageBus/Handling/Handlers.php index b55ba5f..9aacd48 100644 --- a/src/MessageBus/Handling/Handlers.php +++ b/src/MessageBus/Handling/Handlers.php @@ -54,7 +54,6 @@ public function with(string $messageClass, callable $handler, string $qualifier } $copy = clone $this; - /** @phpstan-ignore assign.propertyType */ $copy->handlers[$messageClass][$qualifier] = $handler; return $copy;