Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,22 @@ services:
environment:
IN_CONTAINER: true
HISTFILE: /workspace/var/.docker_history
THESIS_PGMQ_DSN: host=pgmq user=postgres password=postgres
volumes:
- .:/workspace
- ~/.composer/cache:/composer/cache
tty: true
init: true
depends_on:
pgmq:
condition: service_healthy

pgmq:
image: ghcr.io/pgmq/pg18-pgmq:v1.11.1
environment:
POSTGRES_PASSWORD: postgres
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 2s
timeout: 5s
retries: 20
1 change: 0 additions & 1 deletion src/MessageBus/Handling/Handlers.php
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ public function with(string $messageClass, callable $handler, string $qualifier
}

$copy = clone $this;
/** @phpstan-ignore assign.propertyType */
$copy->handlers[$messageClass][$qualifier] = $handler;

return $copy;
Expand Down
82 changes: 80 additions & 2 deletions src/MessageBus/Pgmq/PgmqTransport.php
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,61 @@ public function createQueue(string $name): void
Pgmq\createExtension($this->postgres);
Pgmq\createQueue($this->postgres, $name);

// Consumers are woken up by pgmq_dispatch_v1() itself (see below), not by pgmq's insert trigger. The trigger
// updates the queue's row in pgmq.notify_insert_throttle on every insert, and that row lock is held until the
// sender commits: concurrent senders to one queue wait for each other, and two transactions sending to two
// queues in opposite order deadlock. Undo an earlier setup, without stalling a live queue:
//
// * the throttle row goes first: without it the trigger updates nothing and locks nothing. Deleting it waits
// only for senders holding that row, not for the whole table;
// * the trigger itself is dropped with a short lock_timeout: DROP TRIGGER takes an ACCESS EXCLUSIVE lock that
// waits for every open transaction on the queue and blocks its readers and senders meanwhile. If the queue is
// busy, it is skipped with a warning; the trigger is harmless without its row, and the next setup retries.
//
// Both steps only read the catalog when there is nothing to undo, so repeated setups run no DDL.
$this->postgres->query(\sprintf(
<<<'SQL'
DO $undo$
DECLARE
v_queue text := %s;
v_lock_timeout text := current_setting('lock_timeout');
BEGIN
PERFORM set_config('lock_timeout', '2s', true);

BEGIN
DELETE FROM pgmq.notify_insert_throttle WHERE queue_name = v_queue;
EXCEPTION WHEN lock_not_available THEN
RAISE WARNING 'pgmq queue "%%" is busy: its insert-notify throttle row is left for the next setup', v_queue;
END;

IF EXISTS (
SELECT 1
FROM pg_trigger tg
JOIN pg_class c ON c.oid = tg.tgrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'pgmq'
AND c.relname = pgmq.format_table_name(v_queue, 'q')
AND tg.tgname = 'trigger_notify_queue_insert_listeners'
) THEN
BEGIN
PERFORM pgmq.disable_notify_insert(v_queue);
EXCEPTION WHEN lock_not_available THEN
RAISE WARNING 'pgmq queue "%%" is busy: its insert-notify trigger is left for the next setup', v_queue;
END;
END IF;

PERFORM set_config('lock_timeout', v_lock_timeout, true);
END
$undo$
SQL,
$this->postgres->quoteLiteral($name),
));

$this->postgres->query('create schema if not exists thesis_message_bus;');
// Wakes consumers with pg_notify() on the channel pgmq's insert trigger used, so listeners need no change.
// No row is locked: a notification is delivered on commit (never for a rolled-back send), and identical
// notifications of one transaction are folded into one — a natural throttle per queue and transaction.
// Delayed messages are not announced: they are invisible until the delay passes, and the poll picks them up.
$this->postgres->query(
<<<'SQL'
create or replace function thesis_message_bus.pgmq_dispatch_v1(
Expand All @@ -75,12 +129,28 @@ public function createQueue(string $name): void
as $$
declare
index integer;
target text;
begin
for index in 1..coalesce(array_length(operations, 1), 0) loop
if operations[index] = 'send' then
return query select id from pgmq.send(addresses[index], payloads[index], headers[index], delays[index]) as id;

if delays[index] <= 0 then
perform pg_notify('pgmq.' || pgmq.format_table_name(addresses[index], 'q') || '.INSERT', null);
end if;
elsif operations[index] = 'publish' then
return next pgmq.send_topic(addresses[index], payloads[index], headers[index], delays[index])::bigint;

if delays[index] <= 0 then
-- The same queues pgmq.send_topic() has just sent to.
for target in
select distinct tb.queue_name
from pgmq.topic_bindings tb
where addresses[index] ~ tb.compiled_regex
loop
perform pg_notify('pgmq.' || pgmq.format_table_name(target, 'q') || '.INSERT', null);
end loop;
end if;
else
raise exception 'unknown message bus operation: %', operations[index];
end if;
Expand Down Expand Up @@ -189,14 +259,16 @@ public function startConsumer(string $queue, ConsumerHandler $handler): PgmqCons
$timeoutWatcher,
new ChannelWatcher(
$polls,
$this->postgres->listen($queue->enableNotifyInsert()),
// pgmq_dispatch_v1() notifies this channel; see createQueue() for why the insert trigger is not used. Lower case:
// the SQL side names the channel after the queue table, and pgmq lower-cases table names.
$this->postgres->listen(Pgmq\channelName(strtolower($queue->name))),
$timeoutWatcher,
),
]);

$iterator = $polls->iterate();

EventLoop::queue(function () use ($queue, $handler, $completion, $watcher, $iterator): void {
EventLoop::queue(function () use ($queue, $handler, $completion, $watcher, $iterator, $polls): void {
$watcher->watch();

try {
Expand Down Expand Up @@ -238,6 +310,12 @@ public function startConsumer(string $queue, ConsumerHandler $handler): PgmqCons
} finally {
EventLoop::cancel($updateVisibilityId);
}

// A full batch means the queue may hold more messages. Inserts that already happened will not
// notify again, so waiting for the next signal would leave the backlog idle for up to $pollInterval.
if (\count($messages) === $this->batchSize && !$polls->isComplete()) {
$polls->pushAsync(null)->ignore();
}
}

$completion->complete();
Expand Down
235 changes: 235 additions & 0 deletions tests/MessageBus/Pgmq/PgmqDispatchNotifyTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
<?php

declare(strict_types=1);

namespace Thesis\MessageBus\Pgmq;

use Amp\Future;
use Amp\Postgres\PostgresConfig;
use Amp\Postgres\PostgresConnection;
use Amp\Postgres\PostgresConnectionPool;
use Amp\TimeoutCancellation;
use Testo\Assert;
use Testo\Assert\ExpectNoAssertions;
use Testo\Codecov\Covers;
use Testo\Test;
use Thesis\Headers;
use Thesis\MessageBus\Transport\Operation;
use Thesis\MessageBus\Transport\OutboundEnvelope;
use Thesis\Pgmq;
use Thesis\Time\TimeSpan;
use function Amp\async;
use function Amp\delay;

#[Test]
#[Covers(PgmqTransport::class)]
final readonly class PgmqDispatchNotifyTest
{
#[ExpectNoAssertions]
public function sendersToOneQueueDoNotWaitForEachOther(): void
{
$postgres = self::postgres();
$transport = new PgmqTransport($postgres);
$queue = self::createQueue($transport);
// A live consumer, as in production: with insert notifications done by pgmq's trigger it is the consumer start
// that sets the trigger up.
$consumer = $transport->startConsumer($queue, new SignallingHandler());

// A sender keeps its transaction open after sending, as a handler does while it finishes its unit of work.
$first = $postgres->beginTransaction();
$transport->dispatchInTransaction($first, [self::send($queue)]);

try {
/** @var Future<null> $second */
$second = async(static function () use ($transport, $queue): void {
$transport->dispatch([self::send($queue)]);
});
$second->await(new TimeoutCancellation(1));
} finally {
$first->rollback();
$consumer->stop();
$consumer->awaitCompletion();
Pgmq\dropQueue($postgres, $queue);
}
}

public function crosswiseSendsToTwoQueuesDoNotDeadlock(): void
{
$postgres = self::postgres();
$transport = new PgmqTransport($postgres);
$one = self::createQueue($transport);
$two = self::createQueue($transport);
$consumers = [
$transport->startConsumer($one, new SignallingHandler()),
$transport->startConsumer($two, new SignallingHandler()),
];

$a = $postgres->beginTransaction();
$b = $postgres->beginTransaction();
$failures = [];

try {
$transport->dispatchInTransaction($a, [self::send($one)]);
$transport->dispatchInTransaction($b, [self::send($two)]);

$aToTwo = async(static fn() => $transport->dispatchInTransaction($a, [self::send($two)]));
delay(0.1);
$bToOne = async(static fn() => $transport->dispatchInTransaction($b, [self::send($one)]));

foreach ([$aToTwo, $bToOne] as $future) {
try {
$future->await(new TimeoutCancellation(3));
} catch (\Throwable $e) {
$failures[] = strtok($e->getMessage(), "\n");
}
}

foreach ([$a, $b] as $transaction) {
if ($transaction->isActive()) {
$transaction->commit();
}
}
} finally {
foreach ($consumers as $consumer) {
$consumer->stop();
$consumer->awaitCompletion();
}
Pgmq\dropQueue($postgres, $one);
Pgmq\dropQueue($postgres, $two);
}

Assert::same($failures, []);
}

public function publishWakesSubscribedConsumerOnCommitOnly(): void
{
$postgres = self::postgres();
$transport = new PgmqTransport($postgres, pollInterval: TimeSpan::fromSeconds(10));
$queue = self::createQueue($transport);
$topic = 'test.' . $queue;
$transport->subscribe($queue, [$topic]);

$handler = new SignallingHandler();
$consumer = $transport->startConsumer($queue, $handler);

try {
// Let the consumer make its initial (empty) read first, so only a notification can deliver the message.
delay(0.2);
$tx = $postgres->beginTransaction();
$transport->dispatchInTransaction($tx, [self::publish($topic)]);

// Not committed yet: nothing is announced.
delay(0.5);
Assert::false($handler->received->isComplete());

$tx->commit();
$handler->received->getFuture()->await(new TimeoutCancellation(2));
} finally {
$consumer->stop();
$consumer->awaitCompletion();
Pgmq\dropQueue($postgres, $queue);
}

Assert::true($handler->received->isComplete());
}

public function createQueueDropsTheInsertTriggerOfAnEarlierSetup(): void
{
$postgres = self::postgres();
$transport = new PgmqTransport($postgres);
$queue = self::createQueue($transport);
$postgres->execute('SELECT pgmq.enable_notify_insert(?, 30)', [$queue]);

try {
$transport->createQueue($queue);

/** @var array{triggers: int, rows: int} $row */
$row = $postgres->execute(
<<<'SQL'
SELECT
(SELECT count(*)::int FROM pg_trigger tg JOIN pg_class c ON c.oid = tg.tgrelid
WHERE c.relname = pgmq.format_table_name(:queue, 'q')
AND tg.tgname = 'trigger_notify_queue_insert_listeners') AS triggers,
(SELECT count(*)::int FROM pgmq.notify_insert_throttle WHERE queue_name = :queue) AS rows
SQL,
['queue' => $queue],
)->fetchRow();
} finally {
Pgmq\dropQueue($postgres, $queue);
}

Assert::same($row, ['triggers' => 0, 'rows' => 0]);
}

public function createQueueDoesNotStallABusyQueueWhileUndoingAnEarlierSetup(): void
{
$postgres = self::postgres();
$transport = new PgmqTransport($postgres);
$queue = self::createQueue($transport);
$postgres->execute('SELECT pgmq.enable_notify_insert(?, 30)', [$queue]);

// A long transaction has read the queue and is still open. Its lock does not stop pgmq.create() of the existing
// queue, but DROP TRIGGER (ACCESS EXCLUSIVE) would wait for it, and every reader and sender behind that.
$sender = $postgres->beginTransaction();
$sender->query(\sprintf('SELECT count(*) FROM pgmq.%s', 'q_' . $queue));

try {
$startedAt = microtime(true);
$transport->createQueue($queue);
$took = microtime(true) - $startedAt;

// The throttle row is gone even though the trigger could not be dropped: a send no longer waits for anyone.
/** @var array{rows: int} $row */
$row = $postgres->execute('SELECT count(*)::int AS rows FROM pgmq.notify_insert_throttle WHERE queue_name = ?', [$queue])->fetchRow();
$other = $postgres->beginTransaction();
$transport->dispatchInTransaction($other, [self::send($queue)]);
$third = async(static fn() => $transport->dispatch([self::send($queue)]));
$third->await(new TimeoutCancellation(1));
$other->rollback();
} finally {
$sender->rollback();
Pgmq\dropQueue($postgres, $queue);
}

Assert::true($took < 4.0);
Assert::same($row, ['rows' => 0]);
}

private static function postgres(): PostgresConnection
{
$dsn = getenv('THESIS_PGMQ_DSN');

if (!\is_string($dsn) || $dsn === '') {
throw new \LogicException('Set the THESIS_PGMQ_DSN environment variable (see compose.yaml).');
}

return new PostgresConnectionPool(PostgresConfig::fromString($dsn));
}

/**
* @return non-empty-string
*/
private static function createQueue(PgmqTransport $transport): string
{
$queue = 'q' . bin2hex(random_bytes(8));
$transport->createQueue($queue);

return $queue;
}

/**
* @param non-empty-string $queue
*/
private static function send(string $queue): OutboundEnvelope
{
return new OutboundEnvelope(operation: Operation::Send, address: $queue, payload: '{}', headers: new Headers());
}

/**
* @param non-empty-string $topic
*/
private static function publish(string $topic): OutboundEnvelope
{
return new OutboundEnvelope(operation: Operation::Publish, address: $topic, payload: '{}', headers: new Headers());
}
}
Loading
Loading