From b70cadb00fcf4ec526ff6f65cbbd1d0ca44a77db Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 16 Nov 2026 11:20:18 -0500 Subject: [PATCH] Improve tokenizer performance by scanning character runs --- src/Connection/ImapTokenizer.php | 85 +++++++++++---------- tests/Unit/Connection/ImapTokenizerTest.php | 63 +++++++++++++++ 2 files changed, 109 insertions(+), 39 deletions(-) diff --git a/src/Connection/ImapTokenizer.php b/src/Connection/ImapTokenizer.php index f43a7353..11f4c38f 100644 --- a/src/Connection/ImapTokenizer.php +++ b/src/Connection/ImapTokenizer.php @@ -20,6 +20,13 @@ class ImapTokenizer { + /** + * Atom delimiters, control characters (0x00–0x1F), and DEL (0x7F). + */ + protected const ATOM_DELIMITERS = ' ()[]{}<>' + ."\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F" + ."\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x7F"; + /** * The current position in the buffer. */ @@ -134,24 +141,13 @@ protected function skipWhitespace(): void while (true) { $this->ensureBuffer(1); - $char = $this->currentChar(); - - // Break on EOF. - if ($char === null || $char === '') { - break; - } - - // Break on CRLF. - if ($char === "\r" || $char === "\n") { - break; - } + $length = strspn($this->buffer, " \t", $this->position); - // Break on non-whitespace. - if ($char !== ' ' && $char !== "\t") { - break; + if ($length === 0) { + return; } - $this->advance(); + $this->advance($length); } } @@ -170,6 +166,17 @@ protected function readQuotedString(): QuotedString while (true) { $this->ensureBuffer(1); + // Consume ordinary characters up to the next delimiter. + $length = strcspn($this->buffer, '"\\', $this->position); + + if ($length > 0) { + $value .= substr($this->buffer, $this->position, $length); + + $this->advance($length); + + continue; + } + $char = $this->currentChar(); if ($char === null) { @@ -198,15 +205,9 @@ protected function readQuotedString(): QuotedString continue; } - if ($char === '"') { - $this->advance(); // Skip the closing quote. - - break; - } + $this->advance(); // Skip the closing quote. - $value .= $char; - - $this->advance(); + break; } return new QuotedString($value); @@ -372,19 +373,20 @@ protected function readAtom(): Atom while (true) { $this->ensureBuffer(1); - $char = $this->currentChar(); + // Consume ordinary characters up to the next delimiter. + $length = strcspn($this->buffer, static::ATOM_DELIMITERS, $this->position); - if ($char === null) { - break; - } + if ($length > 0) { + $value .= substr($this->buffer, $this->position, $length); - if (! $this->isValidAtomCharacter($char)) { - break; + $this->advance($length); + + continue; } - $value .= $char; + $char = $this->currentChar(); - $this->advance(); + break; } // If no value was read, we will throw an exception since @@ -422,21 +424,26 @@ protected function readEmailAddress(): ?EmailAddress while (true) { $this->ensureBuffer(1); - $char = $this->currentChar(); + // Consume ordinary characters up to the next delimiter. + $length = strcspn($this->buffer, '>', $this->position); - if ($char === null) { - throw new ImapParserException('Unterminated email address, expected ">"'); + if ($length > 0) { + $value .= substr($this->buffer, $this->position, $length); + + $this->advance($length); + + continue; } - if ($char === '>') { - $this->advance(); // Skip the closing '>'. + $char = $this->currentChar(); - break; + if ($char === null) { + throw new ImapParserException('Unterminated email address, expected ">"'); } - $value .= $char; + $this->advance(); // Skip the closing '>'. - $this->advance(); + break; } return new EmailAddress($value); diff --git a/tests/Unit/Connection/ImapTokenizerTest.php b/tests/Unit/Connection/ImapTokenizerTest.php index 58e676b8..796284c0 100644 --- a/tests/Unit/Connection/ImapTokenizerTest.php +++ b/tests/Unit/Connection/ImapTokenizerTest.php @@ -498,3 +498,66 @@ public function fgets(): string|false "{3}\r\nFoo", // Literal '"Bar"', // QuotedString ]); + +test('tokenizer reads runs across buffer boundaries', function (int $chunkSize) { + $stream = new FakeStream; + $stream->open(); + + $input = " \t \tNIL atom\x80\xff \"hello \\\"world\\\" \\\\ end\" \r\n"; + + $stream->feedRaw(str_split($input, $chunkSize)); + + $tokenizer = new ImapTokenizer($stream); + + foreach ([ + new Nil('NIL'), + new Atom("atom\x80\xff"), + new QuotedString('hello "world" \\ end'), + new EmailAddress('user@example.com'), + new Crlf("\r\n"), + ] as $expected) { + $token = $tokenizer->nextToken(); + + expect($token)->toBeInstanceOf($expected::class); + expect($token->value)->toBe($expected->value); + } + + expect($tokenizer->nextToken())->toBeNull(); +})->with(range(1, 80)); + +test('tokenizer stops atoms at every control byte', function (int $byte) { + $stream = new FakeStream; + $stream->open(); + $stream->feedRaw('example'.chr($byte)); + + $tokenizer = new ImapTokenizer($stream); + + expect($tokenizer->nextToken()->value)->toBe('example'); +})->with([...range(0, 31), 127]); + +test('tokenizer preserves errors across buffer boundaries', function (string $input, string $message, int $chunkSize) { + $stream = new FakeStream; + $stream->open(); + $stream->feedRaw(str_split($input, $chunkSize)); + + $tokenizer = new ImapTokenizer($stream); + + expect(fn () => $tokenizer->nextToken()) + ->toThrow(ImapParserException::class, $message); +})->with([ + ['"unfinished', 'Unterminated quoted string at buffer offset 0. Buffer: ""'], + ['"unfinished\\', 'Unterminated escape sequence in quoted string'], + ['"'], +])->with(range(1, 15)); + +test('tokenizer handles empty strings and whitespace ending at the buffer boundary', function () { + $stream = new FakeStream; + $stream->open(); + $stream->feedRaw(['""', '<>', " \t"]); + + $tokenizer = new ImapTokenizer($stream); + + expect($tokenizer->nextToken())->toEqual(new QuotedString('')); + expect($tokenizer->nextToken())->toEqual(new EmailAddress('')); + expect($tokenizer->nextToken())->toBeNull(); +});