Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 46 additions & 39 deletions src/Connection/ImapTokenizer.php
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@

class ImapTokenizer
{
/**
* Atom delimiters, control characters (0x00–0x1F), and DEL (0x7F).
*/
protected const ATOM_DELIMITERS = ' ()[]{}<>'
."\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F"
."\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x7F";

/**
* The current position in the buffer.
*/
Expand Down Expand Up @@ -134,24 +141,13 @@ protected function skipWhitespace(): void
while (true) {
$this->ensureBuffer(1);

$char = $this->currentChar();

// Break on EOF.
if ($char === null || $char === '') {
break;
}

// Break on CRLF.
if ($char === "\r" || $char === "\n") {
break;
}
$length = strspn($this->buffer, " \t", $this->position);

// Break on non-whitespace.
if ($char !== ' ' && $char !== "\t") {
break;
if ($length === 0) {
return;
}

$this->advance();
$this->advance($length);
}
}

Expand All @@ -170,6 +166,17 @@ protected function readQuotedString(): QuotedString
while (true) {
$this->ensureBuffer(1);

// Consume ordinary characters up to the next delimiter.
$length = strcspn($this->buffer, '"\\', $this->position);

if ($length > 0) {
$value .= substr($this->buffer, $this->position, $length);

$this->advance($length);

continue;
}

$char = $this->currentChar();

if ($char === null) {
Expand Down Expand Up @@ -198,15 +205,9 @@ protected function readQuotedString(): QuotedString
continue;
}

if ($char === '"') {
$this->advance(); // Skip the closing quote.

break;
}
$this->advance(); // Skip the closing quote.

$value .= $char;

$this->advance();
break;
}

return new QuotedString($value);
Expand Down Expand Up @@ -372,19 +373,20 @@ protected function readAtom(): Atom
while (true) {
$this->ensureBuffer(1);

$char = $this->currentChar();
// Consume ordinary characters up to the next delimiter.
$length = strcspn($this->buffer, static::ATOM_DELIMITERS, $this->position);

if ($char === null) {
break;
}
if ($length > 0) {
$value .= substr($this->buffer, $this->position, $length);

if (! $this->isValidAtomCharacter($char)) {
break;
$this->advance($length);

continue;
}

$value .= $char;
$char = $this->currentChar();

$this->advance();
break;
}

// If no value was read, we will throw an exception since
Expand Down Expand Up @@ -422,21 +424,26 @@ protected function readEmailAddress(): ?EmailAddress
while (true) {
$this->ensureBuffer(1);

$char = $this->currentChar();
// Consume ordinary characters up to the next delimiter.
$length = strcspn($this->buffer, '>', $this->position);

if ($char === null) {
throw new ImapParserException('Unterminated email address, expected ">"');
if ($length > 0) {
$value .= substr($this->buffer, $this->position, $length);

$this->advance($length);

continue;
}

if ($char === '>') {
$this->advance(); // Skip the closing '>'.
$char = $this->currentChar();

break;
if ($char === null) {
throw new ImapParserException('Unterminated email address, expected ">"');
}

$value .= $char;
$this->advance(); // Skip the closing '>'.

$this->advance();
break;
}

return new EmailAddress($value);
Expand Down
63 changes: 63 additions & 0 deletions tests/Unit/Connection/ImapTokenizerTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -498,3 +498,66 @@ public function fgets(): string|false
"{3}\r\nFoo", // Literal
'"Bar"', // QuotedString
]);

test('tokenizer reads runs across buffer boundaries', function (int $chunkSize) {
$stream = new FakeStream;
$stream->open();

$input = " \t \tNIL atom\x80\xff \"hello \\\"world\\\" \\\\ end\" <user@example.com>\r\n";

$stream->feedRaw(str_split($input, $chunkSize));

$tokenizer = new ImapTokenizer($stream);

foreach ([
new Nil('NIL'),
new Atom("atom\x80\xff"),
new QuotedString('hello "world" \\ end'),
new EmailAddress('user@example.com'),
new Crlf("\r\n"),
] as $expected) {
$token = $tokenizer->nextToken();

expect($token)->toBeInstanceOf($expected::class);
expect($token->value)->toBe($expected->value);
}

expect($tokenizer->nextToken())->toBeNull();
})->with(range(1, 80));

test('tokenizer stops atoms at every control byte', function (int $byte) {
$stream = new FakeStream;
$stream->open();
$stream->feedRaw('example'.chr($byte));

$tokenizer = new ImapTokenizer($stream);

expect($tokenizer->nextToken()->value)->toBe('example');
})->with([...range(0, 31), 127]);

test('tokenizer preserves errors across buffer boundaries', function (string $input, string $message, int $chunkSize) {
$stream = new FakeStream;
$stream->open();
$stream->feedRaw(str_split($input, $chunkSize));

$tokenizer = new ImapTokenizer($stream);

expect(fn () => $tokenizer->nextToken())
->toThrow(ImapParserException::class, $message);
})->with([
['"unfinished', 'Unterminated quoted string at buffer offset 0. Buffer: ""'],
['"unfinished\\', 'Unterminated escape sequence in quoted string'],
['<unfinished', 'Unterminated email address, expected ">"'],
])->with(range(1, 15));

test('tokenizer handles empty strings and whitespace ending at the buffer boundary', function () {
$stream = new FakeStream;
$stream->open();
$stream->feedRaw(['""', '<>', " \t"]);

$tokenizer = new ImapTokenizer($stream);

expect($tokenizer->nextToken())->toEqual(new QuotedString(''));
expect($tokenizer->nextToken())->toEqual(new EmailAddress(''));
expect($tokenizer->nextToken())->toBeNull();
});
Loading