diff --git a/src/Token/Tokenizer.php b/src/Token/Tokenizer.php index 8fb0c40..95b64c6 100644 --- a/src/Token/Tokenizer.php +++ b/src/Token/Tokenizer.php @@ -478,17 +478,16 @@ private function lexData(int $limit = 0): void $this->lexEOL($match[0]); } elseif (1 === preg_match('/\S+/', $this->code, $match, 0, $this->cursor)) { $value = $match[0]; + // Stop if cursor reaches the next expression starter. + if ($limit > $this->cursor) { + $value = substr($value, 0, $limit - $this->cursor); + } if (self::STATE_COMMENT === $this->getState()) { $this->pushToken(Token::COMMENT_TEXT_TYPE, $value); } elseif (self::STATE_INLINE_COMMENT === $this->getState()) { $this->pushToken(Token::INLINE_COMMENT_TEXT_TYPE, $value); } else { - // Stop if cursor reaches the next expression starter. - if (0 !== $limit) { - $value = substr($value, 0, $limit - $this->cursor); - } - $this->pushToken(Token::TEXT_TYPE, $value); } } else { diff --git a/tests/Token/Tokenizer/Fixtures/test18.twig b/tests/Token/Tokenizer/Fixtures/test18.twig new file mode 100644 index 0000000..0356265 --- /dev/null +++ b/tests/Token/Tokenizer/Fixtures/test18.twig @@ -0,0 +1 @@ +{#comment without space#} diff --git a/tests/Token/Tokenizer/TokenizerTest.php b/tests/Token/Tokenizer/TokenizerTest.php index 86ed99f..234150d 100644 --- a/tests/Token/Tokenizer/TokenizerTest.php +++ b/tests/Token/Tokenizer/TokenizerTest.php @@ -836,6 +836,21 @@ public static function tokenizeDataProvider(): iterable 14 => Token::EOF_TYPE, ], ]; + + yield [ + __DIR__.'/Fixtures/test18.twig', + [ + 0 => Token::COMMENT_START_TYPE, + 1 => Token::COMMENT_TEXT_TYPE, + 2 => Token::COMMENT_WHITESPACE_TYPE, + 3 => Token::COMMENT_TEXT_TYPE, + 4 => Token::COMMENT_WHITESPACE_TYPE, + 5 => Token::COMMENT_TEXT_TYPE, + 6 => Token::COMMENT_END_TYPE, + 7 => Token::EOL_TYPE, + 8 => Token::EOF_TYPE, + ], + ]; } /**