diff --git a/src/Phug/Lexer/Lexer/AbstractToken.php b/src/Phug/Lexer/Lexer/AbstractToken.php index 9f693bf9..5ff5390e 100644 --- a/src/Phug/Lexer/Lexer/AbstractToken.php +++ b/src/Phug/Lexer/Lexer/AbstractToken.php @@ -42,4 +42,9 @@ public function markAsHandled() { $this->handled = true; } + + public function __toString() + { + return '['.get_class($this).']'; + } } diff --git a/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php b/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php index 36ac9cc6..416f967c 100644 --- a/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php +++ b/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php @@ -122,12 +122,6 @@ public function getLines() public function getFlatLines() { return array_map(function ($line) { - foreach ($line as $chunk) { - if ($chunk instanceof TokenInterface) { - $this->state->throwException('Unexpected '.get_class($chunk).' inside raw text.'); - } - } - return implode('', $line); }, $this->lines); } diff --git a/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php b/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php index 6bf25068..a8d84b3d 100644 --- a/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php +++ b/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php @@ -34,6 +34,7 @@ public function scan(State $state) } yield $state->endToken($token); + $line = $reader->readUntilNewLine(); $lines = $line === '' ? [] : [[$line]]; @@ -41,18 +42,18 @@ public function scan(State $state) $token = $state->createToken(TextToken::class); $analyzer = new LineAnalyzer($state, $reader, $lines); + $analyzer->disallowInterpolation(); $analyzer->analyze(false); $lines = $analyzer->getFlatLines(); if (end($lines) === '') { array_pop($lines); } - $token->setValue(implode("\n", $lines)); - //TODO: As it seems, this is the only TextToken that will actually contain newlines, thus Stat->endToken will - // end up with a wrong line offset. This is why endToken is not applied at all here and only the start - // position will be kept - $token->getSourceLocation()->setOffsetLength(1); //Let it have at least 1 length for debugging + $lines = implode("\n", $lines); + $token->setValue($lines); + $token->getSourceLocation()->setOffsetLength(mb_strlen($lines)); + yield $token; if ($analyzer->hasNewLine()) { diff --git a/src/Phug/Lexer/Lexer/Scanner/InterpolationScanner.php b/src/Phug/Lexer/Lexer/Scanner/InterpolationScanner.php index 0bc737c2..0ae43f19 100644 --- a/src/Phug/Lexer/Lexer/Scanner/InterpolationScanner.php +++ b/src/Phug/Lexer/Lexer/Scanner/InterpolationScanner.php @@ -18,6 +18,35 @@ class InterpolationScanner implements ScannerInterface { + protected $interpolationChars = [ + 'tagInterpolation' => ['[', ']'], + 'interpolation' => ['{', '}'], + ]; + + protected $regExp; + + public function __construct() + { + $interpolations = []; + $backIndex = 2; + + foreach ($this->interpolationChars as $name => list($start, $end)) { + $start = preg_quote($start, '/'); + $end = preg_quote($end, '/'); + $interpolations[] = $start.'(?<'.$name.'>'. + '(?>"(?:\\\\[\\S\\s]|[^"\\\\])*"|\'(?:\\\\[\\S\\s]|[^\'\\\\])*\'|[^'. + $start.$end. + '\'"]++|(?-'.$backIndex.'))*+'. + ')'.$end; + $backIndex++; + } + + $this->regExp = '(?.*?)'. + '(?#|!(?='.preg_quote($this->interpolationChars['interpolation'][0], '/').'))'. + '(?'.implode('|', $interpolations).')'; + } + protected function throwEndOfLineExceptionIf(State $state, $condition) { if ($condition) { @@ -103,21 +132,9 @@ public function scan(State $state) { $reader = $state->getReader(); - //TODO: $state->endToken - while ($reader->match( - '(?.*?)'. - '(?#|!(?=\{))(?'. - '\\[(?'. - '(?>"(?:\\\\[\\S\\s]|[^"\\\\])*"|\'(?:\\\\[\\S\\s]|[^\'\\\\])*\'|[^\\[\\]\'"]++|(?-2))*+'. - ')\\]|'. - '\\{(?'. - '(?>"(?:\\\\[\\S\\s]|[^"\\\\])*"|\'(?:\\\\[\\S\\s]|[^\'\\\\])*\'|[^{}\'"]++|(?-3))*+'. - ')\\}'. - ')' - )) { + while ($reader->match($this->regExp)) { $text = $reader->getMatch('text'); - $text = preg_replace('/\\\\([#!]\\[|#\\{)/', '$1', $text); + $text = preg_replace('/\\\\([#!]\\[|#{)/', '$1', $text); if (mb_strlen($text) > 0) { /** @var TextToken $token */ @@ -142,6 +159,7 @@ public function scan(State $state) /** @var TextToken $token */ $token = $state->createToken(TextToken::class); $token->setValue("\n"); + yield $token; } } diff --git a/tests/Phug/AbstractLexerTest.php b/tests/Phug/AbstractLexerTest.php index 436c6cdb..0fc4cd88 100644 --- a/tests/Phug/AbstractLexerTest.php +++ b/tests/Phug/AbstractLexerTest.php @@ -49,7 +49,7 @@ protected function filterTokenClass($className) } } - protected function assertTokens($expression, array $classNames, Lexer $lexer = null) + protected function assertTokens($expression, array $classNames, Lexer $lexer = null, &$tokens = []) { $lexer = $lexer ?: $this->lexer; $tokens = iterator_to_array($lexer->lex($expression)); diff --git a/tests/Phug/Lexer/Scanner/CommentScannerTest.php b/tests/Phug/Lexer/Scanner/CommentScannerTest.php index 5153b635..12ef6680 100644 --- a/tests/Phug/Lexer/Scanner/CommentScannerTest.php +++ b/tests/Phug/Lexer/Scanner/CommentScannerTest.php @@ -146,6 +146,38 @@ public function testCommentInIndent() ]); } + /** + * @covers \Phug\Lexer\Analyzer\LineAnalyzer::getFlatLines + */ + public function testInterpolationInComment() + { + $comment = implode("\n", [ + '', + ' p.', + ' go to #[a(href=\'\') page]', + ]); + + $this->assertTokens("//$comment", [ + CommentToken::class, + TextToken::class, + ], null, $tokens); + + /** @var TextToken $text */ + $text = $tokens[1]; + + $this->assertSame($comment, $text->getValue()); + + $this->assertTokens("//-$comment", [ + CommentToken::class, + TextToken::class, + ], null, $tokens); + + /** @var TextToken $text */ + $text = $tokens[1]; + + $this->assertSame($comment, $text->getValue()); + } + /** * @covers \Phug\Lexer\Scanner\CommentScanner * @covers \Phug\Lexer\Scanner\CommentScanner::scan diff --git a/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php b/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php index 2a27d0a9..83f77789 100644 --- a/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php +++ b/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php @@ -2,8 +2,6 @@ namespace Phug\Test\Lexer\Scanner; -use Phug\Lexer\Analyzer\LineAnalyzer; -use Phug\Lexer\State; use Phug\Lexer\Token\ExpressionToken; use Phug\Lexer\Token\IndentToken; use Phug\Lexer\Token\InterpolationEndToken; @@ -13,9 +11,7 @@ use Phug\Lexer\Token\TagInterpolationStartToken; use Phug\Lexer\Token\TagToken; use Phug\Lexer\Token\TextToken; -use Phug\Reader; use Phug\Test\AbstractLexerTest; -use Phug\Util\SourceLocation; class InterpolationScannerTest extends AbstractLexerTest { @@ -166,22 +162,6 @@ public function testScan() ]); } - /** - * @covers \Phug\Lexer\Analyzer\LineAnalyzer:: - * @expectedException \Phug\LexerException - * @expectedExceptionMessage Failed to lex: Unexpected Phug\Lexer\Token\InterpolationStartToken inside raw text. - */ - public function testTokenInLineAnalyzer() - { - $input = 'p #{42}'; - $analyzer = new LineAnalyzer(new State($this->lexer, $input, []), new Reader($input), [ - [ - new InterpolationStartToken(new SourceLocation('foo.pug', 12, 43)), - ], - ]); - $analyzer->getFlatLines(); - } - /** * @covers \Phug\Lexer\Scanner\InterpolationScanner::scanTagInterpolation * @covers \Phug\Lexer\Scanner\InterpolationScanner::throwEndOfLineExceptionIf @@ -192,7 +172,7 @@ public function testNewLineInTagInterpolation() $this->expectMessageToBeThrown('End of line was reached with no closing bracket for interpolation.'); $input = "p #[em\n]"; - $tokens = iterator_to_array($this->lexer->lex($input)); + iterator_to_array($this->lexer->lex($input)); } /** diff --git a/tests/Phug/Lexer/Token/InterpolationStartTokenTest.php b/tests/Phug/Lexer/Token/InterpolationStartTokenTest.php index fd1c9802..a237a73f 100644 --- a/tests/Phug/Lexer/Token/InterpolationStartTokenTest.php +++ b/tests/Phug/Lexer/Token/InterpolationStartTokenTest.php @@ -23,4 +23,14 @@ public function testEnd() $start->setEnd($end); self::assertSame($end, $start->getEnd()); } + + /** + * @covers ::__toString + */ + public function testStringification() + { + $start = new InterpolationStartToken(); + + self::assertSame('['.InterpolationStartToken::class.']', "$start"); + } }