From b702f78806b19db105b030055b2256b5904a3235 Mon Sep 17 00:00:00 2001 From: KyleKatarn Date: Sat, 16 May 2020 00:40:56 +0200 Subject: [PATCH 1/3] Ignore interpolations in comments --- .../Lexer/Lexer/Analyzer/LineAnalyzer.php | 8 +++++-- .../Lexer/Lexer/Scanner/CommentScanner.php | 10 ++++---- tests/Phug/AbstractLexerTest.php | 2 +- .../Phug/Lexer/Scanner/CommentScannerTest.php | 23 +++++++++++++++++++ .../Scanner/InterpolationScannerTest.php | 20 ---------------- 5 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php b/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php index 3fc4d149..5b5cbebc 100644 --- a/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php +++ b/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php @@ -169,13 +169,17 @@ public function getLines() public function getFlatLines() { return array_map(function ($line) { + $rawText = ''; + foreach ($line as $chunk) { if ($chunk instanceof TokenInterface) { - $this->state->throwException('Unexpected '.get_class($chunk).' inside raw text.'); + continue; } + + $rawText .= $chunk; } - return implode('', $line); + return $rawText; }, $this->lines); } diff --git a/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php b/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php index 6bf25068..70bd3c93 100644 --- a/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php +++ b/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php @@ -34,6 +34,7 @@ public function scan(State $state) } yield $state->endToken($token); + $line = $reader->readUntilNewLine(); $lines = $line === '' ? [] : [[$line]]; @@ -47,12 +48,11 @@ public function scan(State $state) if (end($lines) === '') { array_pop($lines); } - $token->setValue(implode("\n", $lines)); - //TODO: As it seems, this is the only TextToken that will actually contain newlines, thus Stat->endToken will - // end up with a wrong line offset. This is why endToken is not applied at all here and only the start - // position will be kept - $token->getSourceLocation()->setOffsetLength(1); //Let it have at least 1 length for debugging + $lines = implode("\n", $lines); + $token->setValue($lines); + $token->getSourceLocation()->setOffsetLength(mb_strlen($lines)); + yield $token; if ($analyzer->hasNewLine()) { diff --git a/tests/Phug/AbstractLexerTest.php b/tests/Phug/AbstractLexerTest.php index 436c6cdb..0fc4cd88 100644 --- a/tests/Phug/AbstractLexerTest.php +++ b/tests/Phug/AbstractLexerTest.php @@ -49,7 +49,7 @@ protected function filterTokenClass($className) } } - protected function assertTokens($expression, array $classNames, Lexer $lexer = null) + protected function assertTokens($expression, array $classNames, Lexer $lexer = null, &$tokens = []) { $lexer = $lexer ?: $this->lexer; $tokens = iterator_to_array($lexer->lex($expression)); diff --git a/tests/Phug/Lexer/Scanner/CommentScannerTest.php b/tests/Phug/Lexer/Scanner/CommentScannerTest.php index 7851e04f..188df576 100644 --- a/tests/Phug/Lexer/Scanner/CommentScannerTest.php +++ b/tests/Phug/Lexer/Scanner/CommentScannerTest.php @@ -142,6 +142,29 @@ public function testCommentInIndent() ]); } + public function testInterpolationInComment() + { + $code = implode("\n", [ + '//', + ' p.', + ' go to #[a(href=\'\') page]', + ]); + + $this->assertTokens($code, [ + CommentToken::class, + TextToken::class, + ], null, $tokens); + + /** @var TextToken $text */ + $text = $tokens[1]; + + $this->assertSame(implode("\n", [ + '', + ' p.', + ' go to page', + ]), $text->getValue()); + } + /** * @covers \Phug\Lexer\Scanner\CommentScanner * @covers \Phug\Lexer\Scanner\CommentScanner::scan diff --git a/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php b/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php index 6c206567..e9cedd5a 100644 --- a/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php +++ b/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php @@ -2,8 +2,6 @@ namespace Phug\Test\Lexer\Scanner; -use Phug\Lexer\Analyzer\LineAnalyzer; -use Phug\Lexer\State; use Phug\Lexer\Token\ExpressionToken; use Phug\Lexer\Token\IndentToken; use Phug\Lexer\Token\InterpolationEndToken; @@ -13,9 +11,7 @@ use Phug\Lexer\Token\TagInterpolationStartToken; use Phug\Lexer\Token\TagToken; use Phug\Lexer\Token\TextToken; -use Phug\Reader; use Phug\Test\AbstractLexerTest; -use Phug\Util\SourceLocation; class InterpolationScannerTest extends AbstractLexerTest { @@ -165,20 +161,4 @@ public function testScan() TextToken::class, ]); } - - /** - * @covers \Phug\Lexer\Analyzer\LineAnalyzer:: - * @expectedException \Phug\LexerException - * @expectedExceptionMessage Failed to lex: Unexpected Phug\Lexer\Token\InterpolationStartToken inside raw text. - */ - public function testTokenInLineAnalyzer() - { - $input = 'p #{42}'; - $analyzer = new LineAnalyzer(new State($this->lexer, $input, []), new Reader($input), [ - [ - new InterpolationStartToken(new SourceLocation('foo.pug', 12, 43)), - ], - ]); - $analyzer->getFlatLines(); - } } From fd47ed7ea6063d063512a6a7331427ce8a3f65be Mon Sep 17 00:00:00 2001 From: KyleKatarn Date: Sat, 16 May 2020 01:03:22 +0200 Subject: [PATCH 2/3] Add coverage annotation --- tests/Phug/Lexer/Scanner/CommentScannerTest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Phug/Lexer/Scanner/CommentScannerTest.php b/tests/Phug/Lexer/Scanner/CommentScannerTest.php index 188df576..75418145 100644 --- a/tests/Phug/Lexer/Scanner/CommentScannerTest.php +++ b/tests/Phug/Lexer/Scanner/CommentScannerTest.php @@ -142,6 +142,9 @@ public function testCommentInIndent() ]); } + /** + * @covers \Phug\Lexer\Analyzer\LineAnalyzer::getFlatLines + */ public function testInterpolationInComment() { $code = implode("\n", [ From b53a7171ac5f308f1df22132b1dcb23503771cd1 Mon Sep 17 00:00:00 2001 From: KyleKatarn Date: Sat, 16 May 2020 16:44:26 +0200 Subject: [PATCH 3/3] Disable interpolation in comments --- src/Phug/Lexer/Lexer/AbstractToken.php | 5 ++ .../Lexer/Lexer/Analyzer/LineAnalyzer.php | 12 +---- .../Lexer/Lexer/Scanner/CommentScanner.php | 1 + .../Lexer/Scanner/InterpolationScanner.php | 46 +++++++++++++------ .../Phug/Lexer/Scanner/CommentScannerTest.php | 22 +++++---- .../Scanner/InterpolationScannerTest.php | 2 +- .../Token/InterpolationStartTokenTest.php | 10 ++++ 7 files changed, 64 insertions(+), 34 deletions(-) diff --git a/src/Phug/Lexer/Lexer/AbstractToken.php b/src/Phug/Lexer/Lexer/AbstractToken.php index 9f693bf9..5ff5390e 100644 --- a/src/Phug/Lexer/Lexer/AbstractToken.php +++ b/src/Phug/Lexer/Lexer/AbstractToken.php @@ -42,4 +42,9 @@ public function markAsHandled() { $this->handled = true; } + + public function __toString() + { + return '['.get_class($this).']'; + } } diff --git a/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php b/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php index b52dc243..416f967c 100644 --- a/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php +++ b/src/Phug/Lexer/Lexer/Analyzer/LineAnalyzer.php @@ -122,17 +122,7 @@ public function getLines() public function getFlatLines() { return array_map(function ($line) { - $rawText = ''; - - foreach ($line as $chunk) { - if ($chunk instanceof TokenInterface) { - continue; - } - - $rawText .= $chunk; - } - - return $rawText; + return implode('', $line); }, $this->lines); } diff --git a/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php b/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php index 70bd3c93..a8d84b3d 100644 --- a/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php +++ b/src/Phug/Lexer/Lexer/Scanner/CommentScanner.php @@ -42,6 +42,7 @@ public function scan(State $state) $token = $state->createToken(TextToken::class); $analyzer = new LineAnalyzer($state, $reader, $lines); + $analyzer->disallowInterpolation(); $analyzer->analyze(false); $lines = $analyzer->getFlatLines(); diff --git a/src/Phug/Lexer/Lexer/Scanner/InterpolationScanner.php b/src/Phug/Lexer/Lexer/Scanner/InterpolationScanner.php index 0bc737c2..0ae43f19 100644 --- a/src/Phug/Lexer/Lexer/Scanner/InterpolationScanner.php +++ b/src/Phug/Lexer/Lexer/Scanner/InterpolationScanner.php @@ -18,6 +18,35 @@ class InterpolationScanner implements ScannerInterface { + protected $interpolationChars = [ + 'tagInterpolation' => ['[', ']'], + 'interpolation' => ['{', '}'], + ]; + + protected $regExp; + + public function __construct() + { + $interpolations = []; + $backIndex = 2; + + foreach ($this->interpolationChars as $name => list($start, $end)) { + $start = preg_quote($start, '/'); + $end = preg_quote($end, '/'); + $interpolations[] = $start.'(?<'.$name.'>'. + '(?>"(?:\\\\[\\S\\s]|[^"\\\\])*"|\'(?:\\\\[\\S\\s]|[^\'\\\\])*\'|[^'. + $start.$end. + '\'"]++|(?-'.$backIndex.'))*+'. + ')'.$end; + $backIndex++; + } + + $this->regExp = '(?.*?)'. + '(?#|!(?='.preg_quote($this->interpolationChars['interpolation'][0], '/').'))'. + '(?'.implode('|', $interpolations).')'; + } + protected function throwEndOfLineExceptionIf(State $state, $condition) { if ($condition) { @@ -103,21 +132,9 @@ public function scan(State $state) { $reader = $state->getReader(); - //TODO: $state->endToken - while ($reader->match( - '(?.*?)'. - '(?#|!(?=\{))(?'. - '\\[(?'. - '(?>"(?:\\\\[\\S\\s]|[^"\\\\])*"|\'(?:\\\\[\\S\\s]|[^\'\\\\])*\'|[^\\[\\]\'"]++|(?-2))*+'. - ')\\]|'. - '\\{(?'. - '(?>"(?:\\\\[\\S\\s]|[^"\\\\])*"|\'(?:\\\\[\\S\\s]|[^\'\\\\])*\'|[^{}\'"]++|(?-3))*+'. - ')\\}'. - ')' - )) { + while ($reader->match($this->regExp)) { $text = $reader->getMatch('text'); - $text = preg_replace('/\\\\([#!]\\[|#\\{)/', '$1', $text); + $text = preg_replace('/\\\\([#!]\\[|#{)/', '$1', $text); if (mb_strlen($text) > 0) { /** @var TextToken $token */ @@ -142,6 +159,7 @@ public function scan(State $state) /** @var TextToken $token */ $token = $state->createToken(TextToken::class); $token->setValue("\n"); + yield $token; } } diff --git a/tests/Phug/Lexer/Scanner/CommentScannerTest.php b/tests/Phug/Lexer/Scanner/CommentScannerTest.php index 4aead876..12ef6680 100644 --- a/tests/Phug/Lexer/Scanner/CommentScannerTest.php +++ b/tests/Phug/Lexer/Scanner/CommentScannerTest.php @@ -151,13 +151,13 @@ public function testCommentInIndent() */ public function testInterpolationInComment() { - $code = implode("\n", [ - '//', + $comment = implode("\n", [ + '', ' p.', ' go to #[a(href=\'\') page]', ]); - $this->assertTokens($code, [ + $this->assertTokens("//$comment", [ CommentToken::class, TextToken::class, ], null, $tokens); @@ -165,11 +165,17 @@ public function testInterpolationInComment() /** @var TextToken $text */ $text = $tokens[1]; - $this->assertSame(implode("\n", [ - '', - ' p.', - ' go to page', - ]), $text->getValue()); + $this->assertSame($comment, $text->getValue()); + + $this->assertTokens("//-$comment", [ + CommentToken::class, + TextToken::class, + ], null, $tokens); + + /** @var TextToken $text */ + $text = $tokens[1]; + + $this->assertSame($comment, $text->getValue()); } /** diff --git a/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php b/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php index 3db74c22..83f77789 100644 --- a/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php +++ b/tests/Phug/Lexer/Scanner/InterpolationScannerTest.php @@ -172,7 +172,7 @@ public function testNewLineInTagInterpolation() $this->expectMessageToBeThrown('End of line was reached with no closing bracket for interpolation.'); $input = "p #[em\n]"; - $tokens = iterator_to_array($this->lexer->lex($input)); + iterator_to_array($this->lexer->lex($input)); } /** diff --git a/tests/Phug/Lexer/Token/InterpolationStartTokenTest.php b/tests/Phug/Lexer/Token/InterpolationStartTokenTest.php index fd1c9802..a237a73f 100644 --- a/tests/Phug/Lexer/Token/InterpolationStartTokenTest.php +++ b/tests/Phug/Lexer/Token/InterpolationStartTokenTest.php @@ -23,4 +23,14 @@ public function testEnd() $start->setEnd($end); self::assertSame($end, $start->getEnd()); } + + /** + * @covers ::__toString + */ + public function testStringification() + { + $start = new InterpolationStartToken(); + + self::assertSame('['.InterpolationStartToken::class.']', "$start"); + } }