Add ParsedTokenStream class and use it for expansion

src/Expansion.php

@@ -210,8 +210,10 @@ private function mutate(TokenStream $ts, Ast $context, Engine $engine) : TokenSt
                     if (\count($result->{'args'}) === 0)
                         $cg->this->fail(self::E_EMPTY_EXPANDER_SLICE, $expander->implode(), $expander->tokens()[0]->line());
 
-                    $expansion = TokenStream::fromSlice($result->{'args'});
+                    $expansion = ParsedTokenStream::fromSlice($result->{'args'});
+
                     $mutation = $cg->this->mutate($expansion, $cg->context, $cg->engine);
+                    $mutation->setAst($cg->context);
 
                     $expander = $cg->this->compileCallable('\Yay\Dsl\Expanders\\', $expander, self::E_BAD_EXPANDER);
                     $mutation = $expander($mutation, $cg->engine);
@@ -249,12 +251,15 @@ private function mutate(TokenStream $ts, Ast $context, Engine $engine) : TokenSt
                         if ($key = $result->{'key'}) {
                             $scope[(string) $result->{'key'}] = new Token(T_LNUMBER, (string) $i);
                         }
+
                         $expansion = TokenStream::fromSlice($result->{'expansion'});
+
                         $mutation = $cg->this->mutate(
                             $expansion,
                             (new Ast('', $cg->context->unwrap() + (is_array($scope) ? $scope : [$scope]))),
                             $cg->engine
                         );
+
                         if ($i !== count($context)-1) foreach ($delimiters as $d) $mutation->push($d);
                         $cg->ts->inject($mutation);
                     }

src/ParsedTokenStream.php

@@ -0,0 +1,17 @@
+<?php declare(strict_types=1);
+
+namespace Yay;
+
+class ParsedTokenStream extends TokenStream {
+    protected
+        $ast
+    ;
+
+    function setAst(Ast $ast) {
+        $this->ast = $ast;
+    }
+
+    function getAst() {
+        return $this->ast;
+    }
+}

src/TokenStream.php

@@ -235,7 +235,7 @@ function isEmpty() : bool {
     }
 
     static function fromSourceWithoutOpenTag(string $source) : self {
-        $ts = self::fromSource('<?php ' . $source);
+        $ts = static::fromSource('<?php ' . $source);
         $ts->first->next = $ts->first->next->next;
         $ts->first->next->previous = $ts->first;
         $ts->reset();
@@ -246,7 +246,7 @@ static function fromSourceWithoutOpenTag(string $source) : self {
     static function fromSource(string $source) : self {
         $tokens = \token_get_all($source);
 
-        $ts = new self;
+        $ts = new static;
         $first = new NodeStart;
         $last = new NodeEnd;
 
@@ -293,7 +293,7 @@ static function fromSource(string $source) : self {
     }
 
     static function fromSlice(array $tokens) : self {
-        $ts = new self;
+        $ts = new static;
         $first = new NodeStart;
         $last = new NodeEnd;
 
@@ -321,6 +321,6 @@ static function fromSlice(array $tokens) : self {
     }
 
     static function fromSequence(Token ...$tokens) : self {
-        return self::fromSlice($tokens);
+        return static::fromSlice($tokens);
     }
 }

tests/TokenStreamTest.php

@@ -187,4 +187,14 @@ function testPush() {
         $ts->push(new Token(T_WHITESPACE, ' '));
         $this->assertEquals('<?php A B ', (string) $ts);
     }
+
+    function testHasAst() {
+        $source = new Ast("foo");
+
+        $stream = ParsedTokenStream::fromSlice($source->tokens());
+        $stream->setAst($source);
+
+
+        $this->assertSame($source, $stream->getAst());
+    }
 }

tests/fixtures/expanders.php

@@ -1,6 +1,6 @@
 <?php namespace Yay\tests\fixtures\expanders;
 
-use Yay\{Token, TokenStream};
+use Yay\{Ast, Engine, Token, TokenStream, ParsedTokenStream};
 
 function my_hello_expander(TokenStream $ts) : TokenStream {
     $str = str_replace("'", "\'", (string) $ts);
@@ -13,3 +13,36 @@ function my_hello_expander(TokenStream $ts) : TokenStream {
         )
     ;
 }
+
+function reverse_ast_expander(ParsedTokenStream $stream, Engine $engine) : ParsedTokenStream {
+    $ast = $stream->getAst();
+
+    $leaf = $ast->{"* outer inner 0 0"}->token();
+    $ast->{"outer inner 0 0"} = new Token($leaf->type(), strrev($leaf->value()));
+
+    return ParsedTokenStream::fromSource(
+        join(" ", $ast->tokens())
+    );
+}
+
+function upper_ast_expander(ParsedTokenStream $stream, Engine $engine) : ParsedTokenStream {
+    $ast = $stream->getAst();
+
+    $leaf = $ast->{"* outer inner 0 0"}->token();
+    $ast->{"outer inner 0 0"} = new Token($leaf->type(), strtoupper($leaf->value()));
+
+    return ParsedTokenStream::fromSource(
+        join(" ",  $ast->tokens())
+    );
+}
+
+function wrap_ast_expander(ParsedTokenStream $stream, Engine $engine) : ParsedTokenStream {
+    $ast = $stream->getAst();
+
+    $leaf = $ast->{"* outer inner 0 0"}->token();
+    $ast->{"outer inner 0 0"} = new Token($leaf->type(), "[{$leaf->value()}]");
+
+    return ParsedTokenStream::fromSource(
+        join(" ",  $ast->tokens())
+    );
+}

tests/fixtures/parsers.php

@@ -1,7 +1,7 @@
 <?php namespace Yay\tests\fixtures\parsers;
 
 use Yay\{Parser};
-use function Yay\{token};
+use function Yay\{token, chain, buffer};
 
 function my_custom_parser() : Parser {
 	return token(T_STRING);

tests/phpt/expanders/nested_ast_expander.phpt

@@ -0,0 +1,32 @@
+--TEST--
+Provides an AST to a custom expander
+--FILE--
+<?php
+
+$(macro) {
+    $(
+        chain(
+            chain(
+                buffer("foo")
+            ) as inner
+        ) as outer
+    )
+} >> {
+    $$(\Yay\tests\fixtures\expanders\wrap_ast_expander(
+        $$(\Yay\tests\fixtures\expanders\upper_ast_expander(
+            $$(\Yay\tests\fixtures\expanders\reverse_ast_expander(
+                $(outer)
+            ))
+        ))
+    ))
+}
+
+foo
+
+?>
+--EXPECTF--
+<?php
+
+[OOF]
+
+?>