add Ast::tokens and Ast::flatten + tests

Ast::tokens is yet another casting method and returns an array of all tokens
within the ast. Nested Ast still produces a flat array of tokens.

Ast::flatten behaves the same as Ast::tokens except it returns an Ast instance

src/Ast.php

@@ -71,6 +71,14 @@ function token() {
         $this->failCasting(Token::class);
     }
 
+    function tokens() {
+        $tokens = [];
+        $exposed = $this->array();
+        array_walk_recursive($exposed, function($t) use(&$tokens){ if($t instanceof Token) $tokens[] = $t; });
+
+        return $tokens;
+    }
+
     function null() {
         if (\is_null($this->ast)) return $this->ast;
 
@@ -104,7 +112,11 @@ function list() {
         $isAssociative = \count(array_filter(array_keys($array), 'is_string')) > 0;
 
         foreach ($array as $label => $value)
-            yield new Ast(($isAssociative ? $label : null), $value);
+            yield new self(($isAssociative ? $label : null), $value);
+    }
+
+    function flatten() : self {
+        return new self($this->label, $this->tokens());
     }
 
     function append(self $ast) : self {

tests/AstTest.php

@@ -57,6 +57,7 @@ function providerForTestAstCast() {
             ['* some string', 'string', 'foo'],
             ['* some array', 'array', ['foo', 'bar']],
             ['* some token', 'token', new Token(';')],
+            ['* some tokens', 'tokens', []],
         ];
     }
 
@@ -70,7 +71,8 @@ function testAstCast(string $path, string $castMethod, $expected) {
                 'boolean' => true,
                 'string' => 'foo',
                 'array' => ['foo', 'bar'],
-                'token' => new Token(';')
+                'token' => new Token(';'),
+                'tokens' => ['deep' => ['inside' => []]],
             ]
         ]);
 
@@ -79,4 +81,27 @@ function testAstCast(string $path, string $castMethod, $expected) {
         else
             $this->assertEquals($expected, $ast->{$path}->$castMethod());
     }
+
+    function testAstFlattenning() {
+        $ast = new Ast(null, [
+            'deep' => [
+                'token' => $token1 = new Token(T_STRING, 'foo'),
+                'deeper' => [
+                    'token' => $token2 = new Token(T_STRING, 'bar'),
+                ],
+            ]
+        ]);
+
+        $this->assertEquals([$token1, $token2], $ast->tokens());
+
+        $flattened = $ast->flatten();
+
+        $this->assertInstanceOf(Ast::class, $flattened);
+
+        $this->assertEquals([$token1, $token2], $flattened->tokens());
+
+        $this->assertEquals([$token1, $token2], $flattened->unwrap());
+
+        $this->assertEquals([$token1, $token2], $flattened->array());
+    }
 }