#45 Implement YAML support and refactor PreProcessor

src/StackFormation/PreProcessor/Stage/AbstractStringPreProcessorStage.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace StackFormation\PreProcessor\Stage;
+
+abstract class AbstractStringPreProcessorStage
+{
+    protected $stringPreProcessor; // reference to parent
+
+    /**
+     * @param \StackFormation\PreProcessor\StringPreProcessor $stringPreProcessor
+     */
+    public function __construct(\StackFormation\PreProcessor\StringPreProcessor $stringPreProcessor) {
+        $this->stringPreProcessor = $stringPreProcessor;
+    }
+
+    /**
+     * @param string $content
+     * @throws \StackFormation\Exception\StringPreProcessorException
+     */
+    public function __invoke($content)
+    {
+        try {
+            return $this->invoke($content);
+        } catch (\Exception $e) {
+            throw new \StackFormation\Exception\StringPreProcessorException($content, $e);
+        }
+    }
+
+    /**
+     * @param string $content
+     */
+    abstract function invoke($content);
+}

src/StackFormation/PreProcessor/Stage/AbstractTreePreProcessorStage.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace StackFormation\PreProcessor\Stage;
+
+use StackFormation\Template;
+
+abstract class AbstractTreePreProcessorStage
+{
+    protected $treePreProcessor; // reference to parent
+
+    /**
+     * @param \StackFormation\PreProcessor\TreePreProcessor $treePreProcessor
+     */
+    public function __construct(\StackFormation\PreProcessor\TreePreProcessor $treePreProcessor) {
+        $this->treePreProcessor = $treePreProcessor;
+    }
+
+    /**
+     * @param Template $template
+     * @throws \StackFormation\Exception\TreePreProcessorException
+     */
+    public function __invoke(Template $template)
+    {
+        try {
+            return $this->invoke($template);
+        } catch (\Exception $e) {
+            throw new \StackFormation\Exception\TreePreProcessorException($template, $e);
+        }
+    }
+
+    /**
+     * @param Template $template
+     */
+    abstract function invoke(Template $template);
+}

src/StackFormation/PreProcessor/Stage/String/StripComments.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace StackFormation\PreProcessor\Stage\String;
+
+use StackFormation\PreProcessor\Stage\AbstractStringPreProcessorStage;
+
+class StripComments extends AbstractStringPreProcessorStage {
+
+    /**
+     * @param string $content
+     * @return string
+     */
+    public function invoke($content)
+    {
+        // there's a problem with '"http://example.com"' being converted to '"http:'
+        // $content = preg_replace('~//[^\r\n]*|/\*.*?\*/~s', '', $content);
+
+        // there's a problem with "arn:aws:s3:::my-bucket/*"
+        // $content = preg_replace('~/\*.*?\*/~s', '', $content);
+
+        // quick workaround: don't allow quotes
+        return preg_replace('~/\*[^"]*?\*/~s', '', $content);
+    }
+}

src/StackFormation/PreProcessor/Stage/Tree/ExpandPort.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace StackFormation\PreProcessor\Stage\Tree;
+
+use StackFormation\PreProcessor\Stage\AbstractTreePreProcessorStage;
+use StackFormation\Template;
+
+class ExpandPort extends AbstractTreePreProcessorStage
+{
+    /**
+     * @param Template $template
+     */
+    public function invoke(Template $template)
+    {
+        $tree = $template->getTree();
+        $this->treePreProcessor->searchTreeByExpression('/^Port$/', $tree, function (&$tree, $key, $value) {
+            unset($tree[$key]);
+            $tree['FromPort'] = $value;
+            $tree['ToPort'] = $value;
+        }, 'key');
+        $template->setTree($tree);
+    }
+}

src/StackFormation/PreProcessor/StringPreProcessor.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace StackFormation\PreProcessor;
+
+use StackFormation\Helper\Pipeline;
+
+class StringPreProcessor {
+
+    /**
+     * @param string $content
+     * @return string
+     */
+    public function process($content)
+    {
+        $stageClasses = [
+            '\StackFormation\PreProcessor\Stage\String\StripComments',
+        ];
+
+        $pipeline = new Pipeline();
+        foreach ($stageClasses as $stageClass) {
+            $pipeline->addStage(new $stageClass($this));
+        }
+
+        return $pipeline->process($content);
+    }
+}

src/StackFormation/PreProcessor/TreePreProcessor.php

@@ -0,0 +1,56 @@
+<?php
+
+namespace StackFormation\PreProcessor;
+
+use StackFormation\Template;
+use StackFormation\Helper\Pipeline;
+
+class TreePreProcessor {
+
+    /**
+     * @param Template $template
+     */
+    public function process(Template $template)
+    {
+        $stageClasses = [
+            '\StackFormation\PreProcessor\Stage\Tree\ExpandPort',
+
+            # TODO, check also if we still need that
+            #'\StackFormation\PreProcessor\Stage\Tree\ParseRefInDoubleQuotedStrings',
+            #'\StackFormation\PreProcessor\Stage\Tree\InjectFilecontent',
+            #'\StackFormation\PreProcessor\Stage\Tree\Base64encodedJson',
+            #'\StackFormation\PreProcessor\Stage\Tree\Split',
+            #'\StackFormation\PreProcessor\Stage\Tree\ReplaceFnGetAttr',
+            #'\StackFormation\PreProcessor\Stage\Tree\ReplaceRef',
+            #'\StackFormation\PreProcessor\Stage\Tree\ReplaceMarkers',
+        ];
+
+        $pipeline = new Pipeline();
+        foreach ($stageClasses as $stageClass) {
+            $pipeline->addStage(new $stageClass($this));
+        }
+
+        $pipeline->process($template);
+    }
+
+    /**
+     * @param string $expression
+     * @param array $tree
+     * @param callable $callback
+     * @param string $mode
+     */
+    public function searchTreeByExpression($expression, array &$tree, callable $callback, $mode = '')
+    {
+        foreach ($tree as $key => &$leaf) {
+            if (is_array($leaf)) {
+                $this->searchTreeByExpression($expression, $leaf, $callback, $mode);
+                continue;
+            }
+
+            $subject = ($mode == 'key' ? $key : $leaf);
+            if (preg_match($expression, $subject)) {
+                $callback($tree, $key, $leaf);
+            }
+        }
+    }
+}

src/StackFormation/Template.php

@@ -7,29 +7,50 @@ class Template
 
     protected $filepath;
     protected $cache;
-    protected $preProcessor;
+    protected $stringPreProcessor;
+    protected $treePreProcessor;
+    protected $tree;
 
-    public function __construct($filePath, Preprocessor $preprocessor = null)
+    public function __construct($filePath, PreProcessor\StringPreProcessor $stringPreProcessor = null, PreProcessor\TreePreProcessor $treePreProcessor = null)
     {
         if (!is_file($filePath)) {
             throw new \Symfony\Component\Filesystem\Exception\FileNotFoundException("File '$filePath' not found");
         }
+
         $this->filepath = $filePath;
         $this->cache = new \StackFormation\Helper\Cache();
-        $this->preProcessor = $preprocessor ? $preprocessor : new Preprocessor();
+        $this->stringPreProcessor = $stringPreProcessor ? $stringPreProcessor : new PreProcessor\StringPreProcessor();
+        $this->treePreProcessor = $treePreProcessor ? $treePreProcessor : new PreProcessor\TreePreProcessor();
     }
 
     public function getFilePath()
     {
         return $this->filepath;
     }
 
+    /**
+     * @return array
+     */
+    public function getTree()
+    {
+        return $this->tree;
+    }
+
+    /**
+     * @param array $tree
+     */
+    public function setTree(array $tree)
+    {
+        $this->tree = $tree;
+    }
+
     public function getFileContent()
     {
         return $this->cache->get(
             __METHOD__,
             function () {
-                return file_get_contents($this->filepath);
+                $fileContent = file_get_contents($this->filepath);
+                return $this->stringPreProcessor->process($fileContent);
             }
         );
     }
@@ -39,24 +60,34 @@ public function getProcessedTemplate()
         return $this->cache->get(
             __METHOD__,
             function () {
-                return $this->preProcessor->process($this->getFileContent(), $this->getBasePath());
+                return $this->treePreProcessor->process($this);
             }
         );
     }
 
     public function getData()
     {
         if (!$this->cache->has(__METHOD__)) {
-            $templateBody = $this->getProcessedTemplate();
-            $array = json_decode($templateBody, true);
-            if (!is_array($array)) {
+            $fileContent = $this->getFileContent();
+
+            if (\StackFormation\Helper\Div::isJson($fileContent)) {
+                // TODO Just a workaround (need to be a single line, replace \n would also delete new line char in multiline strings
+                $fileContent = str_replace("\n", "", $fileContent);
+            }
+
+            $yamlParser = new \Symfony\Component\Yaml\Parser();
+            $this->tree = $yamlParser->parse($fileContent);
+
+            $this->getProcessedTemplate();
+
+            if (!is_array($this->tree)) {
                 throw new TemplateDecodeException($this->getFilePath(), sprintf("Error decoding file '%s'", $this->getFilePath()));
             }
-            if ($array['AWSTemplateFormatVersion'] != '2010-09-09') {
+            if ($this->tree['AWSTemplateFormatVersion'] != '2010-09-09') {
                 throw new TemplateInvalidException($this->getFilePath(), 'Invalid AWSTemplateFormatVersion');
             }
 
-            $this->cache->set(__METHOD__, $array);
+            $this->cache->set(__METHOD__, $this->tree);
         }
 
         return $this->cache->get(__METHOD__);

src/StackFormation/TemplateMerger.php

@@ -63,7 +63,9 @@ public function merge(array $templates, $description = null, array $additionalDa
             } catch (TemplateDecodeException $e) {
                 if (Div::isProgramInstalled('jq')) {
                     $tmpfile = tempnam(sys_get_temp_dir(), 'json_validate_');
-                    file_put_contents($tmpfile, $template->getProcessedTemplate());
+                    $yaml = new \Symfony\Component\Yaml\Yaml();
+                    $output = $yaml->dump($template->getProcessedTemplate());
+                    file_put_contents($tmpfile, $output);
                     passthru('jq . ' . $tmpfile);
                     unlink($tmpfile);
                 }
@@ -82,18 +84,19 @@ public function merge(array $templates, $description = null, array $additionalDa
 
         $mergedTemplate = array_merge_recursive($mergedTemplate, $additionalData);
 
-        $json = json_encode($mergedTemplate, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
+        $yaml = new \Symfony\Component\Yaml\Yaml();
+        $output = $yaml->dump($mergedTemplate);
 
         // Check for max template size
-        if (strlen($json) > self::MAX_CF_TEMPLATE_SIZE) {
-            $json = json_encode($mergedTemplate, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
+        if (strlen($output) > self::MAX_CF_TEMPLATE_SIZE) {
+            $output = $yaml->dump($mergedTemplate, 1);
 
             // Re-check for max template size
-            if (strlen($json) > self::MAX_CF_TEMPLATE_SIZE) {
-                throw new \Exception(sprintf('Template too big (%s bytes). Maximum template size is %s bytes.', strlen($json), self::MAX_CF_TEMPLATE_SIZE));
+            if (strlen($output) > self::MAX_CF_TEMPLATE_SIZE) {
+                throw new \Exception(sprintf('Template too big (%s bytes). Maximum template size is %s bytes.', strlen($output), self::MAX_CF_TEMPLATE_SIZE));
             }
         }
 
-        return $json;
+        return $output;
     }
 }

tests/StackFormation/BlueprintTest.php

@@ -394,10 +394,13 @@ public function getPreprocessedTemplate()
         $blueprintFactory = new \StackFormation\BlueprintFactory($config, $valueResolver);
         $blueprint = $blueprintFactory->getBlueprint('fixture1');
         $template = $blueprint->getPreprocessedTemplate();
-        $template = json_decode($template, true);
-        $this->assertArrayHasKey('Resources', $template);
-        $this->assertArrayHasKey('MyResource', $template['Resources']);
-        $this->assertEquals('AWS::CloudFormation::WaitConditionHandle', $template['Resources']['MyResource']['Type']);
+
+        $yamlParser = new \Symfony\Component\Yaml\Parser();
+        $tree = $yamlParser->parse($template);
+
+        $this->assertArrayHasKey('Resources', $tree);
+        $this->assertArrayHasKey('MyResource', $tree['Resources']);
+        $this->assertEquals('AWS::CloudFormation::WaitConditionHandle', $tree['Resources']['MyResource']['Type']);
     }
 
     /**
@@ -411,18 +414,21 @@ public function getPreprocessedTemplateContainsBlueprintReference()
         $blueprintFactory = new \StackFormation\BlueprintFactory($config, $valueResolver);
         $blueprint = $blueprintFactory->getBlueprint('reference-fixture-{env:FOO1}');
         $template = $blueprint->getPreprocessedTemplate();
-        $template = json_decode($template, true);
-        $this->assertArrayHasKey('Metadata', $template);
-        $this->assertArrayHasKey('StackFormation', $template['Metadata']);
-        $this->assertArrayHasKey('Blueprint', $template['Metadata']['StackFormation']);
-        $this->assertEquals('reference-fixture-{env:FOO1}', $template['Metadata']['StackFormation']['Blueprint']);
 
-        $this->assertArrayHasKey('EnvironmentVariables', $template['Metadata']['StackFormation']);
+        $yamlParser = new \Symfony\Component\Yaml\Parser();
+        $tree = $yamlParser->parse($template);
+
+        $this->assertArrayHasKey('Metadata', $tree);
+        $this->assertArrayHasKey('StackFormation', $tree['Metadata']);
+        $this->assertArrayHasKey('Blueprint', $tree['Metadata']['StackFormation']);
+        $this->assertEquals('reference-fixture-{env:FOO1}', $tree['Metadata']['StackFormation']['Blueprint']);
+
+        $this->assertArrayHasKey('EnvironmentVariables', $tree['Metadata']['StackFormation']);
         $this->assertEquals([
             'FOO1' => 'BAR1',
             'FOO2' => 'BAR2',
             'FOO3' => 'BAR3'
-        ], $template['Metadata']['StackFormation']['EnvironmentVariables']);
+        ], $tree['Metadata']['StackFormation']['EnvironmentVariables']);
     }
 
 }