Support for sidecar files

statiqdev · May 18, 2020 · e4cc4da · e4cc4da
1 parent 695a8f8
commit e4cc4da
Show file tree

Hide file tree

Showing 14 changed files with 310 additions and 34 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <!-- Controls whether references are local projects or NuGet packages -->
-    <LocalReferences>true</LocalReferences>    
+    <LocalReferences>false</LocalReferences>    
     <!-- The NuGet version of Statiq that should be referenced if LocalReferences is false -->
     <StatiqFrameworkVersion>1.0.0-beta.11</StatiqFrameworkVersion>
   </PropertyGroup>

diff --git a/RELEASE.md b/RELEASE.md
@@ -1,5 +1,10 @@
 # 1.0.0-alpha.14
 
+- Added support for directory metadata for data files.
+- Added support for front matter in data files.
+- Added support for sidecar files as `_[filename].[json|yaml]`.
+- Added `ProcessSidecarFiles` setting to turn sidecar files off.
+- Added `ApplyDirectoryMetadata` setting to turn directory metadata off.
 - Added better xref error messages.
 
 # 1.0.0-alpha.13

diff --git a/src/Statiq.Web/BootstrapperFactoryExtensions.cs b/src/Statiq.Web/BootstrapperFactoryExtensions.cs
@@ -3,6 +3,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Statiq.App;
 using Statiq.Common;
+using Statiq.Web.Modules;
 
 namespace Statiq.Web
 {
@@ -37,10 +38,8 @@ public static Bootstrapper AddWeb(this Bootstrapper boostrapper) =>
                 .AddSettingsIfNonExisting(new Dictionary<string, object>
                 {
                     { WebKeys.ContentFiles, "**/{!_,}*.{html,cshtml,md}" },
-                    { WebKeys.DataFiles, "**/{!_,}*.{json,yaml,yml}" },
+                    { WebKeys.DataFiles, $"**/{{!_,}}*.{{{string.Join(",", ParseDataContent.SupportedExtensions)}}}" },
                     { WebKeys.DirectoryMetadataFiles, "**/_{d,D}irectory.{json,yaml,yml}" },
-                    { WebKeys.ValidateRelativeLinks, true },
-                    { WebKeys.GenerateSitemap, true },
                     { WebKeys.Xref, Config.FromDocument(doc => doc.GetTitle().Replace(' ', '-')) },
                     { WebKeys.Excluded, Config.FromDocument(doc => doc.GetPublishedDate(false) > DateTime.Today.AddDays(1)) } // Add +1 days so the threshold is midnight on the current day
                 });

diff --git a/src/Statiq.Web/Modules/ApplyDirectoryMetadata.cs b/src/Statiq.Web/Modules/ApplyDirectoryMetadata.cs
@@ -31,7 +31,7 @@ protected override IEnumerable<IDocument> ExecuteContext(IExecutionContext conte
                         .ToImmutableArray());
 
             // Iterate through all input documents in parallel
-            IEnumerable<IDocument> results = context.Inputs.Select(input =>
+            return context.Inputs.Select(input =>
             {
                 IDocument merged = input;
 
@@ -65,7 +65,6 @@ protected override IEnumerable<IDocument> ExecuteContext(IExecutionContext conte
 
                 return merged;
             });
-            return results;
         }
 
         private class DirectoryMetadataData

diff --git a/src/Statiq.Web/Modules/ParseDataContent.cs b/src/Statiq.Web/Modules/ParseDataContent.cs
@@ -0,0 +1,22 @@
+using Statiq.Common;
+using Statiq.Core;
+using Statiq.Yaml;
+
+namespace Statiq.Web.Modules
+{
+    /// <summary>
+    /// Common parsing of data content based on media type.
+    /// </summary>
+    public class ParseDataContent : ForAllDocuments
+    {
+        public static string[] SupportedExtensions { get; } = new[] { "json", "yaml", "yml" };
+
+        public ParseDataContent()
+            : base(
+                new ExecuteSwitch(Config.FromDocument(doc => doc.ContentProvider.MediaType))
+                    .Case(MediaTypes.Json, new ParseJson())
+                    .Case(MediaTypes.Yaml, new ParseYaml()))
+        {
+        }
+    }
+}
diff --git a/src/Statiq.Web/Modules/ProcessMetadata.cs b/src/Statiq.Web/Modules/ProcessMetadata.cs
@@ -1,11 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
+using System.Linq;
 using Statiq.Common;
 using Statiq.Core;
-using Statiq.Html;
-using Statiq.Markdown;
-using Statiq.Razor;
 using Statiq.Yaml;
 
 namespace Statiq.Web.Modules
@@ -17,7 +12,24 @@ public class ProcessMetadata : ForAllDocuments
     {
         public ProcessMetadata()
             : base(
-                new ExtractFrontMatter(new ParseYaml()))
+                new ExecuteIf(Config.FromSetting(WebKeys.ApplyDirectoryMetadata, true))
+                {
+                    new ApplyDirectoryMetadata()
+                },
+                new ExecuteIf(Config.FromSetting(WebKeys.ProcessSidecarFiles, true))
+                {
+                    new ProcessSidecarFile(Config.FromDocument((doc, ctx) =>
+                        doc.Source.IsNull
+                            ? NormalizedPath.Null
+                            : ParseDataContent.SupportedExtensions
+                                .Select(x => doc.Source.InsertPrefix("_").ChangeExtension(x))
+                                .FirstOrDefault(x => ctx.FileSystem.GetInputFile(x).Exists)))
+                    {
+                        new ParseDataContent()
+                    }
+                },
+                new ExtractFrontMatter(new ParseYaml()),
+                new ParseDataContent())
         {
         }
     }

diff --git a/src/Statiq.Web/Pipelines/Content.cs b/src/Statiq.Web/Pipelines/Content.cs
@@ -23,10 +23,7 @@ public Content(Templates templates)
                 // Concat all documents from externally declared dependencies (exclude explicit dependencies above like "Data")
                 new ConcatDocuments(Config.FromContext<IEnumerable<IDocument>>(ctx => ctx.Outputs.FromPipelines(ctx.Pipeline.GetAllDependencies(ctx).Except(Dependencies).ToArray()))),
 
-                // Apply directory metadata
-                new ApplyDirectoryMetadata(),
-
-                // Process front matter and sidecar files
+                // Process directory metadata, sidecar files, and front matter
                 new ProcessMetadata(),
 
                 // Filter out excluded documents

diff --git a/src/Statiq.Web/Pipelines/Data.cs b/src/Statiq.Web/Pipelines/Data.cs
@@ -23,20 +23,16 @@ public Data()
                 // Concat all documents from externally declared dependencies (exclude explicit dependencies above)
                 new ConcatDocuments(Config.FromContext<IEnumerable<IDocument>>(ctx => ctx.Outputs.FromPipelines(ctx.Pipeline.GetAllDependencies(ctx).Except(Dependencies).ToArray()))),
 
-                // Apply directory metadata
-                new ApplyDirectoryMetadata(),
-
-                // Parse the content into metadata depending on the content type
-                new ExecuteSwitch(Config.FromDocument(doc => doc.ContentProvider.MediaType))
-                    .Case(MediaTypes.Json, new ParseJson())
-                    .Case(MediaTypes.Yaml, new ParseYaml()),
+                // Process directory metadata, sidecar files, front matter, and data content
+                new ProcessMetadata(),
 
                 // Filter out excluded documents
                 new FilterDocuments(Config.FromDocument(doc => !doc.GetBool(WebKeys.Excluded))),
 
                 // Filter out feed documents (they'll get processed by the Feed pipeline)
                 new FilterDocuments(Config.FromDocument(doc => !Feeds.IsFeed(doc))),
 
+                // Set the destination and optimize filenames
                 new SetDestination(),
                 new ExecuteIf(Config.FromSetting(WebKeys.OptimizeDataFileNames, true))
                 {

diff --git a/src/Statiq.Web/Pipelines/DirectoryMetadata.cs b/src/Statiq.Web/Pipelines/DirectoryMetadata.cs
@@ -9,6 +9,7 @@
 using Statiq.Html;
 using Statiq.Markdown;
 using Statiq.Razor;
+using Statiq.Web.Modules;
 using Statiq.Yaml;
 
 namespace Statiq.Web.Pipelines
@@ -24,10 +25,7 @@ public DirectoryMetadata()
 
             ProcessModules = new ModuleList
             {
-                // Parse the content into metadata depending on the content type
-                new ExecuteSwitch(Config.FromDocument(doc => doc.ContentProvider.MediaType))
-                    .Case(MediaTypes.Json, new ParseJson())
-                    .Case(MediaTypes.Yaml, new ParseYaml())
+                new ParseDataContent()
             };
         }
     }

diff --git a/src/Statiq.Web/Pipelines/Feeds.cs b/src/Statiq.Web/Pipelines/Feeds.cs
@@ -4,6 +4,7 @@
 using Statiq.Common;
 using Statiq.Core;
 using Statiq.Feeds;
+using Statiq.Web.Modules;
 using Statiq.Yaml;
 
 namespace Statiq.Web.Pipelines
@@ -21,10 +22,16 @@ public Feeds()
 
             ProcessModules = new ModuleList
             {
-                new ExecuteSwitch(Config.FromDocument(doc => doc.ContentProvider.MediaType))
-                    .Case(MediaTypes.Json, new ParseJson())
-                    .Case(MediaTypes.Yaml, new ParseYaml()),
+                // Process directory metadata, sidecar files, and front matter, and data content
+                new ProcessMetadata(),
+
+                // Filter out excluded documents
+                new FilterDocuments(Config.FromDocument(doc => !doc.GetBool(WebKeys.Excluded))),
+
+                // Limit to feed documents
                 new FilterDocuments(Config.FromDocument(IsFeed)),
+
+                // Generate the feeds
                 new ForEachDocument
                 {
                     new ExecuteConfig(Config.FromDocument(feedDoc =>

diff --git a/src/Statiq.Web/Pipelines/LinkValidation.cs b/src/Statiq.Web/Pipelines/LinkValidation.cs
@@ -19,7 +19,7 @@ public LinkValidation()
             {
                 new ReplaceDocuments(Dependencies.ToArray()),
                 new ValidateLinks()
-                    .ValidateRelativeLinks(Config.FromSetting<bool>(WebKeys.ValidateRelativeLinks))
+                    .ValidateRelativeLinks(Config.FromSetting(WebKeys.ValidateRelativeLinks, true))
                     .ValidateAbsoluteLinks(Config.FromSetting<bool>(WebKeys.ValidateAbsoluteLinks))
                     .AsError(Config.FromSetting<bool>(WebKeys.ValidateLinksAsError))
             };

diff --git a/src/Statiq.Web/Pipelines/Sitemap.cs b/src/Statiq.Web/Pipelines/Sitemap.cs
@@ -14,7 +14,7 @@ public Sitemap()
 
             PostProcessModules = new ModuleList
             {
-                new ExecuteIf(Config.FromSetting<bool>(WebKeys.GenerateSitemap))
+                new ExecuteIf(Config.FromSetting(WebKeys.GenerateSitemap, true))
                 {
                     new ConcatDocuments(nameof(Content))
                     {

diff --git a/src/Statiq.Web/WebKeys.cs b/src/Statiq.Web/WebKeys.cs
@@ -23,6 +23,16 @@ public static class WebKeys
 
         public const string OptimizeDataFileNames = nameof(OptimizeDataFileNames);
 
+        /// <summary>
+        /// Set to <c>false</c> to prevent processing directory metadata.
+        /// </summary>
+        public const string ApplyDirectoryMetadata = nameof(ApplyDirectoryMetadata);
+
+        /// <summary>
+        /// Set to <c>false</c> to prevent processing sidecar files.
+        /// </summary>
+        public const string ProcessSidecarFiles = nameof(ProcessSidecarFiles);
+
         /// <summary>
         /// Indicates that a sitemap file should be generated if <c>true</c> (the default).
         /// </summary>