feat: liquid markdown front matter separately from the rest of the co…

…ntent

src/transform/frontmatter/common.ts

@@ -0,0 +1,27 @@
+export type FrontMatter = {
+    [key: string]: unknown;
+    metadata?: Record<string, unknown>[];
+};
+
+export const frontMatterFence = '---';
+
+/**
+ * Temporary workaround to enable parsing YAML metadata from potentially
+ * Liquid-aware source files
+ * @param content Input string which could contain Liquid-style substitution syntax (which clashes with YAML
+ * object syntax)
+ * @returns String with `{}` escaped, ready to be parsed with `js-yaml`
+ */
+export const escapeLiquidSubstitutionSyntax = (content: string): string =>
+    content.replace(/{{/g, '(({{').replace(/}}/g, '}}))');
+
+/**
+ * Inverse of a workaround defined above.
+ * @see `escapeLiquidSubstitutionSyntax`
+ * @param escapedContent Input string with `{}` escaped with backslashes
+ * @returns Unescaped string
+ */
+export const unescapeLiquidSubstitutionSyntax = (escapedContent: string): string =>
+    escapedContent.replace(/\(\({{/g, '{{').replace(/}}\)\)/g, '}}');
+
+export const countLineAmount = (str: string) => str.split(/\r?\n/).length;

src/transform/frontmatter/emplace.ts

@@ -0,0 +1,26 @@
+import {dump} from 'js-yaml';
+
+import {FrontMatter, frontMatterFence, unescapeLiquidSubstitutionSyntax} from './common';
+
+export const serializeFrontMatter = (frontMatter: FrontMatter) => {
+    const dumped = unescapeLiquidSubstitutionSyntax(
+        dump(frontMatter, {forceQuotes: true, lineWidth: -1}).trim(),
+    );
+
+    // This empty object check is a bit naive
+    // The other option would be to check if all own fields are `undefined`,
+    // since we exploit passing in `undefined` to remove a field quite a bit
+    if (dumped === '{}') {
+        return '';
+    }
+
+    return `${frontMatterFence}\n${dumped}\n${frontMatterFence}`;
+};
+
+export const emplaceSerializedFrontMatter = (
+    frontMatterStrippedContent: string,
+    frontMatter: string,
+) => `${frontMatter}${frontMatterStrippedContent}`;
+
+export const emplaceFrontMatter = (frontMatterStrippedContent: string, frontMatter: FrontMatter) =>
+    emplaceSerializedFrontMatter(frontMatterStrippedContent, serializeFrontMatter(frontMatter));

src/transform/frontmatter/extract.ts

@@ -0,0 +1,89 @@
+import {YAMLException, load} from 'js-yaml';
+
+import {log} from '../log';
+
+import {
+    FrontMatter,
+    countLineAmount,
+    escapeLiquidSubstitutionSyntax,
+    frontMatterFence,
+} from './common';
+
+type ParseExistingMetadataReturn = {
+    frontMatter: FrontMatter;
+    frontMatterStrippedContent: string;
+    frontMatterLineCount: number;
+};
+
+const matchMetadata = (fileContent: string) => {
+    if (!fileContent.startsWith(frontMatterFence)) {
+        return null;
+    }
+
+    // Search by format:
+    // ---
+    // metaName1: metaValue1
+    // metaName2: meta value2
+    // incorrectMetadata
+    // ---
+    const regexpMetadata = '(?<=-{3}\\r?\\n)((.*\\r?\\n)*?)(?=-{3}\\r?\\n)';
+    // Search by format:
+    // ---
+    // main content 123
+    const regexpFileContent = '-{3}\\r?\\n((.*[\r?\n]*)*)';
+
+    const regexpParseFileContent = new RegExp(`${regexpMetadata}${regexpFileContent}`, 'gm');
+
+    return regexpParseFileContent.exec(fileContent);
+};
+
+const duplicateKeysCompatibleLoad = (yaml: string, filePath: string | undefined) => {
+    try {
+        return load(yaml);
+    } catch (e) {
+        if (e instanceof YAMLException) {
+            const duplicateKeysDeprecationWarning = `
+                In ${filePath ?? '(unknown)'}: Encountered a YAML parsing exception when processing file metadata: ${e.reason}.
+                It's highly possible the input file contains duplicate mapping keys.
+                Will retry processing with necessary compatibility flags.
+                Please note that this behaviour is DEPRECATED and WILL be removed in a future version
+                without further notice, so the build WILL fail when supplied with YAML-incompatible meta.
+            `
+                .replace(/^\s+/gm, '')
+                .replace(/\n/g, ' ')
+                .trim();
+
+            log.warn(duplicateKeysDeprecationWarning);
+
+            return load(yaml, {json: true});
+        }
+
+        throw e;
+    }
+};
+
+export const separateAndExtractFrontMatter = (
+    fileContent: string,
+    filePath?: string,
+): ParseExistingMetadataReturn => {
+    const matches = matchMetadata(fileContent);
+
+    if (matches && matches.length > 0) {
+        const [, metadata, , metadataStrippedContent] = matches;
+
+        return {
+            frontMatter: duplicateKeysCompatibleLoad(
+                escapeLiquidSubstitutionSyntax(metadata),
+                filePath,
+            ) as FrontMatter,
+            frontMatterStrippedContent: metadataStrippedContent,
+            frontMatterLineCount: countLineAmount(metadata),
+        };
+    }
+
+    return {
+        frontMatter: {},
+        frontMatterStrippedContent: fileContent,
+        frontMatterLineCount: 0,
+    };
+};

src/transform/frontmatter/index.ts

@@ -0,0 +1,4 @@
+export * from './extract';
+export * from './emplace';
+export * from './transformValues';
+export {countLineAmount} from './common';

src/transform/frontmatter/transformValues.ts

@@ -0,0 +1,22 @@
+import {FrontMatter} from './common';
+
+export const transformFrontMatterValues = (
+    frontMatter: FrontMatter,
+    valueMapper: (v: unknown) => unknown,
+): FrontMatter => {
+    const transformInner = (something: unknown): unknown => {
+        if (typeof something === 'object' && something !== null) {
+            return Object.fromEntries(
+                Object.entries(something).map(([k, v]) => [k, transformInner(v)]),
+            );
+        }
+
+        if (Array.isArray(something)) {
+            return something.map((el) => transformInner(el));
+        }
+
+        return valueMapper(something);
+    };
+
+    return transformInner(frontMatter) as FrontMatter;
+};

src/transform/index.ts

@@ -3,7 +3,7 @@ import type {EnvType, OptionsType, OutputType} from './typings';
 import {bold} from 'chalk';
 
 import {log} from './log';
-import liquid from './liquid';
+import liquidSnippet from './liquid';
 import initMarkdownit from './md';
 
 function applyLiquid(input: string, options: OptionsType) {
@@ -15,7 +15,9 @@ function applyLiquid(input: string, options: OptionsType) {
         isLiquided = false,
     } = options;
 
-    return disableLiquid || isLiquided ? input : liquid(input, vars, path, {conditionsInCode});
+    return disableLiquid || isLiquided
+        ? input
+        : liquidSnippet(input, vars, path, {conditionsInCode});
 }
 
 function handleError(error: unknown, path?: string): never {

src/transform/liquid/index.ts

@@ -1,5 +1,13 @@
 import type {Dictionary} from 'lodash';
 
+import {
+    countLineAmount,
+    emplaceSerializedFrontMatter,
+    separateAndExtractFrontMatter,
+    serializeFrontMatter,
+    transformFrontMatterValues,
+} from '../frontmatter';
+
 import applySubstitutions from './substitutions';
 import {prepareSourceMap} from './sourceMap';
 import applyCycles from './cycles';
@@ -66,7 +74,7 @@ function repairCode(str: string, codes: string[]) {
     return replace(fence, fence, (code) => codes[Number(code)], str);
 }
 
-function liquid<
+function liquidSnippet<
     B extends boolean = false,
     C = B extends false ? string : {output: string; sourceMap: Dictionary<string>},
 >(
@@ -141,6 +149,62 @@ function liquid<
     return output as unknown as C;
 }
 
-// 'export default' instead of 'export = ' because of circular dependency with './cycles.ts'.
-// somehow it breaks import in './cycles.ts' and imports nothing
-export default liquid;
+type TransformSourceMapOptions = {
+    emplacedResultOffset: number;
+    emplacedSourceOffset: number;
+};
+
+function transformSourceMap(
+    sourceMap: Dictionary<string>,
+    {emplacedResultOffset, emplacedSourceOffset}: TransformSourceMapOptions,
+) {
+    return Object.fromEntries(
+        Object.entries(sourceMap).map(([lineInResult, lineInSource]) => [
+            (Number(lineInResult) + emplacedResultOffset).toString(),
+            (Number(lineInSource) + emplacedSourceOffset).toString(),
+        ]),
+    );
+}
+
+function liquidDocument<
+    B extends boolean = false,
+    C = B extends false ? string : {output: string; sourceMap: Dictionary<string>},
+>(
+    originInput: string,
+    vars: Record<string, unknown>,
+    path?: string,
+    settings?: ArgvSettings & {withSourceMap?: B},
+): C {
+    const {frontMatter, frontMatterStrippedContent, frontMatterLineCount} =
+        separateAndExtractFrontMatter(originInput, path);
+
+    const transformedFrontMatter = transformFrontMatterValues(frontMatter, (v) =>
+        typeof v === 'string'
+            ? liquidSnippet(v, vars, path, {...settings, withSourceMap: false})
+            : v,
+    );
+    const transformedAndSerialized = serializeFrontMatter(transformedFrontMatter);
+
+    // -1 comes from the fact that the last line in serialized FM is the same as the first line in stripped content
+    const resultFrontMatterOffset = Math.max(0, countLineAmount(transformedAndSerialized) - 1);
+    const sourceFrontMatterOffset = Math.max(0, frontMatterLineCount - 1);
+
+    const liquidProcessedContent = liquidSnippet(frontMatterStrippedContent, vars, path, settings);
+
+    return (typeof liquidProcessedContent === 'string'
+        ? liquidProcessedContent
+        : {
+              output: emplaceSerializedFrontMatter(
+                  frontMatterStrippedContent,
+                  transformedAndSerialized,
+              ),
+              sourceMap: transformSourceMap(liquidProcessedContent.sourceMap, {
+                  emplacedResultOffset: resultFrontMatterOffset,
+                  emplacedSourceOffset: sourceFrontMatterOffset,
+              }),
+          }) as unknown as C;
+}
+
+// both default and named exports for convenience
+export {liquidDocument, liquidSnippet};
+export default liquidDocument;

src/transform/utilsFS.ts

@@ -4,7 +4,7 @@ import {readFileSync, statSync} from 'fs';
 import escapeRegExp from 'lodash/escapeRegExp';
 import {join, parse, relative, resolve, sep} from 'path';
 
-import liquid from './liquid';
+import liquidSnippet from './liquid';
 import {StateCore} from './typings';
 import {defaultTransformLink} from './utils';
 
@@ -68,7 +68,7 @@ export function getFileTokens(path: string, state: StateCore, options: GetFileTo
     let sourceMap;
 
     if (!disableLiquid) {
-        const liquidResult = liquid(content, builtVars, path, {
+        const liquidResult = liquidSnippet(content, builtVars, path, {
             withSourceMap: true,
             conditionsInCode,
         });

test/liquid/cycles.test.ts

@@ -1,6 +1,6 @@
 import dedent from 'ts-dedent';
 
-import liquid from '../../src/transform/liquid';
+import liquidSnippet from '../../src/transform/liquid';
 
 const commentsByPage = [
     {
@@ -28,13 +28,17 @@ describe('Cycles', () => {
     describe('location', () => {
         test('Inline for block', () => {
             expect(
-                liquid('Prefix {% for user in users %} {{user}} {% endfor %} Postfix', vars, ''),
+                liquidSnippet(
+                    'Prefix {% for user in users %} {{user}} {% endfor %} Postfix',
+                    vars,
+                    '',
+                ),
             ).toEqual('Prefix Alice Ivan Petr Postfix');
         });
 
         test('Nested inline for block', () => {
             expect(
-                liquid(
+                liquidSnippet(
                     'Prefix {% for user1 in users %} {% for user2 in users %} {{user1}}+{{user2}} {% endfor %} {% endfor %} Postfix',
                     vars,
                     '',
@@ -46,7 +50,7 @@ describe('Cycles', () => {
 
         test('Multiline for block', () => {
             expect(
-                liquid(
+                liquidSnippet(
                     dedent`
                     Prefix
                     {% for user in users %}
@@ -111,12 +115,12 @@ describe('Cycles', () => {
 
                 Postfix
             `;
-            expect(liquid(input, vars, '')).toEqual(result);
+            expect(liquidSnippet(input, vars, '')).toEqual(result);
         });
 
         test('Multiline nested for block without indent', () => {
             expect(
-                liquid(
+                liquidSnippet(
                     dedent`
                     Prefix
                     {% for user1 in users %}
@@ -148,7 +152,7 @@ describe('Cycles', () => {
     describe('with conditions, filters, substitutions', () => {
         test('Test 1', () => {
             expect(
-                liquid(
+                liquidSnippet(
                     'Prefix {% for user in users2 %}{% if needCapitalize %} {{user | capitalize}}+{{user2}} {% else %} {{user}} {% endif %}{% endfor %} Postfix',
                     vars,
                     '',
@@ -160,7 +164,7 @@ describe('Cycles', () => {
     describe('with code blocks', () => {
         test('code block before cycle block', () => {
             expect(
-                liquid(
+                liquidSnippet(
                     '```\nCode block\n```\n\n {% for user in users %} {{user}} {% endfor %}',
                     vars,
                     '',
@@ -170,7 +174,7 @@ describe('Cycles', () => {
 
         test('cycle block in code block', () => {
             expect(
-                liquid('```\n{% for user in users %} {{user}} {% endfor %}\n```', vars, '', {
+                liquidSnippet('```\n{% for user in users %} {{user}} {% endfor %}\n```', vars, '', {
                     conditionsInCode: true,
                 }),
             ).toEqual('```\nAlice Ivan Petr\n```');