Add computed fields to SQL

src/lib/databaseUtils.ts

@@ -1,10 +1,10 @@
 import { Knex } from "knex";
 import { MddbFile, MddbTag, MddbLink, MddbFileTag, File } from "./schema.js";
 import path from "path";
 import { WikiLink } from "./parseFile.js";
 
 export async function resetDatabaseTables(db: Knex) {
-  const tableNames = [MddbFile, MddbTag, MddbFileTag, MddbLink];
+  const tableNames = [MddbTag, MddbFileTag, MddbLink];
   // Drop and Create tables
   for (const table of tableNames) {
     await table.deleteTable(db);
@@ -12,12 +12,12 @@
   }
 }
 
 export function mapFileToInsert(file: any) {
-  const { _id, file_path, extension, url_path, filetype, metadata } = file;
-  return { _id, file_path, extension, url_path, filetype, metadata };
+  const { tags, links, ...rest } = file;
+  return { ...rest };
 }
 
 export function mapLinksToInsert(filesToInsert: File[], file: any) {
   return file.links.map((link: WikiLink) => {
     let to: string | undefined;
     if (!link.internal) {
@@ -45,12 +45,12 @@
   });
 }
 
 export function isLinkToDefined(link: any) {
   return link.to !== undefined;
 }
 
 export function mapFileTagsToInsert(file: any) {
   return file.tags.map((tag: any) => ({
     file: file._id,
     tag: tag as unknown as string,
   }));
@@ -67,3 +67,17 @@
 
   return uniqueArray;
 }
+
+export function getUniqueProperties(objects: any[]): string[] {
+  const uniqueProperties: string[] = [];
+
+  for (const object of objects) {
+    for (const key of Object.keys(object)) {
+      if (!uniqueProperties.includes(key)) {
+        uniqueProperties.push(key);
+      }
+    }
+  }
+
+  return uniqueProperties;
+}

src/lib/markdowndb.ts

@@ -10,9 +10,11 @@ import {
   isLinkToDefined,
   mapFileTagsToInsert,
   getUniqueValues,
+  getUniqueProperties,
 } from "./databaseUtils.js";
 import fs from "fs";
 import { CustomConfig } from "./CustomConfig.js";
+import { FileInfo } from "./process.js";
 
 const defaultFilePathToUrl = (filePath: string) => {
   let url = filePath
@@ -80,14 +82,36 @@ export class MarkdownDB {
     pathToUrlResolver?: (filePath: string) => string;
     customConfig?: CustomConfig;
   }) {
-    await resetDatabaseTables(this.db);
-
     const fileObjects = indexFolder(
       folderPath,
       pathToUrlResolver,
       customConfig,
       ignorePatterns
     );
+    await this.saveDataToDisk(fileObjects);
+  }
+
+  private async saveDataToDisk(fileObjects: FileInfo[]) {
+    await resetDatabaseTables(this.db);
+    const properties = getUniqueProperties(fileObjects);
+    MddbFile.deleteTable(this.db);
+    const defaultProperties = [
+      "_id",
+      "file_path",
+      "extension",
+      "url_path",
+      "filetype",
+      "metadata",
+      "tags",
+      "links",
+    ];
+    await MddbFile.createTable(
+      this.db,
+      properties.filter(
+        (property) => defaultProperties.indexOf(property) === -1
+      )
+    );
+
     const filesToInsert = fileObjects.map(mapFileToInsert);
     const uniqueTags = getUniqueValues(
       fileObjects.flatMap((file) => file.tags)

src/lib/schema.ts

@@ -25,50 +25,61 @@ interface File {
   url_path: string | null;
   filetype: string | null;
   metadata: MetaData | null;
+  [key: string]: string | null | MetaData | undefined;
 }
 
 class MddbFile {
   static table = Table.Files;
   static supportedExtensions = ["md", "mdx"];
 
-  _id: string;
-  file_path: string;
-  extension: string;
-  url_path: string | null;
-  // TODO there should be a separate table for filetypes
-  // and another one for many-to-many relationship between files and filetypes
-  filetype: string | null;
-  metadata: MetaData | null;
+  file: File = {
+    _id: "",
+    file_path: "",
+    extension: "",
+    url_path: null,
+    filetype: null,
+    metadata: null,
+  };
 
   // TODO type?
   constructor(file: any) {
-    this._id = file._id;
-    this.file_path = file.file_path;
-    this.extension = file.extension;
-    this.url_path = file.url_path;
-    this.filetype = file.filetype;
-    this.metadata = file.metadata ? JSON.parse(file.metadata) : null;
+    this.file._id = file._id;
+    this.file.file_path = file.file_path;
+    this.file.extension = file.extension;
+    this.file.url_path = file.url_path;
+    this.file.filetype = file.filetype;
+    this.file.metadata = file.metadata ? JSON.parse(file.metadata) : null;
+
+    // Assign dynamic properties using index signature to this.file
+    for (const key in file) {
+      if (
+        key !== "_id" &&
+        key !== "file_path" &&
+        key !== "extension" &&
+        key !== "url_path" &&
+        key !== "filetype" &&
+        key !== "metadata"
+      ) {
+        this.file[key] = file[key];
+      }
+    }
   }
 
   toObject(): File {
-    return {
-      _id: this._id,
-      file_path: this.file_path,
-      extension: this.extension,
-      url_path: this.url_path,
-      filetype: this.filetype,
-      metadata: this.metadata,
-    };
+    return { ...this.file };
   }
 
-  static async createTable(db: Knex) {
+  static async createTable(db: Knex, properties: string[]) {
     const creator = (table: Knex.TableBuilder) => {
       table.string("_id").primary();
-      table.string("file_path").unique().notNullable(); // Path relative to process.cwd()
-      table.string("extension").notNullable(); // File extension
-      table.string("url_path"); // Sluggfied path relative to content folder
-      table.string("filetype"); // Type field in frontmatter if it exists
-      table.string("metadata"); // All frontmatter data
+      table.string("file_path").unique().notNullable();
+      table.string("extension").notNullable();
+      table.string("url_path");
+      table.string("filetype");
+      table.string("metadata");
+      for (let index = 0; index < properties.length; index++) {
+        table.string(properties[index]);
+      }
     };
     const tableExists = await db.schema.hasTable(this.table);
 
@@ -88,15 +99,58 @@ class MddbFile {
     if (!areUniqueObjectsByKey(files, "file_path")) {
       throw new Error("Files must have unique file_path");
     }
+
     const serializedFiles = files.map((file) => {
-      return {
-        ...file,
-        metadata: JSON.stringify(file.metadata),
-      };
+      const serializedFile: any = {};
+      const defaultProperties = [
+        "_id",
+        "file_path",
+        "extension",
+        "url_path",
+        "filetype",
+      ];
+
+      for (const key in file) {
+        if (Object.prototype.hasOwnProperty.call(file, key)) {
+          const value = file[key];
+          // If the value is undefined, default it to null
+          serializedFile[key] = defaultProperties.includes(key)
+            ? value
+            : value !== undefined
+            ? JSON.stringify(value)
+            : null;
+        }
+      }
+
+      return serializedFile;
     });
 
     return db.batchInsert(Table.Files, serializedFiles);
   }
+
+  get _id(): string {
+    return this.file._id;
+  }
+
+  get file_path(): string {
+    return this.file.file_path;
+  }
+
+  get extension(): string {
+    return this.file.extension;
+  }
+
+  get url_path(): string | null {
+    return this.file.url_path;
+  }
+
+  get filetype(): string | null {
+    return this.file.filetype;
+  }
+
+  get metadata(): MetaData | null {
+    return this.file.metadata;
+  }
 }
 
 interface Link {