From d0b8d8af14dc700192dd3c626ffed3cc0cf3f09b Mon Sep 17 00:00:00 2001 From: mohamed yahia Date: Fri, 15 Dec 2023 11:23:40 +0200 Subject: [PATCH] Add computed fields to SQL (#92) * Add computed fields to SQL * Refactor code * fix tests * fix conflicts --- src/lib/databaseUtils.ts | 20 +++++++++-- src/lib/indexFolder.ts | 2 +- src/lib/markdowndb.ts | 6 ++-- src/lib/schema.ts | 76 +++++++++++++++++++++++++++------------- 4 files changed, 74 insertions(+), 30 deletions(-) diff --git a/src/lib/databaseUtils.ts b/src/lib/databaseUtils.ts index 5292ff6..d1c1b65 100644 --- a/src/lib/databaseUtils.ts +++ b/src/lib/databaseUtils.ts @@ -4,7 +4,7 @@ import path from "path"; import { WikiLink } from "./parseFile.js"; export async function resetDatabaseTables(db: Knex) { - const tableNames = [MddbFile, MddbTag, MddbFileTag, MddbLink]; + const tableNames = [MddbTag, MddbFileTag, MddbLink]; // Drop and Create tables for (const table of tableNames) { await table.deleteTable(db); @@ -13,8 +13,8 @@ export async function resetDatabaseTables(db: Knex) { } export function mapFileToInsert(file: any) { - const { _id, file_path, extension, url_path, filetype, metadata } = file; - return { _id, file_path, extension, url_path, filetype, metadata }; + const { tags, links, ...rest } = file; + return { ...rest }; } export function mapLinksToInsert(filesToInsert: File[], file: any) { @@ -67,3 +67,17 @@ export function getUniqueValues(inputArray: T[]): T[] { return uniqueArray; } + +export function getUniqueProperties(objects: any[]): string[] { + const uniqueProperties: string[] = []; + + for (const object of objects) { + for (const key of Object.keys(object)) { + if (!uniqueProperties.includes(key)) { + uniqueProperties.push(key); + } + } + } + + return uniqueProperties; +} diff --git a/src/lib/indexFolder.ts b/src/lib/indexFolder.ts index 52a839d..c017df8 100644 --- a/src/lib/indexFolder.ts +++ b/src/lib/indexFolder.ts @@ -40,7 +40,7 @@ export function indexFolder( if (!result.success) { const error: ZodError = (result as any).error; - error.errors.forEach((err) => { + error.errors.forEach((err: any) => { const errorMessage = `Error: In ${ fileObject.file_path } for the ${documentType} schema. \n In "${err.path.join( diff --git a/src/lib/markdowndb.ts b/src/lib/markdowndb.ts index 5803736..1ec25ec 100644 --- a/src/lib/markdowndb.ts +++ b/src/lib/markdowndb.ts @@ -10,6 +10,7 @@ import { isLinkToDefined, mapFileTagsToInsert, getUniqueValues, + getUniqueProperties, } from "./databaseUtils.js"; import fs from "fs"; import { CustomConfig } from "./CustomConfig.js"; @@ -85,8 +86,6 @@ export class MarkdownDB { customConfig?: CustomConfig; watch?: boolean; }) { - await resetDatabaseTables(this.db); - const fileObjects = indexFolder( folderPath, pathToUrlResolver, @@ -152,6 +151,9 @@ export class MarkdownDB { private async saveDataToDisk(fileObjects: FileInfo[]) { await resetDatabaseTables(this.db); + const properties = getUniqueProperties(fileObjects); + MddbFile.deleteTable(this.db); + await MddbFile.createTable(this.db, properties); const filesToInsert = fileObjects.map(mapFileToInsert); const uniqueTags = getUniqueValues( diff --git a/src/lib/schema.ts b/src/lib/schema.ts index b1f3d77..b41f090 100644 --- a/src/lib/schema.ts +++ b/src/lib/schema.ts @@ -25,11 +25,20 @@ interface File { url_path: string | null; filetype: string | null; metadata: MetaData | null; + [key: string]: any; } class MddbFile { static table = Table.Files; static supportedExtensions = ["md", "mdx"]; + static defaultProperties = [ + "_id", + "file_path", + "extension", + "url_path", + "filetype", + "metadata", + ]; _id: string; file_path: string; @@ -39,36 +48,42 @@ class MddbFile { // and another one for many-to-many relationship between files and filetypes filetype: string | null; metadata: MetaData | null; + [key: string]: any; // TODO type? constructor(file: any) { - this._id = file._id; - this.file_path = file.file_path; - this.extension = file.extension; - this.url_path = file.url_path; - this.filetype = file.filetype; - this.metadata = file.metadata ? JSON.parse(file.metadata) : null; + if (!file) { + return; + } + Object.keys(file).forEach((key) => { + if (key === "metadata") { + this[key] = file[key] ? JSON.parse(file[key]) : null; + } else { + this[key] = file[key]; + } + }); } toObject(): File { - return { - _id: this._id, - file_path: this.file_path, - extension: this.extension, - url_path: this.url_path, - filetype: this.filetype, - metadata: this.metadata, - }; + return { ...this.file }; } - static async createTable(db: Knex) { + static async createTable(db: Knex, properties: string[]) { const creator = (table: Knex.TableBuilder) => { table.string("_id").primary(); - table.string("file_path").unique().notNullable(); // Path relative to process.cwd() - table.string("extension").notNullable(); // File extension - table.string("url_path"); // Sluggfied path relative to content folder - table.string("filetype"); // Type field in frontmatter if it exists - table.string("metadata"); // All frontmatter data + table.string("file_path").unique().notNullable(); + table.string("extension").notNullable(); + table.string("url_path"); + table.string("filetype"); + table.string("metadata"); + properties.forEach((property) => { + if ( + MddbFile.defaultProperties.indexOf(property) === -1 && + ["tags", "links"].indexOf(property) === -1 + ) { + table.string(property); + } + }); }; const tableExists = await db.schema.hasTable(this.table); @@ -88,11 +103,24 @@ class MddbFile { if (!areUniqueObjectsByKey(files, "file_path")) { throw new Error("Files must have unique file_path"); } + const serializedFiles = files.map((file) => { - return { - ...file, - metadata: JSON.stringify(file.metadata), - }; + const serializedFile: any = {}; + + Object.keys(file).forEach((key) => { + const value = file[key]; + // If the value is undefined, default it to null + if (value !== undefined) { + const shouldStringify = + key === "metadata" || !MddbFile.defaultProperties.includes(key); + // Stringify all user-defined fields and metadata + serializedFile[key] = shouldStringify ? JSON.stringify(value) : value; + } else { + serializedFile[key] = null; + } + }); + + return serializedFile; }); return db.batchInsert(Table.Files, serializedFiles);