Refactor code and simplify file processing

src/bin/index.ts

@@ -18,12 +18,11 @@ const client = new MarkdownDB({
   },
 });
 
-// Ignore top-level await errors
-//@ts-ignore
 await client.init();
 
-//@ts-ignore
 await client.indexFolder({
   folderPath: contentPath,
   ignorePatterns: ignorePatterns,
 });
+
+process.exit();

src/lib/indexFolderToObjects.ts

@@ -1,48 +1,55 @@
-import { WikiLink, getUniqueValues, recursiveWalkDir } from "../utils/index.js";
-import { markdownToObject } from "./markdownToObject.js";
-import { File, FileTag, Link, Tag } from "./types/schemaTypes.js";
+import { getUniqueValues, recursiveWalkDir } from "../utils/index.js";
+import type { WikiLink } from "../utils/index.js";
+import { extractFileSchemeFromObject } from "../utils/extractFileSchemeFromObject.js";
+import { readLocalMarkdownFileToObject } from "./readLocalMarkdownFileToObject.js";
+import type { File, FileTag, Link, Tag } from "./types/schemaTypes.js";
 
 export function indexFolderToObjects(
   folderPath: string,
-  ignorePatterns?: RegExp[],
-  pathToUrlResolver?: (filePath: string) => string
+  pathToUrlResolver: (filePath: string) => string,
+  ignorePatterns?: RegExp[]
 ) {
   const filePathsToIndex = recursiveWalkDir(folderPath);
   const files: File[] = [];
-  const tags: Tag[] = [];
+  const tags: string[] = [];
   const fileTags: FileTag[] = [];
   const links: Link[] = [];
-  const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) => {
-    return !(
-      ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
-    );
-  });
+  const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) =>
+    shouldIncludeFile(filePath, ignorePatterns)
+  );
 
   for (const filePath of filteredFilePathsToIndex) {
-    const fileObject = markdownToObject(folderPath, filePath, filePathsToIndex);
+    const fileObject = readLocalMarkdownFileToObject(
+      folderPath,
+      filePath,
+      filePathsToIndex,
+      pathToUrlResolver
+    );
+
+    const file = extractFileSchemeFromObject(fileObject);
+    files.push(file);
 
-    files.push(fileObject.file);
     tags.push(...fileObject.tags);
 
     const fileTagsToInsert = fileObject.tags.map((tag) => ({
-      tag: tag.name,
-      file: fileObject.file._id,
+      tag: tag,
+      file: fileObject._id,
     }));
-    const uniqueFileTags = getUniqueValues(fileTagsToInsert);
-    fileTags.push(...uniqueFileTags);
+    fileTags.push(...fileTagsToInsert);
 
     const linksToInsert: Link[] = processWikiLinks(
       fileObject.links,
-      fileObject.file._id,
+      fileObject._id,
       filePathsToIndex
     );
     links.push(...linksToInsert);
   }
 
   const uniqueTags = getUniqueValues(tags);
+  const TagsToInsert = uniqueTags.map((tag) => ({ name: tag }));
   return {
     files: files,
-    tags: uniqueTags,
+    tags: TagsToInsert,
     fileTags: fileTags,
     links: links,
   };
@@ -56,8 +63,17 @@
   return links
     .map((link) => ({
       from: fileId,
       to: filePathsToIndex.find((file) => file === link.linkSrc)!,
       link_type: link.linkType,
     }))
     .filter((link) => link.to !== undefined);
 }
+
+function shouldIncludeFile(
+  filePath: string,
+  ignorePatterns?: RegExp[]
+): boolean {
+  return !(
+    ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
+  );
+}

src/lib/markdowndb.ts

@@ -2,17 +2,9 @@ import knex, { Knex } from "knex";
 
 import { MddbFile, MddbTag, MddbFileTag, MddbLink } from "./schema.js";
 import { DbQueryManager } from "./DbQueryManager.js";
-import { FilesQuery, LinkQuery } from "./types/DbQueryTypes.js";
+import type { FilesQuery, LinkQuery } from "./types/DbQueryTypes.js";
 import { indexFolderToObjects } from "./indexFolderToObjects.js";
-
-export const defaultFilePathToUrl = (filePath: string) => {
-  let url = filePath
-    .replace(/\.(mdx|md)/, "")
-    .replace(/\\/g, "/") // replace windows backslash with forward slash
-    .replace(/(\/)?index$/, ""); // remove index from the end of the permalink
-  url = url.length > 0 ? url : "/"; // for home page
-  return encodeURI(url);
-};
+import { defaultFilePathToUrl } from "../utils/defaultFilePathToUrl.js";
 
 export class MarkdownDB {
   config: Knex.Config;
@@ -42,14 +34,14 @@ export class MarkdownDB {
     await resetDatabaseTables(this.db);
     const { files, tags, fileTags, links } = indexFolderToObjects(
       folderPath,
-      ignorePatterns,
-      pathToUrlResolver
+      pathToUrlResolver,
+      ignorePatterns
     );
 
-    MddbFile.batchInsert(this.db, files);
-    MddbTag.batchInsert(this.db, tags);
-    MddbFileTag.batchInsert(this.db, fileTags);
-    MddbLink.batchInsert(this.db, links);
+    await MddbFile.batchInsert(this.db, files);
+    await MddbTag.batchInsert(this.db, tags);
+    await MddbFileTag.batchInsert(this.db, fileTags);
+    await MddbLink.batchInsert(this.db, links);
   }
 
   async getFileById(id: string): Promise<MddbFile | null> {

src/lib/readLocalMarkdownFileToObject.ts

@@ -0,0 +1,65 @@
+import fs from "fs";
+import path from "path";
+import { getUniqueValues, parseMarkdownContent } from "../utils/index.js";
+import type { FileObject } from "./types/FileObject.js";
+import { MddbFile } from "./schema.js";
+import { generateFileIdFromPath } from "../utils/index.js";
+import { getFileExtensionFromPath } from "../utils/index.js";
+
+export function readLocalMarkdownFileToObject(
+  folderPath: string,
+  filePath: string,
+  filePathsToIndex: string[],
+  pathToUrlResolver: (filePath: string) => string
+): FileObject {
+  const id = generateFileIdFromPath(filePath);
+  const extension = getFileExtensionFromPath(filePath);
+  const fileObject: FileObject = {
+    _id: id,
+    file_path: filePath,
+    extension,
+    url_path: null,
+    filetype: null,
+    metadata: null,
+    tags: [],
+    links: [],
+  };
+
+  // if not md or mdx return this
+  const isExtensionNotSupported =
+    !MddbFile.supportedExtensions.includes(extension);
+  if (isExtensionNotSupported) {
+    return fileObject;
+  }
+
+  // url_path
+  const pathRelativeToFolder = path.relative(folderPath, filePath);
+  const urlPath = pathToUrlResolver(pathRelativeToFolder);
+
+  // metadata, tags, links
+  let source: string, metadata, links;
+  try {
+    source = fs.readFileSync(filePath, {
+      encoding: "utf8",
+      flag: "r",
+    });
+
+    ({ metadata, links } = parseMarkdownContent(source, {
+      permalinks: filePathsToIndex,
+    }));
+
+    fileObject.url_path = urlPath;
+    fileObject.metadata = metadata;
+    fileObject.filetype = metadata?.type || null;
+    fileObject.links = links;
+    const tags = metadata.tags;
+    if (tags) {
+      fileObject.tags = getUniqueValues(tags);
+    }
+
+    return fileObject;
+  } catch (e) {
+    console.error(`Error processing file ${filePath}: ${e}`);
+    return fileObject;
+  }
+}

src/lib/types/FileObject.ts

@@ -1,8 +1,7 @@
 import { WikiLink } from "../../utils";
-import { File, Tag } from "./schemaTypes";
+import { File } from "./schemaTypes";
 
-export interface FileObject {
-  file: File;
-  tags: Tag[];
+export interface FileObject extends File {
+  tags: string[];
   links: WikiLink[];
 }

src/lib/types/schemaTypes.ts

@@ -31,4 +31,4 @@ export enum Table {
   Tags = "tags",
   FileTags = "file_tags",
   Links = "links",
-}
+}

src/utils/defaultFilePathToUrl.ts

@@ -0,0 +1,8 @@
+export const defaultFilePathToUrl = (filePath: string) => {
+  let url = filePath
+    .replace(/\.(mdx|md)/, "")
+    .replace(/\\/g, "/") // replace windows backslash with forward slash
+    .replace(/(\/)?index$/, ""); // remove index from the end of the permalink
+  url = url.length > 0 ? url : "/"; // for home page
+  return encodeURI(url);
+};

src/utils/extractFileSchemeFromObject.ts

@@ -0,0 +1,12 @@
+import { FileObject } from "../lib/types/FileObject.js";
+
+export function extractFileSchemeFromObject(fileObject: FileObject) {
+  return {
+    _id: fileObject._id,
+    file_path: fileObject.file_path,
+    extension: fileObject.extension,
+    url_path: fileObject.url_path,
+    filetype: fileObject.filetype,
+    metadata: fileObject.metadata,
+  };
+}

src/utils/index.ts

@@ -1,6 +1,8 @@
 export { recursiveWalkDir } from "./recursiveWalkDir.js";
 export { extractWikiLinks, WikiLink } from "./extractWikiLinks.js";
-export { parseFile } from "./parseFile.js";
+export { parseMarkdownContent } from "./parseMarkdownContent.js";
 export { getUniqueValues } from "./getUniqueValues.js";
 export { generateFileIdFromPath } from "./generateFileIdFromPath.js";
 export { getFileExtensionFromPath } from "./getFileExtensionFromPath.js";
+export { defaultFilePathToUrl } from "./defaultFilePathToUrl.js";
+export { extractFileSchemeFromObject } from "./extractFileSchemeFromObject.js";

src/utils/parseFile.spec.ts → src/utils/parseMarkdownContent.spec.ts

@@ -1,4 +1,4 @@
-import { parseFile } from "./parseFile";
+import { parseMarkdownContent } from "./parseMarkdownContent";
 
 const source = `---
 title: Hello World
@@ -25,7 +25,7 @@ describe("parseFile", () => {
       { linkType: "normal", linkSrc: "blog/Some Other Link" },
       { linkType: "embed", linkSrc: "Some Image.png" },
     ];
-    const { metadata, links } = parseFile(source);
+    const { metadata, links } = parseMarkdownContent(source);
     expect(metadata).toEqual(expectedMetadata);
     expect(links).toEqual(expectedLinks);
   });
@@ -48,7 +48,7 @@ describe("parseFile", () => {
       "/some/folder/blog/Some Other Link",
       "/some/folder/Some Image.png",
     ];
-    const { metadata, links } = parseFile(source, { permalinks });
+    const { metadata, links } = parseMarkdownContent(source, { permalinks });
     expect(metadata).toEqual(expectedMetadata);
     expect(links).toEqual(expectedLinks);
   });

src/utils/parseFile.ts → src/utils/parseMarkdownContent.ts

@@ -1,7 +1,7 @@
 import matter from "gray-matter";
 import { extractWikiLinks } from "./extractWikiLinks.js";
 
-export function parseFile(source: string, options?: { permalinks?: string[] }) {
+export function parseMarkdownContent(source: string, options?: { permalinks?: string[] }) {
   // Metadata
   const { data: metadata } = matter(source);
 

tsconfig.lib.json

@@ -6,7 +6,7 @@
     "strict": true,
     "strictPropertyInitialization": false,
     "target": "es2020",
-    "module": "es2020",
+    "module": "esnext",
     "moduleResolution": "node",
     "esModuleInterop": true,
     "types": ["node"]