#45 , Implement hot-reloading

package.json

@@ -45,6 +45,7 @@
   "types": "./dist/src/index.d.ts",
   "dependencies": {
     "@portaljs/remark-wiki-link": "^1.0.4",
+    "chokidar": "^3.5.3",
     "gray-matter": "^4.0.3",
     "knex": "^2.4.2",
     "react-markdown": "^9.0.1",

src/bin/index.js

@@ -5,12 +5,14 @@ import { MarkdownDB } from "../lib/markdowndb.js";
 // TODO get these from markdowndb.config.js or something
 const dbPath = "markdown.db";
 const ignorePatterns = [/Excalidraw/, /\.obsidian/, /DS_Store/];
-const [contentPath] = process.argv.slice(2);
+const [contentPath, watchFlag] = process.argv.slice(2);
 
 if (!contentPath) {
   throw new Error("Invalid/Missing path to markdown content folder");
 }
 
+const watchEnabled = watchFlag && watchFlag === "--watch";
+
 const client = new MarkdownDB({
   client: "sqlite3",
   connection: {
@@ -23,6 +25,9 @@ await client.init();
 await client.indexFolder({
   folderPath: contentPath,
   ignorePatterns: ignorePatterns,
+  watch: watchEnabled,
 });
 
-process.exit();
+if (!watchEnabled) {
+  process.exit();
+}

src/lib/indexFolder.ts

@@ -38,7 +38,7 @@
       const result = schemas[documentType].safeParse(flattenedFileObject);
 
       if (!result.success) {
         const error: ZodError = (result as any).error;
 
         error.errors.forEach((err) => {
           const errorMessage = `Error: In ${
@@ -56,7 +56,7 @@
   return files;
 }
 
-function shouldIncludeFile(
+export function shouldIncludeFile(
   filePath: string,
   ignorePatterns?: RegExp[]
 ): boolean {

src/lib/markdowndb.ts

@@ -2,7 +2,7 @@
 import knex, { Knex } from "knex";
 
 import { MddbFile, MddbTag, MddbLink, MddbFileTag } from "./schema.js";
-import { indexFolder } from "./indexFolder.js";
+import { indexFolder, shouldIncludeFile } from "./indexFolder.js";
 import {
   resetDatabaseTables,
   mapFileToInsert,
@@ -13,6 +13,9 @@
 } from "./databaseUtils.js";
 import fs from "fs";
 import { CustomConfig } from "./CustomConfig.js";
+import { FileInfo, processFile } from "./process.js";
+import chokidar from "chokidar";
+import { recursiveWalkDir } from "./recursiveWalkDir.js";
 
 const defaultFilePathToUrl = (filePath: string) => {
   let url = filePath
@@ -23,7 +26,7 @@
   return encodeURI(url);
 };
 
 const resolveLinkToUrlPath = (link: string, sourceFilePath?: string) => {
   if (!sourceFilePath) {
     return link;
   }
@@ -74,11 +77,13 @@
     ignorePatterns = [],
     pathToUrlResolver = defaultFilePathToUrl,
     customConfig = {},
+    watch = false,
   }: {
     folderPath: string;
     ignorePatterns?: RegExp[];
     pathToUrlResolver?: (filePath: string) => string;
     customConfig?: CustomConfig;
+    watch?: boolean;
   }) {
     await resetDatabaseTables(this.db);
 
@@ -88,6 +93,66 @@
       customConfig,
       ignorePatterns
     );
+    await this.saveDataToDisk(fileObjects);
+
+    if (watch) {
+      const watcher = chokidar.watch(folderPath, {
+        ignoreInitial: true,
+      });
+
+      const filePathsToIndex = recursiveWalkDir(folderPath);
+      const computedFields = customConfig.computedFields || [];
+
+      const handleFileEvent = (event: string, filePath: string) => {
+        if (!shouldIncludeFile(filePath, ignorePatterns)) {
+          return;
+        }
+
+        if (event === "unlink") {
+          const index = fileObjects.findIndex(
+            (obj) => obj.file_path === filePath
+          );
+          if (index !== -1) {
+            fileObjects.splice(index, 1);
+          }
+          console.log(`File ${filePath} has been removed`);
+          return;
+        }
+
+        const fileObject = processFile(
+          folderPath,
+          filePath,
+          pathToUrlResolver,
+          filePathsToIndex,
+          computedFields
+        );
+        const index = fileObjects.findIndex(
+          (obj) => obj.file_path === filePath
+        );
+
+        if (index !== -1) {
+          fileObjects[index] = fileObject;
+        } else {
+          fileObjects.push(fileObject);
+        }
+
+        console.log(
+          `File ${filePath} has been ${event === "add" ? "added" : "updated"}`
+        );
+      };
+
+      watcher
+        .on("add", (filePath) => handleFileEvent("add", filePath))
+        .on("change", (filePath) => handleFileEvent("change", filePath))
+        .on("unlink", (filePath) => handleFileEvent("unlink", filePath))
+        .on("all", () => this.saveDataToDisk(fileObjects))
+        .on("error", (error) => console.error(`Watcher error: ${error}`));
+    }
+  }
+
+  private async saveDataToDisk(fileObjects: FileInfo[]) {
+    await resetDatabaseTables(this.db);
+
     const filesToInsert = fileObjects.map(mapFileToInsert);
     const uniqueTags = getUniqueValues(
       fileObjects.flatMap((file) => file.tags)
@@ -252,7 +317,7 @@
   }
 }
 
 function writeJsonToFile(filePath: string, jsonData: any[]) {
   try {
     const directory = path.dirname(filePath);
     if (!fs.existsSync(directory)) {