From 7cc1656dd2b6a0cee3f74b1c344d214196168117 Mon Sep 17 00:00:00 2001 From: mohamed yahia Date: Fri, 15 Dec 2023 01:02:48 +0200 Subject: [PATCH] [#45][s]: Implement how-reloading with chokidar. --- package-lock.json | 82 +++++++++++++++++++++++++++++++++++------- package.json | 1 + src/bin/index.js | 9 +++-- src/lib/indexFolder.ts | 2 +- src/lib/markdowndb.ts | 67 +++++++++++++++++++++++++++++++++- 5 files changed, 145 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index 13b7e7c..285faf0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "mddb", - "version": "0.6.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mddb", - "version": "0.6.0", + "version": "0.7.0", "license": "MIT", "dependencies": { "@portaljs/remark-wiki-link": "^1.0.4", + "chokidar": "^3.5.3", "gray-matter": "^4.0.3", "knex": "^2.4.2", "react-markdown": "^9.0.1", @@ -2688,7 +2689,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -2913,6 +2913,14 @@ "node": ">=4" } }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "engines": { + "node": ">=8" + } + }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -2931,7 +2939,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -3167,6 +3174,43 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/chownr": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", @@ -4211,7 +4255,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -4307,7 +4350,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "optional": true, "os": [ @@ -5009,6 +5051,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/is-boolean-object": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", @@ -5118,7 +5171,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -5144,7 +5196,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "dependencies": { "is-extglob": "^2.1.1" }, @@ -5183,7 +5234,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "engines": { "node": ">=0.12.0" } @@ -7984,7 +8034,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -8308,7 +8357,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "engines": { "node": ">=8.6" }, @@ -9369,6 +9417,17 @@ "node": ">= 6" } }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/rechoir": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", @@ -10538,7 +10597,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "dependencies": { "is-number": "^7.0.0" }, diff --git a/package.json b/package.json index 367ab3a..8082c49 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "types": "./dist/src/index.d.ts", "dependencies": { "@portaljs/remark-wiki-link": "^1.0.4", + "chokidar": "^3.5.3", "gray-matter": "^4.0.3", "knex": "^2.4.2", "react-markdown": "^9.0.1", diff --git a/src/bin/index.js b/src/bin/index.js index 6459ccc..6e71d78 100755 --- a/src/bin/index.js +++ b/src/bin/index.js @@ -5,12 +5,14 @@ import { MarkdownDB } from "../lib/markdowndb.js"; // TODO get these from markdowndb.config.js or something const dbPath = "markdown.db"; const ignorePatterns = [/Excalidraw/, /\.obsidian/, /DS_Store/]; -const [contentPath] = process.argv.slice(2); +const [contentPath, watchFlag] = process.argv.slice(2); if (!contentPath) { throw new Error("Invalid/Missing path to markdown content folder"); } +const watchEnabled = watchFlag && watchFlag === "--watch"; + const client = new MarkdownDB({ client: "sqlite3", connection: { @@ -23,6 +25,9 @@ await client.init(); await client.indexFolder({ folderPath: contentPath, ignorePatterns: ignorePatterns, + watch: watchEnabled, }); -process.exit(); +if (!watchEnabled) { + process.exit(); +} diff --git a/src/lib/indexFolder.ts b/src/lib/indexFolder.ts index c533844..52a839d 100644 --- a/src/lib/indexFolder.ts +++ b/src/lib/indexFolder.ts @@ -56,7 +56,7 @@ export function indexFolder( return files; } -function shouldIncludeFile( +export function shouldIncludeFile( filePath: string, ignorePatterns?: RegExp[] ): boolean { diff --git a/src/lib/markdowndb.ts b/src/lib/markdowndb.ts index d49d698..5803736 100644 --- a/src/lib/markdowndb.ts +++ b/src/lib/markdowndb.ts @@ -2,7 +2,7 @@ import path from "path"; import knex, { Knex } from "knex"; import { MddbFile, MddbTag, MddbLink, MddbFileTag } from "./schema.js"; -import { indexFolder } from "./indexFolder.js"; +import { indexFolder, shouldIncludeFile } from "./indexFolder.js"; import { resetDatabaseTables, mapFileToInsert, @@ -13,6 +13,9 @@ import { } from "./databaseUtils.js"; import fs from "fs"; import { CustomConfig } from "./CustomConfig.js"; +import { FileInfo, processFile } from "./process.js"; +import chokidar from "chokidar"; +import { recursiveWalkDir } from "./recursiveWalkDir.js"; const defaultFilePathToUrl = (filePath: string) => { let url = filePath @@ -74,11 +77,13 @@ export class MarkdownDB { ignorePatterns = [], pathToUrlResolver = defaultFilePathToUrl, customConfig = {}, + watch = false, }: { folderPath: string; ignorePatterns?: RegExp[]; pathToUrlResolver?: (filePath: string) => string; customConfig?: CustomConfig; + watch?: boolean; }) { await resetDatabaseTables(this.db); @@ -88,6 +93,66 @@ export class MarkdownDB { customConfig, ignorePatterns ); + await this.saveDataToDisk(fileObjects); + + if (watch) { + const watcher = chokidar.watch(folderPath, { + ignoreInitial: true, + }); + + const filePathsToIndex = recursiveWalkDir(folderPath); + const computedFields = customConfig.computedFields || []; + + const handleFileEvent = (event: string, filePath: string) => { + if (!shouldIncludeFile(filePath, ignorePatterns)) { + return; + } + + if (event === "unlink") { + const index = fileObjects.findIndex( + (obj) => obj.file_path === filePath + ); + if (index !== -1) { + fileObjects.splice(index, 1); + } + console.log(`File ${filePath} has been removed`); + return; + } + + const fileObject = processFile( + folderPath, + filePath, + pathToUrlResolver, + filePathsToIndex, + computedFields + ); + const index = fileObjects.findIndex( + (obj) => obj.file_path === filePath + ); + + if (index !== -1) { + fileObjects[index] = fileObject; + } else { + fileObjects.push(fileObject); + } + + console.log( + `File ${filePath} has been ${event === "add" ? "added" : "updated"}` + ); + }; + + watcher + .on("add", (filePath) => handleFileEvent("add", filePath)) + .on("change", (filePath) => handleFileEvent("change", filePath)) + .on("unlink", (filePath) => handleFileEvent("unlink", filePath)) + .on("all", () => this.saveDataToDisk(fileObjects)) + .on("error", (error) => console.error(`Watcher error: ${error}`)); + } + } + + private async saveDataToDisk(fileObjects: FileInfo[]) { + await resetDatabaseTables(this.db); + const filesToInsert = fileObjects.map(mapFileToInsert); const uniqueTags = getUniqueValues( fileObjects.flatMap((file) => file.tags)