feat(console): use log files instead of in-memory logs (#7043)

This changeset refactors the console so it uses a file to store the logs
(previously, logs were stored in memory). The log retrieval mechanism
still reads the whole file, but some code capable of streaming
line-based files was added for future references.

Closes #7024.

apps/wing-console/console/server/src/consoleLogger.ts

@@ -1,4 +1,9 @@
+import { mkdir, open } from "node:fs/promises";
+import path from "node:path";
+import readline from "node:readline/promises";
+
 import { errorMessage } from "@wingconsole/error-message";
+import { throttle } from "@wingconsole/utilities";
 import { nanoid } from "nanoid";
 
 import type { LogInterface } from "./utils/LogInterface.js";
@@ -26,75 +31,122 @@ export interface LogEntry {
 
 export type MessageType = "info" | "title" | "summary" | "success" | "fail";
 
+interface ListMessagesOptions {}
+
 export interface ConsoleLogger {
-  messages: LogEntry[];
-  verbose: (message: string, source?: LogSource, context?: LogContext) => void;
-  log: (message: string, source?: LogSource, context?: LogContext) => void;
-  error: (message: unknown, source?: LogSource, context?: LogContext) => void;
-  warning: (message: string, source?: LogSource, context?: LogContext) => void;
+  listMessages(options?: ListMessagesOptions): Promise<{
+    entries: LogEntry[];
+  }>;
+  close(): Promise<void>;
+  verbose(message: string, source?: LogSource, context?: LogContext): void;
+  log(message: string, source?: LogSource, context?: LogContext): void;
+  error(message: unknown, source?: LogSource, context?: LogContext): void;
+  warning(message: string, source?: LogSource, context?: LogContext): void;
 }
 
 export interface CreateConsoleLoggerOptions {
-  onLog: (logLevel: LogLevel, message: string) => void;
+  logfile: string;
+  onLog(): void;
   log: LogInterface;
 }
 
-export const createConsoleLogger = ({
+export const createConsoleLogger = async ({
+  logfile,
   onLog,
   log,
-}: CreateConsoleLoggerOptions): ConsoleLogger => {
+}: CreateConsoleLoggerOptions): Promise<ConsoleLogger> => {
+  await mkdir(path.dirname(logfile), { recursive: true });
+
+  // Create or truncate the log file. In the future, we might want to use `a+` to append to the file instead.
+  const fileHandle = await open(logfile, "w+");
+
+  // Create an `appendEntry` function that will append log
+  // entries to the log file at a maximum speed of 4 times a second.
+  // Finally, `onLog` will be called to report changes to the log file.
+  const { appendEntry } = (() => {
+    const pendingEntries = new Array<LogEntry>();
+    const flushEntries = throttle(async () => {
+      const [...entries] = pendingEntries;
+      pendingEntries.length = 0;
+      for (const entry of entries) {
+        await fileHandle.appendFile(`${JSON.stringify(entry)}\n`);
+      }
+      onLog();
+    }, 250);
+    const appendEntry = (entry: LogEntry) => {
+      pendingEntries.push(entry);
+      flushEntries();
+    };
+    return { appendEntry };
+  })();
+
   return {
-    messages: new Array<LogEntry>(),
+    async close() {
+      await fileHandle.close();
+    },
+    async listMessages(options) {
+      const fileHandle = await open(logfile, "r+");
+      const entries = new Array<LogEntry>();
+      for await (const line of readline.createInterface({
+        input: fileHandle.createReadStream(),
+      })) {
+        try {
+          entries.push(JSON.parse(line) as LogEntry);
+        } catch (error) {
+          log.error("Failed to parse log entry:");
+          log.error(error);
+        }
+      }
+      await fileHandle.close();
+      return {
+        entries,
+      };
+    },
     verbose(message, source, context) {
       log.info(message);
-      this.messages.push({
+      appendEntry({
         id: `${nanoid()}`,
         timestamp: Date.now(),
         level: "verbose",
         message,
         source: source ?? "console",
         ctx: context,
       });
-      onLog("verbose", message);
     },
     log(message, source, context) {
       log.info(message);
-      this.messages.push({
+      appendEntry({
         id: `${nanoid()}`,
         timestamp: Date.now(),
         level: "info",
         message,
         source: source ?? "console",
         ctx: context,
       });
-      onLog("info", message);
     },
     warning(message, source, context) {
       log.warning(message);
-      this.messages.push({
+      appendEntry({
         id: `${nanoid()}`,
         timestamp: Date.now(),
         level: "warn",
         message,
         source: source ?? "console",
         ctx: context,
       });
-      onLog("warn", message);
     },
     error(error, source, context) {
       log.error(error);
-      const message = errorMessage(error);
       if (source === "user") {
-        this.messages.push({
+        appendEntry({
           id: `${nanoid()}`,
           timestamp: Date.now(),
           level: "error",
-          message,
+          message: errorMessage(error),
           source,
           ctx: context,
         });
       }
-      onLog("error", message);
     },
   };
 };

apps/wing-console/console/server/src/index.ts

@@ -1,3 +1,5 @@
+import path from "node:path";
+
 import type { inferRouterInputs } from "@trpc/server";
 import { throttle } from "@wingconsole/utilities";
 import Emittery from "emittery";
@@ -115,11 +117,15 @@ export const createConsoleServer = async ({
     invalidateQuery("app.logs");
   }, 300);
 
-  const consoleLogger: ConsoleLogger = createConsoleLogger({
+  const logfile = `${path.dirname(wingfile)}/target/${path.basename(
+    wingfile,
+  )}sim/.console/logs.jsonl`;
+  const consoleLogger = await createConsoleLogger({
+    logfile,
+    log,
     onLog: () => {
       invalidateLogs();
     },
-    log,
   });
 
   const compiler = createCompiler({
@@ -322,6 +328,7 @@ export const createConsoleServer = async ({
         simulator.stop(),
         testRunner.forceStop(),
       ]);
+      await consoleLogger.close();
     } catch (error) {
       log.error(error);
     } finally {

apps/wing-console/console/server/src/router/app.ts

@@ -110,7 +110,8 @@ export const createAppRouter = () => {
         const lowerCaseText = filters.text?.toLowerCase();
         let noVerboseLogsCount = 0;
 
-        const filteredLogs = ctx.logger.messages.filter((entry) => {
+        const { entries } = await ctx.logger.listMessages();
+        const filteredLogs = entries.filter((entry) => {
           // Filter by timestamp
           if (entry.timestamp && entry.timestamp < filters.timestamp) {
             return false;

apps/wing-console/console/ui/src/features/logs-pane/console-logs-filters.tsx

@@ -17,7 +17,11 @@ import classNames from "classnames";
 import { memo, useCallback, useEffect, useMemo, useState } from "react";
 
 export const LOG_LEVELS: LogLevel[] = ["verbose", "info", "warn", "error"];
-export const DEFAULT_LOG_LEVELS: LogLevel[] = ["info", "warn", "error"].sort();
+export const DEFAULT_LOG_LEVELS = [
+  "info",
+  "warn",
+  "error",
+].sort() as LogLevel[];
 
 const logLevelNames = {
   verbose: "Verbose",

apps/wing-console/console/ui/src/features/logs-pane/logs.tsx

@@ -5,7 +5,7 @@ import {
 } from "@wingconsole/design-system";
 import type { LogEntry } from "@wingconsole/server";
 import classNames from "classnames";
-import { useState, useRef, useEffect, useCallback, memo } from "react";
+import { useState, useRef, useEffect, useCallback, memo, useMemo } from "react";
 
 import { trpc } from "../../trpc.js";
 import { useAppLocalStorage } from "../localstorage-context/use-localstorage.js";
@@ -64,6 +64,18 @@ export const LogsWidget = memo(({ onResourceClick }: LogsWidgetProps) => {
     },
   );
 
+  const entries = useMemo(() => {
+    return logs.data?.logs ?? [];
+  }, [logs.data?.logs]);
+
+  const shownLogs = useMemo(() => {
+    return entries.length;
+  }, [entries]);
+
+  const hiddenLogs = useMemo(() => {
+    return logs.data?.hiddenLogs ?? 0;
+  }, [logs.data?.hiddenLogs]);
+
   const scrollableRef = useRef<HTMLDivElement>(null);
   const [scrolledToBottom, setScrolledToBottom] = useState(true);
   const [initialScroll, setInitialScroll] = useState(true);
@@ -127,8 +139,8 @@ export const LogsWidget = memo(({ onResourceClick }: LogsWidgetProps) => {
         selectedResourceTypes={selectedResourceTypes}
         setSelectedResourceTypes={setSelectedResourceTypes}
         onResetFilters={resetFilters}
-        shownLogs={logs.data?.logs.length ?? 0}
-        hiddenLogs={logs.data?.hiddenLogs ?? 0}
+        shownLogs={shownLogs}
+        hiddenLogs={hiddenLogs}
       />
 
       <div className="relative h-full">
@@ -144,8 +156,8 @@ export const LogsWidget = memo(({ onResourceClick }: LogsWidgetProps) => {
           onScrolledToBottomChange={setScrolledToBottom}
         >
           <ConsoleLogs
-            logs={logs.data?.logs ?? []}
-            hiddenLogs={logs.data?.hiddenLogs ?? 0}
+            logs={entries}
+            hiddenLogs={hiddenLogs}
             onResourceClick={onLogClick}
           />
         </ScrollableArea>

apps/wing-console/packages/utilities/src/index.ts

@@ -1,4 +1,5 @@
 export * from "./debounce.js";
 export * from "./escape-html.js";
+export * from "./read-lines.js";
 export * from "./throttle.js";
 export * from "./uniq-by.js";