diff --git a/apps/wing-console/console/server/src/consoleLogger.ts b/apps/wing-console/console/server/src/consoleLogger.ts index 78a958e96da..e3e90c79f2a 100644 --- a/apps/wing-console/console/server/src/consoleLogger.ts +++ b/apps/wing-console/console/server/src/consoleLogger.ts @@ -1,4 +1,9 @@ +import { mkdir, open } from "node:fs/promises"; +import path from "node:path"; +import readline from "node:readline/promises"; + import { errorMessage } from "@wingconsole/error-message"; +import { throttle } from "@wingconsole/utilities"; import { nanoid } from "nanoid"; import type { LogInterface } from "./utils/LogInterface.js"; @@ -26,28 +31,80 @@ export interface LogEntry { export type MessageType = "info" | "title" | "summary" | "success" | "fail"; +interface ListMessagesOptions {} + export interface ConsoleLogger { - messages: LogEntry[]; - verbose: (message: string, source?: LogSource, context?: LogContext) => void; - log: (message: string, source?: LogSource, context?: LogContext) => void; - error: (message: unknown, source?: LogSource, context?: LogContext) => void; - warning: (message: string, source?: LogSource, context?: LogContext) => void; + listMessages(options?: ListMessagesOptions): Promise<{ + entries: LogEntry[]; + }>; + close(): Promise; + verbose(message: string, source?: LogSource, context?: LogContext): void; + log(message: string, source?: LogSource, context?: LogContext): void; + error(message: unknown, source?: LogSource, context?: LogContext): void; + warning(message: string, source?: LogSource, context?: LogContext): void; } export interface CreateConsoleLoggerOptions { - onLog: (logLevel: LogLevel, message: string) => void; + logfile: string; + onLog(): void; log: LogInterface; } -export const createConsoleLogger = ({ +export const createConsoleLogger = async ({ + logfile, onLog, log, -}: CreateConsoleLoggerOptions): ConsoleLogger => { +}: CreateConsoleLoggerOptions): Promise => { + await mkdir(path.dirname(logfile), { recursive: true }); + + // Create or truncate the log file. In the future, we might want to use `a+` to append to the file instead. + const fileHandle = await open(logfile, "w+"); + + // Create an `appendEntry` function that will append log + // entries to the log file at a maximum speed of 4 times a second. + // Finally, `onLog` will be called to report changes to the log file. + const { appendEntry } = (() => { + const pendingEntries = new Array(); + const flushEntries = throttle(async () => { + const [...entries] = pendingEntries; + pendingEntries.length = 0; + for (const entry of entries) { + await fileHandle.appendFile(`${JSON.stringify(entry)}\n`); + } + onLog(); + }, 250); + const appendEntry = (entry: LogEntry) => { + pendingEntries.push(entry); + flushEntries(); + }; + return { appendEntry }; + })(); + return { - messages: new Array(), + async close() { + await fileHandle.close(); + }, + async listMessages(options) { + const fileHandle = await open(logfile, "r+"); + const entries = new Array(); + for await (const line of readline.createInterface({ + input: fileHandle.createReadStream(), + })) { + try { + entries.push(JSON.parse(line) as LogEntry); + } catch (error) { + log.error("Failed to parse log entry:"); + log.error(error); + } + } + await fileHandle.close(); + return { + entries, + }; + }, verbose(message, source, context) { log.info(message); - this.messages.push({ + appendEntry({ id: `${nanoid()}`, timestamp: Date.now(), level: "verbose", @@ -55,11 +112,10 @@ export const createConsoleLogger = ({ source: source ?? "console", ctx: context, }); - onLog("verbose", message); }, log(message, source, context) { log.info(message); - this.messages.push({ + appendEntry({ id: `${nanoid()}`, timestamp: Date.now(), level: "info", @@ -67,11 +123,10 @@ export const createConsoleLogger = ({ source: source ?? "console", ctx: context, }); - onLog("info", message); }, warning(message, source, context) { log.warning(message); - this.messages.push({ + appendEntry({ id: `${nanoid()}`, timestamp: Date.now(), level: "warn", @@ -79,22 +134,19 @@ export const createConsoleLogger = ({ source: source ?? "console", ctx: context, }); - onLog("warn", message); }, error(error, source, context) { log.error(error); - const message = errorMessage(error); if (source === "user") { - this.messages.push({ + appendEntry({ id: `${nanoid()}`, timestamp: Date.now(), level: "error", - message, + message: errorMessage(error), source, ctx: context, }); } - onLog("error", message); }, }; }; diff --git a/apps/wing-console/console/server/src/index.ts b/apps/wing-console/console/server/src/index.ts index 97e88b45574..cce3170b540 100644 --- a/apps/wing-console/console/server/src/index.ts +++ b/apps/wing-console/console/server/src/index.ts @@ -1,3 +1,5 @@ +import path from "node:path"; + import type { inferRouterInputs } from "@trpc/server"; import { throttle } from "@wingconsole/utilities"; import Emittery from "emittery"; @@ -115,11 +117,15 @@ export const createConsoleServer = async ({ invalidateQuery("app.logs"); }, 300); - const consoleLogger: ConsoleLogger = createConsoleLogger({ + const logfile = `${path.dirname(wingfile)}/target/${path.basename( + wingfile, + )}sim/.console/logs.jsonl`; + const consoleLogger = await createConsoleLogger({ + logfile, + log, onLog: () => { invalidateLogs(); }, - log, }); const compiler = createCompiler({ @@ -322,6 +328,7 @@ export const createConsoleServer = async ({ simulator.stop(), testRunner.forceStop(), ]); + await consoleLogger.close(); } catch (error) { log.error(error); } finally { diff --git a/apps/wing-console/console/server/src/router/app.ts b/apps/wing-console/console/server/src/router/app.ts index 84c0b26342f..461962c7395 100644 --- a/apps/wing-console/console/server/src/router/app.ts +++ b/apps/wing-console/console/server/src/router/app.ts @@ -110,7 +110,8 @@ export const createAppRouter = () => { const lowerCaseText = filters.text?.toLowerCase(); let noVerboseLogsCount = 0; - const filteredLogs = ctx.logger.messages.filter((entry) => { + const { entries } = await ctx.logger.listMessages(); + const filteredLogs = entries.filter((entry) => { // Filter by timestamp if (entry.timestamp && entry.timestamp < filters.timestamp) { return false; diff --git a/apps/wing-console/console/ui/src/features/logs-pane/console-logs-filters.tsx b/apps/wing-console/console/ui/src/features/logs-pane/console-logs-filters.tsx index 8b712dace59..996dab1ee95 100644 --- a/apps/wing-console/console/ui/src/features/logs-pane/console-logs-filters.tsx +++ b/apps/wing-console/console/ui/src/features/logs-pane/console-logs-filters.tsx @@ -17,7 +17,11 @@ import classNames from "classnames"; import { memo, useCallback, useEffect, useMemo, useState } from "react"; export const LOG_LEVELS: LogLevel[] = ["verbose", "info", "warn", "error"]; -export const DEFAULT_LOG_LEVELS: LogLevel[] = ["info", "warn", "error"].sort(); +export const DEFAULT_LOG_LEVELS = [ + "info", + "warn", + "error", +].sort() as LogLevel[]; const logLevelNames = { verbose: "Verbose", diff --git a/apps/wing-console/console/ui/src/features/logs-pane/logs.tsx b/apps/wing-console/console/ui/src/features/logs-pane/logs.tsx index aa8833f9c4d..87893983245 100644 --- a/apps/wing-console/console/ui/src/features/logs-pane/logs.tsx +++ b/apps/wing-console/console/ui/src/features/logs-pane/logs.tsx @@ -5,7 +5,7 @@ import { } from "@wingconsole/design-system"; import type { LogEntry } from "@wingconsole/server"; import classNames from "classnames"; -import { useState, useRef, useEffect, useCallback, memo } from "react"; +import { useState, useRef, useEffect, useCallback, memo, useMemo } from "react"; import { trpc } from "../../trpc.js"; import { useAppLocalStorage } from "../localstorage-context/use-localstorage.js"; @@ -64,6 +64,18 @@ export const LogsWidget = memo(({ onResourceClick }: LogsWidgetProps) => { }, ); + const entries = useMemo(() => { + return logs.data?.logs ?? []; + }, [logs.data?.logs]); + + const shownLogs = useMemo(() => { + return entries.length; + }, [entries]); + + const hiddenLogs = useMemo(() => { + return logs.data?.hiddenLogs ?? 0; + }, [logs.data?.hiddenLogs]); + const scrollableRef = useRef(null); const [scrolledToBottom, setScrolledToBottom] = useState(true); const [initialScroll, setInitialScroll] = useState(true); @@ -127,8 +139,8 @@ export const LogsWidget = memo(({ onResourceClick }: LogsWidgetProps) => { selectedResourceTypes={selectedResourceTypes} setSelectedResourceTypes={setSelectedResourceTypes} onResetFilters={resetFilters} - shownLogs={logs.data?.logs.length ?? 0} - hiddenLogs={logs.data?.hiddenLogs ?? 0} + shownLogs={shownLogs} + hiddenLogs={hiddenLogs} />
@@ -144,8 +156,8 @@ export const LogsWidget = memo(({ onResourceClick }: LogsWidgetProps) => { onScrolledToBottomChange={setScrolledToBottom} > diff --git a/apps/wing-console/packages/utilities/src/index.ts b/apps/wing-console/packages/utilities/src/index.ts index 9cfaf8dd6c5..cf321f69a88 100644 --- a/apps/wing-console/packages/utilities/src/index.ts +++ b/apps/wing-console/packages/utilities/src/index.ts @@ -1,4 +1,5 @@ export * from "./debounce.js"; export * from "./escape-html.js"; +export * from "./read-lines.js"; export * from "./throttle.js"; export * from "./uniq-by.js"; diff --git a/apps/wing-console/packages/utilities/src/read-lines.test.ts b/apps/wing-console/packages/utilities/src/read-lines.test.ts new file mode 100644 index 00000000000..7da07eeb38e --- /dev/null +++ b/apps/wing-console/packages/utilities/src/read-lines.test.ts @@ -0,0 +1,180 @@ +import { open, mkdtemp, writeFile } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; +import { tmpdir } from "node:os"; + +import { expect, test as vitest } from "vitest"; + +import { readLines } from "./read-lines.js"; + +const test = vitest.extend<{ + createTemporaryFile: (content: string) => Promise; +}>({ + async createTemporaryFile({}, use) { + const files = new Array(); + await use(async (content: string) => { + const directory = await mkdtemp(`${tmpdir()}/`); + await writeFile(`${directory}/file.jsonl`, content); + const handle = await open(`${directory}/file.jsonl`, "r"); + files.push(handle); + return handle; + }); + for (const file of files) { + await file.close(); + } + }, +}); + +test("reads empty file", async ({ createTemporaryFile }) => { + const fileHandle = await createTemporaryFile(""); + + await expect( + readLines(fileHandle, { direction: "forward" }), + ).resolves.toEqual({ + lines: [], + position: 0, + }); + + await expect( + readLines(fileHandle, { direction: "backward" }), + ).resolves.toEqual({ + lines: [], + position: 0, + }); +}); + +test("reads file with many empty lines", async ({ createTemporaryFile }) => { + const fileHandle = await createTemporaryFile("\n\n\n\n"); + + await expect( + readLines(fileHandle, { + direction: "forward", + }), + ).resolves.toEqual({ + lines: [], + position: 4, + }); + + await expect( + readLines(fileHandle, { + direction: "backward", + }), + ).resolves.toEqual({ + lines: [], + position: 0, + }); +}); + +test("reads small file in a single pass", async ({ createTemporaryFile }) => { + const fileHandle = await createTemporaryFile("1\n2\n"); + + await expect( + readLines(fileHandle, { + direction: "forward", + }), + ).resolves.toEqual({ + lines: ["1", "2"], + position: 4, + }); + + await expect( + readLines(fileHandle, { + direction: "backward", + }), + ).resolves.toEqual({ + lines: ["1", "2"], + position: 0, + }); +}); + +test("reads some lines and returns the new position", async ({ + createTemporaryFile, +}) => { + const fileHandle = await createTemporaryFile("100\n200\n300\n400\n"); + + await expect( + readLines(fileHandle, { + direction: "forward", + buffer: Buffer.alloc(11), + }), + ).resolves.toEqual({ + lines: ["100", "200"], + position: 8, + }); + + await expect( + readLines(fileHandle, { + direction: "forward", + buffer: Buffer.alloc(11), + position: 8, + }), + ).resolves.toEqual({ + lines: ["300", "400"], + position: 16, + }); + + await expect( + readLines(fileHandle, { + direction: "backward", + buffer: Buffer.alloc(11), + }), + ).resolves.toEqual({ + lines: ["300", "400"], + position: 7, + }); + + await expect( + readLines(fileHandle, { + direction: "backward", + buffer: Buffer.alloc(11), + position: 7, + }), + ).resolves.toEqual({ + lines: ["100", "200"], + position: 0, + }); +}); + +test("reads partial lines if they are too big for the buffer size", async ({ + createTemporaryFile, +}) => { + await expect( + readLines(await createTemporaryFile("123456789\n"), { + direction: "forward", + buffer: Buffer.alloc(4), + }), + ).resolves.toEqual({ + lines: [{ partialLine: "1234", start: 0, end: 10 }], + position: 10, + }); + + await expect( + readLines(await createTemporaryFile("123456789\n"), { + direction: "backward", + buffer: Buffer.alloc(4), + }), + ).resolves.toEqual({ + lines: [{ partialLine: "1234", start: 0, end: 10 }], + position: 0, + }); + + await expect( + readLines(await createTemporaryFile("1\n23456789\n"), { + direction: "forward", + buffer: Buffer.alloc(4), + position: 2, + }), + ).resolves.toEqual({ + lines: [{ partialLine: "2345", start: 2, end: 11 }], + position: 11, + }); + + await expect( + readLines(await createTemporaryFile("1\n23456789\n"), { + direction: "backward", + buffer: Buffer.alloc(4), + }), + ).resolves.toEqual({ + lines: [{ partialLine: "2345", start: 2, end: 11 }], + position: 2, + }); +}); diff --git a/apps/wing-console/packages/utilities/src/read-lines.ts b/apps/wing-console/packages/utilities/src/read-lines.ts new file mode 100644 index 00000000000..a0eaf9f6b1a --- /dev/null +++ b/apps/wing-console/packages/utilities/src/read-lines.ts @@ -0,0 +1,205 @@ +import type { FileHandle } from "node:fs/promises"; + +/** + * The default buffer size to use when reading from the file. + */ +const DEFAULT_BUFFER_SIZE = 1024; + +/** + * The character that separates lines in the file. + */ +const SEPARATOR_CHARACTER = "\n"; + +const getFileSize = async (fileHandle: FileHandle): Promise => { + const stats = await fileHandle.stat(); + return stats.size; +}; + +/** + * Reads a chunk of data from the file handle into a buffer. + * + * The chunk is read starting at the given position, in the direction + * specified by the `forward` parameter. + */ +const readChunk = async ( + fileHandle: FileHandle, + buffer: Buffer, + position: number, + forward: boolean, +) => { + const start = forward + ? position + : Math.max(0, position - buffer.byteLength - 1); + const length = forward + ? buffer.byteLength + : Math.min(position, buffer.byteLength); + const { bytesRead } = await fileHandle.read(buffer, 0, length, start); + return { + text: buffer.toString("utf8", 0, bytesRead), + start, + end: start + bytesRead, + }; +}; + +/** + * Attempts to extract lines from a chunk of text. + * + * If the chunk does not contain a separator character, the function + * will return `undefined` instead of an array of lines. + */ +const extractLines = ( + text: string, + start: number, + end: number, + forward: boolean, +) => { + if (forward) { + const separator = text.lastIndexOf(SEPARATOR_CHARACTER); + + if (separator === -1) { + return { + lines: undefined, + position: end, + }; + } + + return { + lines: text + .slice(0, separator) + .split(SEPARATOR_CHARACTER) + .filter((line) => line.length > 0), + position: start + separator + 1, + }; + } + + const separator = start === 0 ? 0 : text.indexOf(SEPARATOR_CHARACTER); + + if (separator === -1) { + return { + lines: undefined, + position: Math.max(0, start - 1), + }; + } + + return { + lines: text + .slice(separator) + .split(SEPARATOR_CHARACTER) + .filter((line) => line.length > 0), + // position: Math.max(0, start + separator - 1), + position: Math.max(0, start + separator), + }; +}; + +/** + * Find the position of the next end of line character in the file. + */ +const findNextEndOfLinePosition = async ( + fileHandle: FileHandle, + buffer: Buffer, + position: number, + forward: boolean, +) => { + while (true) { + const start = forward + ? position + : Math.max(0, position - buffer.byteLength - 1); + const length = forward + ? buffer.byteLength + : Math.min(position, buffer.byteLength); + const { bytesRead } = await fileHandle.read(buffer, 0, length, start); + + if (!forward && position === 0) { + return 0; + } + + if (bytesRead === 0) { + return position; + } + + const text = buffer.toString("utf8", 0, bytesRead); + const separator = forward + ? text.indexOf(SEPARATOR_CHARACTER) + : text.lastIndexOf(SEPARATOR_CHARACTER); + if (separator !== -1) { + return start + separator + 1; + } + position = forward ? start + bytesRead : start; + } +}; + +const readPartialChunk = async ( + fileHandle: FileHandle, + buffer: Buffer, + position: number, + forward: boolean, + chunk: { start: number; end: number }, +) => { + const newPosition = await findNextEndOfLinePosition( + fileHandle, + buffer, + position, + forward, + ); + const newStart = forward ? chunk.start : newPosition; + const newEnd = forward ? newPosition : chunk.end + 1; + const newChunk = await readChunk(fileHandle, buffer, newStart, true); + return { + lines: [ + { + partialLine: newChunk.text, + start: newStart, + end: newEnd, + }, + ], + position: newPosition, + }; +}; + +export interface ReadLinesOptions { + position?: number; + direction?: "forward" | "backward"; + buffer?: Buffer; +} + +export interface PartialLine { + partialLine: string; + start: number; + end: number; +} + +export interface ReadLinesResult { + lines: (string | PartialLine)[]; + position: number; +} + +/** + * Read lines from a file starting at a given position. + * + * If no buffer is passed, a temporary one is created and it's size is {@link DEFAULT_BUFFER_SIZE}. + */ +export const readLines = async ( + fileHandle: FileHandle, + options?: ReadLinesOptions, +): Promise => { + const forward = options?.direction !== "backward"; + const position = + options?.position ?? (forward ? 0 : await getFileSize(fileHandle)); + const buffer = options?.buffer ?? Buffer.alloc(DEFAULT_BUFFER_SIZE); + + const chunk = await readChunk(fileHandle, buffer, position, forward); + if (chunk.text.length === 0) { + return { + lines: [], + position: 0, + }; + } + + const lines = extractLines(chunk.text, chunk.start, chunk.end, forward); + + if (lines.lines === undefined) { + return await readPartialChunk(fileHandle, buffer, position, forward, chunk); + } + + return lines; +};