feat(sdk): containers logs and improved simulator stability (#6408)

## Container improvements

Revisit how we are running containers in `sim.Container` to improve robustness and capture logs from docker build/pull commands and from running containers.

To do that we've changed how container lifecycle is managed by running it without `--detach` and monitoring the child process for exit codes, errors and stdio capture.

## Simulator stability improvements

Additionally, this fixes a few long standing issues in the simulator which caused it to hang when one of the resources failed to start (e.g. due to unresolvable token). To fix this, we track failures and if there were any failing resources, we stop all the started resources and throw an error. Additionally, in the CLI we needed to catch this error, stop the spinner and bail out from the test execution routine. I believe this fixes #6052 and #5426.

## Logging improvements

Add some initial support for log levels, because, come on, currently errors emitted from the simulator or resources are hidden by default because they are treated as "verbose". This includes the relevant Console fixes.

Tweak a little how logs are displayed when `DEBUG=1` so that log entries are separated by a newline and removed the indentation of the first line (basically like jest/vitest is doing).

## Other improvements

Hide the `cloud.Endpoint` nodes from the graph because it was a mistake (mine) to suggest to show them.

## Checklist

- [x] Title matches [Winglang's style guide](https://www.winglang.io/contributing/start-here/pull_requests#how-are-pull-request-titles-formatted)
- [x] Description explains motivation and solution
- [x] Tests added (always)
- [x] Docs updated (only required for features)
- [x] Added `pr/e2e-full` label if this feature requires end-to-end testing

*By submitting this pull request, I confirm that my contribution is made under the terms of the [Wing Cloud Contribution License](https://github.com/winglang/wing/blob/main/CONTRIBUTION_LICENSE.md)*.

apps/wing-console/console/app/src/index.ts

@@ -133,6 +133,7 @@ export const createConsoleApp = async (options: CreateConsoleAppOptions) => {
     log: options.log ?? {
       info() {},
       error: console.error,
+      warning() {},
       verbose() {},
     },
     config: options.config ?? {

apps/wing-console/console/server/src/consoleLogger.ts

@@ -31,6 +31,7 @@ export interface ConsoleLogger {
   verbose: (message: string, source?: LogSource, context?: LogContext) => void;
   log: (message: string, source?: LogSource, context?: LogContext) => void;
   error: (message: unknown, source?: LogSource, context?: LogContext) => void;
+  warning: (message: string, source?: LogSource, context?: LogContext) => void;
 }
 
 export interface CreateConsoleLoggerOptions {
@@ -68,6 +69,18 @@ export const createConsoleLogger = ({
       });
       onLog("info", message);
     },
+    warning(message, source, context) {
+      log.warning(message);
+      this.messages.push({
+        id: `${nanoid()}`,
+        timestamp: Date.now(),
+        level: "warn",
+        message,
+        source: source ?? "console",
+        ctx: context,
+      });
+      onLog("warn", message);
+    },
     error(error, source, context) {
       log.error(error);
       const message = errorMessage(error);

apps/wing-console/console/server/src/index.ts

@@ -7,6 +7,7 @@ import Emittery from "emittery";
 import type { Express } from "express";
 
 import type { Config } from "./config.js";
+import type { LogSource } from "./consoleLogger.js";
 import { type ConsoleLogger, createConsoleLogger } from "./consoleLogger.js";
 import { createExpressServer } from "./expressServer.js";
 import type { HostUtils } from "./hostUtils.js";
@@ -218,23 +219,44 @@ export const createConsoleServer = async ({
     const message = `${
       trace.data.message ?? JSON.stringify(trace.data, undefined, 2)
     }`;
-    if (trace.type === "log") {
-      consoleLogger.log(message, "simulator", {
-        sourceType: trace.sourceType,
-        sourcePath: trace.sourcePath,
-      });
-    } else {
-      consoleLogger.verbose(message, "simulator", {
-        sourceType: trace.sourceType,
-        sourcePath: trace.sourcePath,
-      });
-    }
-    if (trace.data.status === "failure") {
-      let output = await formatTraceError(trace.data.error);
-      consoleLogger.error(output, "user", {
-        sourceType: trace.sourceType,
-        sourcePath: trace.sourcePath,
-      });
+
+    const source = logSourceFromTraceType(trace);
+
+    switch (trace.level) {
+      case "verbose": {
+        consoleLogger.verbose(message, source, {
+          sourceType: trace.sourceType,
+          sourcePath: trace.sourcePath,
+        });
+        break;
+      }
+
+      case "info": {
+        consoleLogger.log(message, source, {
+          sourceType: trace.sourceType,
+          sourcePath: trace.sourcePath,
+        });
+        break;
+      }
+
+      case "warning": {
+        consoleLogger.warning(message, source, {
+          sourceType: trace.sourceType,
+          sourcePath: trace.sourcePath,
+        });
+        break;
+      }
+
+      case "error": {
+        const output = trace.data.error
+          ? await formatTraceError(trace.data.error)
+          : message;
+
+        consoleLogger.error(output, source, {
+          sourceType: trace.sourceType,
+          sourcePath: trace.sourcePath,
+        });
+      }
     }
 
     if (
@@ -322,3 +344,17 @@ export const createConsoleServer = async ({
     close,
   };
 };
+
+function logSourceFromTraceType(trace: Trace): LogSource {
+  switch (trace.type) {
+    case "simulator": {
+      return "simulator";
+    }
+
+    case "resource": {
+      return "compiler";
+    }
+  }
+
+  return "user";
+}

apps/wing-console/console/server/src/utils/LogInterface.ts

@@ -1,5 +1,6 @@
 export interface LogInterface {
   info(message?: any, ...optionalParameters: any[]): void;
   verbose(message?: any, ...optionalParameters: any[]): void;
+  warning(message?: any, ...optionalParameters: any[]): void;
   error(message?: any, ...optionalParameters: any[]): void;
 }

apps/wing/src/commands/test/__snapshots__/test.test.ts.snap

@@ -1,19 +1,20 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`printing test reports > resource traces are not shown if debug mode is disabled 1`] = `
+exports[`printing test reports > verbose traces are not shown if debug mode is disabled 1`] = `
 "fail ┌ hello.w » root/env0/test:test
      │ sleeping for 500 ms
+     │ Subscriber error - returning 1 messages to queue: Missing environment variable: QUEUE_HANDLE_7164aec4
      └ Error: Object does not exist (key=file.txt)"
 `;
 
-exports[`printing test reports > resource traces are shown if debug mode is enabled 1`] = `
+exports[`printing test reports > verbose traces are shown if debug mode is enabled 1`] = `
 "fail ┌ hello.w » root/env0/test:test
-     │ [trace] Push (message=cool).
+     │ Push (message=cool).
      │ sleeping for 500 ms
-     │ [trace] Sending messages (messages=[\\"cool\\"], subscriber=sim-4).
-     │ [trace] Invoke (payload=\\"{\\\\\\"messages\\\\\\":[\\\\\\"cool\\\\\\"]}\\").
-     │ [trace] Subscriber error - returning 1 messages to queue: Missing environment variable: QUEUE_HANDLE_7164aec4
-     │ [trace] Get (key=file.txt).
-     │ [trace] Invoke (payload=\\"\\").
+     │ Sending messages (messages=[\\"cool\\"], subscriber=sim-4).
+     │ Invoke (payload=\\"{\\\\\\"messages\\\\\\":[\\\\\\"cool\\\\\\"]}\\").
+     │ Subscriber error - returning 1 messages to queue: Missing environment variable: QUEUE_HANDLE_7164aec4
+     │ Get (key=file.txt).
+     │ Invoke (payload=\\"\\").
      └ Error: Object does not exist (key=file.txt)"
 `;

apps/wing/src/commands/test/test.test.ts

@@ -3,7 +3,7 @@ import fsPromises from "fs/promises";
 import { tmpdir } from "os";
 import { join } from "path";
 import { BuiltinPlatform } from "@winglang/compiler";
-import { TestResult, TraceType } from "@winglang/sdk/lib/std";
+import { LogLevel, TestResult, TraceType } from "@winglang/sdk/lib/std";
 import chalk from "chalk";
 import { describe, test, expect, beforeEach, afterEach, vi, SpyInstance } from "vitest";
 import { filterTests, renderTestReport, collectTestFiles, test as wingTest } from ".";
@@ -23,14 +23,14 @@ describe("printing test reports", () => {
     process.chdir(cwd);
   });
 
-  test("resource traces are not shown if debug mode is disabled", async () => {
+  test("verbose traces are not shown if debug mode is disabled", async () => {
     const testReport = await renderTestReport("hello.w", EXAMPLE_TEST_RESULTS);
 
     expect(testReport).toMatchSnapshot();
     expect(testReport).not.toContain("Push (message=cool)");
   });
 
-  test("resource traces are shown if debug mode is enabled", async () => {
+  test("verbose traces are shown if debug mode is enabled", async () => {
     const oldDebug = process.env.DEBUG;
     process.env.DEBUG = "1";
 
@@ -369,31 +369,33 @@ const EXAMPLE_TEST_RESULTS: Array<TestResult> = [
       {
         data: { message: "Push (message=cool).", status: "success" },
         type: TraceType.RESOURCE,
+        level: LogLevel.VERBOSE,
         sourcePath: "root/env0/MyProcessor/cloud.Queue",
         sourceType: "@winglang/sdk.cloud.Queue",
         timestamp: "2023-05-15T16:20:46.886Z",
       },
       {
         data: { message: "sleeping for 500 ms" },
         type: TraceType.LOG,
+        level: LogLevel.INFO,
         sourcePath: "root/env0/test:test/Handler",
         sourceType: "@winglang/sdk.cloud.Function",
         timestamp: "2023-05-15T16:20:46.887Z",
       },
       {
         type: TraceType.RESOURCE,
         data: { message: 'Sending messages (messages=["cool"], subscriber=sim-4).' },
+        level: LogLevel.VERBOSE,
         sourcePath: "root/env0/MyProcessor/cloud.Queue",
         sourceType: "@winglang/sdk.cloud.Queue",
         timestamp: "2023-05-15T16:20:46.961Z",
       },
       {
         data: {
           message: 'Invoke (payload="{\\"messages\\":[\\"cool\\"]}").',
-          status: "failure",
-          error: {},
         },
         type: TraceType.RESOURCE,
+        level: LogLevel.VERBOSE,
         sourcePath: "root/env0/MyProcessor/cloud.Queue-AddConsumer-0088483a",
         sourceType: "@winglang/sdk.cloud.Function",
         timestamp: "2023-05-15T16:20:46.966Z",
@@ -406,18 +408,21 @@ const EXAMPLE_TEST_RESULTS: Array<TestResult> = [
         sourcePath: "root/env0/MyProcessor/cloud.Queue",
         sourceType: "@winglang/sdk.cloud.Queue",
         type: TraceType.RESOURCE,
+        level: LogLevel.ERROR,
         timestamp: "2023-05-15T16:20:46.966Z",
       },
       {
-        data: { message: "Get (key=file.txt).", status: "failure", error: {} },
+        data: { message: "Get (key=file.txt)." },
         type: TraceType.RESOURCE,
+        level: LogLevel.VERBOSE,
         sourcePath: "root/env0/MyProcessor/Bucket",
         sourceType: "@winglang/sdk.cloud.Bucket",
         timestamp: "2023-05-15T16:20:47.388Z",
       },
       {
-        data: { message: 'Invoke (payload="").', status: "failure", error: {} },
+        data: { message: 'Invoke (payload="").' },
         type: TraceType.RESOURCE,
+        level: LogLevel.VERBOSE,
         sourcePath: "root/env0/test:test/Handler",
         sourceType: "@winglang/sdk.cloud.Function",
         timestamp: "2023-05-15T16:20:47.388Z",