update from latest studio code

cspell.config.yaml

@@ -103,6 +103,7 @@ words:
   - velodyne
   - waabi
   - webp
+  - xcdr
   - zstandard
   - zstd
   - zustand

typescript/support/package.json

@@ -52,17 +52,20 @@
     "typescript": "4.9.5"
   },
   "dependencies": {
-    "@foxglove/message-definition": "^0.2.0",
+    "@foxglove/message-definition": "^0.3.0",
+    "@foxglove/omgidl-parser": "^1.0.0",
+    "@foxglove/omgidl-serialization": "^1.0.0",
     "@foxglove/protobufjs": "0.0.1-toobject-bigint.1",
+    "@foxglove/ros2idl-parser": "^0.3.0",
     "@foxglove/rosmsg": "^4.2.2",
-    "@foxglove/rosmsg-serialization": "^2.0.1",
+    "@foxglove/rosmsg-serialization": "^2.0.2",
     "@foxglove/rosmsg2-serialization": "^2.0.2",
-    "@foxglove/schemas": "^1.3.1",
+    "@foxglove/schemas": "^1.5.1",
     "@foxglove/wasm-bz2": "^0.1.1",
     "@foxglove/wasm-lz4": "^1.0.2",
     "@foxglove/wasm-zstd": "^1.0.1",
     "@protobufjs/base64": "^1.1.2",
     "flatbuffers": "^23.5.26",
-    "flatbuffers_reflection": "^0.0.6"
+    "flatbuffers_reflection": "^0.0.7"
   }
 }

typescript/support/src/TempBuffer.ts

@@ -0,0 +1,42 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/
+
+import type { IWritable, McapTypes } from "@mcap/core";
+
+/**
+ * In-memory buffer used for reading and writing MCAP files in tests. Can be used as both an IReadable and an IWritable.
+ */
+export class TempBuffer implements McapTypes.IReadable, IWritable {
+  #buffer = new ArrayBuffer(1024);
+  #size = 0;
+
+  public position(): bigint {
+    return BigInt(this.#size);
+  }
+
+  public async write(data: Uint8Array): Promise<void> {
+    if (this.#size + data.byteLength > this.#buffer.byteLength) {
+      const newBuffer = new ArrayBuffer(this.#size + data.byteLength);
+      new Uint8Array(newBuffer).set(new Uint8Array(this.#buffer));
+      this.#buffer = newBuffer;
+    }
+    new Uint8Array(this.#buffer, this.#size).set(data);
+    this.#size += data.byteLength;
+  }
+
+  public async size(): Promise<bigint> {
+    return BigInt(this.#size);
+  }
+
+  public async read(offset: bigint, size: bigint): Promise<Uint8Array> {
+    if (offset < 0n || offset + size > BigInt(this.#buffer.byteLength)) {
+      throw new Error("read out of range");
+    }
+    return new Uint8Array(this.#buffer, Number(offset), Number(size));
+  }
+
+  public get(): Uint8Array {
+    return new Uint8Array(this.#buffer, 0, this.#size);
+  }
+}

typescript/support/src/decompressHandlers.ts

@@ -1,4 +1,4 @@
-import { McapTypes } from "@mcap/core";
+import type { McapTypes } from "@mcap/core";
 
 let handlersPromise: Promise<McapTypes.DecompressHandlers> | undefined;
 export async function loadDecompressHandlers(): Promise<McapTypes.DecompressHandlers> {

typescript/support/src/index.ts

@@ -3,3 +3,4 @@ export * from "./protobufDefinitionsToDatatypes";
 export * from "./protobufDescriptors";
 export * from "./parseChannel";
 export * from "./decompressHandlers";
+export * from "./TempBuffer";

typescript/support/src/parseChannel.test.ts

@@ -88,4 +88,25 @@ describe("parseChannel", () => {
     const obj = channel.deserialize(new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0]));
     expect(obj).toEqual({ data: "ABCD" });
   });
+  it("works with omgidl xcdr2", () => {
+    const channel = parseChannel({
+      messageEncoding: "cdr",
+      schema: {
+        name: "foo_msgs::Bar",
+        encoding: "omgidl",
+        data: new TextEncoder().encode(`
+        enum Color {RED, GREEN, BLUE};
+        module foo_msgs {
+          struct NonRootBar {string data;};
+          struct Bar {foo_msgs::NonRootBar data; Color color;};
+        };
+        `),
+      },
+    });
+
+    const obj = channel.deserialize(
+      new Uint8Array([0, 1, 0, 0, 5, 0, 0, 0, 65, 66, 67, 68, 0, 0, 0, 0, 2, 0, 0, 0]),
+    );
+    expect(obj).toEqual({ data: { data: "ABCD" }, color: 2 });
+  });
 });

typescript/support/src/parseChannel.ts

@@ -1,5 +1,8 @@
-import { MessageDefinition } from "@foxglove/message-definition";
-import { parse as parseMessageDefinition, parseRos2idl } from "@foxglove/rosmsg";
+import { MessageDefinition, MessageDefinitionField } from "@foxglove/message-definition";
+import { IDLMessageDefinition, parseIDL } from "@foxglove/omgidl-parser";
+import { MessageReader as OmgidlMessageReader } from "@foxglove/omgidl-serialization";
+import { parseRos2idl } from "@foxglove/ros2idl-parser";
+import { parse as parseMessageDefinition } from "@foxglove/rosmsg";
 import { MessageReader } from "@foxglove/rosmsg-serialization";
 import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serialization";
 
@@ -18,13 +21,41 @@ export type ParsedChannel = {
   datatypes: MessageDefinitionMap;
 };
 
+function parseIDLDefinitionsToDatatypes(
+  parsedDefinitions: IDLMessageDefinition[],
+  rootName?: string,
+) {
+  //  The only IDL definition non-conformant-to-MessageDefinition is unions
+  const convertUnionToMessageDefinition = (definition: IDLMessageDefinition): MessageDefinition => {
+    if (definition.aggregatedKind === "union") {
+      const innerDefs: MessageDefinitionField[] = definition.cases.map((caseDefinition) => ({
+        ...caseDefinition.type,
+        predicates: caseDefinition.predicates,
+      }));
+
+      if (definition.defaultCase != undefined) {
+        innerDefs.push(definition.defaultCase);
+      }
+      const { name } = definition;
+      return {
+        name,
+        definitions: innerDefs,
+      };
+    }
+    return definition;
+  };
+
+  const standardDefs: MessageDefinition[] = parsedDefinitions.map(convertUnionToMessageDefinition);
+  return parsedDefinitionsToDatatypes(standardDefs, rootName);
+}
+
 function parsedDefinitionsToDatatypes(
   parsedDefinitions: MessageDefinition[],
-  rootName: string,
+  rootName?: string,
 ): MessageDefinitionMap {
   const datatypes: MessageDefinitionMap = new Map();
   parsedDefinitions.forEach(({ name, definitions }, index) => {
-    if (index === 0) {
+    if (rootName != undefined && index === 0) {
       datatypes.set(rootName, { name: rootName, definitions });
     } else if (name != undefined) {
       datatypes.set(name, { name, definitions });
@@ -118,7 +149,11 @@ export function parseChannel(channel: Channel): ParsedChannel {
   }
 
   if (channel.messageEncoding === "cdr") {
-    if (channel.schema?.encoding !== "ros2msg" && channel.schema?.encoding !== "ros2idl") {
+    if (
+      channel.schema?.encoding !== "ros2msg" &&
+      channel.schema?.encoding !== "ros2idl" &&
+      channel.schema?.encoding !== "omgidl"
+    ) {
       throw new Error(
         `Message encoding ${channel.messageEncoding} with ${
           channel.schema == undefined
@@ -128,17 +163,28 @@ export function parseChannel(channel: Channel): ParsedChannel {
       );
     }
     const schema = new TextDecoder().decode(channel.schema.data);
-    const isIdl = channel.schema.encoding === "ros2idl";
+    if (channel.schema.encoding === "omgidl") {
+      const parsedDefinitions = parseIDL(schema);
+      const reader = new OmgidlMessageReader(channel.schema.name, parsedDefinitions);
+      const datatypes = parseIDLDefinitionsToDatatypes(parsedDefinitions);
+      return {
+        datatypes,
+        deserialize: (data) => reader.readMessage(data),
+      };
+    } else {
+      const isIdl = channel.schema.encoding === "ros2idl";
 
-    const parsedDefinitions = isIdl
-      ? parseRos2idl(schema)
-      : parseMessageDefinition(schema, { ros2: true });
+      const parsedDefinitions = isIdl
+        ? parseRos2idl(schema)
+        : parseMessageDefinition(schema, { ros2: true });
 
-    const reader = new ROS2MessageReader(parsedDefinitions);
-    return {
-      datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name),
-      deserialize: (data) => reader.readMessage(data),
-    };
+      const reader = new ROS2MessageReader(parsedDefinitions);
+
+      return {
+        datatypes: parsedDefinitionsToDatatypes(parsedDefinitions, channel.schema.name),
+        deserialize: (data) => reader.readMessage(data),
+      };
+    }
   }
 
   throw new Error(`Unsupported encoding ${channel.messageEncoding}`);

typescript/support/src/parseFlatbufferSchema.test.ts

@@ -351,11 +351,11 @@ describe("parseFlatbufferSchema", () => {
     builder.finish(Type.endType(builder));
 
     expect(deserialize(builder.asUint8Array())).toEqual({
-      base_size: 4n,
+      base_size: 4,
       base_type: 7,
-      element: 0n,
-      element_size: 0n,
-      fixed_length: 0n,
+      element: 0,
+      element_size: 0,
+      fixed_length: 0,
       index: 123,
     });
   });

typescript/support/src/parseFlatbufferSchema.ts

@@ -140,7 +140,7 @@ function typeForField(schema: SchemaT, field: FieldT): MessageDefinitionField[]
     case BaseType.Array:
     case BaseType.MaxBaseType:
     case undefined:
-      throw new Error("Unions and Arrays are not supported in @mcap/support currently.");
+      throw new Error("Unions and Arrays are not currently supported");
   }
   return fields;
 }

typescript/support/src/parseJsonSchema.test.ts

@@ -1,5 +1,5 @@
-import * as protobufjs from "@foxglove/protobufjs";
 import { foxgloveMessageSchemas, generateJsonSchema } from "@foxglove/schemas/internal";
+import * as protobufjs from "protobufjs";
 
 import { parseJsonSchema } from "./parseJsonSchema";
 

typescript/support/src/parseProtobufSchema.test.ts

@@ -204,7 +204,7 @@ describe("parseProtobufSchema", () => {
           {
             "isArray": false,
             "name": "sec",
-            "type": "int64",
+            "type": "int32",
           },
           {
             "isArray": false,
@@ -220,7 +220,7 @@ describe("parseProtobufSchema", () => {
           {
             "isArray": false,
             "name": "sec",
-            "type": "int64",
+            "type": "int32",
           },
           {
             "isArray": false,

typescript/support/src/parseProtobufSchema.ts

@@ -25,32 +25,26 @@ export function parseProtobufSchema(
   // {sec: number, nsec: number}, compatible with the rest of Studio. The standard Protobuf types
   // use different names (`seconds` and `nanos`), and `seconds` is an `int64`, which would be
   // deserialized as a bigint by default.
+  //
+  // protobufDefinitionsToDatatypes also has matching logic to rename the fields.
   const fixTimeType = (type: protobufjs.ReflectionObject | null) => {
     if (!type || !(type instanceof protobufjs.Type)) {
       return;
     }
-    // Rename fields so that protobufDefinitionsToDatatypes uses the new names
-    for (const field of type.fieldsArray) {
-      if (field.name === "seconds") {
-        field.name = "sec";
-      } else if (field.name === "nanos") {
-        field.name = "nsec";
-      }
-    }
     type.setup(); // ensure the original optimized toObject has been created
     const prevToObject = type.toObject; // eslint-disable-line @typescript-eslint/unbound-method
     const newToObject: typeof prevToObject = (message, options) => {
       const result = prevToObject.call(type, message, options);
-      const { sec, nsec } = result as { sec: bigint; nsec: number };
-      if (typeof sec !== "bigint" || typeof nsec !== "number") {
+      const { seconds, nanos } = result as { seconds: bigint; nanos: number };
+      if (typeof seconds !== "bigint" || typeof nanos !== "number") {
         return result;
       }
-      if (sec > BigInt(Number.MAX_SAFE_INTEGER)) {
+      if (seconds > BigInt(Number.MAX_SAFE_INTEGER)) {
         throw new Error(
-          `Timestamps with seconds greater than 2^53-1 are not supported (found seconds=${sec}, nanos=${nsec})`,
+          `Timestamps with seconds greater than 2^53-1 are not supported (found seconds=${seconds}, nanos=${nanos})`,
         );
       }
-      return { sec: Number(sec), nsec };
+      return { sec: Number(seconds), nsec: nanos };
     };
     type.toObject = newToObject;
   };

typescript/support/src/protobufDefinitionsToDatatypes.ts

@@ -70,6 +70,15 @@ export function protobufDefinitionsToDatatypes(
         throw new Error("Repeated bytes are not currently supported");
       }
       definitions.push({ type: "uint8", name: field.name, isArray: true });
+    } else if (
+      type.fullName === ".google.protobuf.Timestamp" ||
+      type.fullName === ".google.protobuf.Duration"
+    ) {
+      definitions.push({
+        type: "int32",
+        name: field.name === "seconds" ? "sec" : "nsec",
+        isArray: field.repeated,
+      });
     } else {
       definitions.push({
         type: protobufScalarToRosPrimitive(field.type),