diff --git a/typescript/support/src/parseChannel.test.ts b/typescript/support/src/parseChannel.test.ts index ca8044ab4b..d5a513e771 100644 --- a/typescript/support/src/parseChannel.test.ts +++ b/typescript/support/src/parseChannel.test.ts @@ -34,13 +34,25 @@ describe("parseChannel", () => { }); it("works with protobuf", () => { + const processRootType = jest.fn().mockImplementation((x: unknown) => x); + const processMessageDefinitions = jest.fn().mockImplementation((x: unknown) => x); const fds = FileDescriptorSet.encode(FileDescriptorSet.root.toDescriptor("proto3")).finish(); - const channel = parseChannel({ - messageEncoding: "protobuf", - schema: { name: "google.protobuf.FileDescriptorSet", encoding: "protobuf", data: fds }, - }); + const channel = parseChannel( + { + messageEncoding: "protobuf", + schema: { name: "google.protobuf.FileDescriptorSet", encoding: "protobuf", data: fds }, + }, + { + protobuf: { + processRootType, + processMessageDefinitions, + }, + }, + ); const deserialized = channel.deserialize(fds) as IFileDescriptorSet; expect(deserialized.file[0]!.name).toEqual("google_protobuf.proto"); + expect(processRootType).toHaveBeenCalled(); + expect(processMessageDefinitions).toHaveBeenCalled(); }); it("works with ros1", () => { diff --git a/typescript/support/src/parseChannel.ts b/typescript/support/src/parseChannel.ts index e5fbe071c9..214caaadb8 100644 --- a/typescript/support/src/parseChannel.ts +++ b/typescript/support/src/parseChannel.ts @@ -8,7 +8,7 @@ import { MessageReader as ROS2MessageReader } from "@foxglove/rosmsg2-serializat import { parseFlatbufferSchema } from "./parseFlatbufferSchema"; import { parseJsonSchema } from "./parseJsonSchema"; -import { parseProtobufSchema } from "./parseProtobufSchema"; +import { ParseProtobufSchemaOptions, parseProtobufSchema } from "./parseProtobufSchema"; import { MessageDefinitionMap } from "./types"; type Channel = { @@ -64,6 +64,10 @@ function parsedDefinitionsToDatatypes( return datatypes; } +export type ParseChannelOptions = { + protobuf?: ParseProtobufSchemaOptions; +}; + /** * Process a channel/schema and extract information that can be used to deserialize messages on the * channel, and schemas in the format expected by Studio's RosDatatypes. @@ -72,7 +76,7 @@ function parsedDefinitionsToDatatypes( * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-message-encodings.md * - https://github.com/foxglove/mcap/blob/main/docs/specification/well-known-schema-encodings.md */ -export function parseChannel(channel: Channel): ParsedChannel { +export function parseChannel(channel: Channel, options?: ParseChannelOptions): ParsedChannel { if (channel.messageEncoding === "json") { if (channel.schema != undefined && channel.schema.encoding !== "jsonschema") { throw new Error( @@ -126,7 +130,7 @@ export function parseChannel(channel: Channel): ParsedChannel { } is not supported (expected protobuf)`, ); } - return parseProtobufSchema(channel.schema.name, channel.schema.data); + return parseProtobufSchema(channel.schema.name, channel.schema.data, options?.protobuf); } if (channel.messageEncoding === "ros1") { diff --git a/typescript/support/src/parseProtobufSchema.test.ts b/typescript/support/src/parseProtobufSchema.test.ts index 7f3de0e050..bb0bdbd59a 100644 --- a/typescript/support/src/parseProtobufSchema.test.ts +++ b/typescript/support/src/parseProtobufSchema.test.ts @@ -26,6 +26,77 @@ describe("parseProtobufSchema", () => { ), ); expect(channel.deserialize(Buffer.from("0A0101", "hex"))).toEqual({ data: [1] }); + expect(channel.datatypes).toEqual( + new Map([ + [ + "ExampleMessage", + { + definitions: [ + { isConstant: true, name: "UNKNOWN", type: "int32", value: 0 }, + { isConstant: true, name: "WHATEVER", type: "int32", value: 0 }, + { isConstant: true, name: "FOO", type: "int32", value: 1 }, + { isConstant: true, name: "BAR", type: "int32", value: 2 }, + { name: "data", type: "int32" }, + ], + }, + ], + ]), + ); + }); + + it("allows modifying deserialization and datatypes", () => { + /* + syntax = "proto3"; + + enum ExampleEnum { + option allow_alias = true; + UNKNOWN = 0; + WHATEVER = 0; + FOO = 1; + BAR = 2; + } + + message ExampleMessage { + repeated ExampleEnum data = 1; + } + */ + const channel = parseProtobufSchema( + "ExampleMessage", + Buffer.from( + // cspell:disable-next-line + "0A8D010A156578616D706C655F6D6573736167652E70726F746F222C0A0E4578616D706C654D657373616765121A0A046461746118012003280E320C2E4578616D706C65456E756D2A3E0A0B4578616D706C65456E756D120B0A07554E4B4E4F574E1000120C0A085748415445564552100012070A03464F4F100112070A0342415210021A021001620670726F746F33", + "hex", + ), + { + processRootType(rootType) { + rootType.fieldsById[1]!.name = "renamed_data"; + return rootType; + }, + processMessageDefinitions(definitions) { + definitions + .get("ExampleMessage")! + .definitions.find((def) => def.name === "renamed_data")!.type = "float64"; + return definitions; + }, + }, + ); + expect(channel.deserialize(Buffer.from("0A0101", "hex"))).toEqual({ renamed_data: [1] }); + expect(channel.datatypes).toEqual( + new Map([ + [ + "ExampleMessage", + { + definitions: [ + { isConstant: true, name: "UNKNOWN", type: "int32", value: 0 }, + { isConstant: true, name: "WHATEVER", type: "int32", value: 0 }, + { isConstant: true, name: "FOO", type: "int32", value: 1 }, + { isConstant: true, name: "BAR", type: "int32", value: 2 }, + { name: "renamed_data", type: "float64" }, + ], + }, + ], + ]), + ); }); it("handles protobuf int64 values", () => { @@ -102,149 +173,4 @@ describe("parseProtobufSchema", () => { ], }); }); - - it("converts protobuf time/duration int64 to number rather than bigint", () => { - const poseInFrameChannel = parseProtobufSchema( - "foxglove.PoseInFrame", - Buffer.from( - // cspell:disable-next-line - "CmcKGWZveGdsb3ZlL1F1YXRlcm5pb24ucHJvdG8SCGZveGdsb3ZlIjgKClF1YXRlcm5pb24SCQoBeBgBIAEoARIJCgF5GAIgASgBEgkKAXoYAyABKAESCQoBdxgEIAEoAWIGcHJvdG8zClYKFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8SCGZveGdsb3ZlIioKB1ZlY3RvcjMSCQoBeBgBIAEoARIJCgF5GAIgASgBEgkKAXoYAyABKAFiBnByb3RvMwqyAQoTZm94Z2xvdmUvUG9zZS5wcm90bxIIZm94Z2xvdmUaGWZveGdsb3ZlL1F1YXRlcm5pb24ucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8iVgoEUG9zZRIjCghwb3NpdGlvbhgBIAEoCzIRLmZveGdsb3ZlLlZlY3RvcjMSKQoLb3JpZW50YXRpb24YAiABKAsyFC5mb3hnbG92ZS5RdWF0ZXJuaW9uYgZwcm90bzMK7wEKH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAucHJvdG8SD2dvb2dsZS5wcm90b2J1ZiIrCglUaW1lc3RhbXASDwoHc2Vjb25kcxgBIAEoAxINCgVuYW5vcxgCIAEoBUKFAQoTY29tLmdvb2dsZS5wcm90b2J1ZkIOVGltZXN0YW1wUHJvdG9QAVoyZ29vZ2xlLmdvbGFuZy5vcmcvcHJvdG9idWYvdHlwZXMva25vd24vdGltZXN0YW1wcGL4AQGiAgNHUEKqAh5Hb29nbGUuUHJvdG9idWYuV2VsbEtub3duVHlwZXNiBnByb3RvMwrSAQoaZm94Z2xvdmUvUG9zZUluRnJhbWUucHJvdG8SCGZveGdsb3ZlGhNmb3hnbG92ZS9Qb3NlLnByb3RvGh9nb29nbGUvcHJvdG9idWYvdGltZXN0YW1wLnByb3RvImwKC1Bvc2VJbkZyYW1lEi0KCXRpbWVzdGFtcBgBIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASEAoIZnJhbWVfaWQYAiABKAkSHAoEcG9zZRgDIAEoCzIOLmZveGdsb3ZlLlBvc2ViBnByb3RvMw==", - "base64", - ), - ); - - const poseInFrame = poseInFrameChannel.deserialize( - Buffer.from( - // cspell:disable-next-line - "CgwIx8LcoQYQuJzV6wESA2ZvbxooChsJAAAAAAAA8D8RAAAAAAAAAEAZAAAAAAAACEASCSEAAAAAAADwPw==", - "base64", - ), - ); - expect(poseInFrame).toMatchInlineSnapshot(` - { - "frame_id": "foo", - "pose": { - "orientation": { - "w": 1, - "x": 0, - "y": 0, - "z": 0, - }, - "position": { - "x": 1, - "y": 2, - "z": 3, - }, - }, - "timestamp": { - "nsec": 494227000, - "sec": 1681334599, - }, - } - `); - - expect(() => - poseInFrameChannel.deserialize( - Buffer.from( - // cspell:disable-next-line - "CgsIgICAgICAgBAQARIDZm9vGigKGwkAAAAAAADwPxEAAAAAAAAAQBkAAAAAAAAIQBIJIQAAAAAAAPA/", - "base64", - ), - ), - ).toThrow( - "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", - ); - - const sceneUpdateChannel = parseProtobufSchema( - "foxglove.SceneUpdate", - Buffer.from( - // cspell:disable-next-line - "Cl0KFGZveGdsb3ZlL0NvbG9yLnByb3RvEghmb3hnbG92ZSIzCgVDb2xvchIJCgFyGAEgASgBEgkKAWcYAiABKAESCQoBYhgDIAEoARIJCgFhGAQgASgBYgZwcm90bzMKZwoZZm94Z2xvdmUvUXVhdGVybmlvbi5wcm90bxIIZm94Z2xvdmUiOAoKUXVhdGVybmlvbhIJCgF4GAEgASgBEgkKAXkYAiABKAESCQoBehgDIAEoARIJCgF3GAQgASgBYgZwcm90bzMKVgoWZm94Z2xvdmUvVmVjdG9yMy5wcm90bxIIZm94Z2xvdmUiKgoHVmVjdG9yMxIJCgF4GAEgASgBEgkKAXkYAiABKAESCQoBehgDIAEoAWIGcHJvdG8zCrIBChNmb3hnbG92ZS9Qb3NlLnByb3RvEghmb3hnbG92ZRoZZm94Z2xvdmUvUXVhdGVybmlvbi5wcm90bxoWZm94Z2xvdmUvVmVjdG9yMy5wcm90byJWCgRQb3NlEiMKCHBvc2l0aW9uGAEgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIpCgtvcmllbnRhdGlvbhgCIAEoCzIULmZveGdsb3ZlLlF1YXRlcm5pb25iBnByb3RvMwqHAgodZm94Z2xvdmUvQXJyb3dQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoTZm94Z2xvdmUvUG9zZS5wcm90byKoAQoOQXJyb3dQcmltaXRpdmUSHAoEcG9zZRgBIAEoCzIOLmZveGdsb3ZlLlBvc2USFAoMc2hhZnRfbGVuZ3RoGAIgASgBEhYKDnNoYWZ0X2RpYW1ldGVyGAMgASgBEhMKC2hlYWRfbGVuZ3RoGAQgASgBEhUKDWhlYWRfZGlhbWV0ZXIYBSABKAESHgoFY29sb3IYBiABKAsyDy5mb3hnbG92ZS5Db2xvcmIGcHJvdG8zCuMBChxmb3hnbG92ZS9DdWJlUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8ibgoNQ3ViZVByaW1pdGl2ZRIcCgRwb3NlGAEgASgLMg4uZm94Z2xvdmUuUG9zZRIfCgRzaXplGAIgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIeCgVjb2xvchgDIAEoCzIPLmZveGdsb3ZlLkNvbG9yYgZwcm90bzMKlQIKIGZveGdsb3ZlL0N5bGluZGVyUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8imwEKEUN5bGluZGVyUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEh8KBHNpemUYAiABKAsyES5mb3hnbG92ZS5WZWN0b3IzEhQKDGJvdHRvbV9zY2FsZRgDIAEoARIRCgl0b3Bfc2NhbGUYBCABKAESHgoFY29sb3IYBSABKAsyDy5mb3hnbG92ZS5Db2xvcmIGcHJvdG8zClsKG2ZveGdsb3ZlL0tleVZhbHVlUGFpci5wcm90bxIIZm94Z2xvdmUiKgoMS2V5VmFsdWVQYWlyEgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCWIGcHJvdG8zClQKFWZveGdsb3ZlL1BvaW50My5wcm90bxIIZm94Z2xvdmUiKQoGUG9pbnQzEgkKAXgYASABKAESCQoBeRgCIAEoARIJCgF6GAMgASgBYgZwcm90bzMKpAMKHGZveGdsb3ZlL0xpbmVQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoVZm94Z2xvdmUvUG9pbnQzLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIq8CCg1MaW5lUHJpbWl0aXZlEioKBHR5cGUYASABKA4yHC5mb3hnbG92ZS5MaW5lUHJpbWl0aXZlLlR5cGUSHAoEcG9zZRgCIAEoCzIOLmZveGdsb3ZlLlBvc2USEQoJdGhpY2tuZXNzGAMgASgBEhcKD3NjYWxlX2ludmFyaWFudBgEIAEoCBIgCgZwb2ludHMYBSADKAsyEC5mb3hnbG92ZS5Qb2ludDMSHgoFY29sb3IYBiABKAsyDy5mb3hnbG92ZS5Db2xvchIfCgZjb2xvcnMYByADKAsyDy5mb3hnbG92ZS5Db2xvchIPCgdpbmRpY2VzGAggAygHIjQKBFR5cGUSDgoKTElORV9TVFJJUBAAEg0KCUxJTkVfTE9PUBABEg0KCUxJTkVfTElTVBACYgZwcm90bzMKrgIKHWZveGdsb3ZlL01vZGVsUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8itwEKDk1vZGVsUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEiAKBXNjYWxlGAIgASgLMhEuZm94Z2xvdmUuVmVjdG9yMxIeCgVjb2xvchgDIAEoCzIPLmZveGdsb3ZlLkNvbG9yEhYKDm92ZXJyaWRlX2NvbG9yGAQgASgIEgsKA3VybBgFIAEoCRISCgptZWRpYV90eXBlGAYgASgJEgwKBGRhdGEYByABKAxiBnByb3RvMwrnAQoeZm94Z2xvdmUvU3BoZXJlUHJpbWl0aXZlLnByb3RvEghmb3hnbG92ZRoUZm94Z2xvdmUvQ29sb3IucHJvdG8aE2ZveGdsb3ZlL1Bvc2UucHJvdG8aFmZveGdsb3ZlL1ZlY3RvcjMucHJvdG8icAoPU3BoZXJlUHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEh8KBHNpemUYAiABKAsyES5mb3hnbG92ZS5WZWN0b3IzEh4KBWNvbG9yGAMgASgLMg8uZm94Z2xvdmUuQ29sb3JiBnByb3RvMwr4AQocZm94Z2xvdmUvVGV4dFByaW1pdGl2ZS5wcm90bxIIZm94Z2xvdmUaFGZveGdsb3ZlL0NvbG9yLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIpoBCg1UZXh0UHJpbWl0aXZlEhwKBHBvc2UYASABKAsyDi5mb3hnbG92ZS5Qb3NlEhEKCWJpbGxib2FyZBgCIAEoCBIRCglmb250X3NpemUYAyABKAESFwoPc2NhbGVfaW52YXJpYW50GAQgASgIEh4KBWNvbG9yGAUgASgLMg8uZm94Z2xvdmUuQ29sb3ISDAoEdGV4dBgGIAEoCWIGcHJvdG8zCqYCCiRmb3hnbG92ZS9UcmlhbmdsZUxpc3RQcmltaXRpdmUucHJvdG8SCGZveGdsb3ZlGhRmb3hnbG92ZS9Db2xvci5wcm90bxoVZm94Z2xvdmUvUG9pbnQzLnByb3RvGhNmb3hnbG92ZS9Qb3NlLnByb3RvIqkBChVUcmlhbmdsZUxpc3RQcmltaXRpdmUSHAoEcG9zZRgBIAEoCzIOLmZveGdsb3ZlLlBvc2USIAoGcG9pbnRzGAIgAygLMhAuZm94Z2xvdmUuUG9pbnQzEh4KBWNvbG9yGAMgASgLMg8uZm94Z2xvdmUuQ29sb3ISHwoGY29sb3JzGAQgAygLMg8uZm94Z2xvdmUuQ29sb3ISDwoHaW5kaWNlcxgFIAMoB2IGcHJvdG8zCusBCh5nb29nbGUvcHJvdG9idWYvZHVyYXRpb24ucHJvdG8SD2dvb2dsZS5wcm90b2J1ZiIqCghEdXJhdGlvbhIPCgdzZWNvbmRzGAEgASgDEg0KBW5hbm9zGAIgASgFQoMBChNjb20uZ29vZ2xlLnByb3RvYnVmQg1EdXJhdGlvblByb3RvUAFaMWdvb2dsZS5nb2xhbmcub3JnL3Byb3RvYnVmL3R5cGVzL2tub3duL2R1cmF0aW9ucGL4AQGiAgNHUEKqAh5Hb29nbGUuUHJvdG9idWYuV2VsbEtub3duVHlwZXNiBnByb3RvMwrvAQofZ29vZ2xlL3Byb3RvYnVmL3RpbWVzdGFtcC5wcm90bxIPZ29vZ2xlLnByb3RvYnVmIisKCVRpbWVzdGFtcBIPCgdzZWNvbmRzGAEgASgDEg0KBW5hbm9zGAIgASgFQoUBChNjb20uZ29vZ2xlLnByb3RvYnVmQg5UaW1lc3RhbXBQcm90b1ABWjJnb29nbGUuZ29sYW5nLm9yZy9wcm90b2J1Zi90eXBlcy9rbm93bi90aW1lc3RhbXBwYvgBAaICA0dQQqoCHkdvb2dsZS5Qcm90b2J1Zi5XZWxsS25vd25UeXBlc2IGcHJvdG8zCrIHChpmb3hnbG92ZS9TY2VuZUVudGl0eS5wcm90bxIIZm94Z2xvdmUaHWZveGdsb3ZlL0Fycm93UHJpbWl0aXZlLnByb3RvGhxmb3hnbG92ZS9DdWJlUHJpbWl0aXZlLnByb3RvGiBmb3hnbG92ZS9DeWxpbmRlclByaW1pdGl2ZS5wcm90bxobZm94Z2xvdmUvS2V5VmFsdWVQYWlyLnByb3RvGhxmb3hnbG92ZS9MaW5lUHJpbWl0aXZlLnByb3RvGh1mb3hnbG92ZS9Nb2RlbFByaW1pdGl2ZS5wcm90bxoeZm94Z2xvdmUvU3BoZXJlUHJpbWl0aXZlLnByb3RvGhxmb3hnbG92ZS9UZXh0UHJpbWl0aXZlLnByb3RvGiRmb3hnbG92ZS9UcmlhbmdsZUxpc3RQcmltaXRpdmUucHJvdG8aHmdvb2dsZS9wcm90b2J1Zi9kdXJhdGlvbi5wcm90bxofZ29vZ2xlL3Byb3RvYnVmL3RpbWVzdGFtcC5wcm90byKjBAoLU2NlbmVFbnRpdHkSLQoJdGltZXN0YW1wGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIQCghmcmFtZV9pZBgCIAEoCRIKCgJpZBgDIAEoCRIrCghsaWZldGltZRgEIAEoCzIZLmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbhIUCgxmcmFtZV9sb2NrZWQYBSABKAgSKAoIbWV0YWRhdGEYBiADKAsyFi5mb3hnbG92ZS5LZXlWYWx1ZVBhaXISKAoGYXJyb3dzGAcgAygLMhguZm94Z2xvdmUuQXJyb3dQcmltaXRpdmUSJgoFY3ViZXMYCCADKAsyFy5mb3hnbG92ZS5DdWJlUHJpbWl0aXZlEioKB3NwaGVyZXMYCSADKAsyGS5mb3hnbG92ZS5TcGhlcmVQcmltaXRpdmUSLgoJY3lsaW5kZXJzGAogAygLMhsuZm94Z2xvdmUuQ3lsaW5kZXJQcmltaXRpdmUSJgoFbGluZXMYCyADKAsyFy5mb3hnbG92ZS5MaW5lUHJpbWl0aXZlEjIKCXRyaWFuZ2xlcxgMIAMoCzIfLmZveGdsb3ZlLlRyaWFuZ2xlTGlzdFByaW1pdGl2ZRImCgV0ZXh0cxgNIAMoCzIXLmZveGdsb3ZlLlRleHRQcmltaXRpdmUSKAoGbW9kZWxzGA4gAygLMhguZm94Z2xvdmUuTW9kZWxQcmltaXRpdmViBnByb3RvMwr+AQoiZm94Z2xvdmUvU2NlbmVFbnRpdHlEZWxldGlvbi5wcm90bxIIZm94Z2xvdmUaH2dvb2dsZS9wcm90b2J1Zi90aW1lc3RhbXAucHJvdG8ipAEKE1NjZW5lRW50aXR5RGVsZXRpb24SLQoJdGltZXN0YW1wGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIwCgR0eXBlGAIgASgOMiIuZm94Z2xvdmUuU2NlbmVFbnRpdHlEZWxldGlvbi5UeXBlEgoKAmlkGAMgASgJIiAKBFR5cGUSDwoLTUFUQ0hJTkdfSUQQABIHCgNBTEwQAWIGcHJvdG8zCtgBChpmb3hnbG92ZS9TY2VuZVVwZGF0ZS5wcm90bxIIZm94Z2xvdmUaGmZveGdsb3ZlL1NjZW5lRW50aXR5LnByb3RvGiJmb3hnbG92ZS9TY2VuZUVudGl0eURlbGV0aW9uLnByb3RvImgKC1NjZW5lVXBkYXRlEjAKCWRlbGV0aW9ucxgBIAMoCzIdLmZveGdsb3ZlLlNjZW5lRW50aXR5RGVsZXRpb24SJwoIZW50aXRpZXMYAiADKAsyFS5mb3hnbG92ZS5TY2VuZUVudGl0eWIGcHJvdG8z", - "base64", - ), - ); - - const sceneUpdate = sceneUpdateChannel.deserialize( - // cspell:disable-next-line - Buffer.from("EhwKDAjHwtyhBhC4nNXrASIMCMfC3KEGELic1esB", "base64"), - ); - expect(sceneUpdate).toMatchInlineSnapshot(` - { - "deletions": [], - "entities": [ - { - "arrows": [], - "cubes": [], - "cylinders": [], - "frame_id": "", - "frame_locked": false, - "id": "", - "lifetime": { - "nsec": 494227000, - "sec": 1681334599, - }, - "lines": [], - "metadata": [], - "models": [], - "spheres": [], - "texts": [], - "timestamp": { - "nsec": 494227000, - "sec": 1681334599, - }, - "triangles": [], - }, - ], - } - `); - - expect(sceneUpdateChannel.datatypes.get("google.protobuf.Timestamp")).toMatchInlineSnapshot(` - { - "definitions": [ - { - "isArray": false, - "name": "sec", - "type": "int32", - }, - { - "isArray": false, - "name": "nsec", - "type": "int32", - }, - ], - } - `); - expect(sceneUpdateChannel.datatypes.get("google.protobuf.Duration")).toMatchInlineSnapshot(` - { - "definitions": [ - { - "isArray": false, - "name": "sec", - "type": "int32", - }, - { - "isArray": false, - "name": "nsec", - "type": "int32", - }, - ], - } - `); - - // Duration too large - expect(() => - // cspell:disable-next-line - sceneUpdateChannel.deserialize(Buffer.from("EhMKBAgCEAMiCwiAgICAgICAEBAB", "base64")), - ).toThrow( - "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", - ); - - // Timestamp too large - expect(() => - // cspell:disable-next-line - sceneUpdateChannel.deserialize(Buffer.from("EhMKCwiAgICAgICAEBABIgQIAhAD", "base64")), - ).toThrow( - "Timestamps with seconds greater than 2^53-1 are not supported (found seconds=9007199254740992, nanos=1)", - ); - }); }); diff --git a/typescript/support/src/parseProtobufSchema.ts b/typescript/support/src/parseProtobufSchema.ts index cc312c67a8..bd8d8dfd4d 100644 --- a/typescript/support/src/parseProtobufSchema.ts +++ b/typescript/support/src/parseProtobufSchema.ts @@ -4,6 +4,22 @@ import { FileDescriptorSet } from "@foxglove/protobufjs/ext/descriptor"; import { protobufDefinitionsToDatatypes, stripLeadingDot } from "./protobufDefinitionsToDatatypes"; import { MessageDefinitionMap } from "./types"; +export type ParseProtobufSchemaOptions = { + /** + * A function that will be called with the root type after parsing the FileDescriptorSet. Used by + * Foxglove Studio to modify the deserialization behavior of google.protobuf.Timestamp & + * google.protobuf.Duration. + */ + processRootType?: (rootType: protobufjs.Type) => protobufjs.Type; + + /** + * A function that will be called after producing message definitions from the schema. Used by + * Foxglove Studio to modify the field name of google.protobuf.Timestamp & + * google.protobuf.Duration. + */ + processMessageDefinitions?: (definitions: MessageDefinitionMap) => MessageDefinitionMap; +}; + /** * Parse a Protobuf binary schema (FileDescriptorSet) and produce datatypes and a deserializer * function. @@ -11,6 +27,7 @@ import { MessageDefinitionMap } from "./types"; export function parseProtobufSchema( schemaName: string, schemaData: Uint8Array, + options?: ParseProtobufSchemaOptions, ): { datatypes: MessageDefinitionMap; deserialize: (buffer: ArrayBufferView) => unknown; @@ -19,38 +36,10 @@ export function parseProtobufSchema( const root = protobufjs.Root.fromDescriptor(descriptorSet); root.resolveAll(); - const rootType = root.lookupType(schemaName); - - // Modify the definition of google.protobuf.Timestamp and Duration so they are interpreted as - // {sec: number, nsec: number}, compatible with the rest of Studio. The standard Protobuf types - // use different names (`seconds` and `nanos`), and `seconds` is an `int64`, which would be - // deserialized as a bigint by default. - // - // protobufDefinitionsToDatatypes also has matching logic to rename the fields. - const fixTimeType = (type: protobufjs.ReflectionObject | null) => { - if (!type || !(type instanceof protobufjs.Type)) { - return; - } - type.setup(); // ensure the original optimized toObject has been created - const prevToObject = type.toObject; // eslint-disable-line @typescript-eslint/unbound-method - const newToObject: typeof prevToObject = (message, options) => { - const result = prevToObject.call(type, message, options); - const { seconds, nanos } = result as { seconds: bigint; nanos: number }; - if (typeof seconds !== "bigint" || typeof nanos !== "number") { - return result; - } - if (seconds > BigInt(Number.MAX_SAFE_INTEGER)) { - throw new Error( - `Timestamps with seconds greater than 2^53-1 are not supported (found seconds=${seconds}, nanos=${nanos})`, - ); - } - return { sec: Number(seconds), nsec: nanos }; - }; - type.toObject = newToObject; - }; - - fixTimeType(root.lookup(".google.protobuf.Timestamp")); - fixTimeType(root.lookup(".google.protobuf.Duration")); + let rootType = root.lookupType(schemaName); + if (options?.processRootType) { + rootType = options.processRootType(rootType); + } const deserialize = (data: ArrayBufferView) => { return rootType.toObject( @@ -59,8 +48,11 @@ export function parseProtobufSchema( ); }; - const datatypes: MessageDefinitionMap = new Map(); + let datatypes: MessageDefinitionMap = new Map(); protobufDefinitionsToDatatypes(datatypes, rootType); + if (options?.processMessageDefinitions) { + datatypes = options.processMessageDefinitions(datatypes); + } if (!datatypes.has(schemaName)) { throw new Error( diff --git a/typescript/support/src/protobufDefinitionsToDatatypes.ts b/typescript/support/src/protobufDefinitionsToDatatypes.ts index 4eea790bdd..9b48da07e0 100644 --- a/typescript/support/src/protobufDefinitionsToDatatypes.ts +++ b/typescript/support/src/protobufDefinitionsToDatatypes.ts @@ -45,8 +45,8 @@ export function protobufDefinitionsToDatatypes( for (const field of type.fieldsArray) { if (field.resolvedType instanceof protobufjs.Enum) { for (const [name, value] of Object.entries(field.resolvedType.values)) { - // Note: names from different enums might conflict. The player API will need to be updated - // to associate fields with enums (similar to the __foxglove_enum annotation hack). + // Note: names from different enums might conflict. The @foxglove/message-definition API + // will need to be updated to associate fields with enums. // https://github.com/foxglove/studio/issues/2214 definitions.push({ name, type: "int32", isConstant: true, value }); } @@ -70,15 +70,6 @@ export function protobufDefinitionsToDatatypes( throw new Error("Repeated bytes are not currently supported"); } definitions.push({ type: "uint8", name: field.name, isArray: true }); - } else if ( - type.fullName === ".google.protobuf.Timestamp" || - type.fullName === ".google.protobuf.Duration" - ) { - definitions.push({ - type: "int32", - name: field.name === "seconds" ? "sec" : "nsec", - isArray: field.repeated, - }); } else { definitions.push({ type: protobufScalarToRosPrimitive(field.type),