Avro path schema autocomplete input.

hermes-console/src/utils/json-avro/avroTypes.ts

@@ -0,0 +1,111 @@
+export const recordType = ['record'];
+export const iterableTypes = ['map', 'array'];
+export const primitiveTypes = [
+  'null',
+  'int',
+  'long',
+  'float',
+  'double',
+  'bytes',
+  'string',
+  'boolean',
+  'enum',
+];
+export const knownTypes = [...primitiveTypes, ...recordType, ...iterableTypes];
+
+export type AvroObject = any;
+
+export const propertyExists = (object: AvroObject, property: string): boolean =>
+  Object.prototype.hasOwnProperty.call(object, property);
+
+export const isValidAvroField = (object: AvroObject): boolean =>
+  object == null ||
+  !propertyExists(object, 'type') ||
+  !propertyExists(object, 'name') ||
+  object['name'] === '__metadata';
+
+export function isRootRecord(object: any) {
+  return object.type === 'record';
+}
+
+export function isNonNullableRecord(object: any) {
+  return object.type.type === 'record';
+}
+
+export function isNonNullableArrayRecord(object: any) {
+  return (
+    object.type.type === 'array' && Array.isArray(object.type.items.fields)
+  );
+}
+
+export function isNullableRecord(object: any) {
+  return (
+    Array.isArray(object.type) &&
+    object.type.find(
+      (t: AvroObject) => t && propertyExists(t, 'type') && t.type === 'record',
+    )
+  );
+}
+
+export function isNullablePrimitive(object: any) {
+  return (
+    Array.isArray(object.type) &&
+    object.type.every((v: any) => primitiveTypes.includes(v))
+  );
+}
+
+export function isNonNullableArrayPrimitive(object: any) {
+  return (
+    !Array.isArray(object.type) &&
+    object.type.type === 'array' &&
+    primitiveTypes.includes(object.type.items)
+  );
+}
+
+export function isNullableArrayComplex(object: any) {
+  return (
+    Array.isArray(object.type) &&
+    object.type.find((t: any) => t !== 'null').type === 'array' &&
+    recordType.includes(object.type.find((t: any) => t !== 'null').items.type)
+  );
+}
+
+export function isNullableArrayPrimitiveItems(object: any) {
+  return (
+    Array.isArray(object.type) &&
+    object.type.find((t: any) => t !== 'null').type === 'array' &&
+    primitiveTypes.includes(
+      object.type.find((t: any) => t !== 'null').items.type,
+    )
+  );
+}
+
+export function isNullableArrayPrimitiveSimple(object: any) {
+  return (
+    Array.isArray(object.type) &&
+    object.type.find((t: any) => t !== 'null').type === 'array' &&
+    primitiveTypes.includes(object.type.find((t: any) => t !== 'null').items)
+  );
+}
+
+export function isNonNullableCustomType(object: any) {
+  return (
+    !Array.isArray(object.type) &&
+    object.type.type != null &&
+    !knownTypes.includes(object.type.type)
+  );
+}
+
+export function isNullableCustomType(object: any) {
+  return (
+    Array.isArray(object.type) &&
+    !knownTypes.includes(object.type.find((t: any) => t != 'null'))
+  );
+}
+
+export function isNonNullablePrimitive(object: any) {
+  return (
+    primitiveTypes.includes(object.type) ||
+    (object.type.type != null && primitiveTypes.includes(object.type.type))
+  );
+}

hermes-console/src/utils/json-avro/jsonAvroUtils.spec.ts

@@ -0,0 +1,111 @@
+import { describe, expect } from 'vitest';
+import { flattenAvro } from '@/utils/json-avro/jsonAvroUtils';
+
+describe('Json-Avro tools', () => {
+  it('should find all avro path from avro json example schema', () => {
+    const jsonAvroSchemaObject = {
+      namespace: 'enterprise.thing',
+      type: 'record',
+      name: 'rating_event',
+      fields: [
+        { name: 'type', type: 'string' },
+        {
+          name: 'account',
+          type: {
+            type: 'record',
+            name: 'account',
+            fields: [
+              { name: 'account_id', type: 'string' },
+              { name: 'id', type: 'string' },
+            ],
+          },
+        },
+        {
+          name: 'rating',
+          type: {
+            type: 'record',
+            name: 'rating',
+            fields: [
+              { name: 'rating_type', type: 'string' },
+              {
+                name: 'rating_results',
+                type: {
+                  type: 'array',
+                  items: {
+                    type: 'record',
+                    name: 'rating_results',
+                    fields: [
+                      {
+                        name: 'rating_result',
+                        type: {
+                          type: 'record',
+                          name: 'rating_result',
+                          fields: [
+                            { name: 'name', type: ['null', 'string'] },
+                            { name: 'age', type: 'float' },
+                            { name: 'city_code', type: ['null', 'float'] },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                },
+              },
+              {
+                name: 'related_to',
+                type: [
+                  'null',
+                  {
+                    type: 'record',
+                    name: 'related_to',
+                    fields: [{ name: 'category', type: 'string' }],
+                  },
+                ],
+              },
+            ],
+          },
+        },
+      ],
+    };
+
+    const foundPaths = flattenAvro(jsonAvroSchemaObject);
+
+    const expectedPaths: string[] = [
+      '.type',
+      '.account',
+      '.account.account_id',
+      '.account.id',
+      '.rating',
+      '.rating.rating_type',
+      '.rating.rating_results[*]',
+      '.rating.rating_results[*].rating_result',
+      '.rating.rating_results[*].rating_result.name',
+      '.rating.rating_results[*].rating_result.age',
+      '.rating.rating_results[*].rating_result.city_code',
+      '.rating.related_to',
+      '.rating.related_to.category',
+    ];
+    expect(foundPaths).toEqual(expectedPaths);
+  });
+
+  it('should return empty paths array when object structure is not valid avro scheme', () => {
+    const incorrectAvroSchemaObject = {
+      keyA: 'valueA',
+      name: 'Incorrect object',
+    };
+
+    const foundPaths = flattenAvro(incorrectAvroSchemaObject);
+
+    const expectedPaths: string[] = [];
+    expect(foundPaths).toEqual(expectedPaths);
+  });
+
+  it('should return empty paths array when object is null', () => {
+    const incorrectAvroSchemaObject = null;
+
+    const foundPaths = flattenAvro(incorrectAvroSchemaObject);
+
+    const expectedPaths: string[] = [];
+    expect(foundPaths).toEqual(expectedPaths);
+  });
+});

hermes-console/src/utils/json-avro/jsonAvroUtils.ts

@@ -0,0 +1,149 @@
+import {
+  type AvroObject,
+  isNonNullableArrayPrimitive,
+  isNonNullableArrayRecord,
+  isNonNullableCustomType,
+  isNonNullablePrimitive,
+  isNonNullableRecord,
+  isNullableArrayComplex,
+  isNullableArrayPrimitiveItems,
+  isNullableArrayPrimitiveSimple,
+  isNullableCustomType,
+  isNullablePrimitive,
+  isNullableRecord,
+  isRootRecord,
+  isValidAvroField,
+  propertyExists,
+} from '@/utils/json-avro/avroTypes';
+import { deepSearch, isJsonValid } from '@/utils/json-avro/jsonUtils';
+
+export const getAvroPaths = (schema?: string): string[] => {
+  let avroPaths: string[] = [];
+  try {
+    if (isJsonValid(schema)) {
+      const parsedSchema = JSON.parse(schema!);
+      avroPaths = flattenAvro(parsedSchema);
+    }
+  } catch (_) {
+    console.log('Cannot parse schema and get avro paths.');
+  }
+  return avroPaths;
+};
+
+export function flattenAvro(
+  object: AvroObject,
+  prefix: string = '',
+  fullObject = object,
+): string[] {
+  if (isValidAvroField(object)) return [];
+  return Object.keys(object).reduce((acc: any, key) => {
+    if (key !== 'name') return acc;
+
+    if (isRootRecord(object)) {
+      object.fields.forEach((field: AvroObject) => {
+        const path = getCurrentPath(prefix, object, false, true);
+        return (acc = acc.concat(flattenAvro(field, path, fullObject)));
+      });
+      return acc;
+    }
+
+    if (isNonNullableRecord(object)) {
+      acc.push(getCurrentPath(prefix, object));
+      object.type.fields.forEach((field: AvroObject) => {
+        const path = getCurrentPath(prefix, object, false, false);
+        return (acc = acc.concat(flattenAvro(field, path, fullObject)));
+      });
+      return acc;
+    }
+
+    if (isNonNullableArrayRecord(object)) {
+      acc.push(getCurrentPath(prefix, object, true));
+      object.type.items.fields.forEach((field: AvroObject) => {
+        const path = getCurrentPath(prefix, object, true, false);
+        return (acc = acc.concat(flattenAvro(field, path, fullObject)));
+      });
+      return acc;
+    }
+
+    if (isNullableRecord(object)) {
+      acc.push(getCurrentPath(prefix, object));
+      object.type
+        .find((t: AvroObject) => t.type === 'record')
+        .fields.forEach((field: AvroObject) => {
+          return (acc = acc.concat(
+            flattenAvro(field, getCurrentPath(prefix, object), fullObject),
+          ));
+        });
+      return acc;
+    }
+
+    if (isNonNullableArrayPrimitive(object)) {
+      acc.push(getCurrentPath(prefix, object, true));
+      return acc;
+    }
+
+    if (isNullableArrayComplex(object)) {
+      acc.push(getCurrentPath(prefix, object, true));
+      object.type
+        .find((t: any) => t !== 'null')
+        .items.fields.forEach((field: AvroObject) => {
+          const path = getCurrentPath(prefix, object, true, false);
+          return (acc = acc.concat(flattenAvro(field, path, fullObject)));
+        });
+      return acc;
+    }
+
+    if (isNullableArrayPrimitiveItems(object)) {
+      acc.push(getCurrentPath(prefix, object, true, false));
+      return acc;
+    }
+
+    if (isNullableArrayPrimitiveSimple(object)) {
+      acc.push(getCurrentPath(prefix, object, true));
+      return acc;
+    }
+
+    if (isNullableCustomType(object)) {
+      const customTypeName = object.type.find((t: any) => t != 'null');
+      const foundType = deepSearch(fullObject, 'name', customTypeName);
+      acc = getCustomObjectPaths(acc, prefix, object, foundType, fullObject);
+      return acc;
+    }
+
+    if (isNonNullableCustomType(object)) {
+      const foundType = deepSearch(fullObject, 'name', object.type.type);
+      acc = getCustomObjectPaths(acc, prefix, object, foundType, fullObject);
+      return acc;
+    }
+
+    if (isNullablePrimitive(object) || isNonNullablePrimitive(object)) {
+      acc.push(getCurrentPath(prefix, object));
+      return acc;
+    }
+  }, []);
+}
+
+const getCurrentPath = (
+  prefix: string,
+  object: AvroObject,
+  isArray: boolean = false,
+  isRoot: boolean = false,
+): string =>
+  (isRoot ? '' : prefix + '.' + object['name']) + (isArray ? '[*]' : '');
+
+const getCustomObjectPaths = (
+  acc: any,
+  prefix: string,
+  object: any,
+  foundType: AvroObject,
+  fullObject: AvroObject,
+) => {
+  acc.push(getCurrentPath(prefix, object, false, false));
+  if (foundType && propertyExists(foundType, 'fields')) {
+    foundType.fields.forEach((field: AvroObject) => {
+      const path = getCurrentPath(prefix, object, false, false);
+      return (acc = acc.concat(flattenAvro(field, path, fullObject)));
+    });
+  }
+  return acc;
+};