From bca479c7b9d6d317d0a8124861562670cdf8eed0 Mon Sep 17 00:00:00 2001 From: Sebastian Straburzynski Date: Sat, 26 Oct 2024 14:41:51 +0200 Subject: [PATCH] Avro path schema autocomplete input. --- .../src/utils/json-avro/avroTypes.ts | 111 +++++++++++++ .../src/utils/json-avro/jsonAvroUtils.spec.ts | 111 +++++++++++++ .../src/utils/json-avro/jsonAvroUtils.ts | 149 ++++++++++++++++++ .../src/utils/json-avro/jsonUtils.spec.ts | 88 +++++++++++ .../src/utils/json-avro/jsonUtils.ts | 22 +++ .../views/subscription/SubscriptionView.vue | 5 +- .../subscription/filters-card/FiltersCard.vue | 5 +- .../subscription-form/SubscriptionForm.vue | 4 +- .../SubscriptionPathFilters.vue | 3 + .../SubscriptionPathFiltersDebug.vue | 8 +- .../path-filter-row/PathFilterRow.vue | 31 ++-- .../SubscriptionMetadata.vue | 8 +- 12 files changed, 523 insertions(+), 22 deletions(-) create mode 100644 hermes-console/src/utils/json-avro/avroTypes.ts create mode 100644 hermes-console/src/utils/json-avro/jsonAvroUtils.spec.ts create mode 100644 hermes-console/src/utils/json-avro/jsonAvroUtils.ts create mode 100644 hermes-console/src/utils/json-avro/jsonUtils.spec.ts create mode 100644 hermes-console/src/utils/json-avro/jsonUtils.ts diff --git a/hermes-console/src/utils/json-avro/avroTypes.ts b/hermes-console/src/utils/json-avro/avroTypes.ts new file mode 100644 index 0000000000..771ab863b4 --- /dev/null +++ b/hermes-console/src/utils/json-avro/avroTypes.ts @@ -0,0 +1,111 @@ +export const recordType = ['record']; +export const iterableTypes = ['map', 'array']; +export const primitiveTypes = [ + 'null', + 'int', + 'long', + 'float', + 'double', + 'bytes', + 'string', + 'boolean', + 'enum', +]; +export const knownTypes = [...primitiveTypes, ...recordType, ...iterableTypes]; + +export type AvroObject = any; + +export const propertyExists = (object: AvroObject, property: string): boolean => + Object.prototype.hasOwnProperty.call(object, property); + +export const isValidAvroField = (object: AvroObject): boolean => + object == null || + !propertyExists(object, 'type') || + !propertyExists(object, 'name') || + object['name'] === '__metadata'; + +export function isRootRecord(object: any) { + return object.type === 'record'; +} + +export function isNonNullableRecord(object: any) { + return object.type.type === 'record'; +} + +export function isNonNullableArrayRecord(object: any) { + return ( + object.type.type === 'array' && Array.isArray(object.type.items.fields) + ); +} + +export function isNullableRecord(object: any) { + return ( + Array.isArray(object.type) && + object.type.find( + (t: AvroObject) => t && propertyExists(t, 'type') && t.type === 'record', + ) + ); +} + +export function isNullablePrimitive(object: any) { + return ( + Array.isArray(object.type) && + object.type.every((v: any) => primitiveTypes.includes(v)) + ); +} + +export function isNonNullableArrayPrimitive(object: any) { + return ( + !Array.isArray(object.type) && + object.type.type === 'array' && + primitiveTypes.includes(object.type.items) + ); +} + +export function isNullableArrayComplex(object: any) { + return ( + Array.isArray(object.type) && + object.type.find((t: any) => t !== 'null').type === 'array' && + recordType.includes(object.type.find((t: any) => t !== 'null').items.type) + ); +} + +export function isNullableArrayPrimitiveItems(object: any) { + return ( + Array.isArray(object.type) && + object.type.find((t: any) => t !== 'null').type === 'array' && + primitiveTypes.includes( + object.type.find((t: any) => t !== 'null').items.type, + ) + ); +} + +export function isNullableArrayPrimitiveSimple(object: any) { + return ( + Array.isArray(object.type) && + object.type.find((t: any) => t !== 'null').type === 'array' && + primitiveTypes.includes(object.type.find((t: any) => t !== 'null').items) + ); +} + +export function isNonNullableCustomType(object: any) { + return ( + !Array.isArray(object.type) && + object.type.type != null && + !knownTypes.includes(object.type.type) + ); +} + +export function isNullableCustomType(object: any) { + return ( + Array.isArray(object.type) && + !knownTypes.includes(object.type.find((t: any) => t != 'null')) + ); +} + +export function isNonNullablePrimitive(object: any) { + return ( + primitiveTypes.includes(object.type) || + (object.type.type != null && primitiveTypes.includes(object.type.type)) + ); +} diff --git a/hermes-console/src/utils/json-avro/jsonAvroUtils.spec.ts b/hermes-console/src/utils/json-avro/jsonAvroUtils.spec.ts new file mode 100644 index 0000000000..a3e058bb58 --- /dev/null +++ b/hermes-console/src/utils/json-avro/jsonAvroUtils.spec.ts @@ -0,0 +1,111 @@ +import { describe, expect } from 'vitest'; +import { flattenAvro } from '@/utils/json-avro/jsonAvroUtils'; + +describe('Json-Avro tools', () => { + it('should find all avro path from avro json example schema', () => { + const jsonAvroSchemaObject = { + namespace: 'enterprise.thing', + type: 'record', + name: 'rating_event', + fields: [ + { name: 'type', type: 'string' }, + { + name: 'account', + type: { + type: 'record', + name: 'account', + fields: [ + { name: 'account_id', type: 'string' }, + { name: 'id', type: 'string' }, + ], + }, + }, + { + name: 'rating', + type: { + type: 'record', + name: 'rating', + fields: [ + { name: 'rating_type', type: 'string' }, + { + name: 'rating_results', + type: { + type: 'array', + items: { + type: 'record', + name: 'rating_results', + fields: [ + { + name: 'rating_result', + type: { + type: 'record', + name: 'rating_result', + fields: [ + { name: 'name', type: ['null', 'string'] }, + { name: 'age', type: 'float' }, + { name: 'city_code', type: ['null', 'float'] }, + ], + }, + }, + ], + }, + }, + }, + { + name: 'related_to', + type: [ + 'null', + { + type: 'record', + name: 'related_to', + fields: [{ name: 'category', type: 'string' }], + }, + ], + }, + ], + }, + }, + ], + }; + + const foundPaths = flattenAvro(jsonAvroSchemaObject); + + const expectedPaths: string[] = [ + '.type', + '.account', + '.account.account_id', + '.account.id', + '.rating', + '.rating.rating_type', + '.rating.rating_results[*]', + '.rating.rating_results[*].rating_result', + '.rating.rating_results[*].rating_result.name', + '.rating.rating_results[*].rating_result.age', + '.rating.rating_results[*].rating_result.city_code', + '.rating.related_to', + '.rating.related_to.category', + ]; + expect(foundPaths).toEqual(expectedPaths); + }); + + it('should return empty paths array when object structure is not valid avro scheme', () => { + const incorrectAvroSchemaObject = { + keyA: 'valueA', + name: 'Incorrect object', + }; + + const foundPaths = flattenAvro(incorrectAvroSchemaObject); + + const expectedPaths: string[] = []; + expect(foundPaths).toEqual(expectedPaths); + }); + + it('should return empty paths array when object is null', () => { + const incorrectAvroSchemaObject = null; + + const foundPaths = flattenAvro(incorrectAvroSchemaObject); + + const expectedPaths: string[] = []; + expect(foundPaths).toEqual(expectedPaths); + }); +}); diff --git a/hermes-console/src/utils/json-avro/jsonAvroUtils.ts b/hermes-console/src/utils/json-avro/jsonAvroUtils.ts new file mode 100644 index 0000000000..ef77d9fbe7 --- /dev/null +++ b/hermes-console/src/utils/json-avro/jsonAvroUtils.ts @@ -0,0 +1,149 @@ +import { + type AvroObject, + isNonNullableArrayPrimitive, + isNonNullableArrayRecord, + isNonNullableCustomType, + isNonNullablePrimitive, + isNonNullableRecord, + isNullableArrayComplex, + isNullableArrayPrimitiveItems, + isNullableArrayPrimitiveSimple, + isNullableCustomType, + isNullablePrimitive, + isNullableRecord, + isRootRecord, + isValidAvroField, + propertyExists, +} from '@/utils/json-avro/avroTypes'; +import { deepSearch, isJsonValid } from '@/utils/json-avro/jsonUtils'; + +export const getAvroPaths = (schema?: string): string[] => { + let avroPaths: string[] = []; + try { + if (isJsonValid(schema)) { + const parsedSchema = JSON.parse(schema!); + avroPaths = flattenAvro(parsedSchema); + } + } catch (_) { + console.log('Cannot parse schema and get avro paths.'); + } + return avroPaths; +}; + +export function flattenAvro( + object: AvroObject, + prefix: string = '', + fullObject = object, +): string[] { + if (isValidAvroField(object)) return []; + return Object.keys(object).reduce((acc: any, key) => { + if (key !== 'name') return acc; + + if (isRootRecord(object)) { + object.fields.forEach((field: AvroObject) => { + const path = getCurrentPath(prefix, object, false, true); + return (acc = acc.concat(flattenAvro(field, path, fullObject))); + }); + return acc; + } + + if (isNonNullableRecord(object)) { + acc.push(getCurrentPath(prefix, object)); + object.type.fields.forEach((field: AvroObject) => { + const path = getCurrentPath(prefix, object, false, false); + return (acc = acc.concat(flattenAvro(field, path, fullObject))); + }); + return acc; + } + + if (isNonNullableArrayRecord(object)) { + acc.push(getCurrentPath(prefix, object, true)); + object.type.items.fields.forEach((field: AvroObject) => { + const path = getCurrentPath(prefix, object, true, false); + return (acc = acc.concat(flattenAvro(field, path, fullObject))); + }); + return acc; + } + + if (isNullableRecord(object)) { + acc.push(getCurrentPath(prefix, object)); + object.type + .find((t: AvroObject) => t.type === 'record') + .fields.forEach((field: AvroObject) => { + return (acc = acc.concat( + flattenAvro(field, getCurrentPath(prefix, object), fullObject), + )); + }); + return acc; + } + + if (isNonNullableArrayPrimitive(object)) { + acc.push(getCurrentPath(prefix, object, true)); + return acc; + } + + if (isNullableArrayComplex(object)) { + acc.push(getCurrentPath(prefix, object, true)); + object.type + .find((t: any) => t !== 'null') + .items.fields.forEach((field: AvroObject) => { + const path = getCurrentPath(prefix, object, true, false); + return (acc = acc.concat(flattenAvro(field, path, fullObject))); + }); + return acc; + } + + if (isNullableArrayPrimitiveItems(object)) { + acc.push(getCurrentPath(prefix, object, true, false)); + return acc; + } + + if (isNullableArrayPrimitiveSimple(object)) { + acc.push(getCurrentPath(prefix, object, true)); + return acc; + } + + if (isNullableCustomType(object)) { + const customTypeName = object.type.find((t: any) => t != 'null'); + const foundType = deepSearch(fullObject, 'name', customTypeName); + acc = getCustomObjectPaths(acc, prefix, object, foundType, fullObject); + return acc; + } + + if (isNonNullableCustomType(object)) { + const foundType = deepSearch(fullObject, 'name', object.type.type); + acc = getCustomObjectPaths(acc, prefix, object, foundType, fullObject); + return acc; + } + + if (isNullablePrimitive(object) || isNonNullablePrimitive(object)) { + acc.push(getCurrentPath(prefix, object)); + return acc; + } + }, []); +} + +const getCurrentPath = ( + prefix: string, + object: AvroObject, + isArray: boolean = false, + isRoot: boolean = false, +): string => + (isRoot ? '' : prefix + '.' + object['name']) + (isArray ? '[*]' : ''); + +const getCustomObjectPaths = ( + acc: any, + prefix: string, + object: any, + foundType: AvroObject, + fullObject: AvroObject, +) => { + acc.push(getCurrentPath(prefix, object, false, false)); + if (foundType && propertyExists(foundType, 'fields')) { + foundType.fields.forEach((field: AvroObject) => { + const path = getCurrentPath(prefix, object, false, false); + return (acc = acc.concat(flattenAvro(field, path, fullObject))); + }); + } + return acc; +}; diff --git a/hermes-console/src/utils/json-avro/jsonUtils.spec.ts b/hermes-console/src/utils/json-avro/jsonUtils.spec.ts new file mode 100644 index 0000000000..42de1afc78 --- /dev/null +++ b/hermes-console/src/utils/json-avro/jsonUtils.spec.ts @@ -0,0 +1,88 @@ +import { deepSearch, isJsonValid } from '@/utils/json-avro/jsonUtils'; +import { describe, expect } from 'vitest'; + +describe('Json utils', () => { + const object = { + key1: 'value1', + key2: { + key21: 'value21', + key22: { + key221: 'value221', + key222: 'value222', + }, + }, + array: [ + { + keyA: 'valueA', + keyB: 'valueB', + }, + { + keyC: { + keyC1: 'valueC1', + }, + }, + ], + }; + + it('deepSearch() - should find root object', () => { + const expectedObject = object; + + const foundObject = deepSearch(object, 'key1', 'value1'); + + expect(foundObject).toEqual(expectedObject); + }); + + it('deepSearch() - should find nested object', () => { + const expectedObject = { + key221: 'value221', + key222: 'value222', + }; + + const foundObject = deepSearch(object, 'key222', 'value222'); + + expect(foundObject).toEqual(expectedObject); + }); + + it('deepSearch() - should find object in array', () => { + const expectedObject = { + keyA: 'valueA', + keyB: 'valueB', + }; + + const foundObject = deepSearch(object, 'keyA', 'valueA'); + + expect(foundObject).toEqual(expectedObject); + }); + + it('deepSearch() - should return null when key with value not found', () => { + const expectedObject = null; + + const foundObject = deepSearch(object, 'keyX', 'valueX'); + + expect(foundObject).toEqual(expectedObject); + }); + + it('isJsonValid() - return true when string is valid json object', () => { + const testJsonString = JSON.stringify(object); + + const result = isJsonValid(testJsonString); + + expect(result).toEqual(true); + }); + + it('isJsonValid() - return false when string is null', () => { + const testJsonString = null; + + const result = isJsonValid(testJsonString); + + expect(result).toEqual(false); + }); + + it('isJsonValid() - return false when string invalid', () => { + const testJsonString = 'incorrect json string'; + + const result = isJsonValid(testJsonString); + + expect(result).toEqual(false); + }); +}); diff --git a/hermes-console/src/utils/json-avro/jsonUtils.ts b/hermes-console/src/utils/json-avro/jsonUtils.ts new file mode 100644 index 0000000000..a767917fb4 --- /dev/null +++ b/hermes-console/src/utils/json-avro/jsonUtils.ts @@ -0,0 +1,22 @@ +import { propertyExists } from '@/utils/json-avro/avroTypes'; + +export const isJsonValid = (text?: string | null): boolean => { + if (!text) return false; + try { + return !!JSON.parse(text); + } catch (_) { + return false; + } +}; + +export const deepSearch = (object: any, key: string, value: string): any => { + if (propertyExists(object, key) && object[key] === value) return object; + for (let i = 0; i < Object.keys(object).length; i++) { + const v = object[Object.keys(object)[i]]; + if (typeof v === 'object' && v != null) { + const o = deepSearch(object[Object.keys(object)[i]], key, value); + if (o != null) return o; + } + } + return null; +}; diff --git a/hermes-console/src/views/subscription/SubscriptionView.vue b/hermes-console/src/views/subscription/SubscriptionView.vue index f5bd73b56f..e1e5b4ab88 100644 --- a/hermes-console/src/views/subscription/SubscriptionView.vue +++ b/hermes-console/src/views/subscription/SubscriptionView.vue @@ -6,6 +6,7 @@ import { useRoles } from '@/composables/roles/use-roles/useRoles'; import { useRouter } from 'vue-router'; import { useSubscription } from '@/composables/subscription/use-subscription/useSubscription'; + import { useTopic } from '@/composables/topic/use-topic/useTopic'; import ConfirmationDialog from '@/components/confirmation-dialog/ConfirmationDialog.vue'; import ConsoleAlert from '@/components/console-alert/ConsoleAlert.vue'; import CostsCard from '@/components/costs-card/CostsCard.vue'; @@ -23,7 +24,7 @@ const router = useRouter(); const { groupId, subscriptionId, topicId } = router.currentRoute.value .params as Record; - + const { topic } = useTopic(topicId); const { t } = useI18n(); const { @@ -202,6 +203,7 @@ :subscription="subscription" :owner="owner" :roles="roles" + :schema="topic?.schema" @remove="openRemoveDialog" @suspend="openSuspendDialog" @activate="openActivateDialog" @@ -247,6 +249,7 @@ import { v4 as generateUUID } from 'uuid'; - import { PathFilter } from '@/views/subscription/subscription-form/subscription-basic-filters/types'; + import { getAvroPaths } from '@/utils/json-avro/jsonAvroUtils'; import SubscriptionPathFiltersDebug from '@/views/subscription/subscription-form/subscription-basic-filters/SubscriptionPathFiltersDebug.vue'; import type { MessageFilterSpecification } from '@/api/subscription'; + import type { PathFilter } from '@/views/subscription/subscription-form/subscription-basic-filters/types'; const props = defineProps<{ topic: string; filters: MessageFilterSpecification[]; + schema?: string; }>(); const pathFilters = (filters: MessageFilterSpecification[]): PathFilter[] => { return filters.filter((f) => f.type !== 'header').map((f) => mapFilter(f)); @@ -64,6 +66,7 @@