From b30ff45fc3b9bdd79bd0fe8bfd95231a94c1fca7 Mon Sep 17 00:00:00 2001 From: Stephanie Closson Date: Tue, 10 Mar 2020 20:38:42 -0600 Subject: [PATCH 1/5] Possible fix for variable queries --- Makefile | 3 +++ src/datasource.ts | 33 ++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index c468d709..4a11a466 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,9 @@ test-in-docker: build-container build: go build -o ./dist/${DSNAME}_${OS}_${ARCH}${EXT} -a -tags netgo -ldflags '-w' ./pkg +build-dev: + go build -o ./dist/${DSNAME}_${OS}_${ARCH}${EXT} -a -tags netgo -ldflags ./pkg + build-linux: GOOS=linux go build -o ./dist/${DSNAME}_linux_${ARCH}${EXT} -a -tags netgo -ldflags '-w' ./pkg diff --git a/src/datasource.ts b/src/datasource.ts index 09b2e830..6d2af63e 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { MetricFindValue } from '@grafana/data'; import { ResponseParser, DatabaseItem } from './response_parser'; import QueryBuilder from './query_builder'; import Cache from './cache'; @@ -143,17 +144,37 @@ export class KustoDBDatasource { }); } - metricFindQuery(query: string, optionalOptions: any) { + metricFindQuery(query: string, optionalOptions: any): Promise { return this.getDefaultOrFirstDatabase() .then(database => this.buildQuery(query, optionalOptions, database)) .then(queries => this.backendSrv.datasourceRequest({ url: '/api/tsdb/query', method: 'POST', - queries, + data: { + from: '5m', + to: 'now', + queries, + }, }) ) - .then(response => new ResponseParser().parseToVariables(response)) + .then(response => { + const responseParser = new ResponseParser(); + const processedResponse = responseParser.processQueryResult(response); + const ret: MetricFindValue[] = []; + processedResponse.data.forEach(dataEntry => { + dataEntry.rows.forEach((r: string[] | string) => { + if (Array.isArray(r)) { + const rArray: string[] = r; + rArray.forEach((item: string) => ret.push({ text: item })); + } else { + ret.push({ text: (r as unknown as string) }); + } + }); + }); + + return ret; + }) .catch(err => { console.log('There was an error', err); throw err; @@ -254,12 +275,13 @@ export class KustoDBDatasource { } private buildQuery(query: string, options: any, database: string) { - if (typeof options === 'undefined') { + if (!options) { options = {}; } - if (typeof options.scopedVars === 'undefined') { + if (!options.hasOwnProperty('scopedVars')) { options['scopedVars'] = {}; } + console.log("options", options); const queryBuilder = new QueryBuilder(this.templateSrv.replace(query, options.scopedVars, this.interpolateVariable), options); const url = `${this.baseUrl}/v1/rest/query`; const interpolatedQuery = queryBuilder.interpolate().query; @@ -272,6 +294,7 @@ export class KustoDBDatasource { query: interpolatedQuery, database, }); + console.log("buildQuery returns", queries); return queries; } From c13a9d7ae60b6fa6c316e2369ea7c0f7dd7f1619 Mon Sep 17 00:00:00 2001 From: Stephanie Closson Date: Wed, 11 Mar 2020 09:35:45 -0600 Subject: [PATCH 2/5] Wrote tests --- src/datasource.test.ts | 102 ----------------------------------- src/datasource.ts | 18 +------ src/response_parser.test.ts | 104 ++++++++++++++++++++++++++++++++++++ src/response_parser.ts | 30 ++++++++++- 4 files changed, 134 insertions(+), 120 deletions(-) create mode 100644 src/response_parser.test.ts diff --git a/src/datasource.test.ts b/src/datasource.test.ts index 381a852a..555a5bb4 100644 --- a/src/datasource.test.ts +++ b/src/datasource.test.ts @@ -160,108 +160,6 @@ describe('KustoDBDatasource', () => { }, ], }; - - let queryResults; - - beforeEach(async () => { - ctx.backendSrv.datasourceRequest = options => { - if (options.url.indexOf('rest/mgmt') > -1) { - return ctx.$q.when({ data: databasesResponse, status: 200 }); - } else { - return ctx.$q.when({ data: tableResponseWithOneColumn, status: 200 }); - } - }; - - queryResults = await ctx.ds.metricFindQuery('Activity | distinct Category'); - }); - - it('should return a list of categories in the correct format', () => { - expect(queryResults.length).toBe(2); - expect(queryResults[0].text).toBe('Administrative'); - expect(queryResults[0].value).toBe('Administrative'); - expect(queryResults[1].text).toBe('Policy'); - expect(queryResults[1].value).toBe('Policy'); - }); - }); - - describe('When performing metricFindQuery (two columns)', () => { - const tableResponseWithTwoColumns = { - Tables: [ - { - TableName: 'Table_1', - Columns: [ - { - ColumnName: 'CatagoryId', - ColumnType: 'int', - }, - { - ColumnName: 'CategoryName', - ColumnType: 'string', - }, - ], - Rows: [[12], ['Titanic'], [13], ['Titanic']], - }, - ], - }; - - const databasesResponse = { - Tables: [ - { - TableName: 'Table_0', - Columns: [ - { ColumnName: 'DatabaseName', DataType: 'String' }, - { ColumnName: 'PersistentStorage', DataType: 'String' }, - { ColumnName: 'Version', DataType: 'String' }, - { ColumnName: 'IsCurrent', DataType: 'Boolean' }, - { ColumnName: 'DatabaseAccessMode', DataType: 'String' }, - { ColumnName: 'PrettyName', DataType: 'String' }, - { - ColumnName: 'CurrentUserIsUnrestrictedViewer', - DataType: 'Boolean', - }, - { ColumnName: 'DatabaseId', DataType: 'Guid' }, - ], - Rows: [ - [ - 'Grafana', - 'https://4bukustoragekus86a3c.blob.core.windows.net/grafanamd201806201624130602', - 'v5.2', - false, - 'ReadWrite', - null, - false, - 'a955a3ed-0668-4d00-a2e5-9c4e610ef057', - ], - ], - }, - ], - }; - - let queryResults; - - beforeEach(async () => { - ctx.backendSrv.datasourceRequest = options => { - if (options.url.indexOf('rest/mgmt') > -1) { - return ctx.$q.when({ data: databasesResponse, status: 200 }); - } else { - return ctx.$q.when({ data: tableResponseWithTwoColumns, status: 200 }); - } - }; - - queryResults = await ctx.ds.metricFindQuery('Activity | project __value = CategoryId, __text = CategoryName'); - }); - - it('should return a list of categories in the correct format', () => { - expect(queryResults.length).toBe(4); - expect(queryResults[0].text).toBe(12); - expect(queryResults[0].value).toBe(12); - expect(queryResults[1].text).toBe('Titanic'); - expect(queryResults[1].value).toBe('Titanic'); - expect(queryResults[2].text).toBe(13); - expect(queryResults[2].value).toBe(13); - expect(queryResults[3].text).toBe('Titanic'); - expect(queryResults[3].value).toBe('Titanic'); - }); }); describe('when performing annotations query', () => { diff --git a/src/datasource.ts b/src/datasource.ts index 6d2af63e..e8ab4bcd 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -160,20 +160,8 @@ export class KustoDBDatasource { ) .then(response => { const responseParser = new ResponseParser(); - const processedResponse = responseParser.processQueryResult(response); - const ret: MetricFindValue[] = []; - processedResponse.data.forEach(dataEntry => { - dataEntry.rows.forEach((r: string[] | string) => { - if (Array.isArray(r)) { - const rArray: string[] = r; - rArray.forEach((item: string) => ret.push({ text: item })); - } else { - ret.push({ text: (r as unknown as string) }); - } - }); - }); - - return ret; + const processedResposne = responseParser.processQueryResult(response); + return responseParser.processVariableQueryResult(processedResposne); }) .catch(err => { console.log('There was an error', err); @@ -281,7 +269,6 @@ export class KustoDBDatasource { if (!options.hasOwnProperty('scopedVars')) { options['scopedVars'] = {}; } - console.log("options", options); const queryBuilder = new QueryBuilder(this.templateSrv.replace(query, options.scopedVars, this.interpolateVariable), options); const url = `${this.baseUrl}/v1/rest/query`; const interpolatedQuery = queryBuilder.interpolate().query; @@ -294,7 +281,6 @@ export class KustoDBDatasource { query: interpolatedQuery, database, }); - console.log("buildQuery returns", queries); return queries; } diff --git a/src/response_parser.test.ts b/src/response_parser.test.ts new file mode 100644 index 00000000..cf46e233 --- /dev/null +++ b/src/response_parser.test.ts @@ -0,0 +1,104 @@ +import { ResponseParser } from './response_parser'; + +describe('ResponseParser', () => { + it('Parse query responses for metrics', () => { + const mockedResponse = { + data: [ + { + columns: [ + { + text: 'print_0', + }, + { + text: 'print_1', + }, + { + text: 'print_2', + }, + ], + rows: [['hello', 'doctor', 'name', 'continue', 'yesterday', 'tomorrow']], + type: 'table', + refId: 'A', + meta: { + KustoError: '', + RawQuery: "print 'hello', 'doctor', 'name', 'continue', 'yesterday', 'tomorrow'", + TimeNotASC: false, + }, + }, + ], + }; + + const mockedParsedResponse = [ + { + text: 'hello', + }, + { + text: 'doctor', + }, + { + text: 'name', + }, + { + text: 'continue', + }, + { + text: 'yesterday', + }, + { + text: 'tomorrow', + }, + ]; + + const rp = new ResponseParser(); + const parsedResponse = rp.processVariableQueryResult(mockedResponse); + expect(parsedResponse).toStrictEqual(mockedParsedResponse); + }); + + it('Parse query responses for metrics', () => { + const mockedResponse = { + data: [ + { + columns: [ + { + text: 'cheese', + }, + { + text: 'cities', + }, + ], + rows: [ + ['gouda', 'Paris'], + ['chedder', 'New York'], + ['mozza', 'Rome'], + ['old', 'Warsaw'], + ['squeaky', 'Montreal'], + ], + type: 'table', + refId: 'A', + meta: { + KustoError: '', + RawQuery: "print 'hello', 'doctor', 'name', 'continue', 'yesterday', 'tomorrow'", + TimeNotASC: false, + }, + }, + ], + }; + + const mockedParsedResponse = [ + { text: 'gouda' }, + { text: 'Paris' }, + { text: 'chedder' }, + { text: 'New York' }, + { text: 'mozza' }, + { text: 'Rome' }, + { text: 'old' }, + { text: 'Warsaw' }, + { text: 'squeaky' }, + { text: 'Montreal' }, + ]; + + const rp = new ResponseParser(); + const parsedResponse = rp.processVariableQueryResult(mockedResponse); + expect(parsedResponse).toStrictEqual(mockedParsedResponse); + }); +}); diff --git a/src/response_parser.ts b/src/response_parser.ts index 3d0c38ed..24625090 100644 --- a/src/response_parser.ts +++ b/src/response_parser.ts @@ -1,5 +1,5 @@ import _ from 'lodash'; -import { dateTime } from '@grafana/data'; +import { dateTime, MetricFindValue } from '@grafana/data'; export interface DataTarget { target: string; @@ -327,7 +327,11 @@ export class ResponseParser { return dateTime(dt).valueOf(); } - // TODO(Temp Comment): processQueryResult is for results using the backend plugin + /** + * Translates the response from the backend plugin into a different format + * @todo figure out what that format is? Is it a documented standard? + * @param res The response from the backend plugin + */ processQueryResult(res) { const data: any[] = []; @@ -382,4 +386,26 @@ export class ResponseParser { valueCount: Object.keys(valueMap).length, }; } + + /** + * Processes the response further for MetricFindValues + * @todo Get rid of all the any's! We need to be strongly typing everything. + * @param res The return value from processQueryResult + * @returns An array of MetricFindValues + */ + processVariableQueryResult(res: any): MetricFindValue[] { + const ret: MetricFindValue[] = []; + res.data.forEach(dataEntry => { + dataEntry.rows.forEach((r: string[] | string) => { + if (Array.isArray(r)) { + const rArray: string[] = r; + rArray.forEach((item: string) => ret.push({ text: item })); + } else { + ret.push({ text: (r as unknown as string) }); + } + }); + }); + + return ret; + } } From eb4d17ffd74d5a994b122533668d7f4ecea6d07c Mon Sep 17 00:00:00 2001 From: Stephanie Closson Date: Wed, 11 Mar 2020 09:38:23 -0600 Subject: [PATCH 3/5] Removed old metric find query tests --- src/datasource.test.ts | 50 ------------------------------------------ 1 file changed, 50 deletions(-) diff --git a/src/datasource.test.ts b/src/datasource.test.ts index 555a5bb4..28b384b9 100644 --- a/src/datasource.test.ts +++ b/src/datasource.test.ts @@ -112,56 +112,6 @@ describe('KustoDBDatasource', () => { }); }); - describe('When performing metricFindQuery', () => { - const tableResponseWithOneColumn = { - Tables: [ - { - TableName: 'Table_0', - Columns: [ - { - ColumnName: 'Category', - ColumnType: 'string', - }, - ], - Rows: [['Administrative'], ['Policy']], - }, - ], - }; - - const databasesResponse = { - Tables: [ - { - TableName: 'Table_0', - Columns: [ - { ColumnName: 'DatabaseName', DataType: 'String' }, - { ColumnName: 'PersistentStorage', DataType: 'String' }, - { ColumnName: 'Version', DataType: 'String' }, - { ColumnName: 'IsCurrent', DataType: 'Boolean' }, - { ColumnName: 'DatabaseAccessMode', DataType: 'String' }, - { ColumnName: 'PrettyName', DataType: 'String' }, - { - ColumnName: 'CurrentUserIsUnrestrictedViewer', - DataType: 'Boolean', - }, - { ColumnName: 'DatabaseId', DataType: 'Guid' }, - ], - Rows: [ - [ - 'Grafana', - 'https://4bukustoragekus86a3c.blob.core.windows.net/grafanamd201806201624130602', - 'v5.2', - false, - 'ReadWrite', - null, - false, - 'a955a3ed-0668-4d00-a2e5-9c4e610ef057', - ], - ], - }, - ], - }; - }); - describe('when performing annotations query', () => { const tableResponse = { Tables: [ From 7a242cccf940a581f0894be52e4bfbd291c79546 Mon Sep 17 00:00:00 2001 From: Stephanie Closson Date: Wed, 11 Mar 2020 09:45:03 -0600 Subject: [PATCH 4/5] Fix linting errors --- src/response_parser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/response_parser.ts b/src/response_parser.ts index 24625090..1f4599e5 100644 --- a/src/response_parser.ts +++ b/src/response_parser.ts @@ -401,11 +401,11 @@ export class ResponseParser { const rArray: string[] = r; rArray.forEach((item: string) => ret.push({ text: item })); } else { - ret.push({ text: (r as unknown as string) }); + ret.push({ text: (r as unknown) as string }); } }); }); - + return ret; } } From c498c0e84609ff36aa531adc80ffe933ae9253d5 Mon Sep 17 00:00:00 2001 From: Stephanie Closson Date: Wed, 11 Mar 2020 09:49:51 -0600 Subject: [PATCH 5/5] Updated version and changelog --- CHANGELOG.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e0d6b72..15e60f71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [2.0.5] + - Bugfix for issue #61. This is a temp fix, as a proper fix requires refactoring some of the backend. + ## [2.0.4] - Bugfix for issue #73 diff --git a/package.json b/package.json index 0224cccb..92216a4d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grafana-azure-data-explorer-datasource", - "version": "2.0.4", + "version": "2.0.5", "description": "Grafana data source for Azure Data Explorer", "scripts": { "build": "grafana-toolkit plugin:build && find src | grep '\\.d\\.ts$' | xargs rm -fv",