diff --git a/src/backendResultTransformer.ts b/src/backendResultTransformer.ts index 2873d64..11eaf25 100644 --- a/src/backendResultTransformer.ts +++ b/src/backendResultTransformer.ts @@ -3,7 +3,7 @@ import { DataQueryResponse, DataFrame, isDataFrame, FieldType, QueryResultMeta, import { getDerivedFields } from './getDerivedFields'; import { makeTableFrames } from './makeTableFrames'; import { getHighlighterExpressionsFromQuery } from './queryUtils'; -import { dataFrameHasLokiError } from './responseUtils'; +import { dataFrameHasError } from './responseUtils'; import { DerivedFieldConfig, Query, QueryType } from './types'; function isMetricFrame(frame: DataFrame): boolean { @@ -28,11 +28,9 @@ function processStreamFrame( ): DataFrame { const custom: Record = { ...frame.meta?.custom, // keep the original meta.custom - // used by logsModel - lokiQueryStatKey: 'Summary: total bytes processed', }; - if (dataFrameHasLokiError(frame)) { + if (dataFrameHasError(frame)) { custom.error = 'Error when parsing some of the logs'; } @@ -102,8 +100,6 @@ function groupFrames( } function improveError(error: DataQueryError | undefined, queryMap: Map): DataQueryError | undefined { - // many things are optional in an error-object, we need an error-message to exist, - // and we need to find the loki-query, based on the refId in the error-object. if (error === undefined) { return error; } @@ -121,7 +117,7 @@ function improveError(error: DataQueryError | undefined, queryMap: Map improveError(error, queryMap)).filter((e) => e !== undefined); + return { ...rest, - error: improveError(error, queryMap), + errors: improvedErrors as DataQueryError[], data: [ ...processMetricRangeFrames(metricRangeFrames), ...processMetricInstantFrames(metricInstantFrames), diff --git a/src/components/QueryEditor/QueryEditor.tsx b/src/components/QueryEditor/QueryEditor.tsx index ebf56b1..146e2af 100644 --- a/src/components/QueryEditor/QueryEditor.tsx +++ b/src/components/QueryEditor/QueryEditor.tsx @@ -1,8 +1,9 @@ +import { css } from "@emotion/css"; import { isEqual } from 'lodash'; import React, { useEffect, useState } from 'react'; -import { CoreApp, LoadingState } from '@grafana/data'; -import { Button } from '@grafana/ui'; +import { CoreApp, GrafanaTheme2, LoadingState } from '@grafana/data'; +import { Button, useStyles2 } from '@grafana/ui'; import { Query, VictoriaLogsQueryEditorProps } from "../../types"; @@ -10,6 +11,8 @@ import QueryCodeEditor from "./QueryCodeEditor"; import { getQueryWithDefaults } from "./state"; const QueryEditor = React.memo((props) => { + const styles = useStyles2(getStyles); + const { onChange, onRunQuery, data, app, queries } = props; const [dataIsStale, setDataIsStale] = useState(false); @@ -27,7 +30,10 @@ const QueryEditor = React.memo((props) => { }; return ( - <> +
+
+ +
{app !== CoreApp.Explore && app !== CoreApp.Correlations && ( )}
-
- -
- +
); }); -QueryEditor.displayName = 'LokiQueryEditor'; +const getStyles = (theme: GrafanaTheme2) => { + return { + wrapper: css` + display: flex; + align-items: flex-start; + gap: ${theme.spacing(1)}; + ` + }; +}; + +QueryEditor.displayName = 'QueryEditor'; export default QueryEditor diff --git a/src/components/QueryEditor/QueryField.tsx b/src/components/QueryEditor/QueryField.tsx index 1cdf1f0..da33104 100644 --- a/src/components/QueryEditor/QueryField.tsx +++ b/src/components/QueryEditor/QueryField.tsx @@ -6,12 +6,12 @@ import { VictoriaLogsDatasource } from "../../datasource"; import { Options, Query } from "../../types"; import { MonacoQueryFieldWrapper } from "../monaco-query-field/MonacoQueryFieldWrapper"; -export interface LokiQueryFieldProps extends QueryEditorProps { +export interface QueryFieldProps extends QueryEditorProps { ExtraFieldElement?: React.ReactNode; 'data-testid'?: string; } -const QueryField: React.FC = (props) => { +const QueryField: React.FC = (props) => { const { ExtraFieldElement, query, @@ -50,7 +50,7 @@ const QueryField: React.FC = (props) => { onChange={onChangeQuery} onRunQuery={onRunQuery} initialValue={query.expr ?? ''} - placeholder="Enter a MetricsQL query…" + placeholder="Enter a LogsQL query…" /> diff --git a/src/configuration/HelpfulLinks.tsx b/src/configuration/HelpfulLinks.tsx index 73b3cfa..1f73b14 100644 --- a/src/configuration/HelpfulLinks.tsx +++ b/src/configuration/HelpfulLinks.tsx @@ -3,19 +3,15 @@ import React from "react"; const tips = [ { title: "Datasource", - url: "https://github.com/VictoriaMetrics/grafana-datasource#victoriametrics-data-source-for-grafana", + url: "https://github.com/VictoriaMetrics/grafana-logs-datasource", }, { - title: "Cluster VM", - url: "https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format", + title: "VictoriaLogs", + url: "https://docs.victoriametrics.com/victorialogs/", }, { - title: "Grafana setup", - url: "https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#grafana-setup", - }, - { - title: "MetricsQL", - url: "https://docs.victoriametrics.com/MetricsQL.html", + title: "LogsQL", + url: "https://docs.victoriametrics.com/victorialogs/logsql/", }, { title: "VictoriaMetrics", diff --git a/src/configuration/QuerySettings.tsx b/src/configuration/QuerySettings.tsx index bcc06f6..1078b65 100644 --- a/src/configuration/QuerySettings.tsx +++ b/src/configuration/QuerySettings.tsx @@ -13,11 +13,10 @@ export const QuerySettings = (props: Props) => {
- Loki queries must contain a limit of the maximum number of lines returned (default: 1000). Increase this + VictoriaLogs queries must contain a limit of the maximum number of lines returned (default: 1000). Increase this limit to have a bigger result set for ad-hoc analysis. Decrease this limit if your browser becomes sluggish when displaying the log results. @@ -25,7 +24,6 @@ export const QuerySettings = (props: Props) => { > ) => onMaxLinedChange(event.currentTarget.value)} width={16} diff --git a/src/datasource.ts b/src/datasource.ts index 1cbac9a..f6e6ae0 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -6,33 +6,35 @@ import { DataSourceWithBackend } from '@grafana/runtime'; import { transformBackendResult } from "./backendResultTransformer"; import QueryEditor from "./components/QueryEditor/QueryEditor"; -import { Query, Options } from './types'; - - +import { escapeLabelValueInSelector } from "./languageUtils"; +import { addLabelToQuery, queryHasFilter, removeLabelFromQuery } from "./modifyQuery"; +import { Query, Options, ToggleFilterAction, QueryFilterOptions, FilterActionType } from './types'; export class VictoriaLogsDatasource extends DataSourceWithBackend { maxLines: number; constructor( - private instanceSettings: DataSourceInstanceSettings, - // private readonly templateSrv: TemplateSrv = getTemplateSrv() + instanceSettings: DataSourceInstanceSettings, ) { super(instanceSettings); - // this.languageProvider = new LanguageProvider(this); const settingsData = instanceSettings.jsonData || {}; this.maxLines = parseInt(settingsData.maxLines ?? '0', 10) || 10; this.annotations = { QueryEditor: QueryEditor, }; - // this.variables = new LokiVariableSupport(this); - // this.logContextProvider = new LogContextProvider(this); } query(request: DataQueryRequest): Observable { - const queries = request.targets - .map((q) => ({ ...q, maxLines: q.maxLines ?? this.maxLines })); + const queries = request.targets.filter(q => q.expr).map((q) => { + // include time range in query if not already present + if (!/_time/.test(q.expr)) { + const timerange = `_time:[${request.range.from.toISOString()}, ${request.range.to.toISOString()}]` + q.expr = `${timerange} AND ${q.expr}`; + } + return { ...q, maxLines: q.maxLines ?? this.maxLines } + }); const fixedRequest: DataQueryRequest = { ...request, @@ -51,4 +53,33 @@ export class VictoriaLogsDatasource ) ); } + + toggleQueryFilter(query: Query, filter: ToggleFilterAction): Query { + let expression = query.expr ?? ''; + + if (!filter.options?.key || !filter.options?.value) { + return { ...query, expr: expression }; + } + + const isFilterFor = filter.type === FilterActionType.FILTER_FOR; + const isFilterOut = filter.type === FilterActionType.FILTER_OUT; + const value = escapeLabelValueInSelector(filter.options.value); + const hasFilter = queryHasFilter(expression, filter.options.key, value) + const operator = filter.type === FilterActionType.FILTER_FOR ? 'AND' : 'NOT'; + + if (hasFilter) { + expression = removeLabelFromQuery(expression, filter.options.key, value); + } + + if ((isFilterFor && !hasFilter) || isFilterOut) { + expression = addLabelToQuery(expression, filter.options.key, value, operator); + } + + return { ...query, expr: expression }; + } + + queryHasFilter(query: Query, filter: QueryFilterOptions): boolean { + let expression = query.expr ?? ''; + return queryHasFilter(expression, filter.key, filter.value); + } } diff --git a/src/languageUtils.ts b/src/languageUtils.ts new file mode 100644 index 0000000..0a00682 --- /dev/null +++ b/src/languageUtils.ts @@ -0,0 +1,27 @@ +const REG_METACHARACTERS = /[*+?()|\\.\[\]{}^$]/g; + +export function unescapeLabelValue(labelValue: string): string { + return labelValue.replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\'); +} + +export function isRegexSelector(selector?: string) { + return !!(selector && (selector.includes('=~') || selector.includes('!~'))); +} + +function escapeMetaRegexp(value: string): string { + return value.replace(REG_METACHARACTERS, '\\$&'); +} + +export function escapeLabelValueInExactSelector(labelValue: string): string { + return labelValue.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"'); +} + +export function escapeLabelValueInRegexSelector(labelValue: string): string { + return escapeLabelValueInExactSelector(escapeMetaRegexp(labelValue)); +} + +export function escapeLabelValueInSelector(labelValue: string, selector?: string): string { + return isRegexSelector(selector) + ? escapeLabelValueInRegexSelector(labelValue) + : escapeLabelValueInExactSelector(labelValue); +} diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts new file mode 100644 index 0000000..818f5db --- /dev/null +++ b/src/modifyQuery.ts @@ -0,0 +1,28 @@ +export function queryHasFilter(query: string, key: string, value: string): boolean { + return query.includes(`${key}:${value}`) +} + +export const removeLabelFromQuery = (query: string, key: string, value: string): string => { + const parts = query.split(' ') + const index = parts.findIndex((part) => part.includes(`${key}:${value}`)) + const newParts = removeAtIndexAndBefore(parts, index) + return newParts.join(' ') +} + +export const addLabelToQuery = (query: string, key: string, value: string, operator: string): string => { + return `${query} ${operator} ${key}:${value}` +} + +const removeAtIndexAndBefore = (arr: string[], index: number): string[] => { + if (index < 0 || index >= arr.length) { + return arr; + } + + if (index === 0) { + arr.splice(index, 1); + } else { + arr.splice(index - 1, 2); + } + + return arr; +} diff --git a/src/parsing.ts b/src/parsing.ts new file mode 100644 index 0000000..12aefe1 --- /dev/null +++ b/src/parsing.ts @@ -0,0 +1,9 @@ +export function handleQuotes(string: string) { + if (string[0] === `"` && string[string.length - 1] === `"`) { + return string + .substring(1, string.length - 1) + .replace(/\\"/g, '"') + .replace(/\\\\/g, '\\'); + } + return string.replace(/`/g, ''); +} diff --git a/src/queryUtils.ts b/src/queryUtils.ts index 64e284b..1b9bf48 100644 --- a/src/queryUtils.ts +++ b/src/queryUtils.ts @@ -3,8 +3,6 @@ import { escapeRegExp } from 'lodash'; import { Filter, FilterOp, LineFilter, OrFilter, parser, PipeExact, PipeMatch, String } from "@grafana/lezer-logql" - - export function getNodesFromQuery(query: string, nodeTypes?: number[]): SyntaxNode[] { const nodes: SyntaxNode[] = []; const tree = parser.parse(query); @@ -59,8 +57,6 @@ export function getHighlighterExpressionsFromQuery(input = ''): string[] { // Only filter expressions with |~ operator are treated as regular expressions if (pipeMatch) { - // When using backticks, Loki doesn't require to escape special characters and we can just push regular expression to highlights array - // When using quotes, we have extra backslash escaping and we need to replace \\ with \ resultTerm = backtickedTerm ? unwrappedFilterTerm : unwrappedFilterTerm.replace(/\\\\/g, '\\'); } else { // We need to escape this string so it is not matched as regular expression diff --git a/src/responseUtils.ts b/src/responseUtils.ts index 14fe7a4..3f6d6f2 100644 --- a/src/responseUtils.ts +++ b/src/responseUtils.ts @@ -1,6 +1,6 @@ import { DataFrame, Labels } from "@grafana/data"; -export function dataFrameHasLokiError(frame: DataFrame): boolean { +export function dataFrameHasError(frame: DataFrame): boolean { const labelSets: Labels[] = frame.fields.find((f) => f.name === 'labels')?.values ?? []; return labelSets.some((labels) => labels.__error__ !== undefined); } diff --git a/src/types.ts b/src/types.ts index 771a42f..921e11a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,4 @@ -import { DataSourceJsonData, QueryEditorProps } from '@grafana/data'; +import { DataFrame, DataSourceJsonData, KeyValue, QueryEditorProps } from '@grafana/data'; import { DataQuery } from '@grafana/schema'; import { VictoriaLogsDatasource } from "./datasource"; @@ -22,10 +22,6 @@ export enum SupportingQueryType { LogsVolume = 'logsVolume', } -export enum QueryEditorMode { - Code = 'code', -} - export enum QueryType { Instant = 'instant', Range = 'range', @@ -56,3 +52,17 @@ export type DerivedFieldConfig = { matcherType?: 'label' | 'regex'; }; +export interface QueryFilterOptions extends KeyValue { +} + +export enum FilterActionType { + FILTER_FOR = 'FILTER_FOR', + FILTER_OUT = 'FILTER_OUT', +} + +export interface ToggleFilterAction { + type: FilterActionType; + options: QueryFilterOptions; + frame?: DataFrame; +} +