T2viz UI metrics

CHANGELOG.md

@@ -10,6 +10,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - fix: make sure $schema always added to LLM generated vega json object([#252](https://github.com/opensearch-project/dashboards-assistant/pull/252))
 - feat: added a new visualization type visualization-nlq to support creating visualization from natural language([#264](https://github.com/opensearch-project/dashboards-assistant/pull/264))
 - feat: exposed an API to check if a give agent config name has configured with agent id([#307](https://github.com/opensearch-project/dashboards-assistant/pull/307))
+- feat: report metrics for text to visualization([#312](https://github.com/opensearch-project/dashboards-assistant/pull/312))
 
 ### 📈 Features/Enhancements
 

opensearch_dashboards.json

@@ -17,7 +17,8 @@
   ],
   "optionalPlugins": [
     "dataSource",
-    "dataSourceManagement"
+    "dataSourceManagement",
+    "usageCollection"
   ],
   "requiredBundles": [],
   "configPath": [

public/components/feedback_thumbs.test.tsx

@@ -0,0 +1,79 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import React from 'react';
+import { render, screen, fireEvent, waitFor } from '@testing-library/react';
+import { METRIC_TYPE } from '@osd/analytics';
+
+import { FeedbackThumbs } from './feedback_thumbs';
+
+describe('<FeedbackThumbs />', () => {
+  it('should report thumbs up metric', () => {
+    const usageCollectionMock = {
+      reportUiStats: jest.fn(),
+      METRIC_TYPE,
+    };
+
+    render(<FeedbackThumbs usageCollection={usageCollectionMock} appName="test-app" />);
+    fireEvent.click(screen.getByLabelText('ThumbsUp'));
+    expect(usageCollectionMock.reportUiStats).toHaveBeenCalledWith(
+      'test-app',
+      METRIC_TYPE.CLICK,
+      expect.stringMatching(/thumbs_up.*/)
+    );
+  });
+
+  it('should report thumbs down metric', () => {
+    const usageCollectionMock = {
+      reportUiStats: jest.fn(),
+      METRIC_TYPE,
+    };
+
+    render(<FeedbackThumbs usageCollection={usageCollectionMock} appName="test-app" />);
+    fireEvent.click(screen.getByLabelText('ThumbsDown'));
+    expect(usageCollectionMock.reportUiStats).toHaveBeenCalledWith(
+      'test-app',
+      METRIC_TYPE.CLICK,
+      expect.stringMatching(/thumbs_down.*/)
+    );
+  });
+
+  it('should only report metric only once', () => {
+    const usageCollectionMock = {
+      reportUiStats: jest.fn(),
+      METRIC_TYPE,
+    };
+
+    render(<FeedbackThumbs usageCollection={usageCollectionMock} appName="test-app" />);
+    // click the button two times
+    fireEvent.click(screen.getByLabelText('ThumbsDown'));
+    fireEvent.click(screen.getByLabelText('ThumbsDown'));
+    expect(usageCollectionMock.reportUiStats).toHaveBeenCalledTimes(1);
+  });
+
+  it('should hide thumbs down button after thumbs up been clicked', () => {
+    const usageCollectionMock = {
+      reportUiStats: jest.fn(),
+      METRIC_TYPE,
+    };
+
+    render(<FeedbackThumbs usageCollection={usageCollectionMock} appName="test-app" />);
+
+    fireEvent.click(screen.getByLabelText('ThumbsUp'));
+    expect(screen.queryByLabelText('ThumbsDown')).toBeNull();
+  });
+
+  it('should hide thumbs up button after thumbs down been clicked', () => {
+    const usageCollectionMock = {
+      reportUiStats: jest.fn(),
+      METRIC_TYPE,
+    };
+
+    render(<FeedbackThumbs usageCollection={usageCollectionMock} appName="test-app" />);
+
+    fireEvent.click(screen.getByLabelText('ThumbsDown'));
+    expect(screen.queryByLabelText('ThumbsUp')).toBeNull();
+  });
+});

public/components/feedback_thumbs.tsx

@@ -0,0 +1,59 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import React, { useState } from 'react';
+import { v4 as uuidv4 } from 'uuid';
+
+import { UsageCollectionStart } from '../../../../src/plugins/usage_collection/public';
+
+interface Props {
+  appName: string;
+  usageCollection: UsageCollectionStart;
+  className?: string;
+}
+
+export const FeedbackThumbs = ({ usageCollection, appName, className }: Props) => {
+  const [feedback, setFeedback] = useState<'thumbs_up' | 'thumbs_down' | undefined>();
+
+  const onFeedback = (eventName: 'thumbs_up' | 'thumbs_down') => {
+    // Only send metric if no current feedback set
+    if (!feedback) {
+      usageCollection.reportUiStats(
+        appName,
+        usageCollection.METRIC_TYPE.CLICK,
+        `${eventName}-${uuidv4()}`
+      );
+      setFeedback(eventName);
+    }
+  };
+
+  return (
+    <EuiFlexGroup gutterSize="none" className={className}>
+      {(!feedback || feedback === 'thumbs_up') && (
+        <EuiFlexItem>
+          <EuiButtonIcon
+            size="xs"
+            color={feedback === 'thumbs_up' ? 'primary' : 'text'}
+            iconType="thumbsUp"
+            aria-label="ThumbsUp"
+            onClick={() => onFeedback('thumbs_up')}
+          />
+        </EuiFlexItem>
+      )}
+      {(!feedback || feedback === 'thumbs_down') && (
+        <EuiFlexItem>
+          <EuiButtonIcon
+            size="xs"
+            color={feedback === 'thumbs_down' ? 'primary' : 'text'}
+            iconType="thumbsDown"
+            aria-label="ThumbsDown"
+            onClick={() => onFeedback('thumbs_down')}
+          />
+        </EuiFlexItem>
+      )}
+    </EuiFlexGroup>
+  );
+};

public/components/visualization/text2vega.ts

@@ -38,8 +38,8 @@ export class Text2Vega {
     this.result$ = this.input$
       .pipe(
         filter((v) => v.prompt.length > 0),
-        debounceTime(200),
-        tap(() => this.status$.next('RUNNING'))
+        tap(() => this.status$.next('RUNNING')),
+        debounceTime(200)
       )
       .pipe(
         switchMap((v) =>

public/components/visualization/text2viz.scss

@@ -16,4 +16,11 @@
     padding-top: 15px;
     padding-left: 30px;
   }
+
+  .feedback_thumbs {
+    position: absolute;
+    right: 16px;
+    top: 4px;
+    z-index: 9999;
+  }
 }

public/components/visualization/text2viz.tsx

@@ -18,6 +18,7 @@ import {
 } from '@elastic/eui';
 import React, { useEffect, useMemo, useRef, useState } from 'react';
 import { i18n } from '@osd/i18n';
+import { v4 as uuidv4 } from 'uuid';
 
 import { useCallback } from 'react';
 import { useObservable } from 'react-use';
@@ -46,9 +47,10 @@ import { getIndexPatterns } from '../../services';
 import { NLQ_VISUALIZATION_EMBEDDABLE_TYPE } from './embeddable/nlq_vis_embeddable';
 import { NLQVisualizationInput } from './embeddable/types';
 import { EditorPanel } from './editor_panel';
-import { VIS_NLQ_SAVED_OBJECT } from '../../../common/constants/vis_type_nlq';
+import { VIS_NLQ_APP_ID, VIS_NLQ_SAVED_OBJECT } from '../../../common/constants/vis_type_nlq';
 import { HeaderVariant } from '../../../../../src/core/public';
 import { TEXT2VEGA_INPUT_SIZE_LIMIT } from '../../../common/constants/llm';
+import { FeedbackThumbs } from '../feedback_thumbs';
 
 export const Text2Viz = () => {
   const { savedObjectId } = useParams<{ savedObjectId?: string }>();
@@ -67,9 +69,23 @@ export const Text2Viz = () => {
       data,
       uiSettings,
       savedObjects,
+      usageCollection,
     },
   } = useOpenSearchDashboards<StartServices>();
 
+  /**
+   * Report metrics when the application is loaded
+   */
+  useEffect(() => {
+    if (usageCollection) {
+      usageCollection.reportUiStats(
+        VIS_NLQ_APP_ID,
+        usageCollection.METRIC_TYPE.LOADED,
+        `app_loaded-${uuidv4()}`
+      );
+    }
+  }, [usageCollection]);
+
   const useUpdatedUX = uiSettings.get('home:useNewHomePage');
 
   const [input, setInput] = useState('');
@@ -111,14 +127,23 @@ export const Text2Viz = () => {
           });
         } else {
           setEditorInput(JSON.stringify(result, undefined, 4));
+
+          // Report metric when visualization generated successfully
+          if (usageCollection) {
+            usageCollection.reportUiStats(
+              VIS_NLQ_APP_ID,
+              usageCollection.METRIC_TYPE.LOADED,
+              `generated-${uuidv4()}`
+            );
+          }
         }
       }
     });
 
     return () => {
       subscription.unsubscribe();
     };
-  }, [http, notifications]);
+  }, [http, notifications, usageCollection]);
 
   /**
    * Loads the saved object from id when editing an existing visualization
@@ -198,7 +223,7 @@ export const Text2Viz = () => {
     });
 
     setSubmitting(false);
-  }, [selectedSource, input, status]);
+  }, [selectedSource, input, status, usageCollection, notifications.toasts]);
 
   /**
    * Display the save visualization dialog to persist the current generated visualization
@@ -242,6 +267,15 @@ export const Text2Viz = () => {
             }),
           });
           dialog.close();
+
+          // Report metric when a new visualization is saved.
+          if (usageCollection) {
+            usageCollection.reportUiStats(
+              VIS_NLQ_APP_ID,
+              usageCollection.METRIC_TYPE.LOADED,
+              `saved-${uuidv4()}`
+            );
+          }
         }
       } catch (e) {
         notifications.toasts.addDanger({
@@ -269,7 +303,7 @@ export const Text2Viz = () => {
         />
       )
     );
-  }, [notifications, vegaSpec, input, overlays, selectedSource, savedObjectId]);
+  }, [notifications, vegaSpec, input, overlays, selectedSource, savedObjectId, usageCollection]);
 
   const pageTitle = savedObjectId
     ? i18n.translate('dashboardAssistant.feature.text2viz.breadcrumbs.editVisualization', {
@@ -410,6 +444,13 @@ export const Text2Viz = () => {
                   paddingSize="none"
                   scrollable={false}
                 >
+                  {usageCollection ? (
+                    <FeedbackThumbs
+                      usageCollection={usageCollection}
+                      appName={VIS_NLQ_APP_ID}
+                      className="feedback_thumbs"
+                    />
+                  ) : null}
                   <EmbeddableRenderer factory={factory} input={visInput} />
                 </EuiResizablePanel>
                 <EuiResizableButton />

public/types.ts

@@ -20,6 +20,10 @@ import { AssistantClient } from './services/assistant_client';
 import { UiActionsSetup, UiActionsStart } from '../../../src/plugins/ui_actions/public';
 import { ExpressionsSetup, ExpressionsStart } from '../../../src/plugins/expressions/public';
 import { SavedObjectsStart } from '../../../src/plugins/saved_objects/public';
+import {
+  UsageCollectionStart,
+  UsageCollectionSetup,
+} from '../../../src/plugins/usage_collection/public';
 
 export interface RenderProps {
   props: MessageContentProps;
@@ -46,6 +50,7 @@ export interface AssistantPluginStartDependencies {
   uiActions: UiActionsStart;
   expressions: ExpressionsStart;
   savedObjects: SavedObjectsStart;
+  usageCollection?: UsageCollectionStart;
 }
 
 export interface AssistantPluginSetupDependencies {
@@ -55,6 +60,7 @@ export interface AssistantPluginSetupDependencies {
   dataSourceManagement?: DataSourceManagementPluginSetup;
   uiActions: UiActionsSetup;
   expressions: ExpressionsSetup;
+  usageCollection?: UsageCollectionSetup;
 }
 
 export interface AssistantSetup {