Chore UI improvements select (#11)

* UI Improvements

* cleanup EUI

* more style

* revert

* remove css

* remove outline

* cleanup screen

* Select

* list adjustments

* disabled states

* disabled and numbers style

* context fix and login

* webhooks red trashcan

* session

* changes ui

* wip

* whisp

* wip

* wip

* wip

* almost

* icons

* pre

* pre-release

* remove unused

* remove comment block

* remove console

README.md

@@ -309,7 +309,7 @@ We offer a commercial version that incorporates embedded AI models and provides
 | Mobile Softphone                  | sources        | :green_circle:       |
 | Events                            | :green_circle: | :green_circle:       |
 | Integrations                      | :red_circle:   | :green_circle:       |
-| AI Speech2Text                    | `english`      | `multilanguage`      |
+| AI Speech2Text                    | :green_circle: | :green_circle:       |
 | AI Summarization                  | :red_circle:   | :green_circle:       |
 | AI Sentiment Analysis             | :red_circle:   | :green_circle:       |
 | AI KPI insights                   | :red_circle:   | :green_circle:       |

packages/008/.gitignore

@@ -43,3 +43,7 @@ yarn-error.*
 # @end @expo/electron-adapter
 
 /web/config*.json
+/web/*.webm
+/web/*.wav
+/web/*.ogg
+/web/*.mp3

packages/008/package.json

@@ -25,7 +25,7 @@
     "@expo/webpack-config": "^18.0.1",
     "@microsoft/applicationinsights-react-js": "^17.0.1",
     "@microsoft/applicationinsights-web": "^3.0.3",
-    "@react-native-picker/picker": "^2.5.1",
+    "audiobuffer-to-wav": "^1.0.0",
     "base-64": "^1.0.0",
     "crypto-js": "^4.1.1",
     "elasticlunr": "^0.9.5",
@@ -42,9 +42,11 @@
     "react-dom": "18.2.0",
     "react-icons": "^3.10.0",
     "react-native": "0.71.8",
+    "react-native-select-dropdown": "^3.4.0",
     "react-native-web": "^0.19.8",
     "sip.js": "^0.15.11",
     "vcf": "^2.1.1",
+    "whisper-webgpu": "^0.8.0",
     "zustand": "^4.4.1"
   },
   "devDependencies": {

packages/008/src/008Q.js

@@ -1,9 +1,67 @@
-import { request } from './utils';
+import * as whisper from 'whisper-webgpu';
+import toWav from 'audiobuffer-to-wav';
 
-export const tts = async ({ audio: body }) => {
-  const port = 13003;
-  const endpoint = `http://localhost:${port}/transcribe`;
-  const transcript = await request({ endpoint, body });
+const CACHE = {};
+const S3Q = 'https://kunziteq.s3.gra.perf.cloud.ovh.net';
 
-  return transcript;
+export const wavBytes = async ({ chunks }) => {
+  // TODO: flatten 2 channels
+  let arrayBuffer = await chunks[0].arrayBuffer();
+  const audioContext = new AudioContext();
+  const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+  const wavBlob = new Blob([toWav(audioBuffer)], { type: 'audio/wav' });
+
+  arrayBuffer = await wavBlob.arrayBuffer();
+
+  return new Uint8Array(arrayBuffer);
+};
+
+export const ttsInfer = async ({
+  chunks,
+  url,
+  audio = [],
+  bin = `${S3Q}/ttsb.bin`,
+  data = `${S3Q}/tts.json`
+}) => {
+  const fetchBytes = async url => {
+    if (!CACHE[url]) {
+      const response = await fetch(url);
+      const buffer = await response.arrayBuffer();
+      const bytes = new Uint8Array(buffer);
+
+      CACHE[url] = bytes;
+    }
+
+    return CACHE[url];
+  };
+
+  const tokenizer = await fetchBytes(data);
+  const model = await fetchBytes(bin);
+
+  if (url) audio = await fetchBytes(url);
+  if (chunks) audio = await wavBytes({ chunks });
+
+  await whisper.default();
+  const builder = new whisper.SessionBuilder();
+  const session = await builder.setModel(model).setTokenizer(tokenizer).build();
+
+  const { segments } = await session.run(audio);
+
+  session.free();
+
+  return segments;
+};
+
+export const tts = async ({ audio }) => {
+  const remote = (await ttsInfer({ audio: audio.remote })).map(item => ({
+    ...item,
+    channel: 'remote'
+  }));
+  const local = (await ttsInfer({ audio: audio.local })).map(item => ({
+    ...item,
+    channel: 'local'
+  }));
+  const merged = [...remote, ...local].sort((a, b) => a.start - b.start);
+
+  return merged;
 };

packages/008/src/008QWorker.js

@@ -0,0 +1,33 @@
+import { tts } from './008Q';
+
+let BUSY = false;
+const QUEUE = [];
+
+const process = async () => {
+  console.log('[008Q] Processing...');
+  if (BUSY || !QUEUE.length) return;
+
+  try {
+    BUSY = true;
+
+    const [data] = QUEUE;
+    const { id, audio } = data;
+
+    console.log('[008Q] Transcribing...');
+    const transcript = await tts({ audio });
+
+    self.postMessage({ id, transcript });
+  } catch (err) {
+    console.error(err);
+  } finally {
+    QUEUE.shift();
+    BUSY = false;
+    process();
+  }
+};
+
+self.addEventListener('message', async ({ data }) => {
+  console.log(`[008Q] Queuing job ${data.id}`);
+  QUEUE.push(data);
+  process();
+});

packages/008/src/Events.js

@@ -1,7 +1,6 @@
-import PQueue from 'p-queue';
-
 import { Platform } from 'react-native';
 
+import PQueue from 'p-queue';
 import _ from 'lodash';
 
 import { useStore } from './store/Context';
@@ -42,6 +41,7 @@ export const emit = async ({ type, data: payload }) => {
 
   const data = { ...payload, context };
 
+  console.error(type, data);
   document?.dispatchEvent?.(new CustomEvent(type, { detail: data }));
   window?.parent?.postMessage?.({ type, data }, '*');
   WEBHOOKS.addJob({ event: { type, data } });

packages/008/src/components/Avatars.jsx

@@ -1,36 +1,5 @@
-import { View } from 'react-native';
-
 import { Avatar } from './Basics';
 
-export const Status = ({ color, size = 10, style }) => {
-  const roundstyle = {
-    width: size,
-    height: size,
-    borderRadius: size / 2,
-    backgroundColor: color || 'black'
-  };
-
-  return <View style={{ ...style, ...roundstyle }} />;
-};
-
-export const UserAvatar = ({
-  color,
-  avatar,
-  size = 35,
-  defaultImageUrl = 'avatar.png'
-}) => {
-  return (
-    <View>
-      <Avatar name="" size={size} imageUrl={avatar || defaultImageUrl} />
-      <Status
-        color={color}
-        size={12}
-        style={{ position: 'absolute', bottom: 0, right: 0 }}
-      />
-    </View>
-  );
-};
-
 export const ContactAvatar = ({
   contact = {},
   size = 35,