Offline Speech Recognition #2089

samples/CommunityToolkit.Maui.Sample.sln

@@ -14,7 +14,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		..\.editorconfig = ..\.editorconfig
 		..\Directory.Build.props = ..\Directory.Build.props
 		..\Directory.Build.targets = ..\Directory.Build.targets
-		..\global.json = ..\global.json
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.Core", "..\src\CommunityToolkit.Maui.Core\CommunityToolkit.Maui.Core.csproj", "{6B572EAD-1581-46BE-A08B-1C7F2246BA2E}"
@@ -51,7 +50,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.Camer
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Benchmarks", "Benchmarks", "{ED5A9C0B-D270-442D-BABE-F4FF622926C8}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.Analyzers.Benchmarks.csproj", "..\src\CommunityToolkit.Maui.Analyzers.Benchmarks\CommunityToolkit.Maui.Analyzers.Benchmarks.csproj", "{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.Analyzers.Benchmarks", "..\src\CommunityToolkit.Maui.Analyzers.Benchmarks\CommunityToolkit.Maui.Analyzers.Benchmarks.csproj", "{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

samples/CommunityToolkit.Maui.Sample/AppShell.xaml.cs

@@ -93,6 +93,7 @@ public partial class AppShell : Shell
 		CreateViewModelMapping<FileSaverPage, FileSaverViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 		CreateViewModelMapping<FolderPickerPage, FolderPickerViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 		CreateViewModelMapping<SpeechToTextPage, SpeechToTextViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
+		CreateViewModelMapping<OfflineSpeechToTextPage, OfflineSpeechToTextViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 
 		// Add Extensions View Models
 		CreateViewModelMapping<ColorAnimationExtensionsPage, ColorAnimationExtensionsViewModel, ExtensionsGalleryPage, ExtensionsGalleryViewModel>(),

samples/CommunityToolkit.Maui.Sample/CommunityToolkit.Maui.Sample.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>$(NetVersion)-ios;$(NetVersion)-android;$(NetVersion)-maccatalyst</TargetFrameworks>

samples/CommunityToolkit.Maui.Sample/MauiProgram.cs

@@ -211,6 +211,7 @@ static void RegisterViewsAndViewModels(in IServiceCollection services)
 		services.AddTransientWithShellRoute<FileSaverPage, FileSaverViewModel>();
 		services.AddTransientWithShellRoute<FolderPickerPage, FolderPickerViewModel>();
 		services.AddTransientWithShellRoute<SpeechToTextPage, SpeechToTextViewModel>();
+		services.AddTransientWithShellRoute<OfflineSpeechToTextPage, OfflineSpeechToTextViewModel>();
 
 		// Add Extensions Pages + ViewModels
 		services.AddTransientWithShellRoute<ColorAnimationExtensionsPage, ColorAnimationExtensionsViewModel>();
@@ -262,7 +263,8 @@ static void RegisterEssentials(in IServiceCollection services)
 		services.AddSingleton<IFileSystem>(FileSystem.Current);
 		services.AddSingleton<IFolderPicker>(FolderPicker.Default);
 		services.AddSingleton<IBadge>(Badge.Default);
-		services.AddSingleton<ISpeechToText>(SpeechToText.Default);
+		services.AddKeyedSingleton<ISpeechToText, SpeechToTextImplementation>("Online");
+		services.AddKeyedSingleton<ISpeechToText, OfflineSpeechToTextImplementation>("Offline");
 		services.AddSingleton<ITextToSpeech>(TextToSpeech.Default);
 	}
 

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/OfflineSpeechToTextPage.xaml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<pages:BasePage xmlns="http://schemas.microsoft.com/dotnet/2021/maui"
+                xmlns:x="http://schemas.microsoft.com/winfx/2009/xaml"
+                xmlns:pages="clr-namespace:CommunityToolkit.Maui.Sample.Pages"
+                x:Class="CommunityToolkit.Maui.Sample.Pages.Essentials.OfflineSpeechToTextPage"
+                xmlns:vm="clr-namespace:CommunityToolkit.Maui.Sample.ViewModels.Essentials"
+                xmlns:essentials="clr-namespace:CommunityToolkit.Maui.Sample.Pages.Essentials"
+                x:TypeArguments="vm:OfflineSpeechToTextViewModel"
+                x:DataType="vm:OfflineSpeechToTextViewModel"
+                Title="OfflineSpeechToText">
+
+    <ScrollView>
+        <VerticalStackLayout
+            Spacing="20"
+            Padding="30,0">
+
+            <Label
+                Text="OfflineSpeechToText allows the user to convert speech to text in real time in offline."
+                HorizontalTextAlignment="Center"/>
+
+            <Label
+                Text="State"
+                FontAttributes="Bold"/>
+
+            <Label
+                Text="{Binding State}"
+                FontSize="18"
+                HorizontalOptions="Center"
+                HorizontalTextAlignment="Center"
+                MinimumHeightRequest="100" />
+
+            <Label
+                Text="Language Output"
+                FontAttributes="Bold"/>
+
+            <Label
+                Text="{Binding RecognitionText}"
+                FontSize="18"
+                HorizontalOptions="Center"
+                HorizontalTextAlignment="Center"
+                MinimumHeightRequest="100" />
+
+            <Border
+                StrokeThickness="2"
+                Stroke="#808080"
+                StrokeShape="RoundRectangle 8,8,8,8"
+                Padding="12">
+                <Border.Content>
+                    <Grid RowDefinitions="*,60"
+                          ColumnDefinitions="*,*"
+                          RowSpacing="12"
+                          ColumnSpacing="12">
+
+                        <Button
+                            Grid.Row="0"
+                            Grid.Column="0"
+                            Text="StartListenAsync"
+                            Command="{Binding StartListenCommand}"
+                            HorizontalOptions="End" />
+
+                        <Button
+                            Grid.Row="0"
+                            Grid.Column="1"
+                            Text="StopListenAsync"
+                            Command="{Binding StopListenCommand}"
+                            HorizontalOptions="Start" />
+
+                        <Label
+                            Grid.Row="1"
+                            Grid.ColumnSpan="2"
+                            Text="The `StartListenAsync` API starts the speech-to-text service and shares the results using `RecognitionResultUpdated` event and `RecognitionResultCompleted` event."
+                            HorizontalOptions="Center"
+                            HorizontalTextAlignment="Center"
+                            FontSize="12"/>
+
+                    </Grid>
+                </Border.Content>
+            </Border>
+        </VerticalStackLayout>
+    </ScrollView>
+
+</pages:BasePage>

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/OfflineSpeechToTextPage.xaml.cs

@@ -0,0 +1,11 @@
+using CommunityToolkit.Maui.Sample.ViewModels.Essentials;
+
+namespace CommunityToolkit.Maui.Sample.Pages.Essentials;
+
+public partial class OfflineSpeechToTextPage : BasePage<OfflineSpeechToTextViewModel>
+{
+	public OfflineSpeechToTextPage(OfflineSpeechToTextViewModel viewModel) : base(viewModel)
+	{
+		InitializeComponent();
+	}
+}

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/SpeechToTextPage.xaml

@@ -31,6 +31,17 @@
                 SelectedItem="{Binding CurrentLocale}"
                 ItemDisplayBinding="{Binding ., Converter={StaticResource PickerLocaleDisplayConverter}}"/>
 
+            <Label
+                Text="State"
+                FontAttributes="Bold"/>
+
+            <Label
+                Text="{Binding State}"
+                FontSize="18"
+                HorizontalOptions="Center"
+                HorizontalTextAlignment="Center"
+                MinimumHeightRequest="100" />
+
             <Label
                 Text="Language Output"
                 FontAttributes="Bold"/>
@@ -47,44 +58,6 @@
                 Command="{Binding PlayCommand}"
                 HorizontalOptions="Center" />
 
-            <Border
-                StrokeThickness="2"
-                Stroke="#808080"
-                StrokeShape="RoundRectangle 8,8,8,8"
-                Padding="12">
-                <Border.Content>
-
-                    <Grid RowDefinitions="*,60"
-                          ColumnDefinitions="*,*"
-                          RowSpacing="12"
-                          ColumnSpacing="12">
-
-                        <Button
-                            Grid.Row="0"
-                            Grid.Column="0"
-                            Text="ListenAsync"
-                            Command="{Binding ListenCommand}"
-                            HorizontalOptions="End" />
-
-                        <Button
-                            Grid.Row="0"
-                            Grid.Column="1"
-                            Text="Cancel Token"
-                            Command="{Binding ListenCancelCommand}"
-                            HorizontalOptions="Start" />
-
-                        <Label
-                            Grid.Row="1"
-                            Grid.ColumnSpan="2"
-                            Text="The `ListenAsync` API allows you to await the final speech recognition results using async/await. `ListenAsync` is cancelled via CancellationToken."
-                            HorizontalOptions="Center"
-                            HorizontalTextAlignment="Center"
-                            FontSize="12"/>
-
-                    </Grid>
-                </Border.Content>
-            </Border>
-
             <Border
                 StrokeThickness="2"
                 Stroke="#808080"

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/EssentialsGalleryViewModel.cs

@@ -8,5 +8,6 @@ public class EssentialsGalleryViewModel() : BaseGalleryViewModel(
 	SectionModel.Create<BadgeViewModel>("Badge", "Allows the user to set app icon badge count on the home screen"),
 	SectionModel.Create<FileSaverViewModel>("FileSaver", "Allows the user to save files to the filesystem"),
 	SectionModel.Create<FolderPickerViewModel>("FolderPicker", "Allows picking folders from the file system"),
-	SectionModel.Create<SpeechToTextViewModel>("SpeechToText", "Converts speech to text")
+	SectionModel.Create<SpeechToTextViewModel>("SpeechToText", "Converts speech to text"),
+	SectionModel.Create<OfflineSpeechToTextViewModel>("OfflineSpeechToText", "Converts speech to text offline")
 ]);

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/OfflineSpeechToTextViewModel.cs

@@ -0,0 +1,77 @@
+using System.Globalization;
+using CommunityToolkit.Maui.Alerts;
+using CommunityToolkit.Maui.Media;
+using CommunityToolkit.Mvvm.ComponentModel;
+using CommunityToolkit.Mvvm.Input;
+
+namespace CommunityToolkit.Maui.Sample.ViewModels.Essentials;
+
+public partial class OfflineSpeechToTextViewModel : BaseViewModel
+{
+	readonly ISpeechToText speechToText;
+
+	public SpeechToTextState? State => speechToText.CurrentState;
+
+	[ObservableProperty]
+	string? recognitionText = "Welcome to .NET MAUI Community Toolkit!";
+
+	public OfflineSpeechToTextViewModel()
+	{
+		// For demo purposes. You can resolve dependency from the DI container,
+		speechToText = OfflineSpeechToText.Default;
+
+		speechToText.StateChanged += HandleSpeechToTextStateChanged;
+		speechToText.RecognitionResultCompleted += HandleRecognitionResultCompleted;
+	}
+
+	[RelayCommand]
+	async Task StartListen()
+	{
+		var isGranted = await speechToText.RequestPermissions(CancellationToken.None);
+		if (!isGranted)
+		{
+			await Toast.Make("Permission not granted").Show(CancellationToken.None);
+			return;
+		}
+
+		const string beginSpeakingPrompt = "Begin speaking...";
+
+		RecognitionText = beginSpeakingPrompt;
+
+		speechToText.RecognitionResultUpdated += HandleRecognitionResultUpdated;
+
+		await speechToText.StartListenAsync(new SpeechToTextOptions
+		{
+			Culture = CultureInfo.CurrentCulture,
+			ShouldReportPartialResults = true
+		}, CancellationToken.None);
+
+		if (RecognitionText is beginSpeakingPrompt)
+		{
+			RecognitionText = string.Empty;
+		}
+	}
+
+	[RelayCommand]
+	Task StopListen()
+	{
+		speechToText.RecognitionResultUpdated -= HandleRecognitionResultUpdated;
+
+		return speechToText.StopListenAsync(CancellationToken.None);
+	}
+
+	void HandleRecognitionResultUpdated(object? sender, SpeechToTextRecognitionResultUpdatedEventArgs e)
+	{
+		RecognitionText += e.RecognitionResult;
+	}
+
+	void HandleRecognitionResultCompleted(object? sender, SpeechToTextRecognitionResultCompletedEventArgs e)
+	{
+		RecognitionText = e.RecognitionResult.IsSuccessful ? e.RecognitionResult.Text : e.RecognitionResult.Exception.Message;
+	}
+
+	void HandleSpeechToTextStateChanged(object? sender, SpeechToTextStateChangedEventArgs e)
+	{
+		OnPropertyChanged(nameof(State));
+	}
+}