Offline Speech Recognition #2089

samples/CommunityToolkit.Maui.Sample/AppShell.xaml.cs

@@ -93,6 +93,7 @@ public partial class AppShell : Shell
 		CreateViewModelMapping<FileSaverPage, FileSaverViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 		CreateViewModelMapping<FolderPickerPage, FolderPickerViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 		CreateViewModelMapping<SpeechToTextPage, SpeechToTextViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
+		CreateViewModelMapping<OfflineSpeechToTextPage, OfflineSpeechToTextViewModel, EssentialsGalleryPage, EssentialsGalleryViewModel>(),
 
 		// Add Extensions View Models
 		CreateViewModelMapping<ColorAnimationExtensionsPage, ColorAnimationExtensionsViewModel, ExtensionsGalleryPage, ExtensionsGalleryViewModel>(),

samples/CommunityToolkit.Maui.Sample/CommunityToolkit.Maui.Sample.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>$(NetVersion)-ios;$(NetVersion)-android;$(NetVersion)-maccatalyst</TargetFrameworks>

samples/CommunityToolkit.Maui.Sample/MauiProgram.cs

@@ -211,6 +211,7 @@ static void RegisterViewsAndViewModels(in IServiceCollection services)
 		services.AddTransientWithShellRoute<FileSaverPage, FileSaverViewModel>();
 		services.AddTransientWithShellRoute<FolderPickerPage, FolderPickerViewModel>();
 		services.AddTransientWithShellRoute<SpeechToTextPage, SpeechToTextViewModel>();
+		services.AddTransientWithShellRoute<OfflineSpeechToTextPage, OfflineSpeechToTextViewModel>();
 
 		// Add Extensions Pages + ViewModels
 		services.AddTransientWithShellRoute<ColorAnimationExtensionsPage, ColorAnimationExtensionsViewModel>();
@@ -262,7 +263,8 @@ static void RegisterEssentials(in IServiceCollection services)
 		services.AddSingleton<IFileSystem>(FileSystem.Current);
 		services.AddSingleton<IFolderPicker>(FolderPicker.Default);
 		services.AddSingleton<IBadge>(Badge.Default);
-		services.AddSingleton<ISpeechToText>(SpeechToText.Default);
+		services.AddKeyedSingleton<ISpeechToText, SpeechToTextImplementation>("Online");
+		services.AddKeyedSingleton<ISpeechToText, OfflineSpeechToTextImplementation>("Offline");
 		services.AddSingleton<ITextToSpeech>(TextToSpeech.Default);
 	}
 

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/OfflineSpeechToTextPage.xaml

@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<pages:BasePage xmlns="http://schemas.microsoft.com/dotnet/2021/maui"
+                xmlns:x="http://schemas.microsoft.com/winfx/2009/xaml"
+                xmlns:pages="clr-namespace:CommunityToolkit.Maui.Sample.Pages"
+                x:Class="CommunityToolkit.Maui.Sample.Pages.Essentials.OfflineSpeechToTextPage"
+                xmlns:vm="clr-namespace:CommunityToolkit.Maui.Sample.ViewModels.Essentials"
+                xmlns:essentials="clr-namespace:CommunityToolkit.Maui.Sample.Pages.Essentials"
+                x:TypeArguments="vm:OfflineSpeechToTextViewModel"
+                x:DataType="vm:OfflineSpeechToTextViewModel"
+                Title="OfflineSpeechToText">
+
+    <ContentPage.Resources>
+        <essentials:PickerLocaleDisplayConverter x:Key="PickerLocaleDisplayConverter" />
+    </ContentPage.Resources>
+
+    <ScrollView>
+        <VerticalStackLayout
+            Spacing="20"
+            Padding="30,0">
+
+            <Label
+                Text="OfflineSpeechToText allows the user to convert speech to text in real time in offline"
+                HorizontalTextAlignment="Center"/>
+
+            <Label
+                Text="Locale"
+                FontAttributes="Bold"/>
+
+            <Picker
+                ItemsSource="{Binding Locales}"
+                SelectedItem="{Binding CurrentLocale}"
+                ItemDisplayBinding="{Binding ., Converter={StaticResource PickerLocaleDisplayConverter}}"/>
+
+            <Label
+                Text="State"
+                FontAttributes="Bold"/>
+
+            <Label
+                Text="{Binding State}"
+                FontSize="18"
+                HorizontalOptions="Center"
+                HorizontalTextAlignment="Center"
+                MinimumHeightRequest="100" />
+
+            <Label
+                Text="Language Output"
+                FontAttributes="Bold"/>
+
+            <Label
+                Text="{Binding RecognitionText}"
+                FontSize="18"
+                HorizontalOptions="Center"
+                HorizontalTextAlignment="Center"
+                MinimumHeightRequest="100" />
+
+            <Button
+                Text="Play"
+                Command="{Binding PlayCommand}"
+                HorizontalOptions="Center" />
+
+            <Border
+                StrokeThickness="2"
+                Stroke="#808080"
+                StrokeShape="RoundRectangle 8,8,8,8"
+                Padding="12">
+                <Border.Content>
+                    <Grid RowDefinitions="*,60"
+                          ColumnDefinitions="*,*"
+                          RowSpacing="12"
+                          ColumnSpacing="12">
+
+                        <Button
+                            Grid.Row="0"
+                            Grid.Column="0"
+                            Text="StartListenAsync"
+                            Command="{Binding StartListenCommand}"
+                            HorizontalOptions="End" />
+
+                        <Button
+                            Grid.Row="0"
+                            Grid.Column="1"
+                            Text="StopListenAsync"
+                            Command="{Binding StopListenCommand}"
+                            HorizontalOptions="Start" />
+
+                        <Label
+                            Grid.Row="1"
+                            Grid.ColumnSpan="2"
+                            Text="The `StartListenAsync` API starts the speech-to-text service and shares the results using `RecognitionResultUpdated` event and `RecognitionResultCompleted` event."
+                            HorizontalOptions="Center"
+                            HorizontalTextAlignment="Center"
+                            FontSize="12"/>
+
+                    </Grid>
+                </Border.Content>
+            </Border>
+        </VerticalStackLayout>
+    </ScrollView>
+
+</pages:BasePage>

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/OfflineSpeechToTextPage.xaml.cs

@@ -0,0 +1,20 @@
+using System.Globalization;
+using CommunityToolkit.Maui.Converters;
+using CommunityToolkit.Maui.Sample.ViewModels.Essentials;
+
+namespace CommunityToolkit.Maui.Sample.Pages.Essentials;
+
+public partial class OfflineSpeechToTextPage : BasePage<OfflineSpeechToTextViewModel>
+{
+	public OfflineSpeechToTextPage(OfflineSpeechToTextViewModel viewModel) : base(viewModel)
+	{
+		InitializeComponent();
+	}
+
+	protected override async void OnAppearing()
+	{
+		base.OnAppearing();
+
+		await BindingContext.SetLocalesCommand.ExecuteAsync(null);
+	}
+}

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/SpeechToTextPage.xaml

@@ -31,6 +31,17 @@
                 SelectedItem="{Binding CurrentLocale}"
                 ItemDisplayBinding="{Binding ., Converter={StaticResource PickerLocaleDisplayConverter}}"/>
 
+            <Label
+                Text="State"
+                FontAttributes="Bold"/>
+
+            <Label
+                Text="{Binding State}"
+                FontSize="18"
+                HorizontalOptions="Center"
+                HorizontalTextAlignment="Center"
+                MinimumHeightRequest="100" />
+
             <Label
                 Text="Language Output"
                 FontAttributes="Bold"/>
@@ -47,44 +58,6 @@
                 Command="{Binding PlayCommand}"
                 HorizontalOptions="Center" />
 
-            <Border
-                StrokeThickness="2"
-                Stroke="#808080"
-                StrokeShape="RoundRectangle 8,8,8,8"
-                Padding="12">
-                <Border.Content>
-
-                    <Grid RowDefinitions="*,60"
-                          ColumnDefinitions="*,*"
-                          RowSpacing="12"
-                          ColumnSpacing="12">
-
-                        <Button
-                            Grid.Row="0"
-                            Grid.Column="0"
-                            Text="ListenAsync"
-                            Command="{Binding ListenCommand}"
-                            HorizontalOptions="End" />
-
-                        <Button
-                            Grid.Row="0"
-                            Grid.Column="1"
-                            Text="Cancel Token"
-                            Command="{Binding ListenCancelCommand}"
-                            HorizontalOptions="Start" />
-
-                        <Label
-                            Grid.Row="1"
-                            Grid.ColumnSpan="2"
-                            Text="The `ListenAsync` API allows you to await the final speech recognition results using async/await. `ListenAsync` is cancelled via CancellationToken."
-                            HorizontalOptions="Center"
-                            HorizontalTextAlignment="Center"
-                            FontSize="12"/>
-
-                    </Grid>
-                </Border.Content>
-            </Border>
-
             <Border
                 StrokeThickness="2"
                 Stroke="#808080"

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/EssentialsGalleryViewModel.cs

@@ -8,5 +8,6 @@ public class EssentialsGalleryViewModel() : BaseGalleryViewModel(
 	SectionModel.Create<BadgeViewModel>("Badge", "Allows the user to set app icon badge count on the home screen"),
 	SectionModel.Create<FileSaverViewModel>("FileSaver", "Allows the user to save files to the filesystem"),
 	SectionModel.Create<FolderPickerViewModel>("FolderPicker", "Allows picking folders from the file system"),
-	SectionModel.Create<SpeechToTextViewModel>("SpeechToText", "Converts speech to text")
+	SectionModel.Create<SpeechToTextViewModel>("SpeechToText", "Converts speech to text"),
+	SectionModel.Create<OfflineSpeechToTextViewModel>("OfflineSpeechToText", "Converts speech to text offline")
 ]);

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/OfflineSpeechToTextViewModel.cs

@@ -0,0 +1,145 @@
+using System.Collections.ObjectModel;
+using System.Collections.Specialized;
+using System.Globalization;
+using CommunityToolkit.Maui.Alerts;
+using CommunityToolkit.Maui.Media;
+using CommunityToolkit.Mvvm.ComponentModel;
+using CommunityToolkit.Mvvm.Input;
+
+namespace CommunityToolkit.Maui.Sample.ViewModels.Essentials;
+
+public partial class OfflineSpeechToTextViewModel : BaseViewModel
+{
+	const string defaultLanguage = "en-US";
+	const string defaultLanguageAndroid = "en";
+	const string defaultLanguageTizen = "en_US";
+
+	readonly ITextToSpeech textToSpeech;
+	readonly ISpeechToText speechToText;
+
+	[ObservableProperty]
+	Locale? currentLocale;
+
+	public SpeechToTextState? State => speechToText.CurrentState;
+
+	[ObservableProperty]
+	string? recognitionText = "Welcome to .NET MAUI Community Toolkit!";
+
+	[ObservableProperty, NotifyCanExecuteChangedFor(nameof(StartListenCommand))]
+	bool canStartListenExecute = true;
+
+	[ObservableProperty, NotifyCanExecuteChangedFor(nameof(StopListenCommand))]
+	bool canStopListenExecute;
+
+	public OfflineSpeechToTextViewModel(ITextToSpeech textToSpeech)
+	{
+		this.textToSpeech = textToSpeech;
+		// For demo purposes. You can resolve dependency from the DI container,
+		this.speechToText = OfflineSpeechToText.Default;
+
+		Locales.CollectionChanged += HandleLocalesCollectionChanged;
+		this.speechToText.StateChanged += HandleSpeechToTextStateChanged;
+		this.speechToText.RecognitionResultCompleted += HandleRecognitionResultCompleted;
+	}
+
+	public ObservableCollection<Locale> Locales { get; } = [];
+
+	[RelayCommand]
+	async Task SetLocales(CancellationToken token)
+	{
+		Locales.Clear();
+
+		var locales = await textToSpeech.GetLocalesAsync().WaitAsync(token);
+
+		foreach (var locale in locales.OrderBy(x => x.Language).ThenBy(x => x.Name))
+		{
+			Locales.Add(locale);
+		}
+
+		CurrentLocale = Locales.FirstOrDefault(x => x.Language is defaultLanguage or defaultLanguageAndroid or defaultLanguageTizen) ?? Locales.FirstOrDefault();
+	}
+
+	[RelayCommand]
+	async Task Play(CancellationToken cancellationToken)
+	{
+		var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+
+		try
+		{
+			await textToSpeech.SpeakAsync(RecognitionText ?? "Welcome to .NET MAUI Community Toolkit!", new()
+			{
+				Locale = CurrentLocale,
+				Pitch = 1,
+				Volume = 1
+			}, cancellationToken).WaitAsync(timeoutCancellationTokenSource.Token);
+		}
+		catch (TaskCanceledException)
+		{
+			await Toast.Make("Playback automatically stopped after 5 seconds").Show(cancellationToken);
+#if IOS
+			await Toast.Make("If you did not hear playback, test again on a physical iOS device").Show(cancellationToken);
+#endif
+		}
+	}
+
+	[RelayCommand(CanExecute = nameof(CanStartListenExecute))]
+	async Task StartListen()
+	{
+		CanStartListenExecute = false;
+
+		var isGranted = await speechToText.RequestPermissions(CancellationToken.None);
+		if (!isGranted)
+		{
+			await Toast.Make("Permission not granted").Show(CancellationToken.None);
+			return;
+		}
+
+		const string beginSpeakingPrompt = "Begin speaking...";
+
+		RecognitionText = beginSpeakingPrompt;
+
+		speechToText.RecognitionResultUpdated += HandleRecognitionResultUpdated;
+
+		await speechToText.StartListenAsync(new SpeechToTextOptions()
+		{
+			Culture = CultureInfo.GetCultureInfo(CurrentLocale?.Language ?? defaultLanguage),
+			ShouldReportPartialResults = true
+		}, CancellationToken.None);
+
+		if (RecognitionText is beginSpeakingPrompt)
+		{
+			RecognitionText = string.Empty;
+		}
+	}
+
+	[RelayCommand(CanExecute = nameof(CanStopListenExecute))]
+	Task StopListen()
+	{
+		CanStartListenExecute = true;
+		CanStopListenExecute = false;
+
+		speechToText.RecognitionResultUpdated -= HandleRecognitionResultUpdated;
+
+		return speechToText.StopListenAsync(CancellationToken.None);
+	}
+
+	void HandleRecognitionResultUpdated(object? sender, SpeechToTextRecognitionResultUpdatedEventArgs e)
+	{
+		RecognitionText += e.RecognitionResult;
+	}
+
+	void HandleRecognitionResultCompleted(object? sender, SpeechToTextRecognitionResultCompletedEventArgs e)
+	{
+		RecognitionText = e.RecognitionResult.IsSuccessful ? e.RecognitionResult.Text : e.RecognitionResult.Exception.Message;
+	}
+
+	void HandleSpeechToTextStateChanged(object? sender, SpeechToTextStateChangedEventArgs e)
+	{
+		OnPropertyChanged(nameof(State));
+	}
+
+	void HandleLocalesCollectionChanged(object? sender, NotifyCollectionChangedEventArgs e)
+	{
+		OnPropertyChanged(nameof(CurrentLocale));
+	}
+}