[linux] Implement libiamf-based IAMF decoder (#4221)

Implements a libiamf-based audio decoder.

Decoder specific code is guarded behind a GN flag `enable_iamf_decode,`
which controls the ENABLE_IAMF_DECODE macro. `enable_iamf_decode` is off
by default, so the new code doesn't build.

b/341792042

media/base/starboard_utils.cc

@@ -25,6 +25,7 @@
 #include "base/strings/string_util.h"
 #include "media/base/decoder_buffer.h"
 #include "media/base/decrypt_config.h"
+#include "starboard/audio_sink.h"
 #include "starboard/common/media.h"
 #include "starboard/configuration.h"
 #include "starboard/memory.h"
@@ -55,6 +56,16 @@ int GetBitsPerPixel(const std::string& mime_type) {
   return 8;
 }
 
+int GetMaxChannelCount() {
+  int channels = 2;
+  int index = 0;
+  SbMediaAudioConfiguration configuration;
+  while (SbMediaGetAudioConfiguration(index++, &configuration)) {
+    channels = std::max(configuration.number_of_channels, channels);
+  }
+  return std::min(channels, SbAudioSinkGetMaxChannels());
+}
+
 }  // namespace
 
 SbMediaAudioCodec MediaAudioCodecToSbMediaAudioCodec(AudioCodec codec) {
@@ -91,7 +102,7 @@ SbMediaAudioCodec MediaAudioCodecToSbMediaAudioCodec(AudioCodec codec) {
       return kSbMediaAudioCodecPcm;
 #if SB_API_VERSION >= 15
     case AudioCodec::kIAMF:
-     return kSbMediaAudioCodecIamf;
+      return kSbMediaAudioCodecIamf;
 #endif  // SB_API_VERSION >= 15
     default:
       // Cobalt only supports a subset of audio codecs defined by Chromium.
@@ -142,15 +153,22 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo(
 
 #if SB_API_VERSION < 15
   audio_stream_info.format_tag = 0x00ff;
-#endif // SB_API_VERSION < 15
+#endif  // SB_API_VERSION < 15
   audio_stream_info.number_of_channels =
       ChannelLayoutToChannelCount(audio_decoder_config.channel_layout());
+#if SB_API_VERSION >= 15
+  if (audio_stream_info.codec == kSbMediaAudioCodecIamf) {
+    // IAMF mixes audio signals to the highest available speaker layout.
+    // TODO: Set the number of channels below Starboard.
+    audio_stream_info.number_of_channels = GetMaxChannelCount();
+  }
+#endif  // SB_API_VERSION >= 15
   audio_stream_info.samples_per_second =
       audio_decoder_config.samples_per_second();
 #if SB_API_VERSION < 15
   audio_stream_info.average_bytes_per_second = 1;
   audio_stream_info.block_alignment = 4;
-#endif // SB_API_VERSION < 15
+#endif  // SB_API_VERSION < 15
   audio_stream_info.bits_per_sample = audio_decoder_config.bits_per_channel();
 
   const auto& extra_data = audio_stream_info.codec == kSbMediaAudioCodecAac

starboard/build/config/BUILDCONFIG.gn

@@ -40,6 +40,8 @@ declare_args() {
   build_with_separate_cobalt_toolchain = false
 
   use_xcode_clang = false
+
+  enable_iamf_decode = false
 }
 
 _is_on_pythonpath = exec_script("//starboard/build/is_on_path.py", [], "json")

starboard/linux/shared/BUILD.gn

@@ -440,6 +440,17 @@ static_library("starboard_platform_sources") {
   if (is_debug || is_devel) {
     defines += [ "SB_PLAYER_ENABLE_VIDEO_DUMPER" ]
   }
+
+  if (enable_iamf_decode) {
+    sources += [
+      "//starboard/shared/libiamf/iamf_audio_decoder.cc",
+      "//starboard/shared/libiamf/iamf_audio_decoder.h",
+      "//starboard/shared/libiamf/iamf_decoder_utils.cc",
+      "//starboard/shared/libiamf/iamf_decoder_utils.h",
+    ]
+
+    defines += [ "ENABLE_IAMF_DECODE" ]
+  }
 }
 
 if (current_toolchain == starboard_toolchain) {

starboard/linux/shared/media_is_audio_supported.cc

@@ -18,9 +18,21 @@
 #include "starboard/configuration.h"
 #include "starboard/configuration_constants.h"
 #include "starboard/media.h"
+#include "starboard/shared/starboard/media/iamf_util.h"
 
+using ::starboard::shared::starboard::media::IamfMimeUtil;
+using ::starboard::shared::starboard::media::kIamfProfileBase;
+using ::starboard::shared::starboard::media::kIamfProfileSimple;
+using ::starboard::shared::starboard::media::kIamfSubstreamCodecOpus;
 using ::starboard::shared::starboard::media::MimeType;
 
+bool HasSupportedIamfProfile(const IamfMimeUtil* mime_util) {
+  return mime_util->primary_profile() == kIamfProfileSimple ||
+         mime_util->primary_profile() == kIamfProfileBase ||
+         mime_util->additional_profile() == kIamfProfileSimple ||
+         mime_util->additional_profile() == kIamfProfileBase;
+}
+
 bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec,
                              const MimeType* mime_type,
                              int64_t bitrate) {
@@ -32,6 +44,26 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec,
     return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond;
   }
 
+#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE
+  if (audio_codec == kSbMediaAudioCodecIamf) {
+    if (!mime_type || !mime_type->is_valid()) {
+      return false;
+    }
+    const std::vector<std::string>& codecs = mime_type->GetCodecs();
+    for (auto& codec : codecs) {
+      IamfMimeUtil mime_util(codec);
+      // We support only IAMF Base or Simple profile streams with an Opus
+      // substream.
+      if (mime_util.is_valid() &&
+          mime_util.substream_codec() == kIamfSubstreamCodecOpus &&
+          HasSupportedIamfProfile(&mime_util)) {
+        return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond;
+      }
+    }
+    return false;
+  }
+#endif  // SB_API_VERSION >= 15
+
   if (audio_codec == kSbMediaAudioCodecAc3 ||
       audio_codec == kSbMediaAudioCodecEac3) {
 #if SB_API_VERSION < 15

starboard/linux/shared/player_components_factory.cc

@@ -41,6 +41,10 @@
 #include "starboard/shared/starboard/player/filter/video_render_algorithm_impl.h"
 #include "starboard/shared/starboard/player/filter/video_renderer_sink.h"
 
+#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE
+#include "starboard/shared/libiamf/iamf_audio_decoder.h"
+#endif  // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE
+
 namespace starboard {
 namespace shared {
 namespace starboard {
@@ -86,6 +90,16 @@ class PlayerComponentsFactory : public PlayerComponents::Factory {
                    libfdkaac::LibfdkaacHandle::GetHandle()->IsLoaded()) {
           SB_LOG(INFO) << "Playing audio using FdkAacAudioDecoder.";
           return std::unique_ptr<AudioDecoder>(new FdkAacAudioDecoder());
+#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE
+        } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) {
+          using ::starboard::shared::libiamf::IamfAudioDecoder;
+          std::unique_ptr<IamfAudioDecoder> audio_decoder_impl(
+              new IamfAudioDecoder(audio_stream_info));
+          if (audio_decoder_impl->is_valid()) {
+            SB_LOG(INFO) << "Playing audio using IamfAudioDecoder.";
+            return std::unique_ptr<AudioDecoder>(std::move(audio_decoder_impl));
+          }
+#endif  // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE
         } else {
           std::unique_ptr<FfmpegAudioDecoder> audio_decoder_impl(
               FfmpegAudioDecoder::Create(audio_stream_info));

starboard/linux/shared/test_filters.py

@@ -39,7 +39,16 @@
 _FILTERED_TESTS = {
     'nplb': [
         # TODO(b/286249595): This test crashes when coverage is enabled.
-        'SbMemoryMapTest.CanChangeMemoryProtection'
+        'SbMemoryMapTest.CanChangeMemoryProtection',
+        # The IamfAudioDecoder doesn't support partial audio.
+        'SbPlayerWriteSampleTests/SbPlayerWriteSampleTest.DiscardAllAudio/audio_iamf_*',
+        'SbPlayerWriteSampleTests/SbPlayerWriteSampleTest.PartialAudio/audio_iamf_*',
+    ],
+    'player_filter_tests': [
+        # TODO(b/375238561): Investigate test failures.
+        'AdaptiveAudioDecoderTests/AdaptiveAudioDecoderTest.SingleInput/*iamf*',
+        'AdaptiveAudioDecoderTests/AdaptiveAudioDecoderTest.MultipleInput*iamf*',
+        'AudioDecoderTests/AudioDecoderTest.SingleInvalidInput/*iamf*',
     ],
 }
 if os.getenv('MODULAR_BUILD', '0') == '1':

starboard/linux/x64x11/shared/platform_configuration/BUILD.gn

@@ -30,6 +30,9 @@ config("platform_configuration") {
 
 config("libraries") {
   configs = [ "//starboard/linux/shared/platform_configuration:libraries" ]
+  if (enable_iamf_decode) {
+    libs = [ "//third_party/libiamf/platforms/linux/libiamf.a" ]
+  }
 }
 
 config("linker_flags") {