From 5868085a1c92cc1ca2654e175b0b8d2095d56860 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Thu, 25 Jan 2024 18:16:24 +0000 Subject: [PATCH] [media] Implement IAMF audio parsing (#2279) Cherry-picks https://chromium-review.googlesource.com/c/chromium/src/+/5128755 and adds minor changes to allow it to build. Original CL description follows. Adds an IAMF codec definition and implements IAMF MP4 parsing for v1.0.0. The implementation is disabled by default. Immersive Audio Model and Formats v1.0.0 https://aomediacodec.github.io/iamf/v1.0.0.html b/271301103 --- third_party/chromium/media/BUILD.gn.chromium | 1 + .../chromium/media/base/audio_codecs.cc | 122 +++++++++++++++++ .../chromium/media/base/audio_codecs.h | 17 ++- .../chromium/media/base/eme_constants.h | 4 + .../chromium/media/base/key_systems.cc | 2 + .../chromium/media/base/mime_util_internal.cc | 15 +++ .../chromium/media/base/mime_util_internal.h | 3 +- .../chromium/media/base/mime_util_unittest.cc | 8 ++ .../chromium/media/base/supported_types.cc | 2 + .../android/media_codec_audio_decoder.cc | 1 + .../media/filters/stream_parser_factory.cc | 30 ++++- .../media/formats/mp4/box_definitions.cc | 123 ++++++++++++++++++ .../media/formats/mp4/box_definitions.h | 19 +++ .../chromium/media/formats/mp4/fourccs.h | 3 + .../media/formats/mp4/mp4_stream_parser.cc | 71 +++++++++- .../media/formats/mp4/mp4_stream_parser.h | 9 +- .../formats/mp4/mp4_stream_parser_unittest.cc | 19 +-- third_party/chromium/media/media_buildflags.h | 1 + third_party/chromium/media/media_options.gni | 4 + tools/metrics/histograms/enums.xml | 2 + 20 files changed, 438 insertions(+), 18 deletions(-) diff --git a/third_party/chromium/media/BUILD.gn.chromium b/third_party/chromium/media/BUILD.gn.chromium index d5c158c08f5c..9a9ea3766a03 100644 --- a/third_party/chromium/media/BUILD.gn.chromium +++ b/third_party/chromium/media/BUILD.gn.chromium @@ -42,6 +42,7 @@ buildflag_header("media_buildflags") { "ENABLE_MEDIA_REMOTING=$enable_media_remoting", "ENABLE_MEDIA_REMOTING_RPC=$enable_media_remoting_rpc", "ENABLE_OPENH264=$media_use_openh264", + "ENABLE_PLATFORM_IAMF_AUDIO=$enable_platform_iamf_audio", "ENABLE_PLATFORM_MPEG_H_AUDIO=$enable_platform_mpeg_h_audio", "ENABLE_MSE_MPEG2TS_STREAM_PARSER=$enable_mse_mpeg2ts_stream_parser", "ENABLE_CAST_STREAMING_RENDERER=$enable_cast_streaming_renderer", diff --git a/third_party/chromium/media/base/audio_codecs.cc b/third_party/chromium/media/base/audio_codecs.cc index 901098149db7..5034b09c09d5 100644 --- a/third_party/chromium/media/base/audio_codecs.cc +++ b/third_party/chromium/media/base/audio_codecs.cc @@ -6,6 +6,8 @@ #include +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_split.h" #include "base/strings/string_util.h" namespace media { @@ -37,6 +39,8 @@ std::string GetCodecName(AudioCodec codec) { return "gsm_ms"; case AudioCodec::kOpus: return "opus"; + case AudioCodec::kIAMF: + return "iamf"; case AudioCodec::kPCM_ALAW: return "pcm_alaw"; case AudioCodec::kEAC3: @@ -56,6 +60,10 @@ std::string GetProfileName(AudioCodecProfile profile) { return "unknown"; case AudioCodecProfile::kXHE_AAC: return "xhe-aac"; + case AudioCodecProfile::kIAMF_SIMPLE: + return "iamf-simple"; + case AudioCodecProfile::kIAMF_BASE: + return "iamf-base"; } } @@ -82,6 +90,10 @@ AudioCodec StringToAudioCodec(const std::string& codec_id) { return AudioCodec::kVorbis; if (base::StartsWith(codec_id, "mp4a.40.", base::CompareCase::SENSITIVE)) return AudioCodec::kAAC; + if (codec_id == "iamf" || + base::StartsWith(codec_id, "iamf.", base::CompareCase::SENSITIVE)) { + return AudioCodec::kIAMF; + } return AudioCodec::kUnknown; } @@ -89,4 +101,114 @@ std::ostream& operator<<(std::ostream& os, const AudioCodec& codec) { return os << GetCodecName(codec); } +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) +bool ParseIamfCodecId(std::string_view codec_id, + uint8_t* primary_profilec, + uint8_t* additional_profilec) { + // Reference: Immersive Audio Model and Formats; + // v1.0.0 + // 6.3. Codecs Parameter String + // (https://aomediacodec.github.io/iamf/v1.0.0.html#codecsparameter) + if (!(std::string(codec_id).find("iamf") == 0)) { + return false; + } + + // For test purposes only, just "iamf" is acceptable. + if (codec_id == "iamf") { + return true; + } + + constexpr int kMaxIamfCodecIdLength = + 4 // FOURCC string "iamf". + + 1 // delimiting period. + + 3 // primary_profile as 3 digit string. + + 1 // delimiting period. + + 3 // additional_profile as 3 digit string. + + 1 // delimiting period. + + 9; // The remaining string is one of + // "opus", "mp4a.40.2", "flac", "ipcm". + + if (codec_id.size() > kMaxIamfCodecIdLength) { + DVLOG(4) << __func__ << ": Codec id is too long (" << codec_id << ")"; + return false; + } + + std::vector elem = base::SplitString( + std::string(codec_id), ".", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); + if (elem.size() < 4) { + DVLOG(4) << __func__ << ": invalid IAMF codec id:" << codec_id; + return false; + } + + DCHECK_EQ(elem[0], "iamf"); + + if (StringToAudioCodec(elem[0]) != AudioCodec::kIAMF) { + DVLOG(4) << __func__ << ": invalid IAMF codec id:" << codec_id; + return false; + } + + // The primary profile string should be three digits, and should be between 0 + // and 255 inclusive. + uint32_t primary_profile = 0; + if (elem[1].size() != 3 || !base::StringToUint(elem[1], &primary_profile) || + primary_profile > 0xFF) { + DVLOG(4) << __func__ << ": invalid IAMF primary profile: " << elem[1]; + return false; + } + + // The additional profile string should be three digits, and should be between + // 0 and 255 inclusive. + uint32_t additional_profile = 0; + if (elem[2].size() != 3 || + !base::StringToUint(elem[2], &additional_profile) || + additional_profile > 0xFF) { + DVLOG(4) << __func__ << ": invalid IAMF additional profile: " << elem[2]; + return false; + } + + // The codec string should be one of "opus", "mp4a", "flac", or "ipcm". + std::string codec = base::ToLowerASCII(elem[3]); + if (codec.size() != 4 || ((codec != "opus") && (codec != "mp4a") && + (codec != "flac") && (codec != "ipcm"))) { + DVLOG(4) << __func__ << ": invalid IAMF stream codec: " << elem[3]; + return false; + } + + if (codec == "mp4a") { + if (elem.size() != 6) { + DVLOG(4) << __func__ + << ": incorrect mp4a codec string syntax:" << codec_id; + return false; + } + + // The fields following "mp4a" should be "40" and "2" to signal AAC-LC. + uint32_t object_type_indication = 0; + if (elem[4].size() != 2 || + !base::HexStringToUInt(elem[4], &object_type_indication) || + object_type_indication != 0x40) { + DVLOG(4) << __func__ + << ": invalid mp4a Object Type Indication:" << codec_id; + return false; + } + + uint32_t audio_object_type = 0; + if (elem[5].size() != 1 || + !base::HexStringToUInt(elem[5], &audio_object_type) || + audio_object_type != 0x02) { + DVLOG(4) << __func__ << ": invalid mp4a Audio Object Type:" << codec_id; + return false; + } + } + + if (primary_profilec) { + *primary_profilec = primary_profile; + } + + if (additional_profilec) { + *additional_profilec = additional_profile; + } + + return true; +} +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) } // namespace media diff --git a/third_party/chromium/media/base/audio_codecs.h b/third_party/chromium/media/base/audio_codecs.h index f679513a20b5..7cd2d04f7bb0 100644 --- a/third_party/chromium/media/base/audio_codecs.h +++ b/third_party/chromium/media/base/audio_codecs.h @@ -5,8 +5,13 @@ #ifndef MEDIA_BASE_AUDIO_CODECS_H_ #define MEDIA_BASE_AUDIO_CODECS_H_ +#include + #include +#include + #include "media/base/media_export.h" +#include "media/media_buildflags.h" namespace media { @@ -33,13 +38,14 @@ enum class AudioCodec { kALAC = 15, kAC3 = 16, kMpegHAudio = 17, + kIAMF = 22, // DO NOT ADD RANDOM AUDIO CODECS! // // The only acceptable time to add a new codec is if there is production code // that uses said codec in the same CL. // Must always be equal to the largest entry ever logged. - kMaxValue = kMpegHAudio, + kMaxValue = kIAMF, }; enum class AudioCodecProfile { @@ -49,7 +55,9 @@ enum class AudioCodecProfile { // kMaxValue to equal the new codec. kUnknown = 0, kXHE_AAC = 1, - kMaxValue = kXHE_AAC, + kIAMF_SIMPLE = 2, + kIAMF_BASE = 3, + kMaxValue = kIAMF_BASE, }; std::string MEDIA_EXPORT GetCodecName(AudioCodec codec); @@ -58,6 +66,11 @@ std::string MEDIA_EXPORT GetProfileName(AudioCodecProfile profile); MEDIA_EXPORT std::ostream& operator<<(std::ostream& os, const AudioCodec& codec); MEDIA_EXPORT AudioCodec StringToAudioCodec(const std::string& codec_id); +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) +MEDIA_EXPORT bool ParseIamfCodecId(std::string_view codec_id, + uint8_t* primary_profilec, + uint8_t* additional_profilec); +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) } // namespace media diff --git a/third_party/chromium/media/base/eme_constants.h b/third_party/chromium/media/base/eme_constants.h index 6e47c0ed18a3..f7a1f5c13d5d 100644 --- a/third_party/chromium/media/base/eme_constants.h +++ b/third_party/chromium/media/base/eme_constants.h @@ -40,6 +40,7 @@ enum EmeCodec : uint32_t { EME_CODEC_FLAC = 1 << 13, EME_CODEC_AV1 = 1 << 14, EME_CODEC_HEVC_PROFILE_MAIN10 = 1 << 15, + EME_CODEC_IAMF = 1 << 23, }; // *_ALL values should only be used for masking, do not use them to specify @@ -60,6 +61,9 @@ constexpr SupportedCodecs GetMp4AudioCodecs() { codecs |= EME_CODEC_MPEG_H_AUDIO; #endif // BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO) #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + codecs |= EME_CODEC_IAMF; +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) return codecs; } diff --git a/third_party/chromium/media/base/key_systems.cc b/third_party/chromium/media/base/key_systems.cc index 9205d7a89238..36355ce2a7c7 100644 --- a/third_party/chromium/media/base/key_systems.cc +++ b/third_party/chromium/media/base/key_systems.cc @@ -75,6 +75,8 @@ EmeCodec ToAudioEmeCodec(AudioCodec codec) { return EME_CODEC_EAC3; case AudioCodec::kAC3: return EME_CODEC_AC3; + case AudioCodec::kIAMF: + return EME_CODEC_IAMF; case AudioCodec::kMpegHAudio: return EME_CODEC_MPEG_H_AUDIO; default: diff --git a/third_party/chromium/media/base/mime_util_internal.cc b/third_party/chromium/media/base/mime_util_internal.cc index a08acbcfd47c..b1f1b2fd1e6b 100644 --- a/third_party/chromium/media/base/mime_util_internal.cc +++ b/third_party/chromium/media/base/mime_util_internal.cc @@ -187,6 +187,8 @@ AudioCodec MimeUtilToAudioCodec(MimeUtil::Codec codec) { return AudioCodec::kOpus; case MimeUtil::FLAC: return AudioCodec::kFLAC; + case MimeUtil::IAMF: + return AudioCodec::kIAMF; default: break; } @@ -345,6 +347,10 @@ void MimeUtil::AddSupportedMediaFormats() { mp4_video_codecs.emplace(AV1); #endif +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + mp4_audio_codecs.emplace(IAMF); +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + CodecSet mp4_codecs(mp4_audio_codecs); mp4_codecs.insert(mp4_video_codecs.begin(), mp4_video_codecs.end()); @@ -670,6 +676,8 @@ bool MimeUtil::IsCodecSupportedOnAndroid( #else return false; #endif + case IAMF: + return false; } return false; @@ -863,6 +871,13 @@ bool MimeUtil::ParseCodecHelper(const std::string& mime_type_lower_case, } #endif +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + if (ParseIamfCodecId(codec_id.data(), nullptr, nullptr)) { + out_result->codec = MimeUtil::IAMF; + return true; + } +#endif + DVLOG(2) << __func__ << ": Unrecognized codec id \"" << codec_id << "\""; return false; } diff --git a/third_party/chromium/media/base/mime_util_internal.h b/third_party/chromium/media/base/mime_util_internal.h index f0cf1a848ff8..ebe840d2a851 100644 --- a/third_party/chromium/media/base/mime_util_internal.h +++ b/third_party/chromium/media/base/mime_util_internal.h @@ -52,7 +52,8 @@ class MEDIA_EXPORT MimeUtil { DOLBY_VISION, AV1, MPEG_H_AUDIO, - LAST_CODEC = MPEG_H_AUDIO + IAMF, + LAST_CODEC = IAMF }; // Platform configuration structure. Controls which codecs are supported at diff --git a/third_party/chromium/media/base/mime_util_unittest.cc b/third_party/chromium/media/base/mime_util_unittest.cc index 4dcafd03f610..e6782e3ad35b 100644 --- a/third_party/chromium/media/base/mime_util_unittest.cc +++ b/third_party/chromium/media/base/mime_util_unittest.cc @@ -614,6 +614,10 @@ TEST(IsCodecSupportedOnAndroidTest, EncryptedCodecBehavior) { case MimeUtil::AV1: EXPECT_EQ(BUILDFLAG(ENABLE_AV1_DECODER), result); break; + + case MimeUtil::IAMF: + EXPECT_EQ(HasIamfSupport(), result); + break; } }); } @@ -677,6 +681,10 @@ TEST(IsCodecSupportedOnAndroidTest, ClearCodecBehavior) { case MimeUtil::AV1: EXPECT_EQ(BUILDFLAG(ENABLE_AV1_DECODER), result); break; + + case MimeUtil::IAMF: + EXPECT_EQ(HasIamfSupport(), result); + break; } }); } diff --git a/third_party/chromium/media/base/supported_types.cc b/third_party/chromium/media/base/supported_types.cc index 995e9b5b54e5..1601c1f02648 100644 --- a/third_party/chromium/media/base/supported_types.cc +++ b/third_party/chromium/media/base/supported_types.cc @@ -231,6 +231,7 @@ bool IsAudioCodecProprietary(AudioCodec codec) { return true; case AudioCodec::kFLAC: + case AudioCodec::kIAMF: case AudioCodec::kMP3: case AudioCodec::kOpus: case AudioCodec::kVorbis: @@ -291,6 +292,7 @@ bool IsDefaultSupportedAudioType(const AudioType& type) { case AudioCodec::kALAC: case AudioCodec::kAC3: case AudioCodec::kMpegHAudio: + case AudioCodec::kIAMF: case AudioCodec::kUnknown: return false; } diff --git a/third_party/chromium/media/filters/android/media_codec_audio_decoder.cc b/third_party/chromium/media/filters/android/media_codec_audio_decoder.cc index 1fd4784e08cd..4f9dc84cb615 100644 --- a/third_party/chromium/media/filters/android/media_codec_audio_decoder.cc +++ b/third_party/chromium/media/filters/android/media_codec_audio_decoder.cc @@ -92,6 +92,7 @@ void MediaCodecAudioDecoder::Initialize(const AudioDecoderConfig& config, config.codec() == AudioCodec::kFLAC || config.codec() == AudioCodec::kAAC || config.codec() == AudioCodec::kOpus || + config.codec() == AudioCodec::kIAMF || is_passthrough_; if (!is_codec_supported) { DVLOG(1) << "Unsuported codec " << GetCodecName(config.codec()); diff --git a/third_party/chromium/media/filters/stream_parser_factory.cc b/third_party/chromium/media/filters/stream_parser_factory.cc index 65e5eb67bc08..480ecb7107fb 100644 --- a/third_party/chromium/media/filters/stream_parser_factory.cc +++ b/third_party/chromium/media/filters/stream_parser_factory.cc @@ -65,8 +65,8 @@ struct CodecInfo { HISTOGRAM_FLAC, HISTOGRAM_AV1, HISTOGRAM_MPEG_H_AUDIO, - HISTOGRAM_MAX = - HISTOGRAM_MPEG_H_AUDIO // Must be equal to largest logged entry. + HISTOGRAM_IAMF, + HISTOGRAM_MAX = HISTOGRAM_IAMF // Must be equal to largest logged entry. }; const char* pattern; @@ -236,6 +236,12 @@ static const CodecInfo kMPEG4VP09CodecInfo = { "vp09.*", CodecInfo::VIDEO, nullptr, CodecInfo::HISTOGRAM_VP9}; static const CodecInfo kMPEG4FLACCodecInfo = {"flac", CodecInfo::AUDIO, nullptr, CodecInfo::HISTOGRAM_FLAC}; +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) +static const CodecInfo kIAMFCodecInfo1 = {"iamf", CodecInfo::AUDIO, nullptr, + CodecInfo::HISTOGRAM_IAMF}; +static const CodecInfo kIAMFCodecInfo2 = {"iamf.*", CodecInfo::AUDIO, nullptr, + CodecInfo::HISTOGRAM_IAMF}; +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) static const CodecInfo* const kVideoMP4Codecs[] = {&kMPEG4FLACCodecInfo, &kOpusCodecInfo, @@ -265,6 +271,10 @@ static const CodecInfo* const kVideoMP4Codecs[] = {&kMPEG4FLACCodecInfo, #if BUILDFLAG(ENABLE_AV1_DECODER) &kAV1CodecInfo, #endif +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + &kIAMFCodecInfo1, + &kIAMFCodecInfo2, +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) nullptr}; static const CodecInfo* const kAudioMP4Codecs[] = {&kMPEG4FLACCodecInfo, @@ -285,6 +295,10 @@ static const CodecInfo* const kAudioMP4Codecs[] = {&kMPEG4FLACCodecInfo, &kEAC3CodecInfo3, #endif // BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO) #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + &kIAMFCodecInfo1, + &kIAMFCodecInfo2, +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) nullptr}; static StreamParser* BuildMP4Parser(base::span codecs, @@ -300,6 +314,11 @@ static StreamParser* BuildMP4Parser(base::span codecs, // conditionally expect a FLAC stream, hence |has_flac|. bool has_flac = false; + // Like FLAC, IAMF v1.0.0 + // (https://aomediacodec.github.io/iamf/v1.0.0.html#isobmff) does not define + // an encapsulation using MP4AudioSampleEntry with objectTypeIndication. + bool has_iamf = false; + for (const auto& codec_id : codecs) { if (base::MatchPattern(codec_id, kMPEG4FLACCodecInfo.pattern)) { has_flac = true; @@ -328,10 +347,15 @@ static StreamParser* BuildMP4Parser(base::span codecs, audio_object_types.insert(mp4::kEAC3); #endif // BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO) #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + } else if (base::MatchPattern(codec_id, kIAMFCodecInfo1.pattern) || + base::MatchPattern(codec_id, kIAMFCodecInfo2.pattern)) { + has_iamf = true; +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) } } - return new mp4::MP4StreamParser(audio_object_types, has_sbr, has_flac); + return new mp4::MP4StreamParser(audio_object_types, has_sbr, has_flac, has_iamf); } #if BUILDFLAG(USE_PROPRIETARY_CODECS) static const CodecInfo kADTSCodecInfo = {nullptr, CodecInfo::AUDIO, nullptr, diff --git a/third_party/chromium/media/formats/mp4/box_definitions.cc b/third_party/chromium/media/formats/mp4/box_definitions.cc index 069f22bcd823..b49a3147376f 100644 --- a/third_party/chromium/media/formats/mp4/box_definitions.cc +++ b/third_party/chromium/media/formats/mp4/box_definitions.cc @@ -1438,6 +1438,121 @@ bool OpusSpecificBox::Parse(BoxReader* reader) { return true; } +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) +enum IamfConfigObuType { + // The following enum values are mapped to their respective Config OBU + // values in the IAMF specification. + // https://aomediacodec.github.io/iamf/v1.0.0.html#obu-header-syntax. + kIamfConfigObuTypeCodecConfig = 0, + kIamfConfigObuTypeAudioElement = 1, + kIamfConfigObuTypeMixPresentation = 2, + kIamfConfigObuTypeSequenceHeader = 31, +}; + +IamfSpecificBox::IamfSpecificBox() = default; + +IamfSpecificBox::IamfSpecificBox(const IamfSpecificBox& other) = default; + +IamfSpecificBox::~IamfSpecificBox() = default; + +FourCC IamfSpecificBox::BoxType() const { + return FOURCC_IAMF; +} + +bool IamfSpecificBox::Parse(BoxReader* reader) { + const int obu_bitstream_size = reader->box_size() - reader->pos(); + const uint8_t* buf = reader->buffer() + reader->pos(); + ia_descriptors.assign(buf, buf + obu_bitstream_size); + + RCHECK(reader->SkipBytes(obu_bitstream_size)); + + BufferReader config_reader(ia_descriptors.data(), ia_descriptors.size()); + + while (config_reader.pos() < config_reader.buffer_size()) { + RCHECK(ReadOBU(&config_reader)); + } + + return true; +} + +bool IamfSpecificBox::ReadOBU(BufferReader* reader) { + uint8_t obu_type; + uint32_t obu_size; + RCHECK(ReadOBUHeader(reader, &obu_type, &obu_size)); + const size_t read_stop_pos = reader->pos() + obu_size; + + switch (static_cast(obu_type)) { + case kIamfConfigObuTypeCodecConfig: + case kIamfConfigObuTypeAudioElement: + case kIamfConfigObuTypeMixPresentation: + break; + case kIamfConfigObuTypeSequenceHeader: + uint32_t ia_code; + RCHECK(reader->Read4(&ia_code)); + RCHECK(ia_code == FOURCC_IAMF); + + RCHECK(reader->Read1(&profile)); + RCHECK(profile <= 1); + + break; + default: + DVLOG(1) << "Unhandled IAMF OBU type " << static_cast(obu_type); + return false; + } + + const size_t remaining_size = read_stop_pos - reader->pos(); + RCHECK(reader->SkipBytes(remaining_size)); + return true; +} + +bool IamfSpecificBox::ReadOBUHeader(BufferReader* reader, + uint8_t* obu_type, + uint32_t* obu_size) { + uint8_t header_flags; + RCHECK(reader->Read1(&header_flags)); + *obu_type = (header_flags >> 3) & 0x1f; + + const bool obu_redundant_copy = (header_flags >> 2) & 1; + const bool obu_trimming_status_flag = (header_flags >> 1) & 1; + const bool obu_extension_flag = header_flags & 1; + + redundant_copy |= obu_redundant_copy; + + RCHECK(ReadLeb128Value(reader, obu_size)); + RCHECK(reader->HasBytes(*obu_size)); + + RCHECK(!obu_trimming_status_flag); + if (obu_extension_flag) { + uint32_t extension_header_size; + const int last_reader_pos = reader->pos(); + RCHECK(ReadLeb128Value(reader, &extension_header_size)); + const int num_leb128_bytes_read = reader->pos() - last_reader_pos; + RCHECK(reader->SkipBytes(extension_header_size)); + obu_size -= (num_leb128_bytes_read + extension_header_size); + } + return true; +} + +bool IamfSpecificBox::ReadLeb128Value(BufferReader* reader, + uint32_t* value) const { + DCHECK(reader); + DCHECK(value); + *value = 0; + bool error = true; + for (size_t i = 0; i < sizeof(uint32_t); ++i) { + uint8_t byte; + RCHECK(reader->Read1(&byte)); + *value |= ((byte & 0x7f) << (i * 7)); + if (!(byte & 0x80)) { + error = false; + break; + } + } + + return !error; +} +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + AudioSampleEntry::AudioSampleEntry() : format(FOURCC_NULL), data_reference_index(0), @@ -1467,6 +1582,14 @@ bool AudioSampleEntry::Parse(BoxReader* reader) { // Convert from 16.16 fixed point to integer samplerate >>= 16; +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + if (format == FOURCC_IAMF) { + RCHECK_MEDIA_LOGGED(iamf.Parse(reader), reader->media_log(), + "Failure parsing IamfSpecificBox (iamf)"); + return true; + } +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + RCHECK(reader->ScanChildren()); if (format == FOURCC_ENCA) { // Continue scanning until a recognized protection scheme is found, or until diff --git a/third_party/chromium/media/formats/mp4/box_definitions.h b/third_party/chromium/media/formats/mp4/box_definitions.h index 5a307f0476b0..c65f5de0b424 100644 --- a/third_party/chromium/media/formats/mp4/box_definitions.h +++ b/third_party/chromium/media/formats/mp4/box_definitions.h @@ -378,6 +378,22 @@ struct MEDIA_EXPORT OpusSpecificBox : Box { uint32_t sample_rate; }; +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) +struct MEDIA_EXPORT IamfSpecificBox : Box { + DECLARE_BOX_METHODS(IamfSpecificBox); + bool ReadOBU(BufferReader* reader); + bool ReadOBUHeader(BufferReader* reader, + uint8_t* obu_type, + uint32_t* obu_size); + bool ReadLeb128Value(BufferReader* reader, uint32_t* value) const; + + uint8_t profile; + bool redundant_copy = false; + + std::vector ia_descriptors; +}; +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + struct MEDIA_EXPORT AudioSampleEntry : Box { DECLARE_BOX_METHODS(AudioSampleEntry); @@ -391,6 +407,9 @@ struct MEDIA_EXPORT AudioSampleEntry : Box { ElementaryStreamDescriptor esds; FlacSpecificBox dfla; OpusSpecificBox dops; +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + IamfSpecificBox iamf; +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) }; struct MEDIA_EXPORT SampleDescription : Box { diff --git a/third_party/chromium/media/formats/mp4/fourccs.h b/third_party/chromium/media/formats/mp4/fourccs.h index 41b85ce5c92e..ac3dc308ad5d 100644 --- a/third_party/chromium/media/formats/mp4/fourccs.h +++ b/third_party/chromium/media/formats/mp4/fourccs.h @@ -63,6 +63,9 @@ enum FourCC { FOURCC_HVC1 = 0x68766331, FOURCC_HVCC = 0x68766343, #endif +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + FOURCC_IAMF = 0x69616d66, +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) FOURCC_ID32 = 0x49443332, FOURCC_IODS = 0x696f6473, FOURCC_MDAT = 0x6d646174, diff --git a/third_party/chromium/media/formats/mp4/mp4_stream_parser.cc b/third_party/chromium/media/formats/mp4/mp4_stream_parser.cc index b5f7b3d66511..feaef1078a00 100644 --- a/third_party/chromium/media/formats/mp4/mp4_stream_parser.cc +++ b/third_party/chromium/media/formats/mp4/mp4_stream_parser.cc @@ -83,7 +83,8 @@ gfx::ColorVolumeMetadata ConvertMdcvToColorVolumeMetadata( MP4StreamParser::MP4StreamParser(const std::set& audio_object_types, bool has_sbr, - bool has_flac) + bool has_flac, + bool has_iamf) : state_(kWaitingForInit), moof_head_(0), mdat_tail_(0), @@ -93,7 +94,7 @@ MP4StreamParser::MP4StreamParser(const std::set& audio_object_types, audio_object_types_(audio_object_types), has_sbr_(has_sbr), has_flac_(has_flac), - num_empty_samples_skipped_(0), + has_iamf_(has_iamf), num_invalid_conversions_(0), num_video_keyframe_mismatches_(0) {} @@ -338,6 +339,9 @@ bool MP4StreamParser::ParseMoov(BoxReader* reader) { #if BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO) audio_format != FOURCC_MHM1 && audio_format != FOURCC_MHA1 && #endif +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + audio_format != FOURCC_IAMF && +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) audio_format != FOURCC_MP4A) { MEDIA_LOG(ERROR, media_log_) << "Unsupported audio format 0x" << std::hex << entry.format @@ -352,8 +356,11 @@ bool MP4StreamParser::ParseMoov(BoxReader* reader) { base::TimeDelta seek_preroll; std::vector extra_data; -#if BUILDFLAG(USE_PROPRIETARY_CODECS) +#if BUILDFLAG(USE_PROPRIETARY_CODECS) || BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) AudioCodecProfile profile = AudioCodecProfile::kUnknown; +#endif // BUILDFLAG(USE_PROPRIETARY_CODECS) || + // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) +#if BUILDFLAG(USE_PROPRIETARY_CODECS) std::vector aac_extra_data; #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) @@ -378,6 +385,29 @@ bool MP4StreamParser::ParseMoov(BoxReader* reader) { channel_layout = GuessChannelLayout(entry.channelcount); sample_per_second = entry.samplerate; extra_data = entry.dfla.stream_info; +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + } else if (audio_format == FOURCC_IAMF) { + // ISOBMFF IAMF streams do not use object type indication. + // |audio_format| is sufficient for identifying IAMF. + if (!has_iamf_) { + MEDIA_LOG(ERROR, media_log_) << "IAMF audio stream detected in MP4, " + "mismatching what is specified in " + "the mimetype."; + return false; + } + + codec = AudioCodec::kIAMF; + profile = entry.iamf.profile == 0 ? AudioCodecProfile::kIAMF_SIMPLE + : AudioCodecProfile::kIAMF_BASE; + // The correct values for the channel layout and sample rate can + // be parsed from the descriptor bitstream prepended to each sample. + // They are set to the following values here to create a valid + // AudioDecoderConfig. + // TODO (crbug.com/1513779): Parse the bitstream to set the correct + // values here. + channel_layout = CHANNEL_LAYOUT_STEREO; + sample_per_second = 48000; +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) #if BUILDFLAG(USE_PROPRIETARY_CODECS) #if BUILDFLAG(ENABLE_PLATFORM_MPEG_H_AUDIO) } else if (audio_format == FOURCC_MHM1 || audio_format == FOURCC_MHA1) { @@ -479,6 +509,11 @@ bool MP4StreamParser::ParseMoov(BoxReader* reader) { audio_config.set_aac_extra_data(std::move(aac_extra_data)); } #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + if (codec == AudioCodec::kIAMF) { + audio_config.set_profile(profile); + } +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) DVLOG(1) << "audio_track_id=" << audio_track_id << " config=" << audio_config.AsHumanReadableString(); @@ -718,6 +753,26 @@ bool MP4StreamParser::PrepareAACBuffer( } #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) +bool MP4StreamParser::PrependIADescriptors( + const IamfSpecificBox& iamf_box, + std::vector* frame_buf, + std::vector* subsamples) const { + // Prepend the IA Descriptors to every IA Sample. + frame_buf->insert(frame_buf->begin(), iamf_box.ia_descriptors.begin(), + iamf_box.ia_descriptors.end()); + if (subsamples->empty()) { + subsamples->push_back( + SubsampleEntry(iamf_box.ia_descriptors.size(), + frame_buf->size() - iamf_box.ia_descriptors.size())); + } else { + (*subsamples)[0].clear_bytes += iamf_box.ia_descriptors.size(); + } + + return true; +} +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + ParseResult MP4StreamParser::EnqueueSample(BufferQueueMap* buffers) { DCHECK_EQ(state_, kEmittingSamples); @@ -881,6 +936,16 @@ ParseResult MP4StreamParser::EnqueueSample(BufferQueueMap* buffers) { #else return ParseResult::kError; #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) + } else { +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + if (runs_->audio_description().format == FOURCC_IAMF) { + if (!PrependIADescriptors(runs_->audio_description().iamf, &frame_buf, + &subsamples)) { + MEDIA_LOG(ERROR, media_log_) + << "Failed to prepare IA sample for decode"; + } + } +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) } } diff --git a/third_party/chromium/media/formats/mp4/mp4_stream_parser.h b/third_party/chromium/media/formats/mp4/mp4_stream_parser.h index c04afcb77953..da2397f8a4db 100644 --- a/third_party/chromium/media/formats/mp4/mp4_stream_parser.h +++ b/third_party/chromium/media/formats/mp4/mp4_stream_parser.h @@ -36,7 +36,8 @@ class MEDIA_EXPORT MP4StreamParser : public StreamParser { public: MP4StreamParser(const std::set& audio_object_types, bool has_sbr, - bool has_flac); + bool has_flac, + bool has_iamf); MP4StreamParser(const MP4StreamParser&) = delete; MP4StreamParser& operator=(const MP4StreamParser&) = delete; @@ -93,6 +94,11 @@ class MEDIA_EXPORT MP4StreamParser : public StreamParser { std::vector* frame_buf, std::vector* subsamples) const; #endif +#if BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) + bool PrependIADescriptors(const IamfSpecificBox& iamf_box, + std::vector* frame_buf, + std::vector* subsamples) const; +#endif // BUILDFLAG(ENABLE_PLATFORM_IAMF_AUDIO) ParseResult EnqueueSample(BufferQueueMap* buffers); bool SendAndFlushSamples(BufferQueueMap* buffers); @@ -145,6 +151,7 @@ class MEDIA_EXPORT MP4StreamParser : public StreamParser { const std::set audio_object_types_; const bool has_sbr_; const bool has_flac_; + const bool has_iamf_; // Tracks the number of MEDIA_LOGS for skipping empty trun samples. int num_empty_samples_skipped_; diff --git a/third_party/chromium/media/formats/mp4/mp4_stream_parser_unittest.cc b/third_party/chromium/media/formats/mp4/mp4_stream_parser_unittest.cc index 2c973ffa1efd..ff98d7619581 100644 --- a/third_party/chromium/media/formats/mp4/mp4_stream_parser_unittest.cc +++ b/third_party/chromium/media/formats/mp4/mp4_stream_parser_unittest.cc @@ -81,7 +81,8 @@ class MP4StreamParserTest : public testing::Test { verifying_keyframeness_sequence_(false) { std::set audio_object_types; audio_object_types.insert(kISO_14496_3); - parser_.reset(new MP4StreamParser(audio_object_types, false, false)); + parser_.reset( + new MP4StreamParser(audio_object_types, false, false, false)); } protected: @@ -406,7 +407,8 @@ TEST_F(MP4StreamParserTest, MPEG2_AAC_LC) { InSequence s; std::set audio_object_types; audio_object_types.insert(kISO_13818_7_AAC_LC); - parser_.reset(new MP4StreamParser(audio_object_types, false, false)); + parser_.reset( + new MP4StreamParser(audio_object_types, false, false, false)); auto params = GetDefaultInitParametersExpectations(); params.detected_video_track_count = 0; InitializeParserWithInitParametersExpectations(params); @@ -418,7 +420,8 @@ TEST_F(MP4StreamParserTest, MPEG4_XHE_AAC) { InSequence s; // The keyframeness sequence matters for this test. std::set audio_object_types; audio_object_types.insert(kISO_14496_3); - parser_.reset(new MP4StreamParser(audio_object_types, false, false)); + parser_.reset( + new MP4StreamParser(audio_object_types, false, false, false)); auto params = GetDefaultInitParametersExpectations(); params.detected_video_track_count = 0; @@ -623,7 +626,7 @@ TEST_F(MP4StreamParserTest, NaturalSizeWithPASP) { TEST_F(MP4StreamParserTest, DemuxingAC3) { std::set audio_object_types; audio_object_types.insert(kAC3); - parser_.reset(new MP4StreamParser(audio_object_types, false, false)); + parser_.reset(new MP4StreamParser(audio_object_types, false, false, false)); #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO) bool expect_success = true; @@ -647,7 +650,7 @@ TEST_F(MP4StreamParserTest, DemuxingAC3) { TEST_F(MP4StreamParserTest, DemuxingEAC3) { std::set audio_object_types; audio_object_types.insert(kEAC3); - parser_.reset(new MP4StreamParser(audio_object_types, false, false)); + parser_.reset(new MP4StreamParser(audio_object_types, false, false, false)); #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO) bool expect_success = true; @@ -669,7 +672,7 @@ TEST_F(MP4StreamParserTest, DemuxingEAC3) { } TEST_F(MP4StreamParserTest, Flac) { - parser_.reset(new MP4StreamParser(std::set(), false, true)); + parser_.reset(new MP4StreamParser(std::set(), false, true, false)); auto params = GetDefaultInitParametersExpectations(); params.detected_video_track_count = 0; @@ -680,7 +683,7 @@ TEST_F(MP4StreamParserTest, Flac) { } TEST_F(MP4StreamParserTest, Flac192kHz) { - parser_.reset(new MP4StreamParser(std::set(), false, true)); + parser_.reset(new MP4StreamParser(std::set(), false, true, false)); auto params = GetDefaultInitParametersExpectations(); params.detected_video_track_count = 0; @@ -834,7 +837,7 @@ class MP4StreamParserRotationMatrixEvaluatorTest MP4StreamParserRotationMatrixEvaluatorTest() { std::set audio_object_types; audio_object_types.insert(kISO_14496_3); - parser_.reset(new MP4StreamParser(audio_object_types, false, false)); + parser_.reset(new MP4StreamParser(audio_object_types, false, false, false)); } protected: diff --git a/third_party/chromium/media/media_buildflags.h b/third_party/chromium/media/media_buildflags.h index be0bd01f9ae4..0971cc878b96 100644 --- a/third_party/chromium/media/media_buildflags.h +++ b/third_party/chromium/media/media_buildflags.h @@ -13,6 +13,7 @@ #define BUILDFLAG_INTERNAL_ALTERNATE_CDM_STORAGE_ID_KEY() ("") #define BUILDFLAG_INTERNAL_CDM_PLATFORM_SPECIFIC_PATH() ("_platform_specific/linux_x64") #define BUILDFLAG_INTERNAL_ENABLE_PLATFORM_AC3_EAC3_AUDIO() (1) +#define BUILDFLAG_INTERNAL_ENABLE_PLATFORM_IAMF_AUDIO() (1) #define BUILDFLAG_INTERNAL_ENABLE_CAST_AUDIO_RENDERER() (0) #define BUILDFLAG_INTERNAL_ENABLE_CDM_HOST_VERIFICATION() (0) #define BUILDFLAG_INTERNAL_ENABLE_CDM_STORAGE_ID() (0) diff --git a/third_party/chromium/media/media_options.gni b/third_party/chromium/media/media_options.gni index 3574b2aa056d..65efc38745d3 100644 --- a/third_party/chromium/media/media_options.gni +++ b/third_party/chromium/media/media_options.gni @@ -47,6 +47,10 @@ declare_args() { # pass-through to HDMI sink on Chromecast. enable_platform_ac3_eac3_audio = proprietary_codecs && is_chromecast + # Enables IAMF audio handling in chromium. This includes demuxing, + # on-device decoding and bitstream passthrough as supported by device. + enable_platform_iamf_audio = false + enable_platform_mpeg_h_audio = proprietary_codecs && is_chromecast enable_mse_mpeg2ts_stream_parser = diff --git a/tools/metrics/histograms/enums.xml b/tools/metrics/histograms/enums.xml index 09454033d343..adbffcb72355 100644 --- a/tools/metrics/histograms/enums.xml +++ b/tools/metrics/histograms/enums.xml @@ -5671,6 +5671,8 @@ others/histograms.xml --> + +