From 3b9d7eb6f5a576ae948cff92175c2f0ac2c1863d Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Thu, 8 Dec 2022 18:18:12 +0000 Subject: [PATCH] cefsrc: Apply running time on audio buffers Also add an audio meta to buffers, for good measure, and set discont flag when needed. This should help with A/V sync issues reported in #59. --- gstcefdemux.cc | 67 ++++++++++++++++++++++++++++++++++++++++---------- gstcefdemux.h | 4 ++- gstcefsrc.cc | 9 ------- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/gstcefdemux.cc b/gstcefdemux.cc index 5cba7fa..4073d95 100644 --- a/gstcefdemux.cc +++ b/gstcefdemux.cc @@ -64,6 +64,7 @@ gst_cef_demux_push_events (GstCefDemux *demux) "channels", G_TYPE_INT, 2, "layout", G_TYPE_STRING, "interleaved", NULL); + gst_audio_info_from_caps (&demux->audio_info, audio_caps); gst_pad_push_event (demux->asrcpad, gst_event_new_caps (audio_caps)); gst_caps_unref (audio_caps); @@ -144,10 +145,21 @@ gst_element_get_current_running_time (GstElement * element) static gboolean gst_cef_demux_push_audio_buffer (GstBuffer **buffer, guint idx, AudioPushData *push_data) { - GST_BUFFER_PTS (*buffer) += push_data->demux->ts_offset; + push_data->demux->last_audio_time = gst_element_get_current_running_time (GST_ELEMENT_CAST (push_data->demux)); + GST_BUFFER_DTS (*buffer) = push_data->demux->last_audio_time; + GST_BUFFER_PTS (*buffer) = push_data->demux->last_audio_time; + + gst_buffer_add_audio_meta (*buffer, &push_data->demux->audio_info, gst_buffer_get_size (*buffer), NULL); + + GST_BUFFER_FLAG_UNSET (*buffer, GST_BUFFER_FLAG_DISCONT); + if (push_data->demux->need_discont) { + GST_BUFFER_FLAG_SET (*buffer, GST_BUFFER_FLAG_DISCONT); + push_data->demux->need_discont = FALSE; + } + push_data->combined = gst_flow_combiner_update_pad_flow (push_data->flow_combiner, push_data->demux->asrcpad, gst_pad_push (push_data->demux->asrcpad, *buffer)); - push_data->demux->last_audio_time = GST_BUFFER_PTS (*buffer) + GST_BUFFER_DURATION (*buffer); + *buffer = NULL; return TRUE; } @@ -185,11 +197,6 @@ gst_cef_demux_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) gst_cef_demux_push_events (demux); - - if (!GST_CLOCK_TIME_IS_VALID (demux->ts_offset)) { - demux->ts_offset = GST_BUFFER_PTS (buffer); - } - for (tmp = demux->cef_audio_stream_start_events; tmp; tmp = tmp->next) { const GstStructure *s = gst_event_get_structure ((GstEvent *) tmp->data); @@ -215,12 +222,18 @@ gst_cef_demux_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) ret = gst_flow_combiner_update_pad_flow (demux->flow_combiner, demux->vsrcpad, gst_pad_push (demux->vsrcpad, buffer)); - if (demux->last_audio_time < GST_BUFFER_PTS (buffer)) { - GstEvent *gap; + if (!GST_CLOCK_TIME_IS_VALID(demux->last_audio_time) || demux->last_audio_time < GST_BUFFER_PTS (buffer)) { + GstClockTime duration, timestamp; - gap = gst_event_new_gap (demux->last_audio_time, GST_BUFFER_PTS (buffer) - demux->last_audio_time); + if (!GST_CLOCK_TIME_IS_VALID(demux->last_audio_time)) { + timestamp = GST_BUFFER_PTS (buffer); + duration = GST_BUFFER_DURATION (buffer); + } else { + timestamp = demux->last_audio_time; + duration = GST_BUFFER_PTS (buffer) - demux->last_audio_time; + } - gst_pad_push_event (demux->asrcpad, gap); + gst_pad_push_event (demux->asrcpad, gst_event_new_gap (timestamp, duration)); demux->last_audio_time = GST_BUFFER_PTS (buffer); } @@ -286,6 +299,30 @@ gst_cef_demux_sink_query (GstPad *pad, GstObject *parent, GstQuery *query) return ret; } +static GstStateChangeReturn +gst_cef_demux_change_state (GstElement * element, GstStateChange transition) +{ + GstStateChangeReturn result; + GstCefDemux *demux = (GstCefDemux *) element; + + GST_DEBUG_OBJECT (element, "%s", gst_state_change_get_name (transition)); + result = GST_CALL_PARENT_WITH_DEFAULT (GST_ELEMENT_CLASS , change_state, (element, transition), GST_STATE_CHANGE_FAILURE); + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + gst_flow_combiner_reset (demux->flow_combiner); + break; + case GST_STATE_CHANGE_PLAYING_TO_PAUSED: + demux->need_discont = TRUE; + break; + default: + break; + } + + return result; +} + + static void gst_cef_demux_init (GstCefDemux * demux) { @@ -306,11 +343,13 @@ gst_cef_demux_init (GstCefDemux * demux) gst_element_add_pad (GST_ELEMENT (demux), demux->asrcpad); gst_flow_combiner_add_pad (demux->flow_combiner, demux->asrcpad); + gst_audio_info_init (&demux->audio_info); + demux->need_stream_start = TRUE; demux->need_caps = TRUE; demux->need_segment = TRUE; - demux->last_audio_time = 0; - demux->ts_offset = GST_CLOCK_TIME_NONE; + demux->need_discont = TRUE; + demux->last_audio_time = GST_CLOCK_TIME_NONE; } static void @@ -331,6 +370,8 @@ gst_cef_demux_class_init (GstCefDemuxClass * klass) gobject_class->finalize = gst_cef_demux_finalize; + gstelement_class->change_state = gst_cef_demux_change_state; + gst_element_class_set_static_metadata (gstelement_class, "Chromium Embedded Framework demuxer", "Demuxer/Audio/Video", "Demuxes audio and video from cefsrc", "Mathieu Duponchelle "); diff --git a/gstcefdemux.h b/gstcefdemux.h index 4f6ccf4..ff9b8c9 100644 --- a/gstcefdemux.h +++ b/gstcefdemux.h @@ -3,6 +3,7 @@ #include #include +#include G_BEGIN_DECLS @@ -26,13 +27,14 @@ struct _GstCefDemux { gboolean need_stream_start; gboolean need_caps; gboolean need_segment; + gboolean need_discont; GstPad *vsrcpad; GstPad *asrcpad; GList *cef_audio_stream_start_events; GstEvent *vcaps_event; GstFlowCombiner *flow_combiner; GstClockTime last_audio_time; - GstClockTime ts_offset; + GstAudioInfo audio_info; }; struct _GstCefDemuxClass { diff --git a/gstcefsrc.cc b/gstcefsrc.cc index b06897e..47e4037 100644 --- a/gstcefsrc.cc +++ b/gstcefsrc.cc @@ -182,7 +182,6 @@ class AudioHandler : public CefAudioHandler mRate = params.sample_rate; mChannels = channels; - mCurrentTime = GST_CLOCK_TIME_NONE; GST_OBJECT_LOCK (mElement); mElement->audio_events = g_list_append (mElement->audio_events, event); @@ -214,14 +213,7 @@ class AudioHandler : public CefAudioHandler GST_OBJECT_LOCK (mElement); - if (!GST_CLOCK_TIME_IS_VALID (mCurrentTime)) { - mCurrentTime = gst_util_uint64_scale (mElement->n_frames, - mElement->vinfo.fps_d * GST_SECOND, mElement->vinfo.fps_n); - } - - GST_BUFFER_PTS (buf) = mCurrentTime; GST_BUFFER_DURATION (buf) = gst_util_uint64_scale (frames, GST_SECOND, mRate); - mCurrentTime += GST_BUFFER_DURATION (buf); if (!mElement->audio_buffers) { mElement->audio_buffers = gst_buffer_list_new(); @@ -245,7 +237,6 @@ class AudioHandler : public CefAudioHandler private: GstCefSrc *mElement; - GstClockTime mCurrentTime; gint mRate; gint mChannels; IMPLEMENT_REFCOUNTING(AudioHandler);