Skip to content

Commit

Permalink
cefsrc: Apply running time on audio buffers
Browse files Browse the repository at this point in the history
Also add an audio meta to buffers, for good measure, and set discont flag when
needed. This should help with A/V sync issues reported in #59.
  • Loading branch information
philn authored and MathieuDuponchelle committed May 1, 2023
1 parent 998fc32 commit 79fbffc
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 23 deletions.
67 changes: 54 additions & 13 deletions gstcefdemux.cc
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ gst_cef_demux_push_events (GstCefDemux *demux)
"channels", G_TYPE_INT, 2,
"layout", G_TYPE_STRING, "interleaved",
NULL);
gst_audio_info_from_caps (&demux->audio_info, audio_caps);
gst_pad_push_event (demux->asrcpad, gst_event_new_caps (audio_caps));
gst_caps_unref (audio_caps);

Expand Down Expand Up @@ -144,10 +145,21 @@ gst_element_get_current_running_time (GstElement * element)
static gboolean
gst_cef_demux_push_audio_buffer (GstBuffer **buffer, guint idx, AudioPushData *push_data)
{
GST_BUFFER_PTS (*buffer) += push_data->demux->ts_offset;
push_data->demux->last_audio_time = gst_element_get_current_running_time (GST_ELEMENT_CAST (push_data->demux));
GST_BUFFER_DTS (*buffer) = push_data->demux->last_audio_time;
GST_BUFFER_PTS (*buffer) = push_data->demux->last_audio_time;

gst_buffer_add_audio_meta (*buffer, &push_data->demux->audio_info, gst_buffer_get_size (*buffer), NULL);

GST_BUFFER_FLAG_UNSET (*buffer, GST_BUFFER_FLAG_DISCONT);
if (push_data->demux->need_discont) {
GST_BUFFER_FLAG_SET (*buffer, GST_BUFFER_FLAG_DISCONT);
push_data->demux->need_discont = FALSE;
}

push_data->combined = gst_flow_combiner_update_pad_flow (push_data->flow_combiner, push_data->demux->asrcpad,
gst_pad_push (push_data->demux->asrcpad, *buffer));
push_data->demux->last_audio_time = GST_BUFFER_PTS (*buffer) + GST_BUFFER_DURATION (*buffer);

*buffer = NULL;
return TRUE;
}
Expand Down Expand Up @@ -185,11 +197,6 @@ gst_cef_demux_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer)

gst_cef_demux_push_events (demux);


if (!GST_CLOCK_TIME_IS_VALID (demux->ts_offset)) {
demux->ts_offset = GST_BUFFER_PTS (buffer);
}

for (tmp = demux->cef_audio_stream_start_events; tmp; tmp = tmp->next) {
const GstStructure *s = gst_event_get_structure ((GstEvent *) tmp->data);

Expand All @@ -215,12 +222,18 @@ gst_cef_demux_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer)
ret = gst_flow_combiner_update_pad_flow (demux->flow_combiner, demux->vsrcpad,
gst_pad_push (demux->vsrcpad, buffer));

if (demux->last_audio_time < GST_BUFFER_PTS (buffer)) {
GstEvent *gap;
if (!GST_CLOCK_TIME_IS_VALID(demux->last_audio_time) || demux->last_audio_time < GST_BUFFER_PTS (buffer)) {
GstClockTime duration, timestamp;

gap = gst_event_new_gap (demux->last_audio_time, GST_BUFFER_PTS (buffer) - demux->last_audio_time);
if (!GST_CLOCK_TIME_IS_VALID(demux->last_audio_time)) {
timestamp = GST_BUFFER_PTS (buffer);
duration = GST_BUFFER_DURATION (buffer);
} else {
timestamp = demux->last_audio_time;
duration = GST_BUFFER_PTS (buffer) - demux->last_audio_time;
}

gst_pad_push_event (demux->asrcpad, gap);
gst_pad_push_event (demux->asrcpad, gst_event_new_gap (timestamp, duration));

demux->last_audio_time = GST_BUFFER_PTS (buffer);
}
Expand Down Expand Up @@ -286,6 +299,30 @@ gst_cef_demux_sink_query (GstPad *pad, GstObject *parent, GstQuery *query)
return ret;
}

static GstStateChangeReturn
gst_cef_demux_change_state (GstElement * element, GstStateChange transition)
{
GstStateChangeReturn result;
GstCefDemux *demux = (GstCefDemux *) element;

GST_DEBUG_OBJECT (element, "%s", gst_state_change_get_name (transition));
result = GST_CALL_PARENT_WITH_DEFAULT (GST_ELEMENT_CLASS , change_state, (element, transition), GST_STATE_CHANGE_FAILURE);

switch (transition) {
case GST_STATE_CHANGE_PAUSED_TO_READY:
gst_flow_combiner_reset (demux->flow_combiner);
break;
case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
demux->need_discont = TRUE;
break;
default:
break;
}

return result;
}


static void
gst_cef_demux_init (GstCefDemux * demux)
{
Expand All @@ -306,11 +343,13 @@ gst_cef_demux_init (GstCefDemux * demux)
gst_element_add_pad (GST_ELEMENT (demux), demux->asrcpad);
gst_flow_combiner_add_pad (demux->flow_combiner, demux->asrcpad);

gst_audio_info_init (&demux->audio_info);

demux->need_stream_start = TRUE;
demux->need_caps = TRUE;
demux->need_segment = TRUE;
demux->last_audio_time = 0;
demux->ts_offset = GST_CLOCK_TIME_NONE;
demux->need_discont = TRUE;
demux->last_audio_time = GST_CLOCK_TIME_NONE;
}

static void
Expand All @@ -331,6 +370,8 @@ gst_cef_demux_class_init (GstCefDemuxClass * klass)

gobject_class->finalize = gst_cef_demux_finalize;

gstelement_class->change_state = gst_cef_demux_change_state;

gst_element_class_set_static_metadata (gstelement_class,
"Chromium Embedded Framework demuxer", "Demuxer/Audio/Video",
"Demuxes audio and video from cefsrc", "Mathieu Duponchelle <[email protected]>");
Expand Down
4 changes: 3 additions & 1 deletion gstcefdemux.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

#include <gst/gst.h>
#include <gst/base/gstflowcombiner.h>
#include <gst/audio/audio.h>

G_BEGIN_DECLS

Expand All @@ -26,13 +27,14 @@ struct _GstCefDemux {
gboolean need_stream_start;
gboolean need_caps;
gboolean need_segment;
gboolean need_discont;
GstPad *vsrcpad;
GstPad *asrcpad;
GList *cef_audio_stream_start_events;
GstEvent *vcaps_event;
GstFlowCombiner *flow_combiner;
GstClockTime last_audio_time;
GstClockTime ts_offset;
GstAudioInfo audio_info;
};

struct _GstCefDemuxClass {
Expand Down
9 changes: 0 additions & 9 deletions gstcefsrc.cc
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,6 @@ class AudioHandler : public CefAudioHandler

mRate = params.sample_rate;
mChannels = channels;
mCurrentTime = GST_CLOCK_TIME_NONE;

GST_OBJECT_LOCK (mElement);
mElement->audio_events = g_list_append (mElement->audio_events, event);
Expand Down Expand Up @@ -214,14 +213,7 @@ class AudioHandler : public CefAudioHandler

GST_OBJECT_LOCK (mElement);

if (!GST_CLOCK_TIME_IS_VALID (mCurrentTime)) {
mCurrentTime = gst_util_uint64_scale (mElement->n_frames,
mElement->vinfo.fps_d * GST_SECOND, mElement->vinfo.fps_n);
}

GST_BUFFER_PTS (buf) = mCurrentTime;
GST_BUFFER_DURATION (buf) = gst_util_uint64_scale (frames, GST_SECOND, mRate);
mCurrentTime += GST_BUFFER_DURATION (buf);

if (!mElement->audio_buffers) {
mElement->audio_buffers = gst_buffer_list_new();
Expand All @@ -245,7 +237,6 @@ class AudioHandler : public CefAudioHandler
private:

GstCefSrc *mElement;
GstClockTime mCurrentTime;
gint mRate;
gint mChannels;
IMPLEMENT_REFCOUNTING(AudioHandler);
Expand Down

0 comments on commit 79fbffc

Please sign in to comment.