gst-decoder.c

/*
 * Copyright (c) 2017 Rob Clark <rclark@redhat.com>
 * Copyright (c) 2017 Carlos Rafael Giani <dv@pseudoterminal.org>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sub license,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <assert.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "common.h"

#include <drm_fourcc.h>

#include <gst/gst.h>
#include <gst/gstmemory.h>
#include <gst/gstpad.h>
#include <gst/allocators/gstdmabuf.h>
#include <gst/app/gstappsink.h>
#include <gst/video/gstvideometa.h>

GST_DEBUG_CATEGORY_EXTERN(kmscube_debug);
#define GST_CAT_DEFAULT kmscube_debug

#define MAX_NUM_PLANES 3

inline static const char *
yesno(int yes)
{
	return yes ? "yes" : "no";
}

struct decoder {
	GMainLoop          *loop;
	GstElement         *pipeline;
	GstElement         *sink;
	pthread_t           gst_thread;

	uint32_t            format;
	GstVideoInfo        info;

	const struct gbm   *gbm;
	const struct egl   *egl;
	unsigned            frame;

	EGLImage            last_frame;
	GstSample          *last_samp;
};

static GstPadProbeReturn
pad_probe(GstPad *pad, GstPadProbeInfo *info, gpointer user_data)
{
	struct decoder *dec = user_data;
	GstEvent *event = GST_PAD_PROBE_INFO_EVENT(info);
	GstCaps *caps;

	(void)pad;

	if (GST_EVENT_TYPE(event) != GST_EVENT_CAPS)
		return GST_PAD_PROBE_OK;

	gst_event_parse_caps(event, &caps);

	if (!caps) {
		GST_ERROR("caps event without caps");
		return GST_PAD_PROBE_OK;
	}

	if (!gst_video_info_from_caps(&dec->info, caps)) {
		GST_ERROR("caps event with invalid video caps");
		return GST_PAD_PROBE_OK;
	}

	switch (GST_VIDEO_INFO_FORMAT(&(dec->info))) {
	case GST_VIDEO_FORMAT_I420:
		dec->format = DRM_FORMAT_YUV420;
		break;
	case GST_VIDEO_FORMAT_NV12:
		dec->format = DRM_FORMAT_NV12;
		break;
	case GST_VIDEO_FORMAT_YUY2:
		dec->format = DRM_FORMAT_YUYV;
		break;
	default:
		GST_ERROR("unknown format\n");
		return GST_PAD_PROBE_OK;
	}

	return GST_PAD_PROBE_OK;
}

static void *
gst_thread_func(void *args)
{
	struct decoder *dec = args;
	g_main_loop_run(dec->loop);
	return NULL;
}

static void
element_added_cb(GstBin *bin, GstElement *element, gpointer user_data)
{
	GstElementFactory *elem_factory;
	gchar const *factory_name;

	(void)user_data;
	(void)bin;

	elem_factory = gst_element_get_factory(element);
	factory_name = gst_plugin_feature_get_name(elem_factory);

	GST_DEBUG("added element %s (created with factory %s)", GST_OBJECT_NAME(element), factory_name);

	/* v4l2 video decoder factories are generated by the GStreamer v4l probe.
	 * The format is v4l2videoNdec, where N is an integer. So, check if the
	 * element's factory name fits this pattern. */
	if (g_str_has_prefix(factory_name, "v4l2video") && g_str_has_suffix(factory_name, "dec")) {
		/* yes, "capture" rather than "output" because v4l2 is bonkers */
		gst_util_set_object_arg(G_OBJECT(element), "capture-io-mode", "dmabuf");
		printf("found GStreamer V4L2 video decoder element with name \"%s\"\n", GST_OBJECT_NAME(element));
	}
}

static gboolean
bus_watch_cb(GstBus *bus, GstMessage *msg, gpointer user_data)
{
	struct decoder *dec = (struct decoder *)user_data;

	(void)bus;

	switch (GST_MESSAGE_TYPE(msg)) {
	case GST_MESSAGE_STATE_CHANGED: {
		gchar *dotfilename;
		GstState old_gst_state, cur_gst_state, pending_gst_state;

		/* Only consider state change messages coming from
		 * the toplevel element. */
		if (GST_MESSAGE_SRC(msg) != GST_OBJECT(dec->pipeline))
			break;

		gst_message_parse_state_changed(msg, &old_gst_state, &cur_gst_state, &pending_gst_state);

		printf(
			"GStreamer state change:  old: %s  current: %s  pending: %s\n",
			gst_element_state_get_name(old_gst_state),
			gst_element_state_get_name(cur_gst_state),
			gst_element_state_get_name(pending_gst_state)
		);

		dotfilename = g_strdup_printf(
			"statechange__old-%s__cur-%s__pending-%s",
			gst_element_state_get_name(old_gst_state),
			gst_element_state_get_name(cur_gst_state),
			gst_element_state_get_name(pending_gst_state)
		);
		GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(dec->pipeline), GST_DEBUG_GRAPH_SHOW_ALL, dotfilename);
		g_free(dotfilename);

		break;
	}
	case GST_MESSAGE_REQUEST_STATE: {
		GstState requested_state;
		gst_message_parse_request_state(msg, &requested_state);
		printf(
			"state change to %s was requested by %s\n",
			gst_element_state_get_name(requested_state),
			GST_MESSAGE_SRC_NAME(msg)
		);
		gst_element_set_state(GST_ELEMENT(dec->pipeline), requested_state);
		break;
	}
	case GST_MESSAGE_LATENCY: {
		printf("redistributing latency\n");
		gst_bin_recalculate_latency(GST_BIN(dec->pipeline));
		break;
	}
	case GST_MESSAGE_INFO:
	case GST_MESSAGE_WARNING:
	case GST_MESSAGE_ERROR: {
		GError *error = NULL;
		gchar *debug_info = NULL;
		gchar const *prefix;

		switch (GST_MESSAGE_TYPE(msg)) {
			case GST_MESSAGE_INFO:
				gst_message_parse_info(msg, &error, &debug_info);
				prefix = "INFO";
				break;
			case GST_MESSAGE_WARNING:
				gst_message_parse_warning(msg, &error, &debug_info);
				prefix = "WARNING";
				break;
			case GST_MESSAGE_ERROR:
				gst_message_parse_error(msg, &error, &debug_info);
				prefix = "ERROR";
				break;
			default:
				g_assert_not_reached();
		}
		printf("GStreamer %s: %s; debug info: %s", prefix, error->message, debug_info);

		g_clear_error(&error);
		g_free(debug_info);

		if (GST_MESSAGE_TYPE(msg) == GST_MESSAGE_ERROR) {
			GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(dec->pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "error");
		}

		// TODO: stop mainloop in case of an error

		break;
	}
	default:
		break;
	}

	return TRUE;
}

static GstPadProbeReturn
appsink_query_cb(GstPad *pad G_GNUC_UNUSED, GstPadProbeInfo *info,
	gpointer user_data G_GNUC_UNUSED)
{
	GstQuery *query = info->data;

	if (GST_QUERY_TYPE (query) != GST_QUERY_ALLOCATION)
	  return GST_PAD_PROBE_OK;

	gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL);

	return GST_PAD_PROBE_HANDLED;
}

struct decoder *
video_init(const struct egl *egl, const struct gbm *gbm, const char *filename)
{
	struct decoder *dec;
	GstElement *src, *decodebin;
	GstPad *pad;
	GstBus *bus;

	if (egl_check(egl, eglCreateImageKHR) ||
	    egl_check(egl, eglDestroyImageKHR))
		return NULL;

	dec = calloc(1, sizeof(*dec));
	dec->loop = g_main_loop_new(NULL, FALSE);
	dec->gbm = gbm;
	dec->egl = egl;

	/* Setup pipeline: */
	static const char *pipeline =
		"filesrc name=\"src\" ! decodebin name=\"decode\" ! video/x-raw ! appsink sync=false name=\"sink\"";
	dec->pipeline = gst_parse_launch(pipeline, NULL);

	dec->sink = gst_bin_get_by_name(GST_BIN(dec->pipeline), "sink");

	/* Implement the allocation query using a pad probe. This probe will
	 * adverstize support for GstVideoMeta, which avoid hardware accelerated
	 * decoder that produce special strides and offsets from having to
	 * copy the buffers.
	 */
	pad = gst_element_get_static_pad(dec->sink, "sink");
	gst_pad_add_probe(pad, GST_PAD_PROBE_TYPE_QUERY_DOWNSTREAM,
		appsink_query_cb, NULL, NULL);
	gst_object_unref(pad);

	src = gst_bin_get_by_name(GST_BIN(dec->pipeline), "src");
	g_object_set(G_OBJECT(src), "location", filename, NULL);
	gst_object_unref(src);

	/* Configure the sink like a video sink (mimic GstVideoSink) */
	gst_base_sink_set_max_lateness(GST_BASE_SINK(dec->sink), 20 * GST_MSECOND);
	gst_base_sink_set_qos_enabled(GST_BASE_SINK(dec->sink), TRUE);

	/* if we don't limit max-buffers then we can let the decoder outrun
	 * vsync and quickly chew up 100's of MB of buffers:
	 */
	g_object_set(G_OBJECT(dec->sink), "max-buffers", 2, NULL);

	gst_pad_add_probe(gst_element_get_static_pad(dec->sink, "sink"),
			GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM,
			pad_probe, dec, NULL);

	/* callback needed to make sure we get dmabuf's from v4l2videoNdec.. */
	decodebin = gst_bin_get_by_name(GST_BIN(dec->pipeline), "decode");
	g_signal_connect(decodebin, "element-added", G_CALLBACK(element_added_cb), dec);

	/* add bus to be able to receive error message, handle latency
	 * requests, produce pipeline dumps, etc. */
	bus = gst_pipeline_get_bus(GST_PIPELINE(dec->pipeline));
	gst_bus_add_watch(bus, bus_watch_cb, dec);
	gst_object_unref(GST_OBJECT(bus));

	/* let 'er rip! */
	gst_element_set_state(dec->pipeline, GST_STATE_PLAYING);

	pthread_create(&dec->gst_thread, NULL, gst_thread_func, dec);

	return dec;
}

static void
set_last_frame(struct decoder *dec, EGLImage frame, GstSample *samp)
{
	if (dec->last_frame)
		dec->egl->eglDestroyImageKHR(dec->egl->display, dec->last_frame);
	dec->last_frame = frame;
	if (dec->last_samp)
		gst_sample_unref(dec->last_samp);
	dec->last_samp = samp;
}

// TODO this could probably be a helper re-used by cube-tex:
static int
buf_to_fd(const struct gbm *gbm, int size, void *ptr)
{
	struct gbm_bo *bo;
	void *map, *map_data = NULL;
	uint32_t stride;
	int fd;

	/* NOTE: do not actually use GBM_BO_USE_WRITE since that gets us a dumb buffer: */
	bo = gbm_bo_create(gbm->dev, size, 1, GBM_FORMAT_R8, GBM_BO_USE_LINEAR);

	map = gbm_bo_map(bo, 0, 0, size, 1, GBM_BO_TRANSFER_WRITE, &stride, &map_data);

	memcpy(map, ptr, size);

	gbm_bo_unmap(bo, map_data);

	fd = gbm_bo_get_fd(bo);

	/* we have the fd now, no longer need the bo: */
	gbm_bo_destroy(bo);

	return fd;
}

static EGLImage
buffer_to_image(struct decoder *dec, GstBuffer *buf)
{
	struct { int fd, offset, stride; } planes[MAX_NUM_PLANES];
	GstVideoMeta *meta = gst_buffer_get_video_meta(buf);
	EGLImage image;
	guint nmems = gst_buffer_n_memory(buf);
	guint nplanes = GST_VIDEO_INFO_N_PLANES(&(dec->info));
	guint i;
	guint width, height;
	gboolean is_dmabuf_mem;
	GstMemory *mem;
	int dmabuf_fd = -1;

	static const EGLint egl_dmabuf_plane_fd_attr[MAX_NUM_PLANES] = {
		EGL_DMA_BUF_PLANE0_FD_EXT,
		EGL_DMA_BUF_PLANE1_FD_EXT,
		EGL_DMA_BUF_PLANE2_FD_EXT,
	};
	static const EGLint egl_dmabuf_plane_offset_attr[MAX_NUM_PLANES] = {
		EGL_DMA_BUF_PLANE0_OFFSET_EXT,
		EGL_DMA_BUF_PLANE1_OFFSET_EXT,
		EGL_DMA_BUF_PLANE2_OFFSET_EXT,
	};
	static const EGLint egl_dmabuf_plane_pitch_attr[MAX_NUM_PLANES] = {
		EGL_DMA_BUF_PLANE0_PITCH_EXT,
		EGL_DMA_BUF_PLANE1_PITCH_EXT,
		EGL_DMA_BUF_PLANE2_PITCH_EXT,
	};

	/* Query gst_is_dmabuf_memory() here, since the gstmemory
	 * block might get merged below by gst_buffer_map(), meaning
	 * that the mem pointer would become invalid */
	mem = gst_buffer_peek_memory(buf, 0);
	is_dmabuf_mem = gst_is_dmabuf_memory(mem);

	if (nmems > 1) {
		if (is_dmabuf_mem) {
			/* this case currently is not defined */

			GST_FIXME("gstbuffers with multiple memory blocks and DMABUF "
			          "memory currently are not supported");
			return EGL_NO_IMAGE_KHR;
		}

		/* if this is not DMABUF memory, then the gst_buffer_map()
		 * call below will automatically merge the memory blocks
		 */
	}

	if (is_dmabuf_mem) {
		dmabuf_fd = dup(gst_dmabuf_memory_get_fd(mem));
	} else {
		GstMapInfo map_info;
		gst_buffer_map(buf, &map_info, GST_MAP_READ);
		dmabuf_fd = buf_to_fd(dec->gbm, map_info.size, map_info.data);
		gst_buffer_unmap(buf, &map_info);
	}

	if (dmabuf_fd < 0) {
		GST_ERROR("could not obtain DMABUF FD");
		return EGL_NO_IMAGE_KHR;
	}

	/* Usually, a videometa should be present, since by using the internal kmscube
	 * video_appsink element instead of the regular appsink, it is guaranteed that
	 * video meta support is declared in the video_appsink's allocation query.
	 * However, this assumes that upstream elements actually look at the allocation
	 * query's contents properly, or that they even send a query at all. If this
	 * is not the case, then upstream might decide to push frames without adding
	 * a meta. It can happen, and in this case, look at the video info data as
	 * a fallback (it is computed out of the input caps).
	 */
	if (meta) {
		for (i = 0; i < nplanes; i++) {
			planes[i].fd = dmabuf_fd;
			planes[i].offset = meta->offset[i];
			planes[i].stride = meta->stride[i];
		}
	} else {
		for (i = 0; i < nplanes; i++) {
			planes[i].fd = dmabuf_fd;
			planes[i].offset = GST_VIDEO_INFO_PLANE_OFFSET(&(dec->info), i);
			planes[i].stride = GST_VIDEO_INFO_PLANE_STRIDE(&(dec->info), i);
		}
	}

	width = GST_VIDEO_INFO_WIDTH(&(dec->info));
	height = GST_VIDEO_INFO_HEIGHT(&(dec->info));

	/* output some information at the beginning (= when the first frame is handled) */
	if (dec->frame == 0) {
		GstVideoFormat pixfmt;
		const char *pixfmt_str;

		pixfmt = GST_VIDEO_INFO_FORMAT(&(dec->info));
		pixfmt_str = gst_video_format_to_string(pixfmt);

		printf("===================================\n");
		printf("GStreamer video stream information:\n");
		printf("  size: %u x %u pixel\n", width, height);
		printf("  pixel format: %s  number of planes: %u\n", pixfmt_str, nplanes);
		printf("  can use zero-copy: %s\n", yesno(is_dmabuf_mem));
		printf("  video meta found: %s\n", yesno(meta != NULL));
		printf("===================================\n");
	}

	{
		/* Initialize the first 6 attributes with values that are
		 * plane invariant (width, height, format) */
		EGLint attr[6 + 6*(MAX_NUM_PLANES) + 1] = {
			EGL_WIDTH, width,
			EGL_HEIGHT, height,
			EGL_LINUX_DRM_FOURCC_EXT, dec->format
		};

		for (i = 0; i < nplanes; i++) {
			attr[6 + 6*i + 0] = egl_dmabuf_plane_fd_attr[i];
			attr[6 + 6*i + 1] = planes[i].fd;
			attr[6 + 6*i + 2] = egl_dmabuf_plane_offset_attr[i];
			attr[6 + 6*i + 3] = planes[i].offset;
			attr[6 + 6*i + 4] = egl_dmabuf_plane_pitch_attr[i];
			attr[6 + 6*i + 5] = planes[i].stride;
		}

		attr[6 + 6*nplanes] = EGL_NONE;

		image = dec->egl->eglCreateImageKHR(dec->egl->display, EGL_NO_CONTEXT,
				EGL_LINUX_DMA_BUF_EXT, NULL, attr);
	}

	/* Cleanup */
	for (unsigned i = 0; i < nplanes; i++)
		close(planes[i].fd);

	return image;
}

EGLImage
video_frame(struct decoder *dec)
{
	GstSample *samp;
	GstBuffer *buf;
	EGLImage   frame = NULL;

	samp = gst_app_sink_pull_sample(GST_APP_SINK(dec->sink));
	if (!samp) {
		GST_DEBUG("got no appsink sample");
		return NULL;
	}

	buf = gst_sample_get_buffer(samp);

	// TODO inline buffer_to_image??
	frame = buffer_to_image(dec, buf);

	// TODO in the zero-copy dmabuf case it would be nice to associate
	// the eglimg w/ the buffer to avoid recreating it every frame..

	set_last_frame(dec, frame, samp);

	dec->frame++;

	return frame;
}

void video_deinit(struct decoder *dec)
{
	set_last_frame(dec, NULL, NULL);
	gst_element_set_state(dec->pipeline, GST_STATE_NULL);
	gst_object_unref(dec->sink);
	gst_object_unref(dec->pipeline);
	g_main_loop_quit(dec->loop);
	g_main_loop_unref(dec->loop);
	pthread_join(dec->gst_thread, 0);
	free(dec);
}