Support GPU format/colourspace conversion

UI/models/multitrack-video.hpp

@@ -56,6 +56,34 @@ NLOHMANN_JSON_SERIALIZE_ENUM(obs_scale_type, {
 						     {OBS_SCALE_AREA, "OBS_SCALE_AREA"},
 					     })
 
+NLOHMANN_JSON_SERIALIZE_ENUM(video_colorspace, {
+						       {VIDEO_CS_DEFAULT, "VIDEO_CS_DEFAULT"},
+						       {VIDEO_CS_601, "VIDEO_CS_601"},
+						       {VIDEO_CS_709, "VIDEO_CS_709"},
+						       {VIDEO_CS_SRGB, "VIDEO_CS_SRGB"},
+						       {VIDEO_CS_2100_PQ, "VIDEO_CS_2100_PQ"},
+						       {VIDEO_CS_2100_HLG, "VIDEO_CS_2100_HLG"},
+					       })
+
+/* This only includes output formats selectable in advanced settings. */
+NLOHMANN_JSON_SERIALIZE_ENUM(video_format, {
+						   {VIDEO_FORMAT_NONE, "VIDEO_FORMAT_NONE"},
+						   {VIDEO_FORMAT_I420, "VIDEO_FORMAT_I420"},
+						   {VIDEO_FORMAT_NV12, "VIDEO_FORMAT_NV12"},
+						   {VIDEO_FORMAT_BGRA, "VIDEO_FORMAT_BGRA"},
+						   {VIDEO_FORMAT_I444, "VIDEO_FORMAT_I444"},
+						   {VIDEO_FORMAT_I010, "VIDEO_FORMAT_I010"},
+						   {VIDEO_FORMAT_P010, "VIDEO_FORMAT_P010"},
+						   {VIDEO_FORMAT_P216, "VIDEO_FORMAT_P216"},
+						   {VIDEO_FORMAT_P416, "VIDEO_FORMAT_P416"},
+					   })
+
+NLOHMANN_JSON_SERIALIZE_ENUM(video_range_type, {
+						       {VIDEO_RANGE_DEFAULT, "VIDEO_RANGE_DEFAULT"},
+						       {VIDEO_RANGE_PARTIAL, "VIDEO_RANGE_PARTIAL"},
+						       {VIDEO_RANGE_FULL, "VIDEO_RANGE_FULL"},
+					       })
+
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(media_frames_per_second, numerator, denominator)
 
 namespace GoLiveApi {
@@ -206,10 +234,13 @@ struct VideoEncoderConfiguration {
 	uint32_t height;
 	optional<media_frames_per_second> framerate;
 	optional<obs_scale_type> gpu_scale_type;
+	optional<video_colorspace> colorspace;
+	optional<video_range_type> range;
+	optional<video_format> format;
 	json settings;
 
 	NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(VideoEncoderConfiguration, type, width, height, framerate,
-						    gpu_scale_type, settings)
+						    gpu_scale_type, colorspace, range, format, settings)
 };
 
 struct AudioEncoderConfiguration {

UI/multitrack-video-output.cpp

@@ -175,9 +175,6 @@ static void adjust_video_encoder_scaling(const obs_video_info &ovi, obs_encoder_
 	auto requested_width = encoder_config.width;
 	auto requested_height = encoder_config.height;
 
-	if (ovi.output_width == requested_width || ovi.output_height == requested_height)
-		return;
-
 	if (ovi.base_width < requested_width || ovi.base_height < requested_height) {
 		blog(LOG_WARNING,
 		     "Requested resolution exceeds canvas/available resolution for encoder %zu: %" PRIu32 "x%" PRIu32
@@ -187,7 +184,9 @@ static void adjust_video_encoder_scaling(const obs_video_info &ovi, obs_encoder_
 
 	obs_encoder_set_scaled_size(video_encoder, requested_width, requested_height);
 	obs_encoder_set_gpu_scale_type(video_encoder, encoder_config.gpu_scale_type.value_or(OBS_SCALE_BICUBIC));
-	obs_encoder_set_preferred_video_format(video_encoder, VIDEO_FORMAT_NV12);
+	obs_encoder_set_preferred_video_format(video_encoder, encoder_config.format.value_or(VIDEO_FORMAT_NV12));
+	obs_encoder_set_preferred_color_space(video_encoder, encoder_config.colorspace.value_or(VIDEO_CS_709));
+	obs_encoder_set_preferred_range(video_encoder, encoder_config.range.value_or(VIDEO_RANGE_PARTIAL));
 }
 
 static uint32_t closest_divisor(const obs_video_info &ovi, const media_frames_per_second &target_fps)

libobs/obs-encoder.c

@@ -203,18 +203,26 @@ static void maybe_set_up_gpu_rescale(struct obs_encoder *encoder)
 	bool create_mix = true;
 	struct obs_video_info ovi;
 	const struct video_output_info *info;
+	uint32_t width, height;
+	enum video_format format;
+	enum video_colorspace space;
+	enum video_range_type range;
 
 	if (!encoder->media)
 		return;
-
-	info = video_output_get_info(encoder->media);
-
 	if (encoder->gpu_scale_type == OBS_SCALE_DISABLE)
 		return;
-
-	if (!encoder->scaled_height && !encoder->scaled_width)
+	if (!encoder->scaled_height && !encoder->scaled_width && encoder->preferred_format == VIDEO_FORMAT_NONE &&
+	    encoder->preferred_space == VIDEO_CS_DEFAULT && encoder->preferred_range == VIDEO_RANGE_DEFAULT)
 		return;
 
+	info = video_output_get_info(encoder->media);
+	width = encoder->scaled_width ? encoder->scaled_width : info->width;
+	height = encoder->scaled_height ? encoder->scaled_height : info->height;
+	format = encoder->preferred_format != VIDEO_FORMAT_NONE ? encoder->preferred_format : info->format;
+	space = encoder->preferred_space != VIDEO_CS_DEFAULT ? encoder->preferred_space : info->colorspace;
+	range = encoder->preferred_range != VIDEO_RANGE_DEFAULT ? encoder->preferred_range : info->range;
+
 	current_mix = get_mix_for_video(encoder->media);
 	if (!current_mix)
 		return;
@@ -226,10 +234,13 @@ static void maybe_set_up_gpu_rescale(struct obs_encoder *encoder)
 		if (current_mix->view != current->view)
 			continue;
 
-		if (voi->width != encoder->scaled_width || voi->height != encoder->scaled_height)
+		if (current->ovi.scale_type != encoder->gpu_scale_type)
+			continue;
+
+		if (voi->width != width || voi->height != height)
 			continue;
 
-		if (voi->format != info->format || voi->colorspace != info->colorspace || voi->range != info->range)
+		if (voi->format != format || voi->colorspace != space || voi->range != range)
 			continue;
 
 		current->encoder_refs += 1;
@@ -245,12 +256,12 @@ static void maybe_set_up_gpu_rescale(struct obs_encoder *encoder)
 
 	ovi = current_mix->ovi;
 
-	ovi.output_format = info->format;
-	ovi.colorspace = info->colorspace;
-	ovi.range = info->range;
+	ovi.output_format = format;
+	ovi.colorspace = space;
+	ovi.range = range;
 
-	ovi.output_height = encoder->scaled_height;
-	ovi.output_width = encoder->scaled_width;
+	ovi.output_height = height;
+	ovi.output_width = width;
 	ovi.scale_type = encoder->gpu_scale_type;
 
 	ovi.gpu_conversion = true;
@@ -272,10 +283,13 @@ static void maybe_set_up_gpu_rescale(struct obs_encoder *encoder)
 		if (current->view != current_mix->view)
 			continue;
 
-		if (voi->width != encoder->scaled_width || voi->height != encoder->scaled_height)
+		if (current->ovi.scale_type != encoder->gpu_scale_type)
+			continue;
+
+		if (voi->width != width || voi->height != height)
 			continue;
 
-		if (voi->format != info->format || voi->colorspace != info->colorspace || voi->range != info->range)
+		if (voi->format != format || voi->colorspace != space || voi->range != range)
 			continue;
 
 		obs_encoder_set_video(encoder, current->video);
@@ -1776,6 +1790,38 @@ enum video_format obs_encoder_get_preferred_video_format(const obs_encoder_t *en
 	return encoder->preferred_format;
 }
 
+void obs_encoder_set_preferred_color_space(obs_encoder_t *encoder, enum video_colorspace colorspace)
+{
+	if (!encoder || encoder->info.type != OBS_ENCODER_VIDEO)
+		return;
+
+	encoder->preferred_space = colorspace;
+}
+
+enum video_colorspace obs_encoder_get_preferred_color_space(const obs_encoder_t *encoder)
+{
+	if (!encoder || encoder->info.type != OBS_ENCODER_VIDEO)
+		return VIDEO_CS_DEFAULT;
+
+	return encoder->preferred_space;
+}
+
+void obs_encoder_set_preferred_range(obs_encoder_t *encoder, enum video_range_type range)
+{
+	if (!encoder || encoder->info.type != OBS_ENCODER_VIDEO)
+		return;
+
+	encoder->preferred_range = range;
+}
+
+enum video_range_type obs_encoder_get_preferred_range(const obs_encoder_t *encoder)
+{
+	if (!encoder || encoder->info.type != OBS_ENCODER_VIDEO)
+		return VIDEO_RANGE_DEFAULT;
+
+	return encoder->preferred_range;
+}
+
 void obs_encoder_release(obs_encoder_t *encoder)
 {
 	if (!encoder)
@@ -2068,3 +2114,15 @@ void obs_encoder_group_destroy(obs_encoder_group_t *group)
 
 	obs_encoder_group_actually_destroy(group);
 }
+
+bool obs_encoder_video_tex_active(const obs_encoder_t *encoder, enum video_format format)
+{
+	struct obs_core_video_mix *mix = get_mix_for_video(encoder->media);
+
+	if (format == VIDEO_FORMAT_NV12)
+		return mix->using_nv12_tex;
+	if (format == VIDEO_FORMAT_P010)
+		return mix->using_p010_tex;
+
+	return false;
+}

libobs/obs-internal.h

@@ -1239,6 +1239,8 @@ struct obs_encoder {
 	uint32_t scaled_width;
 	uint32_t scaled_height;
 	enum video_format preferred_format;
+	enum video_colorspace preferred_space;
+	enum video_range_type preferred_range;
 
 	volatile bool active;
 	volatile bool paused;

libobs/obs.h

@@ -831,8 +831,8 @@ EXPORT uint64_t obs_get_frame_interval_ns(void);
 EXPORT uint32_t obs_get_total_frames(void);
 EXPORT uint32_t obs_get_lagged_frames(void);
 
-EXPORT bool obs_nv12_tex_active(void);
-EXPORT bool obs_p010_tex_active(void);
+OBS_DEPRECATED EXPORT bool obs_nv12_tex_active(void);
+OBS_DEPRECATED EXPORT bool obs_p010_tex_active(void);
 
 EXPORT void obs_apply_private_data(obs_data_t *settings);
 EXPORT void obs_set_private_data(obs_data_t *settings);
@@ -2252,10 +2252,29 @@ EXPORT size_t obs_encoder_get_mixer_index(const obs_encoder_t *encoder);
  *
  * If the format is set to VIDEO_FORMAT_NONE, will revert to the default
  * functionality of converting only when absolutely necessary.
+ *
+ * If GPU scaling is enabled conversion will happen on the GPU.
  */
 EXPORT void obs_encoder_set_preferred_video_format(obs_encoder_t *encoder, enum video_format format);
 EXPORT enum video_format obs_encoder_get_preferred_video_format(const obs_encoder_t *encoder);
 
+/**
+ * Sets the preferred colorspace for an encoder, e.g. to simultaneous SDR and
+ * HDR output.
+ * 
+ * Only supported when GPU scaling is enabled
+ */
+EXPORT void obs_encoder_set_preferred_color_space(obs_encoder_t *encoder, enum video_colorspace colorspace);
+EXPORT enum video_colorspace obs_encoder_get_preferred_color_space(const obs_encoder_t *encoder);
+
+/**
+ * Sets the preferred range for an encoder.
+ * 
+ * Only supported when GPU scaling is enabled.
+ */
+EXPORT void obs_encoder_set_preferred_range(obs_encoder_t *encoder, enum video_range_type range);
+EXPORT enum video_range_type obs_encoder_get_preferred_range(const obs_encoder_t *encoder);
+
 /** Gets the default settings for an encoder type */
 EXPORT obs_data_t *obs_encoder_defaults(const char *id);
 EXPORT obs_data_t *obs_encoder_get_defaults(const obs_encoder_t *encoder);
@@ -2300,6 +2319,9 @@ EXPORT video_t *obs_encoder_video(const obs_encoder_t *encoder);
  */
 EXPORT video_t *obs_encoder_parent_video(const obs_encoder_t *encoder);
 
+/** Returns if the encoder's video output context supports shared textures for the specified video format. */
+EXPORT bool obs_encoder_video_tex_active(const obs_encoder_t *encoder, enum video_format format);
+
 /**
  * Returns the audio output context used with this encoder, or NULL if not
  * a audio context

plugins/obs-ffmpeg/texture-amf.cpp

@@ -1055,9 +1055,10 @@ static void check_texture_encode_capability(obs_encoder_t *encoder, amf_codec_ty
 		throw "Encoder scaling is active";
 
 	if (hevc || av1) {
-		if (!obs_nv12_tex_active() && !obs_p010_tex_active())
+		if (!obs_encoder_video_tex_active(encoder, VIDEO_FORMAT_NV12) &&
+		    !obs_encoder_video_tex_active(encoder, VIDEO_FORMAT_P010))
 			throw "NV12/P010 textures aren't active";
-	} else if (!obs_nv12_tex_active()) {
+	} else if (!obs_encoder_video_tex_active(encoder, VIDEO_FORMAT_NV12)) {
 		throw "NV12 textures aren't active";
 	}
 

plugins/obs-nvenc/nvenc-cuda.c

@@ -99,7 +99,7 @@ void cuda_ctx_free(struct nvenc_data *enc)
 
 static bool cuda_surface_init(struct nvenc_data *enc, struct nv_cuda_surface *nvsurf)
 {
-	const bool p010 = obs_p010_tex_active();
+	const bool p010 = obs_encoder_video_tex_active(enc->encoder, VIDEO_FORMAT_P010);
 	CUDA_ARRAY3D_DESCRIPTOR desc;
 	desc.Width = enc->cx;
 	desc.Height = enc->cy;

plugins/obs-nvenc/nvenc-d3d11.c

@@ -93,7 +93,7 @@ void d3d11_free(struct nvenc_data *enc)
 
 static bool d3d11_texture_init(struct nvenc_data *enc, struct nv_texture *nvtex)
 {
-	const bool p010 = obs_p010_tex_active();
+	const bool p010 = obs_encoder_video_tex_active(enc->encoder, VIDEO_FORMAT_P010);
 
 	D3D11_TEXTURE2D_DESC desc = {0};
 	desc.Width = enc->cx;

plugins/obs-nvenc/nvenc-opengl.c

@@ -96,7 +96,7 @@ bool cuda_opengl_encode(void *data, struct encoder_texture *tex, int64_t pts, ui
 	struct nvenc_data *enc = data;
 	struct nv_cuda_surface *surf;
 	struct nv_bitstream *bs;
-	const bool p010 = obs_p010_tex_active();
+	const bool p010 = obs_encoder_video_tex_active(enc->encoder, VIDEO_FORMAT_P010);
 	GLuint input_tex[2];
 
 	if (tex == NULL || tex->tex[0] == NULL) {

plugins/obs-nvenc/nvenc.c

@@ -209,7 +209,8 @@ static inline NV_ENC_MULTI_PASS get_nv_multipass(const char *multipass)
 
 static bool is_10_bit(const struct nvenc_data *enc)
 {
-	return enc->non_texture ? enc->in_format == VIDEO_FORMAT_P010 : obs_p010_tex_active();
+	return enc->non_texture ? enc->in_format == VIDEO_FORMAT_P010
+				: obs_encoder_video_tex_active(enc->encoder, VIDEO_FORMAT_P010);
 }
 
 static bool init_encoder_base(struct nvenc_data *enc, obs_data_t *settings)
@@ -877,7 +878,8 @@ static void *nvenc_create_base(enum codec_type codec, obs_data_t *settings, obs_
 		}
 	}
 
-	if (texture && !obs_p010_tex_active() && !obs_nv12_tex_active()) {
+	if (texture && !obs_encoder_video_tex_active(encoder, VIDEO_FORMAT_NV12) &&
+	    !obs_encoder_video_tex_active(encoder, VIDEO_FORMAT_P010)) {
 		blog(LOG_INFO, "[obs-nvenc] nv12/p010 not active, falling back to "
 			       "non-texture encoder");
 		goto reroute;
@@ -1187,8 +1189,9 @@ bool nvenc_encode_base(struct nvenc_data *enc, struct nv_bitstream *bs, void *pi
 	if (enc->non_texture) {
 		params.bufferFmt = enc->surface_format;
 	} else {
-		params.bufferFmt = obs_p010_tex_active() ? NV_ENC_BUFFER_FORMAT_YUV420_10BIT
-							 : NV_ENC_BUFFER_FORMAT_NV12;
+		params.bufferFmt = obs_encoder_video_tex_active(enc->encoder, VIDEO_FORMAT_P010)
+					   ? NV_ENC_BUFFER_FORMAT_YUV420_10BIT
+					   : NV_ENC_BUFFER_FORMAT_NV12;
 	}
 
 	/* Add ROI map if enabled */

plugins/obs-qsv11/obs-qsv11.c

@@ -513,7 +513,7 @@ static void update_params(struct obs_qsv *obsqsv, obs_data_t *settings)
 		codec = "HEVC";
 		if (astrcmpi(profile, "main") == 0) {
 			obsqsv->params.nCodecProfile = MFX_PROFILE_HEVC_MAIN;
-			if (obs_p010_tex_active()) {
+			if (obs_encoder_video_tex_active(obsqsv->encoder, VIDEO_FORMAT_P010)) {
 				blog(LOG_WARNING, "[qsv encoder] Forcing main10 for P010");
 				obsqsv->params.nCodecProfile = MFX_PROFILE_HEVC_MAIN10;
 			}
@@ -884,10 +884,10 @@ static void *obs_qsv_create_tex(enum qsv_codec codec, obs_data_t *settings, obs_
 		return obs_encoder_create_rerouted(encoder, (const char *)fallback_id);
 	}
 
-	bool gpu_texture_active = obs_nv12_tex_active();
+	bool gpu_texture_active = obs_encoder_video_tex_active(encoder, VIDEO_FORMAT_NV12);
 
 	if (codec != QSV_CODEC_AVC)
-		gpu_texture_active = gpu_texture_active || obs_p010_tex_active();
+		gpu_texture_active = gpu_texture_active || obs_encoder_video_tex_active(encoder, VIDEO_FORMAT_P010);
 
 	if (!gpu_texture_active) {
 		blog(LOG_INFO, ">>> gpu tex not active, fall back to old qsv encoder");