diff --git a/Cargo.lock b/Cargo.lock index dee78313ee9a..21ce251d1c4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8494,6 +8494,7 @@ version = "0.1.0" dependencies = [ "chrono", "ffmpeg-sys-next", + "image", "libc", "tempfile", "thiserror", diff --git a/core/src/location/non_indexed.rs b/core/src/location/non_indexed.rs index c4ace38cc2c6..5c93ff650838 100644 --- a/core/src/location/non_indexed.rs +++ b/core/src/location/non_indexed.rs @@ -242,7 +242,7 @@ pub async fn walk( date_modified: entry.metadata.modified_or_now().into(), size_in_bytes_bytes: entry.metadata.len().to_be_bytes().to_vec(), }, - has_created_thumbnail: false, + has_created_thumbnail: true, })) .await?; } diff --git a/core/src/object/media/old_thumbnail/process.rs b/core/src/object/media/old_thumbnail/process.rs index 000f368efc17..728f3cf7ceae 100644 --- a/core/src/object/media/old_thumbnail/process.rs +++ b/core/src/object/media/old_thumbnail/process.rs @@ -470,9 +470,14 @@ async fn generate_video_thumbnail( file_path: impl AsRef, output_path: impl AsRef, ) -> Result<(), ThumbnailerError> { - use sd_ffmpeg::to_thumbnail; + use sd_ffmpeg::{to_thumbnail, ThumbnailSize}; - to_thumbnail(file_path, output_path, 256, TARGET_QUALITY) - .await - .map_err(Into::into) + to_thumbnail( + file_path, + output_path, + ThumbnailSize::Scale(256), + TARGET_QUALITY, + ) + .await + .map_err(Into::into) } diff --git a/crates/ffmpeg/Cargo.toml b/crates/ffmpeg/Cargo.toml index 3b8a77abdef6..e4949b9e6a54 100644 --- a/crates/ffmpeg/Cargo.toml +++ b/crates/ffmpeg/Cargo.toml @@ -11,13 +11,15 @@ edition = { workspace = true } [dependencies] chrono = { workspace = true } -ffmpeg-sys-next = "6.0.1" +image = { workspace = true } libc = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["fs", "rt"] } tracing = { workspace = true } webp = { workspace = true } +ffmpeg-sys-next = "6.0.1" + [dev-dependencies] tempfile = { workspace = true } tokio = { workspace = true, features = ["fs", "rt", "macros"] } diff --git a/crates/ffmpeg/src/filter_graph.rs b/crates/ffmpeg/src/filter_graph.rs index ab2941b7519d..3525abc9704a 100644 --- a/crates/ffmpeg/src/filter_graph.rs +++ b/crates/ffmpeg/src/filter_graph.rs @@ -12,7 +12,6 @@ use ffmpeg_sys_next::{ avfilter_graph_create_filter, avfilter_graph_free, avfilter_link, AVFilterContext, AVFilterGraph, AVRational, }; - pub(crate) struct FFmpegFilterGraph(*mut AVFilterGraph); impl<'a> FFmpegFilterGraph { @@ -39,7 +38,6 @@ impl<'a> FFmpegFilterGraph { size: Option, time_base: &AVRational, codec_ctx: &FFmpegCodecContext, - rotation_angle: f64, interlaced_frame: bool, pixel_aspect_ratio: &AVRational, maintain_aspect_ratio: bool, @@ -115,55 +113,14 @@ impl<'a> FFmpegFilterGraph { "Failed to create format filter", )?; - let mut rotate_filter = ptr::null_mut(); - if rotation_angle < -135.0 { - filter_graph.setup_filter( - &mut rotate_filter, - c"rotate", - c"thumb_rotate", - Some(c"PI"), - "Failed to create rotate filter", - )?; - } else if rotation_angle > 45.0 && rotation_angle < 135.0 { - filter_graph.setup_filter( - &mut rotate_filter, - c"transpose", - c"thumb_transpose", - Some(c"2"), - "Failed to create transpose filter", - )?; - } else if rotation_angle < -45.0 && rotation_angle > -135.0 { - filter_graph.setup_filter( - &mut rotate_filter, - c"transpose", - c"thumb_transpose", - Some(c"1"), - "Failed to create transpose filter", - )?; - } - Self::link( - if rotate_filter.is_null() { - format_filter - } else { - rotate_filter - }, + format_filter, 0, filter_sink_ctx, 0, "Failed to link final filter", )?; - if !rotate_filter.is_null() { - Self::link( - format_filter, - 0, - rotate_filter, - 0, - "Failed to link format filter", - )?; - } - Self::link( scale_filter, 0, @@ -253,7 +210,7 @@ fn thumb_scale_filter_args( ) -> Result { let (width, height) = match size { Some(ThumbnailSize::Dimensions { width, height }) => (width, Some(height)), - Some(ThumbnailSize::Size(width)) => (width, None), + Some(ThumbnailSize::Scale(width)) => (width, None), None => return Ok("w=0:h=0".to_string()), }; diff --git a/crates/ffmpeg/src/format_ctx.rs b/crates/ffmpeg/src/format_ctx.rs index 12e2704dd728..36ea484b72a2 100644 --- a/crates/ffmpeg/src/format_ctx.rs +++ b/crates/ffmpeg/src/format_ctx.rs @@ -8,9 +8,9 @@ use crate::{ use ffmpeg_sys_next::{ av_cmp_q, av_display_rotation_get, av_read_frame, av_reduce, av_stream_get_side_data, - avformat_close_input, avformat_find_stream_info, avformat_open_input, AVCodecID, AVDictionary, - AVFormatContext, AVMediaType, AVPacket, AVPacketSideDataType, AVRational, AVStream, - AV_DISPOSITION_ATTACHED_PIC, AV_DISPOSITION_CAPTIONS, AV_DISPOSITION_CLEAN_EFFECTS, + avformat_close_input, avformat_find_stream_info, avformat_open_input, AVChapter, AVCodecID, + AVDictionary, AVFormatContext, AVMediaType, AVPacket, AVPacketSideDataType, AVRational, + AVStream, AV_DISPOSITION_ATTACHED_PIC, AV_DISPOSITION_CAPTIONS, AV_DISPOSITION_CLEAN_EFFECTS, AV_DISPOSITION_COMMENT, AV_DISPOSITION_DEFAULT, AV_DISPOSITION_DEPENDENT, AV_DISPOSITION_DESCRIPTIONS, AV_DISPOSITION_DUB, AV_DISPOSITION_FORCED, AV_DISPOSITION_HEARING_IMPAIRED, AV_DISPOSITION_KARAOKE, AV_DISPOSITION_LYRICS, @@ -116,22 +116,22 @@ impl FFmpegFormatContext { } } - pub(crate) fn read_frame(&mut self, packet: *mut AVPacket) -> Result<(), Error> { + pub(crate) fn read_frame(&mut self, packet: *mut AVPacket) -> Result<&mut Self, Error> { check_error( unsafe { av_read_frame(self.as_mut(), packet) }, "Fail to read the next frame of a media file", )?; - Ok(()) + Ok(self) } - pub(crate) fn find_stream_info(&mut self) -> Result<(), Error> { + pub(crate) fn find_stream_info(&mut self) -> Result<&mut Self, Error> { check_error( unsafe { avformat_find_stream_info(self.as_mut(), ptr::null_mut()) }, "Fail to read packets of a media file to get stream information", )?; - Ok(()) + Ok(self) } pub(crate) fn find_preferred_video_stream( @@ -232,30 +232,12 @@ impl FFmpegFormatContext { fn chapters(&self) -> Vec { let chapters_ptr = self.as_ref().chapters; - - let Ok(num_chapters) = isize::try_from(self.as_ref().nb_chapters) else { - return vec![]; - }; - (!chapters_ptr.is_null()) .then(|| { - (0..num_chapters) - .filter_map(|id| { - unsafe { (*(chapters_ptr.offset(id))).as_ref() }.map(|chapter| { - MediaChapter { - // Note: id is guaranteed to be a valid u32 because it was calculated from a u32 - id: id as u32, - start: chapter.start, - end: chapter.end, - time_base_num: chapter.time_base.num, - time_base_den: chapter.time_base.den, - metadata: unsafe { chapter.metadata.as_mut() } - .map(|metadata| FFmpegDict::new(Some(metadata)).into()) - .unwrap_or_else(MediaMetadata::default), - } - }) - }) - .collect::>() + (0..isize::try_from(self.as_ref().nb_chapters).unwrap_or(0)) + .filter_map(|id| unsafe { (*(chapters_ptr.offset(id))).as_ref() }) + .map(|chapter| chapter.into()) + .collect() }) .unwrap_or(vec![]) } @@ -266,37 +248,29 @@ impl FFmpegFormatContext { let mut programs = (!programs_ptr.is_null()) .then(|| { - let Ok(num_programs) = isize::try_from(self.as_ref().nb_programs) else { - return vec![]; - }; - - (0..num_programs) - .filter_map(|id| { - unsafe { (*(programs_ptr.offset(id))).as_ref() }.map(|program| { - let (metadata, name) = - extract_name_and_convert_metadata(program.metadata); - - let streams = (0..num_programs) - .filter_map(|index| { - unsafe { program.stream_index.offset(index).as_ref() }.and_then( - |stream_index| { - self.stream(*stream_index).map(|stream| { - visited_streams.insert(*stream_index); - (&*stream).into() - }) - }, - ) - }) - .collect::>(); - - MediaProgram { - // Note: id is guaranteed to be a valid u32 because it was calculated from a u32 - id: id as u32, - name, - streams, - metadata, - } - }) + (0..isize::try_from(self.as_ref().nb_programs).unwrap_or(0)) + .filter_map(|id| unsafe { (*(programs_ptr.offset(id))).as_ref() }) + .map(|program| { + let (metadata, name) = extract_name_and_convert_metadata(program.metadata); + + let streams = (0..isize::try_from(program.nb_stream_indexes).unwrap_or(0)) + .filter_map(|index| unsafe { + program.stream_index.offset(index).as_ref() + }) + .copied() + .filter_map(|stream_index| { + visited_streams.insert(stream_index); + self.stream(stream_index) + }) + .map(|stream| (&*stream).into()) + .collect::>(); + + MediaProgram { + id: program.id.unsigned_abs(), + name, + streams, + metadata, + } }) .collect::>() }) @@ -322,8 +296,8 @@ impl FFmpegFormatContext { } fn metadata(&self) -> Option { - unsafe { (*(self.0)).metadata.as_mut() } - .map(|metadata| FFmpegDict::new(Some(metadata)).into()) + let fmt_ctx = self.as_ref(); + unsafe { fmt_ctx.metadata.as_mut() }.map(|metadata| FFmpegDict::new(Some(metadata)).into()) } } @@ -350,6 +324,21 @@ impl From<&FFmpegFormatContext> for MediaInfo { } } +impl From<&AVChapter> for MediaChapter { + fn from(chapter: &AVChapter) -> Self { + MediaChapter { + id: chapter.id.unsigned_abs(), + start: chapter.start, + end: chapter.end, + time_base_num: chapter.time_base.num, + time_base_den: chapter.time_base.den, + metadata: unsafe { chapter.metadata.as_mut() } + .map(|metadata| FFmpegDict::new(Some(metadata)).into()) + .unwrap_or_else(MediaMetadata::default), + } + } +} + impl From<&AVStream> for MediaStream { fn from(stream: &AVStream) -> Self { let (metadata, name) = extract_name_and_convert_metadata(stream.metadata); diff --git a/crates/ffmpeg/src/frame_decoder.rs b/crates/ffmpeg/src/frame_decoder.rs index 636c6f6bf798..d67bd57e0570 100644 --- a/crates/ffmpeg/src/frame_decoder.rs +++ b/crates/ffmpeg/src/frame_decoder.rs @@ -3,7 +3,6 @@ use crate::{ error::{Error, FFmpegError}, filter_graph::FFmpegFilterGraph, format_ctx::FFmpegFormatContext, - probe::probe, utils::{check_error, from_path}, video_frame::FFmpegFrame, }; @@ -20,8 +19,8 @@ use ffmpeg_sys_next::{ #[derive(Debug, Clone, Copy)] pub enum ThumbnailSize { + Scale(u32), Dimensions { width: u32, height: u32 }, - Size(u32), } #[derive(Debug)] @@ -29,6 +28,7 @@ pub(crate) struct VideoFrame { pub data: Vec, pub width: u32, pub height: u32, + pub rotation: f64, } pub struct FrameDecoder { @@ -49,9 +49,6 @@ impl FrameDecoder { ) -> Result { let filename = filename.as_ref(); - // TODO: Remove this, just here to test and so clippy stops complaining about it being unused - let _ = probe(filename); - let mut format_context = FFmpegFormatContext::open_file(from_path(filename)?.as_c_str())?; format_context.find_stream_info()?; @@ -163,16 +160,10 @@ impl FrameDecoder { av_guess_sample_aspect_ratio(self.format_ctx.as_mut(), stream_ptr, self.frame.as_mut()) }; - let rotation_angle = self - .format_ctx - .get_stream_rotation_angle(self.preferred_stream_id) - .round(); - let (_guard, filter_source, filter_sink) = FFmpegFilterGraph::thumbnail_graph( size, &time_base, &self.codec_ctx, - rotation_angle, (self.frame.as_mut().flags & AV_FRAME_FLAG_INTERLACED) != 0, &pixel_aspect_ratio, maintain_aspect_ratio, @@ -195,17 +186,23 @@ impl FrameDecoder { } check_error(get_frame_errno, "Failed to get buffer from filter")?; - let height = new_frame.as_ref().height; - let line_size = new_frame.as_ref().linesize[0]; - let mut data = Vec::with_capacity(usize::try_from(line_size * height)?); + let width = new_frame.as_ref().width.unsigned_abs(); + let height = new_frame.as_ref().height.unsigned_abs(); + let line_size = usize::try_from(new_frame.as_ref().linesize[0])?; + + let mut data = Vec::with_capacity(line_size * usize::try_from(height)?); data.extend_from_slice(unsafe { std::slice::from_raw_parts(new_frame.as_ref().data[0], data.capacity()) }); Ok(VideoFrame { - height: u32::try_from(height)?, - width: new_frame.as_ref().width.try_into()?, data, + width, + height, + rotation: self + .format_ctx + .get_stream_rotation_angle(self.preferred_stream_id) + .round(), }) } diff --git a/crates/ffmpeg/src/lib.rs b/crates/ffmpeg/src/lib.rs index 5ba6b9a72bb2..f5200c89c770 100644 --- a/crates/ffmpeg/src/lib.rs +++ b/crates/ffmpeg/src/lib.rs @@ -1,7 +1,12 @@ -use crate::frame_decoder::{FrameDecoder, ThumbnailSize}; +use crate::{ + format_ctx::FFmpegFormatContext, frame_decoder::FrameDecoder, model::MediaInfo, + utils::from_path, +}; use std::path::Path; +use ffmpeg_sys_next::{av_log_set_level, AV_LOG_FATAL}; + mod codec_ctx; mod dict; mod error; @@ -9,21 +14,49 @@ mod filter_graph; mod format_ctx; mod frame_decoder; mod model; -mod probe; mod thumbnailer; mod utils; mod video_frame; pub use error::Error; -pub use thumbnailer::{Thumbnailer, ThumbnailerBuilder}; +pub use frame_decoder::ThumbnailSize; +pub use thumbnailer::ThumbnailerBuilder; + +/// Helper function to generate retrieve media data from from a video/audio file +pub fn probe(filename: impl AsRef) -> Result { + let filename = filename.as_ref(); + + // Reduce the amount of logs generated by FFmpeg + unsafe { av_log_set_level(AV_LOG_FATAL) }; + + // Dictionary to store format options + // let mut format_opts = FFmpegDict::new(None); + // Some MPEGTS specific option (copied from ffprobe) + // let scan_all_pmts = c"scan_all_pmts"; + // format_opts.set(scan_all_pmts, c"1")?; + + // Open an input stream, read the header and allocate the format context + let mut fmt_ctx = FFmpegFormatContext::open_file(from_path(filename)?.as_c_str())?; + + // // Reset MPEGTS specific option + // format_opts.remove(scan_all_pmts)?; + + // Read packets of media file to get stream information. + fmt_ctx.find_stream_info()?; + + Ok((&fmt_ctx).into()) +} /// Helper function to generate a thumbnail file from a video file with reasonable defaults pub async fn to_thumbnail( video_file_path: impl AsRef, output_thumbnail_path: impl AsRef, - size: u32, + size: ThumbnailSize, quality: f32, ) -> Result<(), Error> { + // Reduce the amount of logs generated by FFmpeg + unsafe { av_log_set_level(AV_LOG_FATAL) }; + ThumbnailerBuilder::new() .size(size) .quality(quality)? @@ -79,7 +112,7 @@ mod tests { ]; for (input, output) in video_file_path.iter().zip(actual_webp_files.iter()) { - if let Err(e) = to_thumbnail(input, output, 128, 100.0).await { + if let Err(e) = to_thumbnail(input, output, ThumbnailSize::Scale(128), 100.0).await { eprintln!("Error: {e}; Input: {}", input.display()); panic!("{}", e); } diff --git a/crates/ffmpeg/src/model.rs b/crates/ffmpeg/src/model.rs index cc7879189710..196f03b17e77 100644 --- a/crates/ffmpeg/src/model.rs +++ b/crates/ffmpeg/src/model.rs @@ -28,7 +28,7 @@ pub struct MediaMetadata { } pub struct MediaChapter { - pub id: u32, + pub id: u64, pub start: i64, pub end: i64, pub time_base_den: i32, diff --git a/crates/ffmpeg/src/probe.rs b/crates/ffmpeg/src/probe.rs deleted file mode 100644 index 48219c937d43..000000000000 --- a/crates/ffmpeg/src/probe.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::{error::Error, format_ctx::FFmpegFormatContext, model::MediaInfo, utils::from_path}; - -use ffmpeg_sys_next::{av_log_set_level, AV_LOG_FATAL}; - -use std::path::Path; - -pub fn probe(filename: impl AsRef) -> Result { - let filename = filename.as_ref(); - - // Reduce the amount of logs generated by FFmpeg - unsafe { av_log_set_level(AV_LOG_FATAL) }; - - // Dictionary to store format options - // let mut format_opts = FFmpegDict::new(None); - // Some MPEGTS specific option (copied from ffprobe) - // let scan_all_pmts = c"scan_all_pmts"; - // format_opts.set(scan_all_pmts, c"1")?; - - // Open an input stream, read the header and allocate the format context - let mut fmt_ctx = FFmpegFormatContext::open_file(from_path(filename)?.as_c_str())?; - - // // Reset MPEGTS specific option - // format_opts.remove(scan_all_pmts)?; - - // Read packets of media file to get stream information. - fmt_ctx.find_stream_info()?; - - Ok((&fmt_ctx).into()) -} diff --git a/crates/ffmpeg/src/thumbnailer.rs b/crates/ffmpeg/src/thumbnailer.rs index 2cd9b8d7d2cf..520cc9fc84db 100644 --- a/crates/ffmpeg/src/thumbnailer.rs +++ b/crates/ffmpeg/src/thumbnailer.rs @@ -1,7 +1,8 @@ -use crate::{Error, FrameDecoder, ThumbnailSize}; +use crate::{frame_decoder::ThumbnailSize, Error, FrameDecoder}; use std::{io, ops::Deref, path::Path}; +use image::{DynamicImage, RgbImage}; use tokio::{fs, task::spawn_blocking}; use tracing::error; use webp::Encoder; @@ -15,7 +16,7 @@ pub struct Thumbnailer { impl Thumbnailer { /// Processes an video input file and write to file system a thumbnail with webp format - pub async fn process( + pub(crate) async fn process( &self, video_file_path: impl AsRef, output_thumbnail_path: impl AsRef, @@ -38,7 +39,7 @@ impl Thumbnailer { } /// Processes an video input file and returns a webp encoded thumbnail as bytes - pub async fn process_to_webp_bytes( + async fn process_to_webp_bytes( &self, video_file_path: impl AsRef, ) -> Result, Error> { @@ -50,9 +51,13 @@ impl Thumbnailer { let quality = self.builder.quality; spawn_blocking(move || -> Result, Error> { - // TODO: Allow_seek should be false, for remote files - let mut decoder = - FrameDecoder::new(video_file_path.clone(), prefer_embedded_metadata, true)?; + let mut decoder = FrameDecoder::new( + video_file_path.clone(), + // TODO: allow_seek should be false for remote files + true, + prefer_embedded_metadata, + )?; + // We actually have to decode a frame to get some metadata before we can start decoding for real decoder.decode_video_frame()?; @@ -62,30 +67,50 @@ impl Thumbnailer { .ok_or(Error::NoVideoDuration) .and_then(|duration| { decoder.seek( + // This conversion are ok because we don't worry much about precision here (duration.num_seconds() as f64 * f64::from(seek_percentage)).round() as i64, ) }); if let Err(err) = result { - error!("Failed to seek: {err:#?}"); - // seeking failed, try the first frame again - decoder = FrameDecoder::new(video_file_path, prefer_embedded_metadata, false)?; + error!( + "Failed to seek {}: {err:#?}", + video_file_path.to_string_lossy() + ); + // Seeking failed, try first frame again + // Reinstantiating decoder to avoid possible segfault + // https://github.com/dirkvdb/ffmpegthumbnailer/commit/da292ccb51a526ebc833f851a388ca308d747289 + decoder = FrameDecoder::new(video_file_path, false, prefer_embedded_metadata)?; decoder.decode_video_frame()?; } } let video_frame = decoder.get_scaled_video_frame(Some(size), maintain_aspect_ratio)?; + let image = DynamicImage::ImageRgb8( + RgbImage::from_raw(video_frame.width, video_frame.height, video_frame.data) + .ok_or(Error::CorruptVideo)?, + ); + + let image = if video_frame.rotation < -135.0 { + image.rotate180() + } else if video_frame.rotation > 45.0 && video_frame.rotation < 135.0 { + image.rotate270() + } else if video_frame.rotation < -45.0 && video_frame.rotation > -135.0 { + image.rotate90() + } else { + image + }; + // Type WebPMemory is !Send, which makes the Future in this function !Send, // this make us `deref` to have a `&[u8]` and then `to_owned` to make a Vec // which implies on a unwanted clone... - Ok( - Encoder::from_rgb(&video_frame.data, video_frame.width, video_frame.height) - .encode(quality) - .deref() - .to_vec(), - ) + Ok(Encoder::from_image(&image) + .expect("Should not fail as the underlining DynamicImage is an RgbImage") + .encode(quality) + .deref() + .to_vec()) }) .await? } @@ -107,7 +132,7 @@ impl Default for ThumbnailerBuilder { fn default() -> Self { Self { maintain_aspect_ratio: true, - size: ThumbnailSize::Size(128), + size: ThumbnailSize::Scale(128), seek_percentage: 0.1, quality: 80.0, prefer_embedded_metadata: true, @@ -133,14 +158,8 @@ impl ThumbnailerBuilder { } /// To set a thumbnail size, respecting or not its aspect ratio, according to `maintain_aspect_ratio` value - pub const fn size(mut self, size: u32) -> Self { - self.size = ThumbnailSize::Size(size); - self - } - - /// To specify width and height of the thumbnail - pub const fn width_and_height(mut self, width: u32, height: u32) -> Self { - self.size = ThumbnailSize::Dimensions { width, height }; + pub const fn size(mut self, size: ThumbnailSize) -> Self { + self.size = size; self }