diff --git a/.typos.toml b/.typos.toml index bf6fb7aa..9d361b20 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,6 +1,7 @@ [default.extend-words] gir = "gir" inout = "inout" +numer = "numer" # Short for numerator in GStreamer [type.po] extend-glob = ["*.po"] diff --git a/Cargo.lock b/Cargo.lock index c72c7de1..24905bdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1531,6 +1531,7 @@ dependencies = [ "glycin", "gst-plugin-gtk4", "gstreamer", + "gstreamer-app", "gstreamer-base", "gstreamer-pbutils", "gstreamer-play", @@ -2165,6 +2166,34 @@ dependencies = [ "thiserror", ] +[[package]] +name = "gstreamer-app" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c86915cc4cdfa030532301a46c725e0ce0c6c2b57a68c44ce9b34db587e552" +dependencies = [ + "futures-core", + "futures-sink", + "glib", + "gstreamer", + "gstreamer-app-sys", + "gstreamer-base", + "libc", +] + +[[package]] +name = "gstreamer-app-sys" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37066c1b93ba57aa070ebc1e0a564bc1a9adda78fb0850e624861fad46fd1448" +dependencies = [ + "glib-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps 7.0.1", +] + [[package]] name = "gstreamer-audio" version = "0.23.0" diff --git a/Cargo.toml b/Cargo.toml index 300d5b3e..46c2cee3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ webp = "0.3" adw = { package = "libadwaita", version = "0.7", features = ["v1_5"] } glycin = { version = "2.0.0-beta", default-features = false, features = ["tokio", "gdk4"] } gst = { version = "0.23", package = "gstreamer" } +gst_app = { version = "0.23", package = "gstreamer-app" } gst_base = { version = "0.23", package = "gstreamer-base" } gst_gtk = { version = "0.13", package = "gst-plugin-gtk4" } gst_pbutils = { version = "0.23", package = "gstreamer-pbutils" } diff --git a/meson.build b/meson.build index 995af893..1440d4ad 100644 --- a/meson.build +++ b/meson.build @@ -30,6 +30,7 @@ dependency( # Please keep these dependencies sorted. dependency('gstreamer-1.0', version: '>= 1.20') +dependency('gstreamer-app-1.0', version: '>= 1.20') dependency('gstreamer-base-1.0', version: '>= 1.20') dependency('gstreamer-pbutils-1.0', version: '>= 1.20') dependency('gstreamer-play-1.0', version: '>= 1.20') diff --git a/src/session/view/content/room_history/message_toolbar/mod.rs b/src/session/view/content/room_history/message_toolbar/mod.rs index c1e34e12..07096af3 100644 --- a/src/session/view/content/room_history/message_toolbar/mod.rs +++ b/src/session/view/content/room_history/message_toolbar/mod.rs @@ -41,7 +41,8 @@ use crate::{ utils::{ matrix::AT_ROOM, media::{ - filename_for_mime, image::ImageInfoLoader, load_audio_info, load_file, load_video_info, + filename_for_mime, image::ImageInfoLoader, load_audio_info, load_file, + video::load_video_info, }, template_callbacks::TemplateCallbacks, Location, LocationError, TokioDrop, @@ -912,9 +913,9 @@ impl MessageToolbar { (AttachmentInfo::Image(info), thumbnail) } mime::VIDEO => { - let mut info = load_video_info(&file).await; + let (mut info, thumbnail) = load_video_info(&file).await; info.size = size; - (AttachmentInfo::Video(info), None) + (AttachmentInfo::Video(info), thumbnail) } mime::AUDIO => { let mut info = load_audio_info(&file).await; diff --git a/src/utils/media/image.rs b/src/utils/media/image.rs index be79ceb7..fb3b82ec 100644 --- a/src/utils/media/image.rs +++ b/src/utils/media/image.rs @@ -234,38 +234,7 @@ impl Frame { let image = DynamicImage::from_decoder(self).ok()?; let thumbnail = image.thumbnail(THUMBNAIL_DEFAULT_WIDTH, THUMBNAIL_DEFAULT_HEIGHT); - // Convert to RGB8/RGBA8 since it's the only format supported by webp. - let thumbnail: DynamicImage = match &thumbnail { - DynamicImage::ImageLuma8(_) - | DynamicImage::ImageRgb8(_) - | DynamicImage::ImageLuma16(_) - | DynamicImage::ImageRgb16(_) - | DynamicImage::ImageRgb32F(_) => thumbnail.into_rgb8().into(), - DynamicImage::ImageLumaA8(_) - | DynamicImage::ImageRgba8(_) - | DynamicImage::ImageLumaA16(_) - | DynamicImage::ImageRgba16(_) - | DynamicImage::ImageRgba32F(_) => thumbnail.into_rgba8().into(), - _ => return None, - }; - - let encoder = webp::Encoder::from_image(&thumbnail).ok()?; - let thumbnail_bytes = encoder.encode(WEBP_DEFAULT_QUALITY).to_vec(); - - let thumbnail_content_type = mime::Mime::from_str(WEBP_CONTENT_TYPE) - .expect("image should provide a valid content type"); - - let thumbnail_info = BaseThumbnailInfo { - width: Some(thumbnail.width().into()), - height: Some(thumbnail.height().into()), - size: thumbnail_bytes.len().try_into().ok(), - }; - - Some(Thumbnail { - data: thumbnail_bytes, - content_type: thumbnail_content_type, - info: Some(thumbnail_info), - }) + prepare_thumbnail_for_sending(thumbnail) } } @@ -336,6 +305,66 @@ struct ImageDimensions { height: Option, } +/// Compute the dimensions of the thumbnail while preserving the aspect ratio of +/// the image. +/// +/// Returns `None` if the dimensions are smaller than the wanted dimensions. +pub(super) fn thumbnail_dimensions(width: u32, height: u32) -> Option<(u32, u32)> { + if width <= (THUMBNAIL_DEFAULT_WIDTH + THUMBNAIL_DIMENSIONS_THRESHOLD) + && height <= (THUMBNAIL_DEFAULT_HEIGHT + THUMBNAIL_DIMENSIONS_THRESHOLD) + { + return None; + } + + let w_ratio = width as f64 / THUMBNAIL_DEFAULT_WIDTH as f64; + let h_ratio = height as f64 / THUMBNAIL_DEFAULT_HEIGHT as f64; + + if w_ratio > h_ratio { + let new_height = height as f64 / w_ratio; + Some((THUMBNAIL_DEFAULT_WIDTH, new_height as u32)) + } else { + let new_width = width as f64 / h_ratio; + Some((new_width as u32, THUMBNAIL_DEFAULT_HEIGHT)) + } +} + +/// Prepare the given thumbnail to send it. +pub(super) fn prepare_thumbnail_for_sending(thumbnail: image::DynamicImage) -> Option { + // Convert to RGB8/RGBA8 since those are the only formats supported by WebP. + let thumbnail: DynamicImage = match &thumbnail { + DynamicImage::ImageLuma8(_) + | DynamicImage::ImageRgb8(_) + | DynamicImage::ImageLuma16(_) + | DynamicImage::ImageRgb16(_) + | DynamicImage::ImageRgb32F(_) => thumbnail.into_rgb8().into(), + DynamicImage::ImageLumaA8(_) + | DynamicImage::ImageRgba8(_) + | DynamicImage::ImageLumaA16(_) + | DynamicImage::ImageRgba16(_) + | DynamicImage::ImageRgba32F(_) => thumbnail.into_rgba8().into(), + _ => return None, + }; + + // Encode to WebP. + let encoder = webp::Encoder::from_image(&thumbnail).ok()?; + let thumbnail_bytes = encoder.encode(WEBP_DEFAULT_QUALITY).to_vec(); + + let thumbnail_content_type = + mime::Mime::from_str(WEBP_CONTENT_TYPE).expect("image should provide a valid content type"); + + let thumbnail_info = BaseThumbnailInfo { + width: Some(thumbnail.width().into()), + height: Some(thumbnail.height().into()), + size: thumbnail_bytes.len().try_into().ok(), + }; + + Some(Thumbnail { + data: thumbnail_bytes, + content_type: thumbnail_content_type, + info: Some(thumbnail_info), + }) +} + impl From for BaseImageInfo { fn from(value: ImageDimensions) -> Self { let ImageDimensions { width, height } = value; diff --git a/src/utils/media/mod.rs b/src/utils/media/mod.rs index 752bb7a3..1c3e7d39 100644 --- a/src/utils/media/mod.rs +++ b/src/utils/media/mod.rs @@ -4,10 +4,11 @@ use std::{cell::Cell, str::FromStr, sync::Mutex}; use gettextrs::gettext; use gtk::{gio, glib, prelude::*}; -use matrix_sdk::attachment::{BaseAudioInfo, BaseVideoInfo}; +use matrix_sdk::attachment::BaseAudioInfo; use mime::Mime; pub mod image; +pub mod video; /// Get a default filename for a mime type. /// @@ -118,34 +119,6 @@ async fn load_gstreamer_media_info(file: &gio::File) -> Option BaseVideoInfo { - let mut info = BaseVideoInfo { - duration: None, - width: None, - height: None, - size: None, - blurhash: None, - }; - - let Some(media_info) = load_gstreamer_media_info(file).await else { - return info; - }; - - info.duration = media_info.duration().map(Into::into); - - if let Some(stream_info) = media_info - .video_streams() - .first() - .and_then(|s| s.downcast_ref::()) - { - info.width = Some(stream_info.width().into()); - info.height = Some(stream_info.height().into()); - } - - info -} - /// Load information for the audio in the given file. pub async fn load_audio_info(file: &gio::File) -> BaseAudioInfo { let mut info = BaseAudioInfo { diff --git a/src/utils/media/video.rs b/src/utils/media/video.rs new file mode 100644 index 00000000..fd2b80a7 --- /dev/null +++ b/src/utils/media/video.rs @@ -0,0 +1,262 @@ +//! Collection of methods for videos. + +use std::sync::{Arc, Mutex}; + +use futures_channel::oneshot; +use gst::prelude::*; +use gst_video::prelude::*; +use gtk::{gio, glib, glib::clone, prelude::*}; +use image::GenericImageView; +use matrix_sdk::attachment::{BaseVideoInfo, Thumbnail}; +use tracing::warn; + +use super::{ + image::{prepare_thumbnail_for_sending, thumbnail_dimensions}, + load_gstreamer_media_info, +}; + +/// A channel sender to send the result of a video thumbnail. +type ThumbnailResultSender = oneshot::Sender>; + +/// Load information and try to generate a thumbnail for the video in the given +/// file. +pub async fn load_video_info(file: &gio::File) -> (BaseVideoInfo, Option) { + let mut info = BaseVideoInfo { + duration: None, + width: None, + height: None, + size: None, + blurhash: None, + }; + + let Some(media_info) = load_gstreamer_media_info(file).await else { + return (info, None); + }; + + info.duration = media_info.duration().map(Into::into); + + if let Some(stream_info) = media_info + .video_streams() + .first() + .and_then(|s| s.downcast_ref::()) + { + info.width = Some(stream_info.width().into()); + info.height = Some(stream_info.height().into()); + } + + let thumbnail = generate_video_thumbnail(file).await; + + (info, thumbnail) +} + +/// Generate a thumbnail for the video in the given file. +async fn generate_video_thumbnail(file: &gio::File) -> Option { + let (sender, receiver) = oneshot::channel(); + let sender = Arc::new(Mutex::new(Some(sender))); + + let pipeline = match create_thumbnailer_pipeline(&file.uri(), sender.clone()) { + Ok(pipeline) => pipeline, + Err(error) => { + warn!("Could not create pipeline for video thumbnail: {error}"); + return None; + } + }; + + if pipeline.set_state(gst::State::Paused).is_err() { + warn!("Could not initialize pipeline for video thumbnail"); + return None; + } + + let bus = pipeline.bus().expect("Pipeline has a bus"); + + let mut started = false; + let _bus_guard = bus + .add_watch(clone!( + #[weak] + pipeline, + #[upgrade_or] + glib::ControlFlow::Break, + move |_, message| { + match message.view() { + gst::MessageView::AsyncDone(_) => { + if !started { + // AsyncDone means that the pipeline has started now. + if pipeline.set_state(gst::State::Playing).is_err() { + warn!("Could not start pipeline for video thumbnail"); + send_video_thumbnail_result(&sender, Err(())); + + return glib::ControlFlow::Break; + }; + + started = true; + } + + glib::ControlFlow::Continue + } + gst::MessageView::Eos(_) => { + // We have the thumbnail or we cannot have one. + glib::ControlFlow::Break + } + gst::MessageView::Error(error) => { + warn!("Could not generate video thumbnail: {error}"); + send_video_thumbnail_result(&sender, Err(())); + + glib::ControlFlow::Break + } + _ => glib::ControlFlow::Continue, + } + } + )) + .expect("Setting bus watch succeeds"); + + let thumbnail = receiver.await; + + // Clean up. + let _ = pipeline.set_state(gst::State::Null); + bus.set_flushing(true); + + thumbnail.ok().transpose().ok().flatten() +} + +/// Create a GStreamer pipeline to get a thumbnail of the first frame. +fn create_thumbnailer_pipeline( + uri: &str, + sender: Arc>>, +) -> Result { + // Create our pipeline from a pipeline description string. + let pipeline = gst::parse::launch(&format!( + "uridecodebin uri={uri} ! videoconvert ! appsink name=sink" + ))? + .downcast::() + .expect("Element is a pipeline"); + + let appsink = pipeline + .by_name("sink") + .expect("Sink element is in the pipeline") + .downcast::() + .expect("Sink element is an appsink"); + + // Don't synchronize on the clock, we only want a snapshot asap. + appsink.set_property("sync", false); + + // Tell the appsink what format we want, for simplicity we only accept 8-bit + // RGB. + appsink.set_caps(Some( + &gst_video::VideoCapsBuilder::new() + .format(gst_video::VideoFormat::Rgbx) + .build(), + )); + + let mut got_snapshot = false; + + // Listen to callbacks to get the data. + appsink.set_callbacks( + gst_app::AppSinkCallbacks::builder() + .new_sample(move |appsink| { + // Pull the sample out of the buffer. + let sample = appsink.pull_sample().map_err(|_| gst::FlowError::Eos)?; + let Some(buffer) = sample.buffer() else { + warn!("Could not get buffer from appsink"); + send_video_thumbnail_result(&sender, Err(())); + + return Err(gst::FlowError::Error); + }; + + // Make sure that we only get a single buffer. + if got_snapshot { + return Err(gst::FlowError::Eos); + } + got_snapshot = true; + + let Some(caps) = sample.caps() else { + warn!("Got video sample without caps"); + send_video_thumbnail_result(&sender, Err(())); + + return Err(gst::FlowError::Error); + }; + let Ok(info) = gst_video::VideoInfo::from_caps(caps) else { + warn!("Could not parse video caps"); + send_video_thumbnail_result(&sender, Err(())); + + return Err(gst::FlowError::Error); + }; + + let frame = gst_video::VideoFrameRef::from_buffer_ref_readable(buffer, &info) + .map_err(|_| { + warn!("Could not map video buffer readable"); + send_video_thumbnail_result(&sender, Err(())); + + gst::FlowError::Error + })?; + + // Create a FlatSamples around the borrowed video frame data from GStreamer with + // the correct stride. + let img = image::FlatSamples::<&[u8]> { + samples: frame.plane_data(0).unwrap(), + layout: image::flat::SampleLayout { + channels: 3, // RGB + channel_stride: 1, // 1 byte from component to component + width: frame.width(), + width_stride: 4, // 4 bytes from pixel to pixel + height: frame.height(), + height_stride: frame.plane_stride()[0] as usize, // stride from line to line + }, + color_hint: Some(image::ColorType::Rgb8), + }; + + let Ok(view) = img.as_view::>() else { + warn!("Could not parse frame as view"); + send_video_thumbnail_result(&sender, Err(())); + + return Err(gst::FlowError::Error); + }; + + // Reduce the dimensions if the thumbnail is bigger than the wanted size. + let thumbnail = if let Some((target_width, target_height)) = thumbnail_dimensions( + frame.width() * info.par().numer() as u32, + frame.height() * info.par().denom() as u32, + ) { + image::imageops::thumbnail(&view, target_width, target_height) + } else { + image::ImageBuffer::from_fn(view.width(), view.height(), |x, y| { + view.get_pixel(x, y) + }) + }; + + // Prepare it. + if let Some(thumbnail) = prepare_thumbnail_for_sending(thumbnail.into()) { + send_video_thumbnail_result(&sender, Ok(thumbnail)); + + Err(gst::FlowError::Eos) + } else { + warn!("Failed to convert video thumbnail"); + send_video_thumbnail_result(&sender, Err(())); + + Err(gst::FlowError::Error) + } + }) + .build(), + ); + + Ok(pipeline) +} + +/// Try to send the given video thumbnail result through the given sender. +fn send_video_thumbnail_result( + sender: &Mutex>, + result: Result, +) { + let mut sender = match sender.lock() { + Ok(sender) => sender, + Err(error) => { + warn!("Failed to lock video thumbnail mutex: {error}"); + return; + } + }; + + if let Some(sender) = sender.take() { + if sender.send(result).is_err() { + warn!("Failed to send video thumbnail result through channel"); + } + } +}