utils: Generate thumbnails for videos

.typos.toml

@@ -1,6 +1,7 @@
 [default.extend-words]
 gir = "gir"
 inout = "inout"
+numer = "numer" # Short for numerator in GStreamer
 
 [type.po]
 extend-glob = ["*.po"]

Cargo.lock

@@ -1531,6 +1531,7 @@ dependencies = [
  "glycin",
  "gst-plugin-gtk4",
  "gstreamer",
+ "gstreamer-app",
  "gstreamer-base",
  "gstreamer-pbutils",
  "gstreamer-play",
@@ -2165,6 +2166,34 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "gstreamer-app"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0c86915cc4cdfa030532301a46c725e0ce0c6c2b57a68c44ce9b34db587e552"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "glib",
+ "gstreamer",
+ "gstreamer-app-sys",
+ "gstreamer-base",
+ "libc",
+]
+
+[[package]]
+name = "gstreamer-app-sys"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37066c1b93ba57aa070ebc1e0a564bc1a9adda78fb0850e624861fad46fd1448"
+dependencies = [
+ "glib-sys",
+ "gstreamer-base-sys",
+ "gstreamer-sys",
+ "libc",
+ "system-deps 7.0.1",
+]
+
 [[package]]
 name = "gstreamer-audio"
 version = "0.23.0"

Cargo.toml

@@ -55,6 +55,7 @@ webp = "0.3"
 adw = { package = "libadwaita", version = "0.7", features = ["v1_5"] }
 glycin = { version = "2.0.0-beta", default-features = false, features = ["tokio", "gdk4"] }
 gst = { version = "0.23", package = "gstreamer" }
+gst_app = { version = "0.23", package = "gstreamer-app" }
 gst_base = { version = "0.23", package = "gstreamer-base" }
 gst_gtk = { version = "0.13", package = "gst-plugin-gtk4" }
 gst_pbutils = { version = "0.23", package = "gstreamer-pbutils" }

meson.build

@@ -30,6 +30,7 @@ dependency(
 
 # Please keep these dependencies sorted.
 dependency('gstreamer-1.0', version: '>= 1.20')
+dependency('gstreamer-app-1.0', version: '>= 1.20')
 dependency('gstreamer-base-1.0', version: '>= 1.20')
 dependency('gstreamer-pbutils-1.0', version: '>= 1.20')
 dependency('gstreamer-play-1.0', version: '>= 1.20')

src/session/view/content/room_history/message_toolbar/mod.rs

@@ -41,7 +41,8 @@ use crate::{
     utils::{
         matrix::AT_ROOM,
         media::{
-            filename_for_mime, image::ImageInfoLoader, load_audio_info, load_file, load_video_info,
+            filename_for_mime, image::ImageInfoLoader, load_audio_info, load_file,
+            video::load_video_info,
         },
         template_callbacks::TemplateCallbacks,
         Location, LocationError, TokioDrop,
@@ -912,9 +913,9 @@ impl MessageToolbar {
                 (AttachmentInfo::Image(info), thumbnail)
             }
             mime::VIDEO => {
-                let mut info = load_video_info(&file).await;
+                let (mut info, thumbnail) = load_video_info(&file).await;
                 info.size = size;
-                (AttachmentInfo::Video(info), None)
+                (AttachmentInfo::Video(info), thumbnail)
             }
             mime::AUDIO => {
                 let mut info = load_audio_info(&file).await;

src/utils/media/image.rs

@@ -234,38 +234,7 @@ impl Frame {
         let image = DynamicImage::from_decoder(self).ok()?;
         let thumbnail = image.thumbnail(THUMBNAIL_DEFAULT_WIDTH, THUMBNAIL_DEFAULT_HEIGHT);
 
-        // Convert to RGB8/RGBA8 since it's the only format supported by webp.
+        prepare_thumbnail_for_sending(thumbnail)
-        let thumbnail: DynamicImage = match &thumbnail {
-            DynamicImage::ImageLuma8(_)
-            | DynamicImage::ImageRgb8(_)
-            | DynamicImage::ImageLuma16(_)
-            | DynamicImage::ImageRgb16(_)
-            | DynamicImage::ImageRgb32F(_) => thumbnail.into_rgb8().into(),
-            DynamicImage::ImageLumaA8(_)
-            | DynamicImage::ImageRgba8(_)
-            | DynamicImage::ImageLumaA16(_)
-            | DynamicImage::ImageRgba16(_)
-            | DynamicImage::ImageRgba32F(_) => thumbnail.into_rgba8().into(),
-            _ => return None,
-        };
-
-        let encoder = webp::Encoder::from_image(&thumbnail).ok()?;
-        let thumbnail_bytes = encoder.encode(WEBP_DEFAULT_QUALITY).to_vec();
-
-        let thumbnail_content_type = mime::Mime::from_str(WEBP_CONTENT_TYPE)
-            .expect("image should provide a valid content type");
-
-        let thumbnail_info = BaseThumbnailInfo {
-            width: Some(thumbnail.width().into()),
-            height: Some(thumbnail.height().into()),
-            size: thumbnail_bytes.len().try_into().ok(),
-        };
-
-        Some(Thumbnail {
-            data: thumbnail_bytes,
-            content_type: thumbnail_content_type,
-            info: Some(thumbnail_info),
-        })
     }
 }
 
@@ -336,6 +305,66 @@ struct ImageDimensions {
     height: Option<u32>,
 }
 
+/// Compute the dimensions of the thumbnail while preserving the aspect ratio of
+/// the image.
+///
+/// Returns `None` if the dimensions are smaller than the wanted dimensions.
+pub(super) fn thumbnail_dimensions(width: u32, height: u32) -> Option<(u32, u32)> {
+    if width <= (THUMBNAIL_DEFAULT_WIDTH + THUMBNAIL_DIMENSIONS_THRESHOLD)
+        && height <= (THUMBNAIL_DEFAULT_HEIGHT + THUMBNAIL_DIMENSIONS_THRESHOLD)
+    {
+        return None;
+    }
+
+    let w_ratio = width as f64 / THUMBNAIL_DEFAULT_WIDTH as f64;
+    let h_ratio = height as f64 / THUMBNAIL_DEFAULT_HEIGHT as f64;
+
+    if w_ratio > h_ratio {
+        let new_height = height as f64 / w_ratio;
+        Some((THUMBNAIL_DEFAULT_WIDTH, new_height as u32))
+    } else {
+        let new_width = width as f64 / h_ratio;
+        Some((new_width as u32, THUMBNAIL_DEFAULT_HEIGHT))
+    }
+}
+
+/// Prepare the given thumbnail to send it.
+pub(super) fn prepare_thumbnail_for_sending(thumbnail: image::DynamicImage) -> Option<Thumbnail> {
+    // Convert to RGB8/RGBA8 since those are the only formats supported by WebP.
+    let thumbnail: DynamicImage = match &thumbnail {
+        DynamicImage::ImageLuma8(_)
+        | DynamicImage::ImageRgb8(_)
+        | DynamicImage::ImageLuma16(_)
+        | DynamicImage::ImageRgb16(_)
+        | DynamicImage::ImageRgb32F(_) => thumbnail.into_rgb8().into(),
+        DynamicImage::ImageLumaA8(_)
+        | DynamicImage::ImageRgba8(_)
+        | DynamicImage::ImageLumaA16(_)
+        | DynamicImage::ImageRgba16(_)
+        | DynamicImage::ImageRgba32F(_) => thumbnail.into_rgba8().into(),
+        _ => return None,
+    };
+
+    // Encode to WebP.
+    let encoder = webp::Encoder::from_image(&thumbnail).ok()?;
+    let thumbnail_bytes = encoder.encode(WEBP_DEFAULT_QUALITY).to_vec();
+
+    let thumbnail_content_type =
+        mime::Mime::from_str(WEBP_CONTENT_TYPE).expect("image should provide a valid content type");
+
+    let thumbnail_info = BaseThumbnailInfo {
+        width: Some(thumbnail.width().into()),
+        height: Some(thumbnail.height().into()),
+        size: thumbnail_bytes.len().try_into().ok(),
+    };
+
+    Some(Thumbnail {
+        data: thumbnail_bytes,
+        content_type: thumbnail_content_type,
+        info: Some(thumbnail_info),
+    })
+}
+
 impl From<ImageDimensions> for BaseImageInfo {
     fn from(value: ImageDimensions) -> Self {
         let ImageDimensions { width, height } = value;

src/utils/media/mod.rs

@@ -4,10 +4,11 @@ use std::{cell::Cell, str::FromStr, sync::Mutex};
 
 use gettextrs::gettext;
 use gtk::{gio, glib, prelude::*};
-use matrix_sdk::attachment::{BaseAudioInfo, BaseVideoInfo};
+use matrix_sdk::attachment::BaseAudioInfo;
 use mime::Mime;
 
 pub mod image;
+pub mod video;
 
 /// Get a default filename for a mime type.
 ///
@@ -118,34 +119,6 @@ async fn load_gstreamer_media_info(file: &gio::File) -> Option<gst_pbutils::Disc
     Some(media_info)
 }
 
-/// Load information for the video in the given file.
-pub async fn load_video_info(file: &gio::File) -> BaseVideoInfo {
-    let mut info = BaseVideoInfo {
-        duration: None,
-        width: None,
-        height: None,
-        size: None,
-        blurhash: None,
-    };
-
-    let Some(media_info) = load_gstreamer_media_info(file).await else {
-        return info;
-    };
-
-    info.duration = media_info.duration().map(Into::into);
-
-    if let Some(stream_info) = media_info
-        .video_streams()
-        .first()
-        .and_then(|s| s.downcast_ref::<gst_pbutils::DiscovererVideoInfo>())
-    {
-        info.width = Some(stream_info.width().into());
-        info.height = Some(stream_info.height().into());
-    }
-
-    info
-}
-
 /// Load information for the audio in the given file.
 pub async fn load_audio_info(file: &gio::File) -> BaseAudioInfo {
     let mut info = BaseAudioInfo {

src/utils/media/video.rs

@@ -0,0 +1,262 @@
+//! Collection of methods for videos.
+
+use std::sync::{Arc, Mutex};
+
+use futures_channel::oneshot;
+use gst::prelude::*;
+use gst_video::prelude::*;
+use gtk::{gio, glib, glib::clone, prelude::*};
+use image::GenericImageView;
+use matrix_sdk::attachment::{BaseVideoInfo, Thumbnail};
+use tracing::warn;
+
+use super::{
+    image::{prepare_thumbnail_for_sending, thumbnail_dimensions},
+    load_gstreamer_media_info,
+};
+
+/// A channel sender to send the result of a video thumbnail.
+type ThumbnailResultSender = oneshot::Sender<Result<Thumbnail, ()>>;
+
+/// Load information and try to generate a thumbnail for the video in the given
+/// file.
+pub async fn load_video_info(file: &gio::File) -> (BaseVideoInfo, Option<Thumbnail>) {
+    let mut info = BaseVideoInfo {
+        duration: None,
+        width: None,
+        height: None,
+        size: None,
+        blurhash: None,
+    };
+
+    let Some(media_info) = load_gstreamer_media_info(file).await else {
+        return (info, None);
+    };
+
+    info.duration = media_info.duration().map(Into::into);
+
+    if let Some(stream_info) = media_info
+        .video_streams()
+        .first()
+        .and_then(|s| s.downcast_ref::<gst_pbutils::DiscovererVideoInfo>())
+    {
+        info.width = Some(stream_info.width().into());
+        info.height = Some(stream_info.height().into());
+    }
+
+    let thumbnail = generate_video_thumbnail(file).await;
+
+    (info, thumbnail)
+}
+
+/// Generate a thumbnail for the video in the given file.
+async fn generate_video_thumbnail(file: &gio::File) -> Option<Thumbnail> {
+    let (sender, receiver) = oneshot::channel();
+    let sender = Arc::new(Mutex::new(Some(sender)));
+
+    let pipeline = match create_thumbnailer_pipeline(&file.uri(), sender.clone()) {
+        Ok(pipeline) => pipeline,
+        Err(error) => {
+            warn!("Could not create pipeline for video thumbnail: {error}");
+            return None;
+        }
+    };
+
+    if pipeline.set_state(gst::State::Paused).is_err() {
+        warn!("Could not initialize pipeline for video thumbnail");
+        return None;
+    }
+
+    let bus = pipeline.bus().expect("Pipeline has a bus");
+
+    let mut started = false;
+    let _bus_guard = bus
+        .add_watch(clone!(
+            #[weak]
+            pipeline,
+            #[upgrade_or]
+            glib::ControlFlow::Break,
+            move |_, message| {
+                match message.view() {
+                    gst::MessageView::AsyncDone(_) => {
+                        if !started {
+                            // AsyncDone means that the pipeline has started now.
+                            if pipeline.set_state(gst::State::Playing).is_err() {
+                                warn!("Could not start pipeline for video thumbnail");
+                                send_video_thumbnail_result(&sender, Err(()));
+
+                                return glib::ControlFlow::Break;
+                            };
+
+                            started = true;
+                        }
+
+                        glib::ControlFlow::Continue
+                    }
+                    gst::MessageView::Eos(_) => {
+                        // We have the thumbnail or we cannot have one.
+                        glib::ControlFlow::Break
+                    }
+                    gst::MessageView::Error(error) => {
+                        warn!("Could not generate video thumbnail: {error}");
+                        send_video_thumbnail_result(&sender, Err(()));
+
+                        glib::ControlFlow::Break
+                    }
+                    _ => glib::ControlFlow::Continue,
+                }
+            }
+        ))
+        .expect("Setting bus watch succeeds");
+
+    let thumbnail = receiver.await;
+
+    // Clean up.
+    let _ = pipeline.set_state(gst::State::Null);
+    bus.set_flushing(true);
+
+    thumbnail.ok().transpose().ok().flatten()
+}
+
+/// Create a GStreamer pipeline to get a thumbnail of the first frame.
+fn create_thumbnailer_pipeline(
+    uri: &str,
+    sender: Arc<Mutex<Option<ThumbnailResultSender>>>,
+) -> Result<gst::Pipeline, glib::Error> {
+    // Create our pipeline from a pipeline description string.
+    let pipeline = gst::parse::launch(&format!(
+        "uridecodebin uri={uri} ! videoconvert ! appsink name=sink"
+    ))?
+    .downcast::<gst::Pipeline>()
+    .expect("Element is a pipeline");
+
+    let appsink = pipeline
+        .by_name("sink")
+        .expect("Sink element is in the pipeline")
+        .downcast::<gst_app::AppSink>()
+        .expect("Sink element is an appsink");
+
+    // Don't synchronize on the clock, we only want a snapshot asap.
+    appsink.set_property("sync", false);
+
+    // Tell the appsink what format we want, for simplicity we only accept 8-bit
+    // RGB.
+    appsink.set_caps(Some(
+        &gst_video::VideoCapsBuilder::new()
+            .format(gst_video::VideoFormat::Rgbx)
+            .build(),
+    ));
+
+    let mut got_snapshot = false;
+
+    // Listen to callbacks to get the data.
+    appsink.set_callbacks(
+        gst_app::AppSinkCallbacks::builder()
+            .new_sample(move |appsink| {
+                // Pull the sample out of the buffer.
+                let sample = appsink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
+                let Some(buffer) = sample.buffer() else {
+                    warn!("Could not get buffer from appsink");
+                    send_video_thumbnail_result(&sender, Err(()));
+
+                    return Err(gst::FlowError::Error);
+                };
+
+                // Make sure that we only get a single buffer.
+                if got_snapshot {
+                    return Err(gst::FlowError::Eos);
+                }
+                got_snapshot = true;
+
+                let Some(caps) = sample.caps() else {
+                    warn!("Got video sample without caps");
+                    send_video_thumbnail_result(&sender, Err(()));
+
+                    return Err(gst::FlowError::Error);
+                };
+                let Ok(info) = gst_video::VideoInfo::from_caps(caps) else {
+                    warn!("Could not parse video caps");
+                    send_video_thumbnail_result(&sender, Err(()));
+
+                    return Err(gst::FlowError::Error);
+                };
+
+                let frame = gst_video::VideoFrameRef::from_buffer_ref_readable(buffer, &info)
+                    .map_err(|_| {
+                        warn!("Could not map video buffer readable");
+                        send_video_thumbnail_result(&sender, Err(()));
+
+                        gst::FlowError::Error
+                    })?;
+
+                // Create a FlatSamples around the borrowed video frame data from GStreamer with
+                // the correct stride.
+                let img = image::FlatSamples::<&[u8]> {
+                    samples: frame.plane_data(0).unwrap(),
+                    layout: image::flat::SampleLayout {
+                        channels: 3,       // RGB
+                        channel_stride: 1, // 1 byte from component to component
+                        width: frame.width(),
+                        width_stride: 4, // 4 bytes from pixel to pixel
+                        height: frame.height(),
+                        height_stride: frame.plane_stride()[0] as usize, // stride from line to line
+                    },
+                    color_hint: Some(image::ColorType::Rgb8),
+                };
+
+                let Ok(view) = img.as_view::<image::Rgb<u8>>() else {
+                    warn!("Could not parse frame as view");
+                    send_video_thumbnail_result(&sender, Err(()));
+
+                    return Err(gst::FlowError::Error);
+                };
+
+                // Reduce the dimensions if the thumbnail is bigger than the wanted size.
+                let thumbnail = if let Some((target_width, target_height)) = thumbnail_dimensions(
+                    frame.width() * info.par().numer() as u32,
+                    frame.height() * info.par().denom() as u32,
+                ) {
+                    image::imageops::thumbnail(&view, target_width, target_height)
+                } else {
+                    image::ImageBuffer::from_fn(view.width(), view.height(), |x, y| {
+                        view.get_pixel(x, y)
+                    })
+                };
+
+                // Prepare it.
+                if let Some(thumbnail) = prepare_thumbnail_for_sending(thumbnail.into()) {
+                    send_video_thumbnail_result(&sender, Ok(thumbnail));
+
+                    Err(gst::FlowError::Eos)
+                } else {
+                    warn!("Failed to convert video thumbnail");
+                    send_video_thumbnail_result(&sender, Err(()));
+
+                    Err(gst::FlowError::Error)
+                }
+            })
+            .build(),
+    );
+
+    Ok(pipeline)
+}
+
+/// Try to send the given video thumbnail result through the given sender.
+fn send_video_thumbnail_result(
+    sender: &Mutex<Option<ThumbnailResultSender>>,
+    result: Result<Thumbnail, ()>,
+) {
+    let mut sender = match sender.lock() {
+        Ok(sender) => sender,
+        Err(error) => {
+            warn!("Failed to lock video thumbnail mutex: {error}");
+            return;
+        }
+    };
+
+    if let Some(sender) = sender.take() {
+        if sender.send(result).is_err() {
+            warn!("Failed to send video thumbnail result through channel");
+        }
+    }
+}