diff --git a/Cargo.lock b/Cargo.lock index 417789b6..d3acaa4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1005,6 +1005,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide 0.5.4", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1047,12 +1057,13 @@ dependencies = [ "gst-plugin-gtk4", "gstreamer", "gstreamer-base", + "gstreamer-pbutils", "gstreamer-player", "gstreamer-video", "gtk-macros", "gtk4", "html2pango", - "image", + "image 0.23.14", "indexmap", "libadwaita", "libsecret", @@ -1653,6 +1664,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "gstreamer-audio-sys" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34258fb53c558c0f41dad194037cbeaabf49d347570df11b8bd1c4897cf7d7c" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + [[package]] name = "gstreamer-base" version = "0.18.0" @@ -1680,6 +1705,35 @@ dependencies = [ "system-deps", ] +[[package]] +name = "gstreamer-pbutils" +version = "0.18.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330684c49f79775d7acce8bef5a7a7475f02374c9c6cead39ced3ad423fc8ea9" +dependencies = [ + "bitflags", + "glib", + "gstreamer", + "gstreamer-pbutils-sys", + "libc", + "thiserror", +] + +[[package]] +name = "gstreamer-pbutils-sys" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f79839066fbcc6d1a8690b2f85d5cc5cdc0984f36d4054f5cc67a7ad3ab72d" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-audio-sys", + "gstreamer-sys", + "gstreamer-video-sys", + "libc", + "system-deps", +] + [[package]] name = "gstreamer-player" version = "0.18.0" @@ -2077,13 +2131,31 @@ dependencies = [ "byteorder", "color_quant", "gif", - "jpeg-decoder", + "jpeg-decoder 0.1.22", "num-iter", "num-rational 0.3.2", "num-traits", - "png", + "png 0.16.8", "scoped_threadpool", - "tiff", + "tiff 0.6.1", +] + +[[package]] +name = "image" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd8e4fb07cf672b1642304e731ef8a6a4c7891d67bb4fd4f5ce58cd6ed86803c" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder 0.2.6", + "num-rational 0.4.1", + "num-traits", + "png 0.17.6", + "scoped_threadpool", + "tiff 0.7.3", ] [[package]] @@ -2163,6 +2235,15 @@ dependencies = [ "rayon", ] +[[package]] +name = "jpeg-decoder" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.60" @@ -2478,6 +2559,7 @@ dependencies = [ "futures-signals", "futures-util", "http", + "image 0.24.4", "matrix-sdk-base", "matrix-sdk-common", "matrix-sdk-indexeddb", @@ -2713,6 +2795,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miniz_oxide" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.4" @@ -3290,6 +3381,18 @@ dependencies = [ "miniz_oxide 0.3.7", ] +[[package]] +name = "png" +version = "0.17.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0e7f4c94ec26ff209cee506314212639d6c91b80afb82984819fafce9df01c" +dependencies = [ + "bitflags", + "crc32fast", + "flate2", + "miniz_oxide 0.5.4", +] + [[package]] name = "polling" version = "2.3.0" @@ -3434,7 +3537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16d2f1455f3630c6e5107b4f2b94e74d76dea80736de0981fd27644216cff57f" dependencies = [ "checked_int_cast", - "image", + "image 0.23.14", ] [[package]] @@ -3660,7 +3763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fa79947f53b20adb909a323d828d0fd744fa9d854792df07913b083bcd4d63b" dependencies = [ "g2p", - "image", + "image 0.23.14", "lru 0.6.6", ] @@ -4295,11 +4398,22 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" dependencies = [ - "jpeg-decoder", + "jpeg-decoder 0.1.22", "miniz_oxide 0.4.4", "weezl", ] +[[package]] +name = "tiff" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7259662e32d1e219321eb309d5f9d898b779769d81b76e762c07c8e5d38fcb65" +dependencies = [ + "flate2", + "jpeg-decoder 0.2.6", + "weezl", +] + [[package]] name = "tinyvec" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 8a255d9a..07680938 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ gst_base = { version = "0.18", package = "gstreamer-base" } gst_video = { version = "0.18", package = "gstreamer-video" } gst_player = { version = "0.18", package = "gstreamer-player" } gst_gtk = { version = "0.1.0", package = "gst-plugin-gtk4" } +gst_pbutils = { version = "0.18", package = "gstreamer-pbutils" } image = { version = "0.23", default-features = false, features = ["png"] } regex = "1.5.4" mime_guess = "2.0.3" @@ -74,7 +75,14 @@ version = "0.1.1" [dependencies.matrix-sdk] version = "0.6.0" -features = ["socks", "sso-login", "markdown", "qrcode", "experimental-timeline"] +features = [ + "socks", + "sso-login", + "markdown", + "qrcode", + "experimental-timeline", + "image-rayon", +] [dependencies.ruma] version = "0.7.4" diff --git a/src/session/content/room_history/mod.rs b/src/session/content/room_history/mod.rs index b4794456..665861be 100644 --- a/src/session/content/room_history/mod.rs +++ b/src/session/content/room_history/mod.rs @@ -23,14 +23,17 @@ use gtk::{ CompositeTemplate, }; use log::{error, warn}; -use matrix_sdk::ruma::{ - events::{ - room::message::{ - EmoteMessageEventContent, FormattedBody, MessageType, TextMessageEventContent, +use matrix_sdk::{ + attachment::{AttachmentInfo, BaseFileInfo, BaseImageInfo}, + ruma::{ + events::{ + room::message::{ + EmoteMessageEventContent, FormattedBody, MessageType, TextMessageEventContent, + }, + AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent, }, - AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent, + EventId, }, - EventId, }; use ruma::events::{ room::message::{ForwardThread, LocationMessageEventContent, RoomMessageEventContent}, @@ -52,7 +55,10 @@ use crate::{ user::UserExt, }, spawn, spawn_tokio, toast, - utils::{media::filename_for_mime, template_callbacks::TemplateCallbacks}, + utils::{ + media::{filename_for_mime, get_audio_info, get_image_info, get_video_info}, + template_callbacks::TemplateCallbacks, + }, }; #[derive(Debug, Hash, Eq, PartialEq, Clone, Copy, glib::Enum)] @@ -969,11 +975,15 @@ impl RoomHistory { } if let Some(room) = self.room() { - room.send_attachment( - image.save_to_png_bytes().to_vec(), - mime::IMAGE_PNG, - &filename, - ); + let bytes = image.save_to_png_bytes(); + let info = AttachmentInfo::Image(BaseImageInfo { + width: Some((image.width() as u32).into()), + height: Some((image.height() as u32).into()), + size: Some((bytes.len() as u32).into()), + blurhash: None, + }); + + room.send_attachment(bytes.to_vec(), mime::IMAGE_PNG, &filename, info); } } @@ -1008,6 +1018,7 @@ impl RoomHistory { let attributes: &[&str] = &[ *gio::FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE, *gio::FILE_ATTRIBUTE_STANDARD_DISPLAY_NAME, + *gio::FILE_ATTRIBUTE_STANDARD_SIZE, ]; // Read mime type. @@ -1025,10 +1036,15 @@ impl RoomHistory { .and_then(|info| info.content_type()) .and_then(|content_type| mime::Mime::from_str(&content_type).ok()) .unwrap_or(mime::APPLICATION_OCTET_STREAM); - let filename = info.map(|info| info.display_name()).map_or_else( + let filename = info.as_ref().map(|info| info.display_name()).map_or_else( || filename_for_mime(Some(mime.as_ref()), None), |name| name.to_string(), ); + let size = info + .as_ref() + .map(|info| info.size()) + .filter(|size| *size > 0) + .map(|size| (size as u32).into()); match file.load_contents_future().await { Ok((bytes, _tag)) => { @@ -1040,7 +1056,26 @@ impl RoomHistory { } if let Some(room) = self.room() { - room.send_attachment(bytes.clone(), mime.clone(), &filename); + let info = match mime.type_() { + mime::IMAGE => { + let mut info = get_image_info(&file).await; + info.size = size; + AttachmentInfo::Image(info) + } + mime::VIDEO => { + let mut info = get_video_info(&file).await; + info.size = size; + AttachmentInfo::Video(info) + } + mime::AUDIO => { + let mut info = get_audio_info(&file).await; + info.size = size; + AttachmentInfo::Audio(info) + } + _ => AttachmentInfo::File(BaseFileInfo { size }), + }; + + room.send_attachment(bytes.clone(), mime.clone(), &filename, info); } } Err(err) => { diff --git a/src/session/room/mod.rs b/src/session/room/mod.rs index efbd5e09..4d171abf 100644 --- a/src/session/room/mod.rs +++ b/src/session/room/mod.rs @@ -10,13 +10,13 @@ mod reaction_list; mod room_type; mod timeline; -use std::{cell::RefCell, path::PathBuf}; +use std::{cell::RefCell, io::Cursor, path::PathBuf}; use gettextrs::{gettext, ngettext}; use gtk::{glib, glib::clone, prelude::*, subclass::prelude::*}; use log::{debug, error, info, warn}; use matrix_sdk::{ - attachment::AttachmentConfig, + attachment::{generate_image_thumbnail, AttachmentConfig, AttachmentInfo, Thumbnail}, deserialized_responses::{JoinedRoom, LeftRoom, SyncTimelineEvent}, room::Room as MatrixRoom, ruma::{ @@ -1569,13 +1569,42 @@ impl Room { Some(()) } - pub fn send_attachment(&self, bytes: Vec, mime: mime::Mime, body: &str) { + pub fn send_attachment( + &self, + bytes: Vec, + mime: mime::Mime, + body: &str, + info: AttachmentInfo, + ) { let matrix_room = self.matrix_room(); if let MatrixRoom::Joined(matrix_room) = matrix_room { let body = body.to_string(); spawn_tokio!(async move { - let config = AttachmentConfig::default(); + // Needed to hold the thumbnail data until it is sent. + let data_slot; + + // The method will filter compatible mime types so we don't need to + // since we ignore errors. + let thumbnail = match generate_image_thumbnail(&mime, Cursor::new(&bytes), None) { + Ok((data, info)) => { + data_slot = data; + Some(Thumbnail { + data: &data_slot, + content_type: &mime::IMAGE_JPEG, + info: Some(info), + }) + } + _ => None, + }; + + let config = if let Some(thumbnail) = thumbnail { + AttachmentConfig::with_thumbnail(thumbnail) + } else { + AttachmentConfig::new() + } + .info(info); + matrix_room // TODO This should be added to pending messages instead of // sending it directly. diff --git a/src/utils/media.rs b/src/utils/media.rs index ba8be9a9..dfc08ca2 100644 --- a/src/utils/media.rs +++ b/src/utils/media.rs @@ -1,6 +1,10 @@ //! Collection of methods for media files. +use std::{cell::Cell, sync::Mutex}; + use gettextrs::gettext; +use gtk::{gdk_pixbuf, gio, prelude::*}; +use matrix_sdk::attachment::{BaseAudioInfo, BaseImageInfo, BaseVideoInfo}; use ruma::events::room::MediaSource; /// Get the unique id of the given `MediaSource`. @@ -56,3 +60,88 @@ pub fn filename_for_mime(mime_type: Option<&str>, fallback: Option) .map(|extension| format!("{}.{}", name, extension)) .unwrap_or(name) } + +pub async fn get_image_info(file: &gio::File) -> BaseImageInfo { + let mut info = BaseImageInfo { + width: None, + height: None, + size: None, + blurhash: None, + }; + + let path = match file.path() { + Some(path) => path, + None => return info, + }; + + if let Ok(Some((_format, w, h))) = gdk_pixbuf::Pixbuf::file_info_future(path).await { + info.width = Some((w as u32).into()); + info.height = Some((h as u32).into()); + } + + info +} + +async fn get_gstreamer_media_info(file: &gio::File) -> Option { + let timeout = gst::ClockTime::from_seconds(15); + let discoverer = gst_pbutils::Discoverer::new(timeout).ok()?; + + let (sender, receiver) = futures::channel::oneshot::channel(); + let sender = Mutex::new(Cell::new(Some(sender))); + discoverer.connect_discovered(move |_, info, _| { + if let Some(sender) = sender.lock().unwrap().take() { + sender.send(info.clone()).unwrap(); + } + }); + + discoverer.start(); + discoverer.discover_uri_async(&file.uri()).ok()?; + + let media_info = receiver.await.unwrap(); + discoverer.stop(); + + Some(media_info) +} + +pub async fn get_video_info(file: &gio::File) -> BaseVideoInfo { + let mut info = BaseVideoInfo { + duration: None, + width: None, + height: None, + size: None, + blurhash: None, + }; + + let media_info = match get_gstreamer_media_info(file).await { + Some(media_info) => media_info, + None => return info, + }; + + info.duration = media_info.duration().map(Into::into); + + if let Some(stream_info) = media_info + .video_streams() + .get(0) + .and_then(|s| s.downcast_ref::()) + { + info.width = Some(stream_info.width().into()); + info.height = Some(stream_info.height().into()); + } + + info +} + +pub async fn get_audio_info(file: &gio::File) -> BaseAudioInfo { + let mut info = BaseAudioInfo { + duration: None, + size: None, + }; + + let media_info = match get_gstreamer_media_info(file).await { + Some(media_info) => media_info, + None => return info, + }; + + info.duration = media_info.duration().map(Into::into); + info +}