Browse Source

session: Send media info with attachments

Generate thumbnails for images.

Part-of: <https://gitlab.gnome.org/GNOME/fractal/-/merge_requests/1173>
merge-requests/1327/merge
Kévin Commaille 4 years ago committed by Marge Bot
parent
commit
dd1f5b8246
  1. 128
      Cargo.lock
  2. 10
      Cargo.toml
  3. 63
      src/session/content/room_history/mod.rs
  4. 37
      src/session/room/mod.rs
  5. 89
      src/utils/media.rs

128
Cargo.lock generated

@ -1005,6 +1005,16 @@ dependencies = [
"rustc_version",
]
[[package]]
name = "flate2"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
dependencies = [
"crc32fast",
"miniz_oxide 0.5.4",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -1047,12 +1057,13 @@ dependencies = [
"gst-plugin-gtk4",
"gstreamer",
"gstreamer-base",
"gstreamer-pbutils",
"gstreamer-player",
"gstreamer-video",
"gtk-macros",
"gtk4",
"html2pango",
"image",
"image 0.23.14",
"indexmap",
"libadwaita",
"libsecret",
@ -1653,6 +1664,20 @@ dependencies = [
"thiserror",
]
[[package]]
name = "gstreamer-audio-sys"
version = "0.18.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a34258fb53c558c0f41dad194037cbeaabf49d347570df11b8bd1c4897cf7d7c"
dependencies = [
"glib-sys",
"gobject-sys",
"gstreamer-base-sys",
"gstreamer-sys",
"libc",
"system-deps",
]
[[package]]
name = "gstreamer-base"
version = "0.18.0"
@ -1680,6 +1705,35 @@ dependencies = [
"system-deps",
]
[[package]]
name = "gstreamer-pbutils"
version = "0.18.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330684c49f79775d7acce8bef5a7a7475f02374c9c6cead39ced3ad423fc8ea9"
dependencies = [
"bitflags",
"glib",
"gstreamer",
"gstreamer-pbutils-sys",
"libc",
"thiserror",
]
[[package]]
name = "gstreamer-pbutils-sys"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36f79839066fbcc6d1a8690b2f85d5cc5cdc0984f36d4054f5cc67a7ad3ab72d"
dependencies = [
"glib-sys",
"gobject-sys",
"gstreamer-audio-sys",
"gstreamer-sys",
"gstreamer-video-sys",
"libc",
"system-deps",
]
[[package]]
name = "gstreamer-player"
version = "0.18.0"
@ -2077,13 +2131,31 @@ dependencies = [
"byteorder",
"color_quant",
"gif",
"jpeg-decoder",
"jpeg-decoder 0.1.22",
"num-iter",
"num-rational 0.3.2",
"num-traits",
"png",
"png 0.16.8",
"scoped_threadpool",
"tiff",
"tiff 0.6.1",
]
[[package]]
name = "image"
version = "0.24.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd8e4fb07cf672b1642304e731ef8a6a4c7891d67bb4fd4f5ce58cd6ed86803c"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"gif",
"jpeg-decoder 0.2.6",
"num-rational 0.4.1",
"num-traits",
"png 0.17.6",
"scoped_threadpool",
"tiff 0.7.3",
]
[[package]]
@ -2163,6 +2235,15 @@ dependencies = [
"rayon",
]
[[package]]
name = "jpeg-decoder"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b"
dependencies = [
"rayon",
]
[[package]]
name = "js-sys"
version = "0.3.60"
@ -2478,6 +2559,7 @@ dependencies = [
"futures-signals",
"futures-util",
"http",
"image 0.24.4",
"matrix-sdk-base",
"matrix-sdk-common",
"matrix-sdk-indexeddb",
@ -2713,6 +2795,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
dependencies = [
"adler",
]
[[package]]
name = "mio"
version = "0.8.4"
@ -3290,6 +3381,18 @@ dependencies = [
"miniz_oxide 0.3.7",
]
[[package]]
name = "png"
version = "0.17.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f0e7f4c94ec26ff209cee506314212639d6c91b80afb82984819fafce9df01c"
dependencies = [
"bitflags",
"crc32fast",
"flate2",
"miniz_oxide 0.5.4",
]
[[package]]
name = "polling"
version = "2.3.0"
@ -3434,7 +3537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16d2f1455f3630c6e5107b4f2b94e74d76dea80736de0981fd27644216cff57f"
dependencies = [
"checked_int_cast",
"image",
"image 0.23.14",
]
[[package]]
@ -3660,7 +3763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fa79947f53b20adb909a323d828d0fd744fa9d854792df07913b083bcd4d63b"
dependencies = [
"g2p",
"image",
"image 0.23.14",
"lru 0.6.6",
]
@ -4295,11 +4398,22 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437"
dependencies = [
"jpeg-decoder",
"jpeg-decoder 0.1.22",
"miniz_oxide 0.4.4",
"weezl",
]
[[package]]
name = "tiff"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7259662e32d1e219321eb309d5f9d898b779769d81b76e762c07c8e5d38fcb65"
dependencies = [
"flate2",
"jpeg-decoder 0.2.6",
"weezl",
]
[[package]]
name = "tinyvec"
version = "1.6.0"

10
Cargo.toml

@ -44,6 +44,7 @@ gst_base = { version = "0.18", package = "gstreamer-base" }
gst_video = { version = "0.18", package = "gstreamer-video" }
gst_player = { version = "0.18", package = "gstreamer-player" }
gst_gtk = { version = "0.1.0", package = "gst-plugin-gtk4" }
gst_pbutils = { version = "0.18", package = "gstreamer-pbutils" }
image = { version = "0.23", default-features = false, features = ["png"] }
regex = "1.5.4"
mime_guess = "2.0.3"
@ -74,7 +75,14 @@ version = "0.1.1"
[dependencies.matrix-sdk]
version = "0.6.0"
features = ["socks", "sso-login", "markdown", "qrcode", "experimental-timeline"]
features = [
"socks",
"sso-login",
"markdown",
"qrcode",
"experimental-timeline",
"image-rayon",
]
[dependencies.ruma]
version = "0.7.4"

63
src/session/content/room_history/mod.rs

@ -23,14 +23,17 @@ use gtk::{
CompositeTemplate,
};
use log::{error, warn};
use matrix_sdk::ruma::{
events::{
room::message::{
EmoteMessageEventContent, FormattedBody, MessageType, TextMessageEventContent,
use matrix_sdk::{
attachment::{AttachmentInfo, BaseFileInfo, BaseImageInfo},
ruma::{
events::{
room::message::{
EmoteMessageEventContent, FormattedBody, MessageType, TextMessageEventContent,
},
AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent,
},
AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent,
EventId,
},
EventId,
};
use ruma::events::{
room::message::{ForwardThread, LocationMessageEventContent, RoomMessageEventContent},
@ -52,7 +55,10 @@ use crate::{
user::UserExt,
},
spawn, spawn_tokio, toast,
utils::{media::filename_for_mime, template_callbacks::TemplateCallbacks},
utils::{
media::{filename_for_mime, get_audio_info, get_image_info, get_video_info},
template_callbacks::TemplateCallbacks,
},
};
#[derive(Debug, Hash, Eq, PartialEq, Clone, Copy, glib::Enum)]
@ -969,11 +975,15 @@ impl RoomHistory {
}
if let Some(room) = self.room() {
room.send_attachment(
image.save_to_png_bytes().to_vec(),
mime::IMAGE_PNG,
&filename,
);
let bytes = image.save_to_png_bytes();
let info = AttachmentInfo::Image(BaseImageInfo {
width: Some((image.width() as u32).into()),
height: Some((image.height() as u32).into()),
size: Some((bytes.len() as u32).into()),
blurhash: None,
});
room.send_attachment(bytes.to_vec(), mime::IMAGE_PNG, &filename, info);
}
}
@ -1008,6 +1018,7 @@ impl RoomHistory {
let attributes: &[&str] = &[
*gio::FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE,
*gio::FILE_ATTRIBUTE_STANDARD_DISPLAY_NAME,
*gio::FILE_ATTRIBUTE_STANDARD_SIZE,
];
// Read mime type.
@ -1025,10 +1036,15 @@ impl RoomHistory {
.and_then(|info| info.content_type())
.and_then(|content_type| mime::Mime::from_str(&content_type).ok())
.unwrap_or(mime::APPLICATION_OCTET_STREAM);
let filename = info.map(|info| info.display_name()).map_or_else(
let filename = info.as_ref().map(|info| info.display_name()).map_or_else(
|| filename_for_mime(Some(mime.as_ref()), None),
|name| name.to_string(),
);
let size = info
.as_ref()
.map(|info| info.size())
.filter(|size| *size > 0)
.map(|size| (size as u32).into());
match file.load_contents_future().await {
Ok((bytes, _tag)) => {
@ -1040,7 +1056,26 @@ impl RoomHistory {
}
if let Some(room) = self.room() {
room.send_attachment(bytes.clone(), mime.clone(), &filename);
let info = match mime.type_() {
mime::IMAGE => {
let mut info = get_image_info(&file).await;
info.size = size;
AttachmentInfo::Image(info)
}
mime::VIDEO => {
let mut info = get_video_info(&file).await;
info.size = size;
AttachmentInfo::Video(info)
}
mime::AUDIO => {
let mut info = get_audio_info(&file).await;
info.size = size;
AttachmentInfo::Audio(info)
}
_ => AttachmentInfo::File(BaseFileInfo { size }),
};
room.send_attachment(bytes.clone(), mime.clone(), &filename, info);
}
}
Err(err) => {

37
src/session/room/mod.rs

@ -10,13 +10,13 @@ mod reaction_list;
mod room_type;
mod timeline;
use std::{cell::RefCell, path::PathBuf};
use std::{cell::RefCell, io::Cursor, path::PathBuf};
use gettextrs::{gettext, ngettext};
use gtk::{glib, glib::clone, prelude::*, subclass::prelude::*};
use log::{debug, error, info, warn};
use matrix_sdk::{
attachment::AttachmentConfig,
attachment::{generate_image_thumbnail, AttachmentConfig, AttachmentInfo, Thumbnail},
deserialized_responses::{JoinedRoom, LeftRoom, SyncTimelineEvent},
room::Room as MatrixRoom,
ruma::{
@ -1569,13 +1569,42 @@ impl Room {
Some(())
}
pub fn send_attachment(&self, bytes: Vec<u8>, mime: mime::Mime, body: &str) {
pub fn send_attachment(
&self,
bytes: Vec<u8>,
mime: mime::Mime,
body: &str,
info: AttachmentInfo,
) {
let matrix_room = self.matrix_room();
if let MatrixRoom::Joined(matrix_room) = matrix_room {
let body = body.to_string();
spawn_tokio!(async move {
let config = AttachmentConfig::default();
// Needed to hold the thumbnail data until it is sent.
let data_slot;
// The method will filter compatible mime types so we don't need to
// since we ignore errors.
let thumbnail = match generate_image_thumbnail(&mime, Cursor::new(&bytes), None) {
Ok((data, info)) => {
data_slot = data;
Some(Thumbnail {
data: &data_slot,
content_type: &mime::IMAGE_JPEG,
info: Some(info),
})
}
_ => None,
};
let config = if let Some(thumbnail) = thumbnail {
AttachmentConfig::with_thumbnail(thumbnail)
} else {
AttachmentConfig::new()
}
.info(info);
matrix_room
// TODO This should be added to pending messages instead of
// sending it directly.

89
src/utils/media.rs

@ -1,6 +1,10 @@
//! Collection of methods for media files.
use std::{cell::Cell, sync::Mutex};
use gettextrs::gettext;
use gtk::{gdk_pixbuf, gio, prelude::*};
use matrix_sdk::attachment::{BaseAudioInfo, BaseImageInfo, BaseVideoInfo};
use ruma::events::room::MediaSource;
/// Get the unique id of the given `MediaSource`.
@ -56,3 +60,88 @@ pub fn filename_for_mime(mime_type: Option<&str>, fallback: Option<mime::Name>)
.map(|extension| format!("{}.{}", name, extension))
.unwrap_or(name)
}
pub async fn get_image_info(file: &gio::File) -> BaseImageInfo {
let mut info = BaseImageInfo {
width: None,
height: None,
size: None,
blurhash: None,
};
let path = match file.path() {
Some(path) => path,
None => return info,
};
if let Ok(Some((_format, w, h))) = gdk_pixbuf::Pixbuf::file_info_future(path).await {
info.width = Some((w as u32).into());
info.height = Some((h as u32).into());
}
info
}
async fn get_gstreamer_media_info(file: &gio::File) -> Option<gst_pbutils::DiscovererInfo> {
let timeout = gst::ClockTime::from_seconds(15);
let discoverer = gst_pbutils::Discoverer::new(timeout).ok()?;
let (sender, receiver) = futures::channel::oneshot::channel();
let sender = Mutex::new(Cell::new(Some(sender)));
discoverer.connect_discovered(move |_, info, _| {
if let Some(sender) = sender.lock().unwrap().take() {
sender.send(info.clone()).unwrap();
}
});
discoverer.start();
discoverer.discover_uri_async(&file.uri()).ok()?;
let media_info = receiver.await.unwrap();
discoverer.stop();
Some(media_info)
}
pub async fn get_video_info(file: &gio::File) -> BaseVideoInfo {
let mut info = BaseVideoInfo {
duration: None,
width: None,
height: None,
size: None,
blurhash: None,
};
let media_info = match get_gstreamer_media_info(file).await {
Some(media_info) => media_info,
None => return info,
};
info.duration = media_info.duration().map(Into::into);
if let Some(stream_info) = media_info
.video_streams()
.get(0)
.and_then(|s| s.downcast_ref::<gst_pbutils::DiscovererVideoInfo>())
{
info.width = Some(stream_info.width().into());
info.height = Some(stream_info.height().into());
}
info
}
pub async fn get_audio_info(file: &gio::File) -> BaseAudioInfo {
let mut info = BaseAudioInfo {
duration: None,
size: None,
};
let media_info = match get_gstreamer_media_info(file).await {
Some(media_info) => media_info,
None => return info,
};
info.duration = media_info.duration().map(Into::into);
info
}

Loading…
Cancel
Save