diff --git a/data/resources/icons/scalable/actions/pause-symbolic.svg b/data/resources/icons/scalable/actions/pause-symbolic.svg new file mode 100644 index 00000000..6ba52e09 --- /dev/null +++ b/data/resources/icons/scalable/actions/pause-symbolic.svg @@ -0,0 +1,2 @@ + + diff --git a/data/resources/icons/scalable/actions/play-symbolic.svg b/data/resources/icons/scalable/actions/play-symbolic.svg new file mode 100644 index 00000000..f3f28d63 --- /dev/null +++ b/data/resources/icons/scalable/actions/play-symbolic.svg @@ -0,0 +1,2 @@ + + diff --git a/data/resources/resources.gresource.xml b/data/resources/resources.gresource.xml index 575694b6..ce2b0019 100644 --- a/data/resources/resources.gresource.xml +++ b/data/resources/resources.gresource.xml @@ -35,6 +35,8 @@ icons/scalable/actions/menu-primary-symbolic.svg icons/scalable/actions/menu-secondary-symbolic.svg icons/scalable/actions/more-symbolic.svg + icons/scalable/actions/pause-symbolic.svg + icons/scalable/actions/play-symbolic.svg icons/scalable/actions/refresh-symbolic.svg icons/scalable/actions/remove-symbolic.svg icons/scalable/actions/restore-symbolic.svg diff --git a/data/resources/stylesheet/_components.scss b/data/resources/stylesheet/_components.scss index 320f37e0..6ae56b44 100644 --- a/data/resources/stylesheet/_components.scss +++ b/data/resources/stylesheet/_components.scss @@ -166,3 +166,9 @@ user-page scrolledwindow > viewport > clamp > box { margin: 12px; border-spacing: 24px; } + +audio-player waveform { + &:focus { + color: var(--accent-color); + } +} diff --git a/po/POTFILES.in b/po/POTFILES.in index 98d65903..8470ddf6 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -28,6 +28,7 @@ src/components/dialogs/room_preview.rs src/components/dialogs/room_preview.blp src/components/dialogs/user_profile.blp src/components/offline_banner.rs +src/components/media/audio_player/mod.rs src/components/media/content_viewer.rs src/components/media/location_viewer.rs src/components/pill/at_room.rs diff --git a/src/components/media/audio_player.blp b/src/components/media/audio_player.blp deleted file mode 100644 index e2dcb65b..00000000 --- a/src/components/media/audio_player.blp +++ /dev/null @@ -1,8 +0,0 @@ -using Gtk 4.0; -using Adw 1; - -template $AudioPlayer: Adw.Bin { - Gtk.MediaControls { - media-stream: bind template.media-file; - } -} diff --git a/src/components/media/audio_player.rs b/src/components/media/audio_player.rs deleted file mode 100644 index 9a986a50..00000000 --- a/src/components/media/audio_player.rs +++ /dev/null @@ -1,105 +0,0 @@ -use adw::{prelude::*, subclass::prelude::*}; -use gtk::{gio, glib}; - -use crate::utils::BoundObject; - -mod imp { - use std::cell::Cell; - - use glib::subclass::InitializingObject; - - use super::*; - - #[derive(Debug, Default, gtk::CompositeTemplate, glib::Properties)] - #[template(resource = "/org/gnome/Fractal/ui/components/media/audio_player.ui")] - #[properties(wrapper_type = super::AudioPlayer)] - pub struct AudioPlayer { - /// The media file to play. - #[property(get, set = Self::set_media_file, explicit_notify, nullable)] - media_file: BoundObject, - /// Whether to play the media automatically. - #[property(get, set = Self::set_autoplay, explicit_notify)] - autoplay: Cell, - } - - #[glib::object_subclass] - impl ObjectSubclass for AudioPlayer { - const NAME: &'static str = "AudioPlayer"; - type Type = super::AudioPlayer; - type ParentType = adw::Bin; - - fn class_init(klass: &mut Self::Class) { - Self::bind_template(klass); - } - - fn instance_init(obj: &InitializingObject) { - obj.init_template(); - } - } - - #[glib::derived_properties] - impl ObjectImpl for AudioPlayer {} - - impl WidgetImpl for AudioPlayer {} - impl BinImpl for AudioPlayer {} - - impl AudioPlayer { - /// Set the media file to play. - fn set_media_file(&self, media_file: Option) { - if self.media_file.obj() == media_file { - return; - } - - self.media_file.disconnect_signals(); - - if let Some(media_file) = media_file { - let mut handlers = Vec::new(); - - if self.autoplay.get() { - let prepared_handler = media_file.connect_prepared_notify(|media_file| { - if media_file.is_prepared() { - media_file.play(); - } - }); - handlers.push(prepared_handler); - } - - self.media_file.set(media_file, handlers); - } - - self.obj().notify_media_file(); - } - - /// Set whether to play the media automatically. - fn set_autoplay(&self, autoplay: bool) { - if self.autoplay.get() == autoplay { - return; - } - - self.autoplay.set(autoplay); - self.obj().notify_autoplay(); - } - } -} - -glib::wrapper! { - /// A widget displaying a video media file. - pub struct AudioPlayer(ObjectSubclass) - @extends gtk::Widget, adw::Bin, - @implements gtk::Accessible, gtk::Buildable, gtk::ConstraintTarget; -} - -impl AudioPlayer { - /// Create a new audio player. - pub fn new() -> Self { - glib::Object::new() - } - - /// Set the file to play. - /// - /// This is a convenience method that calls - /// [`AudioPlayer::set_media_file()`]. - pub(crate) fn set_file(&self, file: Option<&gio::File>) { - self.set_media_file(file.map(gtk::MediaFile::for_file)); - } -} diff --git a/src/components/media/audio_player/mod.blp b/src/components/media/audio_player/mod.blp new file mode 100644 index 00000000..5bb6b249 --- /dev/null +++ b/src/components/media/audio_player/mod.blp @@ -0,0 +1,92 @@ +using Gtk 4.0; +using Adw 1; + +template $AudioPlayer: Adw.BreakpointBin { + margin-start: 6; + margin-end: 6; + width-request: 200; + height-request: 100; + + Gtk.Box { + orientation: vertical; + spacing: 6; + + Gtk.Box { + spacing: 6; + + Gtk.Label position_label { + styles [ + "caption", + ] + } + + Gtk.Overlay { + $Waveform waveform { + hexpand: true; + seek => $seek() swapped; + } + + [overlay] + Adw.Spinner spinner { + visible: false; + height-request: 20; + width-request: 20; + halign: center; + valign: center; + } + + [overlay] + Gtk.Image error_img { + visible: false; + icon-name: "error-symbolic"; + halign: center; + valign: center; + } + } + + Gtk.Label remaining_label { + styles [ + "caption", + ] + } + } + + Gtk.Box bottom_box { + spacing: 6; + + Adw.Bin play_button_bin { + child: Gtk.Button play_button { + halign: center; + clicked => $toggle_playing() swapped; + + styles [ + "flat", + ] + }; + } + + Gtk.Label filename_label { + hexpand: true; + xalign: 0.0; + ellipsize: end; + } + + Gtk.Label position_label_narrow { + visible: false; + halign: end; + label: bind position_label.label; + + styles [ + "caption", + ] + } + } + } +} + +Gtk.SizeGroup { + widgets [ + position_label, + play_button_bin, + ] +} diff --git a/src/components/media/audio_player/mod.rs b/src/components/media/audio_player/mod.rs new file mode 100644 index 00000000..cf16aa12 --- /dev/null +++ b/src/components/media/audio_player/mod.rs @@ -0,0 +1,609 @@ +use std::time::Duration; + +use adw::{prelude::*, subclass::prelude::*}; +use gettextrs::gettext; +use gtk::{gio, glib, glib::clone}; +use tracing::warn; + +mod waveform; +mod waveform_paintable; + +use self::waveform::Waveform; +use crate::{ + session::model::Session, + spawn, + utils::{ + File, LoadingState, + matrix::{AudioMessageExt, MediaMessage, MessageCacheKey}, + media::{ + self, MediaFileError, + audio::{generate_waveform, load_audio_info}, + }, + }, +}; + +mod imp { + use std::cell::{Cell, RefCell}; + + use glib::subclass::InitializingObject; + + use super::*; + + #[derive(Debug, Default, gtk::CompositeTemplate, glib::Properties)] + #[template(resource = "/org/gnome/Fractal/ui/components/media/audio_player/mod.ui")] + #[properties(wrapper_type = super::AudioPlayer)] + pub struct AudioPlayer { + #[template_child] + position_label: TemplateChild, + #[template_child] + waveform: TemplateChild, + #[template_child] + spinner: TemplateChild, + #[template_child] + error_img: TemplateChild, + #[template_child] + remaining_label: TemplateChild, + #[template_child] + bottom_box: TemplateChild, + #[template_child] + play_button: TemplateChild, + #[template_child] + filename_label: TemplateChild, + #[template_child] + position_label_narrow: TemplateChild, + /// The source to play. + source: RefCell>, + /// The API used to play the audio file. + #[property(get)] + media_file: gtk::MediaFile, + /// The audio file that is currently loaded. + /// + /// This is used to keep a strong reference to the temporary file. + file: RefCell>, + /// Whether the audio player is the main widget of the current view. + /// + /// This hides the filename and centers the play button. + #[property(get, set = Self::set_standalone, explicit_notify)] + standalone: Cell, + /// Whether we are in narrow mode. + narrow: Cell, + /// The state of the audio file. + #[property(get, builder(LoadingState::default()))] + state: Cell, + /// The duration of the audio stream, in microseconds. + duration: Cell, + } + + #[glib::object_subclass] + impl ObjectSubclass for AudioPlayer { + const NAME: &'static str = "AudioPlayer"; + type Type = super::AudioPlayer; + type ParentType = adw::BreakpointBin; + + fn class_init(klass: &mut Self::Class) { + Self::bind_template(klass); + Self::bind_template_callbacks(klass); + + klass.set_css_name("audio-player"); + } + + fn instance_init(obj: &InitializingObject) { + obj.init_template(); + } + } + + #[glib::derived_properties] + impl ObjectImpl for AudioPlayer { + fn constructed(&self) { + self.parent_constructed(); + + let breakpoint = adw::Breakpoint::new(adw::BreakpointCondition::new_length( + adw::BreakpointConditionLengthType::MaxWidth, + 360.0, + adw::LengthUnit::Px, + )); + breakpoint.connect_apply(clone!( + #[weak(rename_to = imp)] + self, + move |_| { + imp.set_narrow(true); + } + )); + breakpoint.connect_unapply(clone!( + #[weak(rename_to = imp)] + self, + move |_| { + imp.set_narrow(false); + } + )); + self.obj().add_breakpoint(breakpoint); + + self.media_file.connect_duration_notify(clone!( + #[weak(rename_to = imp)] + self, + move |media_file| { + if !imp.use_media_file_data() { + return; + } + + let duration = Duration::from_micros(media_file.duration().cast_unsigned()); + imp.set_duration(duration); + } + )); + + self.media_file.connect_timestamp_notify(clone!( + #[weak(rename_to = imp)] + self, + move |media_file| { + if !imp.use_media_file_data() { + return; + } + + let mut duration = media_file.duration(); + let timestamp = media_file.timestamp(); + + // The duration should always be bigger than the timestamp, but let's be safe. + if duration != 0 && timestamp > duration { + duration = timestamp; + } + + let position = if duration == 0 { + 0.0 + } else { + (timestamp as f64 / duration as f64) as f32 + }; + + imp.waveform.set_position(position); + } + )); + + self.media_file.connect_playing_notify(clone!( + #[weak(rename_to = imp)] + self, + move |_| { + imp.update_play_button(); + } + )); + + self.media_file.connect_prepared_notify(clone!( + #[weak(rename_to = imp)] + self, + move |media_file| { + if media_file.is_prepared() { + // The media file should only become prepared after the user clicked play, + // so start playing it. + media_file.set_playing(true); + + // If the user selected a position while we didn't have a media file, seek + // to it. + let position = imp.waveform.position(); + if position > 0.0 { + media_file + .seek((media_file.duration() as f64 * f64::from(position)) as i64); + } + } + } + )); + + self.media_file.connect_error_notify(clone!( + #[weak(rename_to = imp)] + self, + move |media_file| { + if let Some(error) = media_file.error() { + warn!("Could not read audio file: {error}"); + imp.set_error(&gettext("Error reading audio file")); + } + } + )); + + self.waveform.connect_position_notify(clone!( + #[weak(rename_to = imp)] + self, + move |_| { + imp.update_position_labels(); + } + )); + + self.update_play_button(); + } + + fn dispose(&self) { + self.media_file.clear(); + } + } + + impl WidgetImpl for AudioPlayer {} + impl BreakpointBinImpl for AudioPlayer {} + + #[gtk::template_callbacks] + impl AudioPlayer { + /// Set the source to play. + pub(super) fn set_source(&self, source: Option) { + let should_reload = source.as_ref().is_none_or(|source| { + self.source + .borrow() + .as_ref() + .is_none_or(|old_source| old_source.should_reload(source)) + }); + + if should_reload { + self.set_state(LoadingState::Initial); + self.media_file.clear(); + self.file.take(); + } + + self.source.replace(source); + + if should_reload { + spawn!(clone!( + #[weak(rename_to = imp)] + self, + async move { + imp.load_source_duration().await; + } + )); + spawn!(clone!( + #[weak(rename_to = imp)] + self, + async move { + imp.load_source_waveform().await; + } + )); + + self.update_source_filename(); + } + + self.update_play_button(); + } + + /// Set whether the audio player is the main widget of the current view. + fn set_standalone(&self, standalone: bool) { + if self.standalone.get() == standalone { + return; + } + + self.standalone.set(standalone); + self.update_layout(); + self.obj().notify_standalone(); + } + + /// Set whether we are in narrow mode. + fn set_narrow(&self, narrow: bool) { + if self.narrow.get() == narrow { + return; + } + + self.narrow.set(narrow); + self.update_layout(); + } + + /// Update the layout for the current state. + fn update_layout(&self) { + let standalone = self.standalone.get(); + let narrow = self.narrow.get(); + + self.position_label.set_visible(!narrow); + self.remaining_label.set_visible(!narrow); + self.filename_label.set_visible(!standalone); + self.position_label_narrow + .set_visible(narrow && !standalone); + + self.bottom_box.set_halign(if standalone { + gtk::Align::Center + } else { + gtk::Align::Fill + }); + } + + /// Set the state of the audio stream. + fn set_state(&self, state: LoadingState) { + if self.state.get() == state { + return; + } + + self.waveform + .set_sensitive(matches!(state, LoadingState::Initial | LoadingState::Ready)); + self.spinner + .set_visible(matches!(state, LoadingState::Loading)); + self.error_img + .set_visible(matches!(state, LoadingState::Error)); + + self.state.set(state); + self.obj().notify_state(); + } + + /// Convenience method to set the state to `Error` with the given error + /// message. + fn set_error(&self, error: &str) { + self.set_state(LoadingState::Error); + self.error_img.set_tooltip_text(Some(error)); + } + + /// Whether we should use the source data rather than the `GtkMediaFile` + /// data. + /// + /// We cannot use the `GtkMediaFile` data if it doesn't have a `GFile` + /// set. + fn use_media_file_data(&self) -> bool { + self.state.get() != LoadingState::Initial + } + + /// Set the duration of the audio stream. + fn set_duration(&self, duration: Duration) { + if self.duration.get() == duration { + return; + } + + self.duration.set(duration); + self.update_duration_labels_width(); + self.update_position_labels(); + } + + /// Update the width of labels presenting a duration. + fn update_duration_labels_width(&self) { + let has_hours = self.duration.get().as_secs() > 60 * 60; + let time_width = if has_hours { 8 } else { 5 }; + + self.position_label.set_width_chars(time_width); + self.remaining_label.set_width_chars(time_width + 1); + } + + /// Load the duration of the current source. + async fn load_source_duration(&self) { + let Some(source) = self.source.borrow().clone() else { + self.set_duration(Duration::default()); + return; + }; + + let duration = source.duration().await; + self.set_duration(duration.unwrap_or_default()); + } + + /// Load the waveform of the current source. + async fn load_source_waveform(&self) { + let Some(source) = self.source.borrow().clone() else { + self.waveform.set_waveform(vec![]); + return; + }; + + let waveform = source.waveform().await; + self.waveform.set_waveform(waveform.unwrap_or_default()); + } + + /// Update the name of the source. + fn update_source_filename(&self) { + let filename = self + .source + .borrow() + .as_ref() + .map(AudioPlayerSource::filename) + .unwrap_or_default(); + + self.filename_label.set_label(&filename); + } + + /// Update the labels displaying the position in the audio stream. + fn update_position_labels(&self) { + let duration = self.duration.get(); + let position = self.waveform.position(); + + let position = duration.mul_f32(position); + let remaining = duration.saturating_sub(position); + + self.position_label + .set_label(&media::time_to_label(&position)); + self.remaining_label + .set_label(&format!("-{}", media::time_to_label(&remaining))); + } + + /// Update the play button. + fn update_play_button(&self) { + let is_playing = self.media_file.is_playing(); + + let (icon_name, tooltip) = if is_playing { + ("pause-symbolic", gettext("Pause")) + } else { + ("play-symbolic", gettext("Play")) + }; + + self.play_button.set_icon_name(icon_name); + self.play_button.set_tooltip_text(Some(&tooltip)); + + if is_playing { + self.set_state(LoadingState::Ready); + } + } + + /// Set the media file to play. + async fn set_file(&self, file: File) { + let gfile = file.as_gfile(); + self.media_file.set_file(Some(&gfile)); + self.file.replace(Some(file)); + + // Reload the waveform if we got it from a message, because we cannot trust the + // sender. + if self + .source + .borrow() + .as_ref() + .is_some_and(|source| matches!(source, AudioPlayerSource::Message(_))) + && let Some(waveform) = generate_waveform(&gfile, None).await + { + self.waveform.set_waveform(waveform); + } + } + + /// Play or pause the media. + #[template_callback] + async fn toggle_playing(&self) { + if self.use_media_file_data() { + self.media_file.set_playing(!self.media_file.is_playing()); + return; + } + + let Some(source) = self.source.borrow().clone() else { + return; + }; + + self.set_state(LoadingState::Loading); + + match source.to_file().await { + Ok(file) => { + self.set_file(file).await; + } + Err(error) => { + warn!("Could not retrieve audio file: {error}"); + self.set_error(&gettext("Could not retrieve audio file")); + } + } + } + + /// Seek to the given relative position. + /// + /// The position must be a value between 0 and 1. + #[template_callback] + fn seek(&self, new_position: f32) { + if self.use_media_file_data() { + let duration = self.duration.get(); + + if !duration.is_zero() { + let timestamp = duration.as_micros() as f64 * f64::from(new_position); + self.media_file.seek(timestamp as i64); + } + } else { + self.waveform.set_position(new_position); + } + } + } +} + +glib::wrapper! { + /// A widget displaying a video media file. + pub struct AudioPlayer(ObjectSubclass) + @extends gtk::Widget, adw::BreakpointBin, + @implements gtk::Accessible, gtk::Buildable, gtk::ConstraintTarget; +} + +impl AudioPlayer { + /// Create a new audio player. + pub fn new() -> Self { + glib::Object::new() + } + + /// Set the source to play. + pub(crate) fn set_source(&self, source: Option) { + self.imp().set_source(source); + } +} + +/// The possible sources accepted by the audio player. +#[derive(Debug, Clone)] +pub(crate) enum AudioPlayerSource { + /// An audio file. + File(gio::File), + /// An audio message. + Message(AudioPlayerMessage), +} + +impl AudioPlayerSource { + /// Get the filename of the source. + fn filename(&self) -> String { + match self { + Self::File(file) => file + .path() + .and_then(|path| path.file_name().map(|s| s.to_string_lossy().into_owned())) + .unwrap_or_default(), + Self::Message(message) => message.message.filename(), + } + } + + /// Whether the source should be reloaded because it has changed. + fn should_reload(&self, new_source: &Self) -> bool { + match (self, new_source) { + (Self::File(file), Self::File(new_file)) => file != new_file, + (Self::Message(message), Self::Message(new_message)) => { + message.cache_key.should_reload(&new_message.cache_key) + } + _ => true, + } + } + + /// Get the duration of this source, if any. + async fn duration(&self) -> Option { + match self { + Self::File(file) => load_audio_info(file).await.duration, + Self::Message(message) => { + if let MediaMessage::Audio(content) = &message.message { + content.info.as_deref().and_then(|info| info.duration) + } else { + None + } + } + } + } + + /// Get the waveform representation of this source, if any. + async fn waveform(&self) -> Option> { + match self { + Self::File(file) => generate_waveform(file, None).await, + Self::Message(message) => { + if let MediaMessage::Audio(content) = &message.message { + content.normalized_waveform() + } else { + None + } + } + } + } + + /// Get a file to play this source. + async fn to_file(&self) -> Result { + match self { + Self::File(file) => Ok(file.clone().into()), + Self::Message(message) => { + let Some(session) = message.session.upgrade() else { + return Err(MediaFileError::NoSession); + }; + + message + .message + .clone() + .into_tmp_file(&session.client()) + .await + } + } + } +} + +/// The data required to play an audio message. +#[derive(Debug, Clone)] +pub(crate) struct AudioPlayerMessage { + /// The audio message. + pub(crate) message: MediaMessage, + /// The session that will be used to load the file. + pub(crate) session: glib::WeakRef, + /// The cache key for the audio message. + /// + /// The audio is only reloaded if the cache key changes. This is to + /// avoid reloading the audio when the local echo is updated to a remote + /// echo. + pub(crate) cache_key: MessageCacheKey, +} + +impl AudioPlayerMessage { + /// Construct a new `AudioPlayerMessage`. + pub(crate) fn new( + message: MediaMessage, + session: &Session, + cache_key: MessageCacheKey, + ) -> Self { + let session_weak = glib::WeakRef::new(); + session_weak.set(Some(session)); + + Self { + message, + session: session_weak, + cache_key, + } + } +} diff --git a/src/components/media/audio_player/waveform.rs b/src/components/media/audio_player/waveform.rs new file mode 100644 index 00000000..378e8951 --- /dev/null +++ b/src/components/media/audio_player/waveform.rs @@ -0,0 +1,453 @@ +use adw::prelude::*; +use gtk::{ + gdk, glib, + glib::{clone, closure_local}, + graphene, gsk, + subclass::prelude::*, +}; +use tracing::error; + +use super::waveform_paintable::WaveformPaintable; + +/// The height of the waveform. +pub(super) const WAVEFORM_HEIGHT: f32 = 60.0; +/// The height of the waveform, as an integer. +pub(super) const WAVEFORM_HEIGHT_I32: i32 = 60; +/// The duration of the animation, in milliseconds. +const ANIMATION_DURATION: u32 = 250; +/// The error margin when comparing two `f32`s. +const F32_ERROR_MARGIN: f32 = 0.0001; + +mod imp { + use std::{ + cell::{Cell, OnceCell, RefCell}, + sync::LazyLock, + }; + + use glib::subclass::Signal; + + use super::*; + + #[derive(Debug, Default, glib::Properties)] + #[properties(wrapper_type = super::Waveform)] + pub struct Waveform { + /// The paintable that draws the waveform. + #[property(get)] + paintable: WaveformPaintable, + /// The current position in the audio stream. + /// + /// Must be a value between 0 and 1. + #[property(get, set = Self::set_position, explicit_notify, minimum = 0.0, maximum = 1.0)] + position: Cell, + /// The animation for the transition between waveforms. + animation: OnceCell, + /// The current hover position, if any. + hover_position: Cell>, + /// The cached paintable. + /// + /// We only need to redraw it when the waveform changes of the widget is + /// resized. + paintable_cache: RefCell>, + } + + #[glib::object_subclass] + impl ObjectSubclass for Waveform { + const NAME: &'static str = "Waveform"; + type Type = super::Waveform; + type ParentType = gtk::Widget; + + fn class_init(klass: &mut Self::Class) { + klass.set_css_name("waveform"); + klass.set_accessible_role(gtk::AccessibleRole::Slider); + } + } + + #[glib::derived_properties] + impl ObjectImpl for Waveform { + fn signals() -> &'static [Signal] { + static SIGNALS: LazyLock> = LazyLock::new(|| { + vec![ + Signal::builder("seek") + .param_types([f32::static_type()]) + .build(), + ] + }); + SIGNALS.as_ref() + } + + fn constructed(&self) { + self.parent_constructed(); + + self.init_event_controllers(); + + let obj = self.obj(); + obj.set_focusable(true); + obj.update_property(&[ + gtk::accessible::Property::ValueMin(0.0), + gtk::accessible::Property::ValueMax(1.0), + gtk::accessible::Property::ValueNow(0.0), + ]); + + self.paintable.connect_invalidate_contents(clone!( + #[weak] + obj, + move |_| { + obj.queue_draw(); + } + )); + } + } + + impl WidgetImpl for Waveform { + fn request_mode(&self) -> gtk::SizeRequestMode { + gtk::SizeRequestMode::HeightForWidth + } + + fn measure(&self, orientation: gtk::Orientation, _for_size: i32) -> (i32, i32, i32, i32) { + if orientation == gtk::Orientation::Vertical { + // The height is fixed. + (WAVEFORM_HEIGHT_I32, WAVEFORM_HEIGHT_I32, -1, -1) + } else { + // We accept any width, the optimal width is the default width of the paintable. + (0, self.paintable.intrinsic_width(), -1, -1) + } + } + + fn size_allocate(&self, width: i32, _height: i32, _baseline: i32) { + if self + .paintable_cache + .borrow() + .as_ref() + .is_some_and(|paintable| width != paintable.intrinsic_width()) + { + // We need to adjust the waveform to the new width. + self.paintable_cache.take(); + self.obj().queue_draw(); + } + } + + fn snapshot(&self, snapshot: >k::Snapshot) { + let obj = self.obj(); + let width = obj.width(); + + if width <= 0 { + return; + } + + let Some(paintable) = self.paintable() else { + return; + }; + + let width = width as f32; + let is_rtl = obj.direction() == gtk::TextDirection::Rtl; + + // Use the waveform as a mask that we will apply to the colored rectangles + // below. + snapshot.push_mask(gsk::MaskMode::Alpha); + snapshot.save(); + + // Invert the paintable horizontally if we are in right-to-left direction. + if is_rtl { + snapshot.translate(&graphene::Point::new(width, 0.0)); + snapshot.scale(-1.0, 1.0); + } + + paintable.snapshot(snapshot, width.into(), WAVEFORM_HEIGHT.into()); + + snapshot.restore(); + snapshot.pop(); + + // Paint three colored rectangles to mark the two positions: + // + // ---------------------------- + // | played | hover | remaining | + // ---------------------------- + // + // The "played" part stops at the first of the `position` or the + // `hover_position` and the "hover" part stops at the last of the + // `position` or the `hover_position`. + // + // The order is inverted in right-to-left direction, and any rectangle that is + // not visible (i.e. has a width of 0) is not drawn. + let (start, end) = if is_rtl { (width, 0.0) } else { (0.0, width) }; + let mut position = self.position.get() * width; + if is_rtl { + position = width - position; + } + let hover_position = self.hover_position.get(); + + let (played_end, hover_end) = if let Some(hover_position) = hover_position { + if (!is_rtl && hover_position > position) || (is_rtl && hover_position < position) { + (position, hover_position) + } else { + (hover_position, position) + } + } else { + (position, position) + }; + + let color = obj.color(); + let is_high_contrast = adw::StyleManager::default().is_high_contrast(); + + if (played_end - start).abs() > F32_ERROR_MARGIN { + let rect = graphene::Rect::new(start, 0.0, played_end - start, WAVEFORM_HEIGHT); + snapshot.append_color(&color, &rect); + } + + if (hover_end - played_end).abs() > F32_ERROR_MARGIN { + let color = color.with_alpha(if is_high_contrast { 0.7 } else { 0.45 }); + + let rect = + graphene::Rect::new(played_end, 0.0, hover_end - played_end, WAVEFORM_HEIGHT); + snapshot.append_color(&color, &rect); + } + + if (hover_end - end).abs() > F32_ERROR_MARGIN { + let color = color.with_alpha(if is_high_contrast { 0.4 } else { 0.2 }); + + let rect = graphene::Rect::new(hover_end, 0.0, end - hover_end, WAVEFORM_HEIGHT); + snapshot.append_color(&color, &rect); + } + + snapshot.pop(); + } + } + + impl Waveform { + /// Set the waveform to display. + /// + /// The values must be normalized between 0 and 1. + pub(super) fn set_waveform(&self, waveform: Vec) { + let animate_transition = self.paintable.set_waveform(waveform); + self.paintable_cache.take(); + + if animate_transition { + self.animation().play(); + } + } + + /// Set the current position in the audio stream. + pub(super) fn set_position(&self, position: f32) { + if (self.position.get() - position).abs() > F32_ERROR_MARGIN { + return; + } + + self.position.set(position); + + let obj = self.obj(); + obj.update_property(&[gtk::accessible::Property::ValueNow(position.into())]); + obj.notify_position(); + obj.queue_draw(); + } + + /// The animation for the waveform change. + fn animation(&self) -> &adw::TimedAnimation { + self.animation.get_or_init(|| { + adw::TimedAnimation::builder() + .widget(&*self.obj()) + .value_to(1.0) + .duration(ANIMATION_DURATION) + .target(&adw::PropertyAnimationTarget::new( + &self.paintable, + "transition-progress", + )) + .easing(adw::Easing::EaseInOutQuad) + .build() + }) + } + + // Get the waveform shape as a monochrome paintable. + // + // If we are not in a transition phase, we cache it because the shape only + // changes if the widget is resized. + fn paintable(&self) -> Option { + let transition_is_ongoing = self + .animation + .get() + .is_some_and(|animation| animation.state() == adw::AnimationState::Playing); + + if !transition_is_ongoing && let Some(paintable) = self.paintable_cache.borrow().clone() + { + return Some(paintable); + } + + let width = self.obj().width() as f32; + let cache_snapshot = gtk::Snapshot::new(); + + self.paintable + .snapshot(&cache_snapshot, width.into(), WAVEFORM_HEIGHT.into()); + let Some(paintable) = + cache_snapshot.to_paintable(Some(&graphene::Size::new(width, WAVEFORM_HEIGHT))) + else { + error!("Could not convert snapshot to paintable"); + return None; + }; + + if !transition_is_ongoing { + self.paintable_cache.replace(Some(paintable.clone())); + } + + Some(paintable) + } + + /// Convert the given x coordinate on the waveform to a relative + /// position. + /// + /// Takes into account the text direction. + /// + /// Returns a value between 0 and 1. + fn x_coord_to_position(&self, x: f64) -> f32 { + let obj = self.obj(); + + let mut position = (x / f64::from(obj.width())) as f32; + + if obj.direction() == gtk::TextDirection::Rtl { + position = 1.0 - position; + } + + position + } + + /// Emit the `seek` signal with the given new position. + fn emit_seek(&self, new_position: f32) { + self.obj().emit_by_name::<()>("seek", &[&new_position]); + } + + /// Initialize the event controllers on the waveform. + fn init_event_controllers(&self) { + let obj = self.obj(); + + // Show mouse hover effect. + let motion = gtk::EventControllerMotion::builder() + .name("waveform-motion") + .build(); + motion.connect_motion(clone!( + #[weak] + obj, + move |_, x, _| { + obj.imp().hover_position.set(Some(x as f32)); + obj.queue_draw(); + } + )); + motion.connect_leave(clone!( + #[weak] + obj, + move |_| { + obj.imp().hover_position.take(); + obj.queue_draw(); + } + )); + obj.add_controller(motion); + + // Handle dragging to seek. This also handles clicks because a click triggers a + // drag begin. + let drag = gtk::GestureDrag::builder() + .name("waveform-drag") + .button(0) + .build(); + drag.connect_drag_begin(clone!( + #[weak] + obj, + move |gesture, x, _| { + gesture.set_state(gtk::EventSequenceState::Claimed); + + if !obj.has_focus() { + obj.grab_focus(); + } + + let imp = obj.imp(); + imp.emit_seek(imp.x_coord_to_position(x)); + } + )); + drag.connect_drag_update(clone!( + #[weak] + obj, + move |gesture, offset_x, _| { + gesture.set_state(gtk::EventSequenceState::Claimed); + + if !obj.has_focus() { + obj.grab_focus(); + } + + let x = gesture + .start_point() + .expect("ongoing drag should have start point") + .0 + + offset_x; + + let imp = obj.imp(); + imp.emit_seek(imp.x_coord_to_position(x)); + } + )); + obj.add_controller(drag); + + // Handle left and right key presses to seek. + let key = gtk::EventControllerKey::builder() + .name("waveform-key") + .build(); + key.connect_key_released(clone!( + #[weak] + obj, + move |_, keyval, _, _| { + let mut delta = match keyval { + gdk::Key::Left | gdk::Key::KP_Left => -0.05, + gdk::Key::Right | gdk::Key::KP_Right => 0.05, + _ => return, + }; + + if obj.direction() == gtk::TextDirection::Rtl { + delta = -delta; + } + + let imp = obj.imp(); + let new_position = imp.position.get() + delta; + + if (0.0..=1.0).contains(&new_position) { + imp.emit_seek(new_position); + } + } + )); + obj.add_controller(key); + } + } +} + +glib::wrapper! { + /// A widget displaying a waveform. + /// + /// This widget supports seeking with the keyboard and mouse. + pub struct Waveform(ObjectSubclass) + @extends gtk::Widget, + @implements gtk::Accessible, gtk::Buildable, gtk::ConstraintTarget; +} + +impl Waveform { + /// Create a new empty `Waveform`. + pub fn new() -> Self { + glib::Object::new() + } + + /// Set the waveform to display. + /// + /// The values must be normalized between 0 and 1. + pub(crate) fn set_waveform(&self, waveform: Vec) { + self.imp().set_waveform(waveform); + } + + /// Connect to the signal emitted when the user seeks another position. + pub fn connect_seek(&self, f: F) -> glib::SignalHandlerId { + self.connect_closure( + "seek", + true, + closure_local!(move |obj: Self, position: f32| { + f(&obj, position); + }), + ) + } +} + +impl Default for Waveform { + fn default() -> Self { + Self::new() + } +} diff --git a/src/components/media/audio_player/waveform_paintable.rs b/src/components/media/audio_player/waveform_paintable.rs new file mode 100644 index 00000000..11d31df6 --- /dev/null +++ b/src/components/media/audio_player/waveform_paintable.rs @@ -0,0 +1,195 @@ +use std::borrow::Cow; + +use gtk::{gdk, glib, graphene, prelude::*, subclass::prelude::*}; + +use super::waveform::{WAVEFORM_HEIGHT, WAVEFORM_HEIGHT_I32}; +use crate::utils::resample_slice; + +/// The width of the bars in the waveform. +const BAR_WIDTH: f32 = 2.0; +/// The horizontal padding around bars in the waveform. +const BAR_HORIZONTAL_PADDING: f32 = 1.0; +/// The full width of a bar, including its padding. +const BAR_FULL_WIDTH: f32 = BAR_WIDTH + 2.0 * BAR_HORIZONTAL_PADDING; +/// The minimum height of the bars in the waveform. +/// +/// We do not want to have holes in the waveform so we restrict the minimum +/// height. +const BAR_MIN_HEIGHT: f32 = 2.0; +/// The waveform used as fallback. +/// +/// It will generate a full waveform. +const WAVEFORM_FALLBACK: &[f32] = &[1.0]; + +mod imp { + use std::cell::{Cell, RefCell}; + + use super::*; + + #[derive(Debug, glib::Properties)] + #[properties(wrapper_type = super::WaveformPaintable)] + pub struct WaveformPaintable { + /// The waveform to display. + /// + /// The values must be normalized between 0 and 1. + waveform: RefCell>, + /// The previous waveform that was displayed, if any. + /// + /// Use for the transition between waveforms. + previous_waveform: RefCell>>, + /// The progress of the transition between waveforms. + #[property(get, set = Self::set_transition_progress, explicit_notify)] + transition_progress: Cell, + } + + impl Default for WaveformPaintable { + fn default() -> Self { + Self { + waveform: RefCell::new(Cow::Borrowed(WAVEFORM_FALLBACK)), + previous_waveform: Default::default(), + transition_progress: Cell::new(1.0), + } + } + } + + #[glib::object_subclass] + impl ObjectSubclass for WaveformPaintable { + const NAME: &'static str = "WaveformPaintable"; + type Type = super::WaveformPaintable; + type Interfaces = (gdk::Paintable,); + } + + #[glib::derived_properties] + impl ObjectImpl for WaveformPaintable {} + + impl PaintableImpl for WaveformPaintable { + fn intrinsic_width(&self) -> i32 { + (self.waveform.borrow().len() as f32 * BAR_FULL_WIDTH) as i32 + } + + fn intrinsic_height(&self) -> i32 { + WAVEFORM_HEIGHT_I32 + } + + fn snapshot(&self, snapshot: &gdk::Snapshot, width: f64, _height: f64) { + if width <= 0.0 { + return; + } + + let exact_samples_needed = width as f32 / BAR_FULL_WIDTH; + + // If the number of samples has a fractional part, compute a padding to center + // the waveform horizontally in the paintable. + let waveform_start_padding = (exact_samples_needed.fract() * BAR_FULL_WIDTH).trunc(); + // We are sure that the number of samples is positive. + #[allow(clippy::cast_sign_loss)] + let samples_needed = exact_samples_needed.trunc() as usize; + + let mut waveform = + resample_slice(self.waveform.borrow().as_ref(), samples_needed).into_owned(); + + // If there is a previous waveform, we have an ongoing transition. + if let Some(previous_waveform) = self.previous_waveform.borrow().as_ref() + && *previous_waveform != waveform + { + let previous_waveform = resample_slice(previous_waveform, samples_needed); + let progress = self.transition_progress.get() as f32; + + // Compute the current waveform for the ongoing transition. + waveform = waveform + .into_iter() + .zip(previous_waveform.iter()) + .map(|(current, &previous)| { + (((current - previous) * progress) + previous).clamp(0.0, 1.0) + }) + .collect(); + } + + for (pos, value) in waveform.into_iter().enumerate() { + if value > 1.0 { + tracing::error!("Waveform sample value is higher than 1: {value}"); + } + + let x = waveform_start_padding + pos as f32 * (BAR_FULL_WIDTH); + let height = (WAVEFORM_HEIGHT * value).max(BAR_MIN_HEIGHT); + // Center the bar vertically. + let y = (WAVEFORM_HEIGHT - height) / 2.0; + + let rect = graphene::Rect::new(x, y, BAR_WIDTH, height); + snapshot.append_color(&gdk::RGBA::WHITE, &rect); + } + } + } + + impl WaveformPaintable { + /// Set the values of the bars to display. + /// + /// The values must be normalized between 0 and 1. + /// + /// Returns whether the waveform changed. + pub(super) fn set_waveform(&self, waveform: Vec) -> bool { + let waveform = if waveform.is_empty() { + Cow::Borrowed(WAVEFORM_FALLBACK) + } else { + Cow::Owned(waveform) + }; + + if *self.waveform.borrow() == waveform { + return false; + } + + let previous = self.waveform.replace(waveform); + self.previous_waveform.replace(Some(previous)); + + self.obj().invalidate_contents(); + + true + } + + /// Set the progress of the transition between waveforms. + fn set_transition_progress(&self, progress: f64) { + if (self.transition_progress.get() - progress).abs() > 0.000_001 { + return; + } + + self.transition_progress.set(progress); + + if (progress - 1.0).abs() > 0.000_001 { + // This is the end of the transition, we can drop the previous waveform. + self.previous_waveform.take(); + } + + let obj = self.obj(); + obj.notify_transition_progress(); + obj.invalidate_contents(); + } + } +} + +glib::wrapper! { + /// A paintable displaying a waveform. + pub struct WaveformPaintable(ObjectSubclass) + @implements gdk::Paintable; +} + +impl WaveformPaintable { + /// Create a new empty `WaveformPaintable`. + pub fn new() -> Self { + glib::Object::new() + } + + /// Set the waveform to display. + /// + /// The values must be normalized between 0 and 1. + /// + /// Returns whether the waveform changed. + pub(crate) fn set_waveform(&self, waveform: Vec) -> bool { + self.imp().set_waveform(waveform) + } +} + +impl Default for WaveformPaintable { + fn default() -> Self { + Self::new() + } +} diff --git a/src/components/media/content_viewer.rs b/src/components/media/content_viewer.rs index 028c7222..c083765d 100644 --- a/src/components/media/content_viewer.rs +++ b/src/components/media/content_viewer.rs @@ -3,7 +3,7 @@ use geo_uri::GeoUri; use gettextrs::gettext; use gtk::{gdk, gio, glib}; -use super::{AnimatedImagePaintable, AudioPlayer, LocationViewer}; +use super::{AnimatedImagePaintable, AudioPlayer, AudioPlayerSource, LocationViewer}; use crate::{ components::ContextMenuBin, prelude::*, @@ -189,16 +189,15 @@ mod imp { audio } else { let audio = AudioPlayer::new(); - audio.add_css_class("toolbar"); - audio.add_css_class("osd"); - audio.set_autoplay(self.autoplay.get()); + audio.set_standalone(true); + audio.set_margin_start(12); + audio.set_margin_end(12); audio.set_valign(gtk::Align::Center); - audio.set_halign(gtk::Align::Center); self.viewer.set_child(Some(&audio)); audio }; - audio.set_file(Some(&file.as_gfile())); + audio.set_source(Some(AudioPlayerSource::File(file.as_gfile()))); self.update_animated_paintable_state(); self.set_visible_child("viewer"); return; diff --git a/src/components/media/mod.rs b/src/components/media/mod.rs index 490a3665..f70d8f76 100644 --- a/src/components/media/mod.rs +++ b/src/components/media/mod.rs @@ -5,9 +5,9 @@ mod location_viewer; mod video_player; mod video_player_renderer; -pub use self::{ +pub(crate) use self::{ animated_image_paintable::AnimatedImagePaintable, - audio_player::AudioPlayer, + audio_player::*, content_viewer::{ContentType, MediaContentViewer}, location_viewer::LocationViewer, video_player::VideoPlayer, diff --git a/src/components/media/video_player.rs b/src/components/media/video_player.rs index bce53943..4602bfb1 100644 --- a/src/components/media/video_player.rs +++ b/src/components/media/video_player.rs @@ -3,7 +3,7 @@ use gtk::{gio, glib, glib::clone}; use tracing::{error, warn}; use super::video_player_renderer::VideoPlayerRenderer; -use crate::utils::LoadingState; +use crate::utils::{LoadingState, media}; mod imp { use std::cell::{Cell, OnceCell, RefCell}; @@ -191,24 +191,7 @@ mod imp { let is_visible = visible_duration.is_some(); if let Some(duration) = visible_duration { - let mut time = duration.seconds(); - - let sec = time % 60; - time -= sec; - let min = (time % (60 * 60)) / 60; - time -= min * 60; - let hour = time / (60 * 60); - - let label = if hour > 0 { - // FIXME: Find how to localize this. - // hour:minutes:seconds - format!("{hour}:{min:02}:{sec:02}") - } else { - // FIXME: Find how to localize this. - // minutes:seconds - format!("{min:02}:{sec:02}") - }; - + let label = media::time_to_label(&duration.into()); self.timestamp.set_label(&label); } diff --git a/src/session/view/content/room_history/message_row/audio.blp b/src/session/view/content/room_history/message_row/audio.blp index ebee5cfc..cd1e8c8f 100644 --- a/src/session/view/content/room_history/message_row/audio.blp +++ b/src/session/view/content/room_history/message_row/audio.blp @@ -1,40 +1,26 @@ using Gtk 4.0; -using Adw 1; -template $ContentMessageAudio: Adw.Bin { - Gtk.Box { - orientation: vertical; - - Gtk.Box { - margin-top: 6; - spacing: 6; - - Gtk.Image { - visible: bind template.compact; - icon-name: "audio-symbolic"; - } - - Gtk.Label { - ellipsize: end; - xalign: 0.0; - hexpand: true; - label: bind template.filename; - } +template $ContentMessageAudio: Gtk.Box { + orientation: vertical; - [end] - Adw.Spinner state_spinner { - height-request: 20; - width-request: 20; - } + Gtk.Box { + visible: bind template.compact; + margin-top: 6; + spacing: 6; - [end] - Gtk.Image state_error { - icon-name: "error-symbolic"; - } + Gtk.Image { + icon-name: "audio-symbolic"; } - $AudioPlayer player { - visible: bind template.compact inverted; + Gtk.Label { + ellipsize: end; + xalign: 0.0; + hexpand: true; + label: bind template.filename; } } + + $AudioPlayer player { + visible: bind template.compact inverted; + } } diff --git a/src/session/view/content/room_history/message_row/audio.rs b/src/session/view/content/room_history/message_row/audio.rs index 1dc3ee9e..228c946f 100644 --- a/src/session/view/content/room_history/message_row/audio.rs +++ b/src/session/view/content/room_history/message_row/audio.rs @@ -1,15 +1,10 @@ -use adw::{prelude::*, subclass::prelude::*}; use gettextrs::gettext; -use gtk::{glib, glib::clone}; -use tracing::warn; +use gtk::{glib, prelude::*, subclass::prelude::*}; -use super::{ContentFormat, content::MessageCacheKey}; +use super::ContentFormat; use crate::{ - components::AudioPlayer, + components::{AudioPlayer, AudioPlayerMessage, AudioPlayerSource}, gettext_f, - session::model::Session, - spawn, - utils::{File, LoadingState, matrix::MediaMessage}, }; mod imp { @@ -27,24 +22,9 @@ mod imp { pub struct MessageAudio { #[template_child] player: TemplateChild, - #[template_child] - state_spinner: TemplateChild, - #[template_child] - state_error: TemplateChild, /// The filename of the audio file. #[property(get)] - filename: RefCell>, - /// The cache key for the current audio message. - /// - /// The audio is only reloaded if the cache key changes. This is to - /// avoid reloading the audio when the local echo is updated to a remote - /// echo. - cache_key: RefCell, - /// The media file. - file: RefCell>, - /// The state of the audio file. - #[property(get, builder(LoadingState::default()))] - state: Cell, + filename: RefCell, /// Whether to display this audio message in a compact format. #[property(get)] compact: Cell, @@ -54,12 +34,10 @@ mod imp { impl ObjectSubclass for MessageAudio { const NAME: &'static str = "ContentMessageAudio"; type Type = super::MessageAudio; - type ParentType = adw::Bin; + type ParentType = gtk::Box; fn class_init(klass: &mut Self::Class) { Self::bind_template(klass); - - klass.set_accessible_role(gtk::AccessibleRole::Group); } fn instance_init(obj: &InitializingObject) { @@ -71,20 +49,22 @@ mod imp { impl ObjectImpl for MessageAudio {} impl WidgetImpl for MessageAudio {} - impl BinImpl for MessageAudio {} + impl BoxImpl for MessageAudio {} impl MessageAudio { /// Set the filename of the audio file. fn set_filename(&self, filename: Option) { + let filename = filename.unwrap_or_default(); + if *self.filename.borrow() == filename { return; } let obj = self.obj(); - let accessible_label = if let Some(filename) = &filename { - gettext_f("Audio: {filename}", &[("filename", filename)]) - } else { + let accessible_label = if filename.is_empty() { gettext("Audio") + } else { + gettext_f("Audio: {filename}", &[("filename", &filename)]) }; obj.update_property(&[gtk::accessible::Property::Label(&accessible_label)]); @@ -97,123 +77,22 @@ mod imp { let obj = self.obj(); self.compact.set(compact); - if compact { - obj.remove_css_class("osd"); - obj.remove_css_class("toolbar"); - } else { - obj.add_css_class("osd"); - obj.add_css_class("toolbar"); - } - obj.notify_compact(); } - /// Set the state of the audio file. - fn set_state(&self, state: LoadingState) { - if self.state.get() == state { - return; - } - - match state { - LoadingState::Loading | LoadingState::Initial => { - self.state_spinner.set_visible(true); - self.state_error.set_visible(false); - } - LoadingState::Ready => { - self.state_spinner.set_visible(false); - self.state_error.set_visible(false); - } - LoadingState::Error => { - self.state_spinner.set_visible(false); - self.state_error.set_visible(true); - } - } - - self.state.set(state); - self.obj().notify_state(); - } - - /// Convenience method to set the state to `Error` with the given error - /// message. - fn set_error(&self, error: &str) { - self.set_state(LoadingState::Error); - self.state_error.set_tooltip_text(Some(error)); - } - - /// Set the cache key with the given value. - /// - /// Returns `true` if the audio should be reloaded. - fn set_cache_key(&self, key: MessageCacheKey) -> bool { - let should_reload = self.cache_key.borrow().should_reload(&key); - self.cache_key.replace(key); - - should_reload - } - /// Display the given `audio` message. - pub(super) fn audio( - &self, - message: MediaMessage, - session: &Session, - format: ContentFormat, - cache_key: MessageCacheKey, - ) { - if !self.set_cache_key(cache_key) { - // We do not need to reload the audio. - return; - } - - self.file.take(); - self.set_filename(Some(message.filename())); + pub(super) fn set_audio_message(&self, message: AudioPlayerMessage, format: ContentFormat) { + self.set_filename(Some(message.message.filename())); let compact = matches!(format, ContentFormat::Compact | ContentFormat::Ellipsized); self.set_compact(compact); + if compact { - self.set_state(LoadingState::Ready); - return; + self.player.set_source(None); + } else { + self.player + .set_source(Some(AudioPlayerSource::Message(message))); } - - self.set_state(LoadingState::Loading); - - let client = session.client(); - - spawn!( - glib::Priority::LOW, - clone!( - #[weak(rename_to = imp)] - self, - async move { - match message.into_tmp_file(&client).await { - Ok(file) => { - imp.display_file(file); - } - Err(error) => { - warn!("Could not retrieve audio file: {error}"); - imp.set_error(&gettext("Could not retrieve audio file")); - } - } - } - ) - ); - } - - fn display_file(&self, file: File) { - let media_file = gtk::MediaFile::for_file(&file.as_gfile()); - - media_file.connect_error_notify(clone!( - #[weak(rename_to = imp)] - self, - move |media_file| { - if let Some(error) = media_file.error() { - warn!("Error reading audio file: {error}"); - imp.set_error(&gettext("Error reading audio file")); - } - } - )); - - self.file.replace(Some(file)); - self.player.set_media_file(Some(media_file)); - self.set_state(LoadingState::Ready); } } } @@ -221,7 +100,7 @@ mod imp { glib::wrapper! { /// A widget displaying an audio message in the timeline. pub struct MessageAudio(ObjectSubclass) - @extends gtk::Widget, adw::Bin, + @extends gtk::Widget, gtk::Box, @implements gtk::Accessible, gtk::Buildable, gtk::ConstraintTarget; } @@ -232,14 +111,8 @@ impl MessageAudio { } /// Display the given `audio` message. - pub(crate) fn audio( - &self, - message: MediaMessage, - session: &Session, - format: ContentFormat, - cache_key: MessageCacheKey, - ) { - self.imp().audio(message, session, format, cache_key); + pub(crate) fn set_audio_message(&self, message: AudioPlayerMessage, format: ContentFormat) { + self.imp().set_audio_message(message, format); } } diff --git a/src/session/view/content/room_history/message_row/content.rs b/src/session/view/content/room_history/message_row/content.rs index 4f248e01..53166c1f 100644 --- a/src/session/view/content/room_history/message_row/content.rs +++ b/src/session/view/content/room_history/message_row/content.rs @@ -10,13 +10,14 @@ use super::{ reply::MessageReply, text::MessageText, visual_media::MessageVisualMedia, }; use crate::{ + components::AudioPlayerMessage, prelude::*, session::{ model::{Event, Member, Room}, view::content::room_history::message_toolbar::MessageEventSource, }, spawn, - utils::matrix::MediaMessage, + utils::matrix::{MediaMessage, MessageCacheKey}, }; #[derive(Debug, Default, Hash, Eq, PartialEq, Clone, Copy, glib::Enum)] @@ -439,7 +440,10 @@ trait MessageContentContainer: ChildPropertyExt { return; }; let widget = self.child_or_default::(); - widget.audio(audio.into(), &session, format, cache_key); + widget.set_audio_message( + AudioPlayerMessage::new(audio.into(), &session, cache_key), + format, + ); } MediaMessage::File(file) => { let widget = self.child_or_default::(); @@ -467,46 +471,3 @@ trait MessageContentContainer: ChildPropertyExt { impl MessageContentContainer for W where W: IsABin {} impl MessageContentContainer for MessageCaption {} - -/// The data used as a cache key for messages. -/// -/// This is used when there is no reliable way to detect if the content of a -/// message changed. For example, the URI of a media file might change between a -/// local echo and a remote echo, but we do not need to reload the media in this -/// case, and we have no other way to know that both URIs point to the same -/// file. -#[derive(Debug, Clone, Default)] -pub(crate) struct MessageCacheKey { - /// The transaction ID of the event. - /// - /// Local echo should keep its transaction ID after the message is sent, so - /// we do not need to reload the message if it did not change. - transaction_id: Option, - /// The global ID of the event. - /// - /// Local echo that was sent and remote echo should have the same event ID, - /// so we do not need to reload the message if it did not change. - pub(crate) event_id: Option, - /// Whether the message is edited. - /// - /// The message must be reloaded when it was edited. - is_edited: bool, -} - -impl MessageCacheKey { - /// Whether the given new `MessageCacheKey` should trigger a reload of the - /// message compared to this one. - pub(super) fn should_reload(&self, new: &MessageCacheKey) -> bool { - if new.is_edited { - return true; - } - - let transaction_id_invalidated = self.transaction_id.is_none() - || new.transaction_id.is_none() - || self.transaction_id != new.transaction_id; - let event_id_invalidated = - self.event_id.is_none() || new.event_id.is_none() || self.event_id != new.event_id; - - transaction_id_invalidated && event_id_invalidated - } -} diff --git a/src/session/view/content/room_history/message_row/visual_media.rs b/src/session/view/content/room_history/message_row/visual_media.rs index c5545244..5dcef9eb 100644 --- a/src/session/view/content/room_history/message_row/visual_media.rs +++ b/src/session/view/content/room_history/message_row/visual_media.rs @@ -4,7 +4,7 @@ use gtk::{gdk, glib, glib::clone}; use ruma::api::client::media::get_content_thumbnail::v3::Method; use tracing::warn; -use super::{ContentFormat, content::MessageCacheKey}; +use super::ContentFormat; use crate::{ Window, components::{AnimatedImagePaintable, VideoPlayer}, @@ -13,7 +13,7 @@ use crate::{ spawn, utils::{ CountedRef, File, LoadingState, TemplateCallbacks, key_bindings, - matrix::{VisualMediaMessage, VisualMediaType}, + matrix::{MessageCacheKey, VisualMediaMessage, VisualMediaType}, media::{ FrameDimensions, image::{ImageRequestPriority, THUMBNAIL_MAX_DIMENSIONS, ThumbnailSettings}, diff --git a/src/session/view/content/room_history/message_toolbar/mod.rs b/src/session/view/content/room_history/message_toolbar/mod.rs index a685a40c..50020aa6 100644 --- a/src/session/view/content/room_history/message_toolbar/mod.rs +++ b/src/session/view/content/room_history/message_toolbar/mod.rs @@ -44,7 +44,7 @@ use crate::{ utils::{ Location, LocationError, TemplateCallbacks, TokioDrop, media::{ - FileInfo, filename_for_mime, image::ImageInfoLoader, load_audio_info, + FileInfo, audio::load_audio_info, filename_for_mime, image::ImageInfoLoader, video::load_video_info, }, }, diff --git a/src/ui-blueprint-resources.in b/src/ui-blueprint-resources.in index 1a7c605f..bd34ac30 100644 --- a/src/ui-blueprint-resources.in +++ b/src/ui-blueprint-resources.in @@ -20,7 +20,7 @@ components/dialogs/room_preview.blp components/dialogs/toastable.blp components/dialogs/user_profile.blp components/loading/bin.blp -components/media/audio_player.blp +components/media/audio_player/mod.blp components/media/content_viewer.blp components/media/location_viewer.blp components/media/video_player.blp diff --git a/src/utils/matrix/media_message.rs b/src/utils/matrix/media_message.rs index ca614a96..9b8f7679 100644 --- a/src/utils/matrix/media_message.rs +++ b/src/utils/matrix/media_message.rs @@ -18,6 +18,7 @@ use crate::{ File, media::{ FrameDimensions, MediaFileError, + audio::normalize_waveform, image::{ Blurhash, Image, ImageError, ImageRequestPriority, ImageSource, ThumbnailDownloader, ThumbnailSettings, @@ -422,3 +423,29 @@ pub(crate) enum VisualMediaType { /// A sticker. Sticker, } + +/// Extension trait for audio messages. +pub(crate) trait AudioMessageExt { + /// Get the normalized waveform in this audio message, if any. + /// + /// A normalized waveform is a waveform containing only values between 0 and + /// 1. + fn normalized_waveform(&self) -> Option>; +} + +impl AudioMessageExt for AudioMessageEventContent { + fn normalized_waveform(&self) -> Option> { + let waveform = &self.audio.as_ref()?.waveform; + + if waveform.is_empty() { + return None; + } + + Some(normalize_waveform( + waveform + .iter() + .map(|amplitude| u64::from(amplitude.get()) as f64) + .collect(), + )) + } +} diff --git a/src/utils/matrix/mod.rs b/src/utils/matrix/mod.rs index 8099be01..64b09f0c 100644 --- a/src/utils/matrix/mod.rs +++ b/src/utils/matrix/mod.rs @@ -16,8 +16,8 @@ use matrix_sdk::{ }; use ruma::{ EventId, IdParseError, MatrixToUri, MatrixUri, MatrixUriError, MilliSecondsSinceUnixEpoch, - OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName, OwnedUserId, - RoomId, RoomOrAliasId, UserId, + OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName, + OwnedTransactionId, OwnedUserId, RoomId, RoomOrAliasId, UserId, events::{AnyStrippedStateEvent, AnySyncTimelineEvent}, html::{ Children, Html, NodeRef, StrTendril, @@ -617,3 +617,46 @@ pub(crate) fn seconds_since_unix_epoch_to_date(secs: i64) -> glib::DateTime { .and_then(|date| date.to_local()) .expect("constructing GDateTime from timestamp should work") } + +/// The data used as a cache key for messages. +/// +/// This is used when there is no reliable way to detect if the content of a +/// message changed. For example, the URI of a media file might change between a +/// local echo and a remote echo, but we do not need to reload the media in this +/// case, and we have no other way to know that both URIs point to the same +/// file. +#[derive(Debug, Clone, Default)] +pub(crate) struct MessageCacheKey { + /// The transaction ID of the event. + /// + /// Local echo should keep its transaction ID after the message is sent, so + /// we do not need to reload the message if it did not change. + pub(crate) transaction_id: Option, + /// The global ID of the event. + /// + /// Local echo that was sent and remote echo should have the same event ID, + /// so we do not need to reload the message if it did not change. + pub(crate) event_id: Option, + /// Whether the message is edited. + /// + /// The message must be reloaded when it was edited. + pub(crate) is_edited: bool, +} + +impl MessageCacheKey { + /// Whether the given new `MessageCacheKey` should trigger a reload of the + /// message compared to this one. + pub(crate) fn should_reload(&self, new: &MessageCacheKey) -> bool { + if new.is_edited { + return true; + } + + let transaction_id_invalidated = self.transaction_id.is_none() + || new.transaction_id.is_none() + || self.transaction_id != new.transaction_id; + let event_id_invalidated = + self.event_id.is_none() || new.event_id.is_none() || self.event_id != new.event_id; + + transaction_id_invalidated && event_id_invalidated + } +} diff --git a/src/utils/media/audio.rs b/src/utils/media/audio.rs new file mode 100644 index 00000000..a2dea4f2 --- /dev/null +++ b/src/utils/media/audio.rs @@ -0,0 +1,192 @@ +//! Collection of methods for audio. + +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use futures_channel::oneshot; +use gst::prelude::*; +use gtk::{gio, glib, prelude::*}; +use matrix_sdk::attachment::BaseAudioInfo; +use tracing::warn; + +use super::load_gstreamer_media_info; +use crate::utils::resample_slice; + +/// Load information for the audio in the given file. +pub(crate) async fn load_audio_info(file: &gio::File) -> BaseAudioInfo { + let mut info = BaseAudioInfo::default(); + + let Some(media_info) = load_gstreamer_media_info(file).await else { + return info; + }; + + info.duration = media_info.duration().map(Into::into); + info +} + +/// Generate a waveform for the given audio file. +/// +/// The returned waveform should contain between 30 and 110 samples with a value +/// between 0 and 1. +pub(crate) async fn generate_waveform( + file: &gio::File, + duration: Option, +) -> Option> { + // According to MSC3246, we want at least 30 values and at most 120 values. It + // should also allow us to have enough samples for drawing our waveform. + let interval = duration + .and_then(|duration| { + // Try to get around 1 sample per second, except if the duration is too short or + // too long. + match duration.as_secs() { + 0..30 => duration.checked_div(30), + 30..110 => Some(Duration::from_secs(1)), + _ => duration.checked_div(110), + } + }) + .unwrap_or_else(|| Duration::from_secs(1)); + + // Create our pipeline from a pipeline description string. + let pipeline = match gst::parse::launch(&format!( + "uridecodebin uri={} ! audioconvert ! audio/x-raw,channels=1 ! level name=level interval={} ! fakesink qos=false sync=false", + file.uri(), + interval.as_nanos() + )) { + Ok(pipeline) => pipeline + .downcast::() + .expect("GstElement should be a GstPipeline"), + Err(error) => { + warn!("Could not create GstPipeline for audio waveform: {error}"); + return None; + } + }; + + let (sender, receiver) = oneshot::channel(); + let sender = Arc::new(Mutex::new(Some(sender))); + let samples = Arc::new(Mutex::new(vec![])); + let bus = pipeline.bus().expect("GstPipeline should have a GstBus"); + + let samples_clone = samples.clone(); + let _bus_guard = bus + .add_watch(move |_, message| { + match message.view() { + gst::MessageView::Eos(_) => { + // We are done collecting the samples. + send_empty_signal(&sender); + glib::ControlFlow::Break + } + gst::MessageView::Error(error) => { + warn!("Could not generate audio waveform: {error}"); + send_empty_signal(&sender); + glib::ControlFlow::Break + } + gst::MessageView::Element(element) => { + if let Some(structure) = element.structure() + && structure.has_name("level") + { + let peaks_array = structure + .get::<&glib::ValueArray>("peak") + .expect("peak value should be a GValueArray"); + let peak = peaks_array[0] + .get::() + .expect("GValueArray value should be a double"); + + match samples_clone.lock() { + Ok(mut samples) => { + let value_db = if peak.is_nan() { 0.0 } else { peak }; + // Convert the decibels to a relative amplitude, to get a value + // between 0 and 1. + let value = 10.0_f64.powf(value_db / 20.0); + + samples.push(value); + } + Err(error) => { + warn!("Failed to lock audio waveform samples mutex: {error}"); + } + } + } + glib::ControlFlow::Continue + } + _ => glib::ControlFlow::Continue, + } + }) + .expect("Adding GstBus watch should succeed"); + + match pipeline.set_state(gst::State::Playing) { + Ok(_) => { + let _ = receiver.await; + } + Err(error) => { + warn!("Could not start GstPipeline for audio waveform: {error}"); + } + } + + // Clean up pipeline. + let _ = pipeline.set_state(gst::State::Null); + bus.set_flushing(true); + + let waveform = match samples.lock() { + Ok(mut samples) => std::mem::take(&mut *samples), + Err(error) => { + warn!("Failed to lock audio waveform samples mutex: {error}"); + return None; + } + }; + + Some(normalize_waveform(waveform)).filter(|waveform| !waveform.is_empty()) +} + +/// Try to send an empty signal through the given sender. +fn send_empty_signal(sender: &Mutex>>) { + let mut sender = match sender.lock() { + Ok(sender) => sender, + Err(error) => { + warn!("Failed to lock audio waveform signal mutex: {error}"); + return; + } + }; + + if let Some(sender) = sender.take() + && sender.send(()).is_err() + { + warn!("Failed to send audio waveform end through channel"); + } +} + +/// Normalize the given waveform to have between 30 and 120 samples with a value +/// between 0 and 1. +/// +/// All the samples in the waveform must be positive or negative. If they are +/// mixed, this will change the waveform because it uses the absolute value of +/// the sample. +/// +/// If the waveform was empty, returns an empty vec. +pub(crate) fn normalize_waveform(waveform: Vec) -> Vec { + if waveform.is_empty() { + return vec![]; + } + + let max = waveform + .iter() + .copied() + .map(f64::abs) + .reduce(f64::max) + .expect("iterator should contain at least one value"); + + // Normalize between 0 and 1, with the highest value as 1. + let mut normalized = waveform + .into_iter() + .map(f64::abs) + .map(|value| if max == 0.0 { value } else { value / max } as f32) + .collect::>(); + + match normalized.len() { + 0..30 => normalized = resample_slice(&normalized, 30).into_owned(), + 30..120 => {} + _ => normalized = resample_slice(&normalized, 120).into_owned(), + } + + normalized +} diff --git a/src/utils/media/image/mod.rs b/src/utils/media/image/mod.rs index b6b8de03..db1a133d 100644 --- a/src/utils/media/image/mod.rs +++ b/src/utils/media/image/mod.rs @@ -950,6 +950,7 @@ impl From for ImageError { match value { MediaFileError::Sdk(_) => Self::Download, MediaFileError::File(_) => Self::File, + MediaFileError::NoSession => Self::Unknown, } } } diff --git a/src/utils/media/mod.rs b/src/utils/media/mod.rs index 0f4905b2..f7e0db7c 100644 --- a/src/utils/media/mod.rs +++ b/src/utils/media/mod.rs @@ -1,13 +1,13 @@ //! Collection of methods for media. -use std::{cell::Cell, str::FromStr, sync::Mutex}; +use std::{cell::Cell, str::FromStr, sync::Mutex, time::Duration}; use gettextrs::gettext; use gtk::{gio, glib, prelude::*}; -use matrix_sdk::attachment::BaseAudioInfo; use mime::Mime; use ruma::UInt; +pub(crate) mod audio; pub(crate) mod image; pub(crate) mod video; @@ -117,18 +117,6 @@ async fn load_gstreamer_media_info(file: &gio::File) -> Option BaseAudioInfo { - let mut info = BaseAudioInfo::default(); - - let Some(media_info) = load_gstreamer_media_info(file).await else { - return info; - }; - - info.duration = media_info.duration().map(Into::into); - info -} - /// All errors that can occur when downloading a media to a file. #[derive(Debug, thiserror::Error)] #[error(transparent)] @@ -137,6 +125,11 @@ pub(crate) enum MediaFileError { Sdk(#[from] matrix_sdk::Error), /// An error occurred when writing the media to a file. File(#[from] std::io::Error), + /// We could not access the Matrix client via the [`Session`]. + /// + /// [`Session`]: crate::session::model::Session + #[error("Could not access session")] + NoSession, } /// The dimensions of a frame. @@ -234,3 +227,25 @@ impl FrameDimensions { Self { width, height } } } + +/// Get the string representation of the given elapsed time to present it in a +/// media player. +pub(crate) fn time_to_label(time: &Duration) -> String { + let mut time = time.as_secs(); + + let sec = time % 60; + time -= sec; + let min = (time % (60 * 60)) / 60; + time -= min * 60; + let hour = time / (60 * 60); + + if hour > 0 { + // FIXME: Find how to localize this. + // hour:minutes:seconds + format!("{hour}:{min:02}:{sec:02}") + } else { + // FIXME: Find how to localize this. + // minutes:seconds + format!("{min:02}:{sec:02}") + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 4141f519..c42146d5 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -699,3 +699,54 @@ impl Drop for AbortableHandle { self.abort(); } } + +/// Resample the given slice to the given length, using linear interpolation. +/// +/// Returns the slice as-is if it is of the correct length. Returns a `Vec` of +/// zeroes if the slice is empty. +pub(crate) fn resample_slice(slice: &[f32], new_len: usize) -> Cow<'_, [f32]> { + let len = slice.len(); + + if len == new_len { + // The slice has the correct length, return it. + return Cow::Borrowed(slice); + } + + if new_len == 0 { + // We do not need values, return an empty slice. + return Cow::Borrowed(&[]); + } + + if len <= 1 + || slice + .iter() + .all(|value| (*value - slice[0]).abs() > 0.000_001) + { + // There is a single value so we do not need to interpolate, return a `Vec` + // containing that value. + let value = slice.first().copied().unwrap_or_default(); + return Cow::Owned(std::iter::repeat_n(value, new_len).collect()); + } + + // We need to interpolate the values. + let mut result = Vec::with_capacity(new_len); + let ratio = (len - 1) as f32 / (new_len - 1) as f32; + + for i in 0..new_len { + let position_abs = i as f32 * ratio; + let position_before = position_abs.floor(); + let position_after = position_abs.ceil(); + let position_rel = position_abs % 1.0; + + // We are sure that the positions are positive. + #[allow(clippy::cast_sign_loss)] + let value_before = slice[position_before as usize]; + #[allow(clippy::cast_sign_loss)] + let value_after = slice[(position_after as usize).min(slice.len().saturating_sub(1))]; + + let value = (1.0 - position_rel) * value_before + position_rel * value_after; + result.push(value); + } + + Cow::Owned(result) +}