diff --git a/src/session/model/room/member_list.rs b/src/session/model/room/member_list.rs
index 4b18af2a..2cc76817 100644
--- a/src/session/model/room/member_list.rs
+++ b/src/session/model/room/member_list.rs
@@ -14,7 +14,7 @@ use matrix_sdk::{
 };
 use tracing::error;
 
-use super::{Member, Membership, Room};
+use super::{Event, Member, Membership, Room};
 use crate::{spawn, spawn_tokio};
 
 mod imp {
@@ -223,6 +223,24 @@ impl MemberList {
                     member.update_from_room_member(room_member);
                 }
             }
+
+            // Restore the members activity according to the known timeline events.
+            for item in self.room().timeline().items().iter::<glib::Object>().rev() {
+                let Ok(item) = item else {
+                    // The iterator is broken, stop.
+                    break;
+                };
+                let Ok(event) = item.downcast::<Event>() else {
+                    continue;
+                };
+                if !event.counts_as_unread() {
+                    continue;
+                }
+
+                if let Some(member) = members.get(&event.sender_id()) {
+                    member.set_latest_activity(event.origin_server_ts_u64());
+                }
+            }
         }
 
         if num_members_added > 0 {
diff --git a/src/session/model/room/timeline/mod.rs b/src/session/model/room/timeline/mod.rs
index 76b36688..f91fe554 100644
--- a/src/session/model/room/timeline/mod.rs
+++ b/src/session/model/room/timeline/mod.rs
@@ -346,6 +346,14 @@ impl Timeline {
                 .event_map
                 .borrow_mut()
                 .insert(event.key(), event.clone());
+
+            // Keep track of the activity of the sender.
+            if event.counts_as_unread() {
+                if let Some(members) = self.room().members() {
+                    let member = members.get_or_create(event.sender_id());
+                    member.set_latest_activity(event.origin_server_ts_u64());
+                }
+            }
         }
 
         item