fix: sending does not work

We were inserting one too many 0xff bytes

Cargo.lock

@@ -198,9 +198,9 @@ dependencies = [
 
 [[package]]
 name = "bindgen"
-version = "0.60.1"
+version = "0.59.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
+checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8"
 dependencies = [
  "bitflags",
  "cexpr",
@@ -295,17 +295,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
 
-[[package]]
-name = "bzip2-sys"
-version = "0.1.11+1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
-]
-
 [[package]]
 name = "cc"
 version = "1.0.73"
@@ -1283,17 +1272,14 @@ dependencies = [
 
 [[package]]
 name = "librocksdb-sys"
-version = "0.8.0+7.4.4"
+version = "6.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d"
+checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d"
 dependencies = [
  "bindgen",
- "bzip2-sys",
  "cc",
  "glob",
  "libc",
- "libz-sys",
- "zstd-sys",
 ]
 
 [[package]]
@@ -1307,17 +1293,6 @@ dependencies = [
  "vcpkg",
 ]
 
-[[package]]
-name = "libz-sys"
-version = "1.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
-dependencies = [
- "cc",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "linked-hash-map"
 version = "0.5.6"
@@ -2053,9 +2028,9 @@ dependencies = [
 
 [[package]]
 name = "rocksdb"
-version = "0.19.0"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc"
+checksum = "7a62eca5cacf2c8261128631bed9f045598d40bfbe4b29f5163f0f802f8f44a7"
 dependencies = [
  "libc",
  "librocksdb-sys",
@@ -3286,13 +3261,3 @@ checksum = "70b40401a28d86ce16a330b863b86fd7dbee4d7c940587ab09ab8c019f9e3fdf"
 dependencies = [
  "num-traits",
 ]
-
-[[package]]
-name = "zstd-sys"
-version = "2.0.1+zstd.1.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
-dependencies = [
- "cc",
- "libc",
-]

Cargo.toml

@@ -76,7 +76,7 @@ crossbeam = { version = "0.8.1", optional = true }
 num_cpus = "1.13.0"
 threadpool = "1.8.1"
 heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true }
-rocksdb = { version = "0.19.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true }
+rocksdb = { version = "0.17.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true }
 
 thread_local = "1.1.3"
 # used for TURN server authentication
@@ -94,7 +94,7 @@ lazy_static = "1.4.0"
 async-trait = "0.1.57"
 
 [features]
-default = ["conduit_bin", "backend_sqlite", "jemalloc"]
+default = ["conduit_bin", "backend_sqlite", "backend_rocksdb", "jemalloc"]
 #backend_sled = ["sled"]
 backend_persy = ["persy", "parking_lot"]
 backend_sqlite = ["sqlite"]

src/database/abstraction/rocksdb.rs

@@ -161,6 +161,7 @@ impl KvTree for RocksDbEngineTree<'_> {
             self.db
                 .rocks
                 .iterator_cf(&self.cf(), rocksdb::IteratorMode::Start)
+                //.map(|r| r.unwrap())
                 .map(|(k, v)| (Vec::from(k), Vec::from(v))),
         )
     }
@@ -184,6 +185,7 @@ impl KvTree for RocksDbEngineTree<'_> {
                         },
                     ),
                 )
+                //.map(|r| r.unwrap())
                 .map(|(k, v)| (Vec::from(k), Vec::from(v))),
         )
     }
@@ -224,6 +226,7 @@ impl KvTree for RocksDbEngineTree<'_> {
                     &self.cf(),
                     rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward),
                 )
+                //.map(|r| r.unwrap())
                 .map(|(k, v)| (Vec::from(k), Vec::from(v)))
                 .take_while(move |(k, _)| k.starts_with(&prefix)),
         )

src/database/key_value/sending.rs

@@ -66,7 +66,6 @@ impl service::sending::Data for KeyValueDatabase {
         let mut keys = Vec::new();
         for (outgoing_kind, event) in requests {
             let mut key = outgoing_kind.get_prefix();
-            key.push(0xff);
             key.extend_from_slice(if let SendingEventType::Pdu(value) = &event {
                 &**value
             } else {
@@ -139,6 +138,7 @@ fn parse_servercurrentevent(
         let event = parts
             .next()
             .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+
         let server = utils::string_from_bytes(server).map_err(|_| {
             Error::bad_database("Invalid server bytes in server_currenttransaction")
         })?;
@@ -169,6 +169,7 @@ fn parse_servercurrentevent(
         let event = parts
             .next()
             .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+
         (
             OutgoingKind::Push(user_id, pushkey_string),
             if value.is_empty() {
@@ -185,6 +186,7 @@ fn parse_servercurrentevent(
         let event = parts
             .next()
             .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+
         let server = utils::string_from_bytes(server).map_err(|_| {
             Error::bad_database("Invalid server bytes in server_currenttransaction")
         })?;

src/database/mod.rs

@@ -556,7 +556,7 @@ impl KeyValueDatabase {
                                 services()
                                     .rooms
                                     .state_compressor
-                                    .load_shortstatehash_info(dbg!(last_roomsstatehash))
+                                    .load_shortstatehash_info(last_roomsstatehash)
                             },
                         )?;
 
@@ -579,7 +579,7 @@ impl KeyValueDatabase {
                             };
 
                         services().rooms.state_compressor.save_state_from_diff(
-                            dbg!(current_sstatehash),
+                            current_sstatehash,
                             statediffnew,
                             statediffremoved,
                             2, // every state change is 2 event changes on average

src/service/sending/mod.rs

@@ -587,7 +587,7 @@ impl Service {
                         .notification_count(&userid, &pdu.room_id)
                         .map_err(|e| (kind.clone(), e))?
                         .try_into()
-                        .expect("notifiation count can't go that high");
+                        .expect("notification count can't go that high");
 
                     let permit = services().sending.maximum_requests.acquire().await;
 
@@ -616,6 +616,7 @@ impl Service {
                                     .get_pdu_json_from_id(pdu_id)
                                     .map_err(|e| (OutgoingKind::Normal(server.clone()), e))?
                                     .ok_or_else(|| {
+                                        error!("event not found: {server} {pdu_id:?}");
                                         (
                                             OutgoingKind::Normal(server.clone()),
                                             Error::bad_database(