Browse Source

refactor: move configuration to it's own crate

and some related changes/improvements
rate-limiting
Matthias Ahouansou 1 week ago
parent
commit
9937b4484e
No known key found for this signature in database
  1. 64
      Cargo.lock
  2. 11
      Cargo.toml
  3. 34
      conduit-config/Cargo.toml
  4. 21
      conduit-config/src/error.rs
  5. 96
      conduit-config/src/lib.rs
  6. 35
      conduit-config/src/proxy.rs
  7. 29
      conduit/Cargo.toml
  8. 3
      conduit/src/api/client_server/voip.rs
  9. 4
      conduit/src/api/server_server.rs
  10. 3
      conduit/src/database/abstraction.rs
  11. 3
      conduit/src/database/abstraction/rocksdb.rs
  12. 3
      conduit/src/database/abstraction/sqlite.rs
  13. 2
      conduit/src/database/key_value/media.rs
  14. 71
      conduit/src/database/mod.rs
  15. 3
      conduit/src/lib.rs
  16. 25
      conduit/src/main.rs
  17. 12
      conduit/src/service/globals/mod.rs
  18. 3
      conduit/src/service/media/data.rs
  19. 15
      conduit/src/service/media/mod.rs
  20. 6
      conduit/src/service/rooms/alias/mod.rs
  21. 12
      conduit/src/utils/error.rs
  22. 25
      conduit/src/utils/mod.rs
  23. 2
      flake.nix
  24. 5
      nix/pkgs/default/default.nix

64
Cargo.lock generated

@ -521,7 +521,7 @@ dependencies = [
"bytesize", "bytesize",
"chrono", "chrono",
"clap", "clap",
"directories", "conduit-config",
"figment", "figment",
"futures-util", "futures-util",
"hex", "hex",
@ -530,7 +530,6 @@ dependencies = [
"http", "http",
"http-body-util", "http-body-util",
"humantime", "humantime",
"humantime-serde",
"hyper", "hyper",
"hyper-util", "hyper-util",
"image", "image",
@ -572,6 +571,19 @@ dependencies = [
"tracing-flame", "tracing-flame",
"tracing-opentelemetry", "tracing-opentelemetry",
"tracing-subscriber", "tracing-subscriber",
]
[[package]]
name = "conduit-config"
version = "0.11.0-alpha"
dependencies = [
"bytesize",
"humantime-serde",
"reqwest",
"ruma",
"rusty-s3",
"serde",
"thiserror 2.0.12",
"url", "url",
] ]
@ -737,27 +749,6 @@ dependencies = [
"subtle", "subtle",
] ]
[[package]]
name = "directories"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
]
[[package]] [[package]]
name = "displaydoc" name = "displaydoc"
version = "0.2.5" version = "0.2.5"
@ -1726,16 +1717,6 @@ dependencies = [
"windows-targets 0.52.6", "windows-targets 0.52.6",
] ]
[[package]]
name = "libredox"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.9.1",
"libc",
]
[[package]] [[package]]
name = "libsqlite3-sys" name = "libsqlite3-sys"
version = "0.33.0" version = "0.33.0"
@ -2119,12 +2100,6 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "option-ext"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]] [[package]]
name = "overload" name = "overload"
version = "0.1.1" version = "0.1.1"
@ -2468,17 +2443,6 @@ dependencies = [
"bitflags 2.9.1", "bitflags 2.9.1",
] ]
[[package]]
name = "redox_users"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b"
dependencies = [
"getrandom 0.2.16",
"libredox",
"thiserror 2.0.12",
]
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.11.1" version = "1.11.1"

11
Cargo.toml

@ -1,6 +1,6 @@
[workspace] [workspace]
default-members = ["conduit"] default-members = ["conduit"]
members = ["conduit"] members = ["conduit", "conduit-config"]
resolver = "2" resolver = "2"
[workspace.lints.rust] [workspace.lints.rust]
@ -19,6 +19,15 @@ homepage = "https://conduit.rs"
repository = "https://gitlab.com/famedly/conduit" repository = "https://gitlab.com/famedly/conduit"
rust-version = "1.85.0" rust-version = "1.85.0"
[workspace.dependencies]
bytesize = "2"
conduit-config.path = "conduit-config"
reqwest = { version = "0.12", default-features = false }
ruma.git = "https://github.com/ruma/ruma.git"
rusty-s3 = "0.8"
serde = "1"
thiserror = "2"
[profile.dev] [profile.dev]
incremental = true incremental = true
lto = 'off' lto = 'off'

34
conduit-config/Cargo.toml

@ -0,0 +1,34 @@
[package]
edition.workspace = true
homepage.workspace = true
name = "conduit-config"
repository.workspace = true
rust-version.workspace = true
version = "0.11.0-alpha"
[dependencies]
# Self explanitory, used for deserializing the configuration
serde.workspace = true
# Parsing media retention policies
bytesize = { workspace = true, features = ["serde"] }
humantime-serde = "1"
# Validating urls
url = { version = "2", features = ["serde"] }
# Error type
thiserror.workspace = true
# Validating S3 config
rusty-s3.workspace = true
# Proxy config
reqwest.workspace = true
# default room version, server name, ignored keys
[dependencies.ruma]
features = ["federation-api"]
workspace = true
[features]
rocksdb = []
sqlite = []
[lints]
workspace = true

21
conduit-config/src/error.rs

@ -0,0 +1,21 @@
use thiserror::Error;
pub type Result<T, E = Error> = std::result::Result<T, E>;
#[derive(Error, Debug)]
pub enum Error {
#[error("The media directory structure depth multiplied by the length is equal to or greater than a sha256 hex hash, please reduce at least one of the two so that their product is less than 64")]
DirectoryStructureLengthDepthTooLarge,
#[error("Invalid S3 config")]
S3,
#[error("Failed to construct proxy config: {source}")]
Proxy {
#[from]
source: reqwest::Error,
},
#[error("Registration token is empty")]
EmptyRegistrationToken,
}

96
conduit/src/config/mod.rs → conduit-config/src/lib.rs

@ -1,6 +1,5 @@
use std::{ use std::{
collections::{BTreeMap, HashMap, HashSet}, collections::{BTreeMap, HashMap, HashSet},
fmt,
net::{IpAddr, Ipv4Addr}, net::{IpAddr, Ipv4Addr},
num::NonZeroU8, num::NonZeroU8,
path::PathBuf, path::PathBuf,
@ -8,14 +7,15 @@ use std::{
}; };
use bytesize::ByteSize; use bytesize::ByteSize;
pub use error::Error;
use ruma::{api::federation::discovery::VerifyKey, serde::Base64, OwnedServerName, RoomVersionId}; use ruma::{api::federation::discovery::VerifyKey, serde::Base64, OwnedServerName, RoomVersionId};
use serde::{de::IgnoredAny, Deserialize}; use serde::{
use tokio::time::{interval, Interval}; de::{Error as _, IgnoredAny},
use tracing::warn; Deserialize,
};
use url::Url; use url::Url;
use crate::Error; pub mod error;
mod proxy; mod proxy;
use self::proxy::ProxyConfig; use self::proxy::ProxyConfig;
@ -30,7 +30,7 @@ pub struct IncompleteConfig {
pub tls: Option<TlsConfig>, pub tls: Option<TlsConfig>,
pub server_name: OwnedServerName, pub server_name: OwnedServerName,
pub database_backend: String, pub database_backend: DatabaseBackend,
pub database_path: String, pub database_path: String,
#[serde(default = "default_db_cache_capacity_mb")] #[serde(default = "default_db_cache_capacity_mb")]
pub db_cache_capacity_mb: f64, pub db_cache_capacity_mb: f64,
@ -54,6 +54,7 @@ pub struct IncompleteConfig {
pub max_fetch_prev_events: u16, pub max_fetch_prev_events: u16,
#[serde(default = "false_fn")] #[serde(default = "false_fn")]
pub allow_registration: bool, pub allow_registration: bool,
#[serde(default, deserialize_with = "forbid_empty_registration_token")]
pub registration_token: Option<String>, pub registration_token: Option<String>,
#[serde(default = "default_openid_token_ttl")] #[serde(default = "default_openid_token_ttl")]
pub openid_token_ttl: u64, pub openid_token_ttl: u64,
@ -109,7 +110,7 @@ pub struct Config {
pub tls: Option<TlsConfig>, pub tls: Option<TlsConfig>,
pub server_name: OwnedServerName, pub server_name: OwnedServerName,
pub database_backend: String, pub database_backend: DatabaseBackend,
pub database_path: String, pub database_path: String,
pub db_cache_capacity_mb: f64, pub db_cache_capacity_mb: f64,
pub enable_lightning_bolt: bool, pub enable_lightning_bolt: bool,
@ -296,6 +297,43 @@ impl From<IncompleteConfig> for Config {
} }
} }
fn forbid_empty_registration_token<'de, D>(de: D) -> Result<Option<String>, D::Error>
where
D: serde::Deserializer<'de>,
{
let opt = Option::<String>::deserialize(de)?;
if opt
.as_ref()
.map(|token| token.is_empty())
.unwrap_or_default()
{
return Err(D::Error::custom(Error::EmptyRegistrationToken));
}
Ok(opt)
}
#[derive(Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum DatabaseBackend {
#[cfg(feature = "sqlite")]
SQLite,
#[cfg(feature = "rocksdb")]
RocksDB,
}
#[cfg(any(feature = "sqlite", feature = "rocksdb"))]
impl std::fmt::Display for DatabaseBackend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let string = match self {
#[cfg(feature = "rocksdb")]
DatabaseBackend::RocksDB => "RocksDB",
#[cfg(feature = "sqlite")]
DatabaseBackend::SQLite => "SQLite",
};
write!(f, "{}", string)
}
}
#[derive(Clone, Debug, Deserialize)] #[derive(Clone, Debug, Deserialize)]
pub struct TlsConfig { pub struct TlsConfig {
pub certs: String, pub certs: String,
@ -356,7 +394,7 @@ pub struct MediaRetentionConfig {
impl MediaRetentionConfig { impl MediaRetentionConfig {
/// Interval for the duration-based retention policies to be checked & enforced /// Interval for the duration-based retention policies to be checked & enforced
pub fn cleanup_interval(&self) -> Option<Interval> { pub fn cleanup_interval(&self) -> Option<Duration> {
self.scoped self.scoped
.values() .values()
.filter_map(|scoped| match (scoped.created, scoped.accessed) { .filter_map(|scoped| match (scoped.created, scoped.accessed) {
@ -369,7 +407,6 @@ impl MediaRetentionConfig {
.max(Duration::from_secs(60).min(Duration::from_secs(60 * 60 * 24))) .max(Duration::from_secs(60).min(Duration::from_secs(60 * 60 * 24)))
}) })
.min() .min()
.map(interval)
} }
} }
@ -559,7 +596,7 @@ impl TryFrom<ShadowDirectoryStructure> for DirectoryStructure {
{ {
Ok(Self::Deep { length, depth }) Ok(Self::Deep { length, depth })
} else { } else {
Err(Error::bad_config("The media directory structure depth multiplied by the depth is equal to or greater than a sha256 hex hash, please reduce at least one of the two so that their product is less than 64")) Err(Error::DirectoryStructureLengthDepthTooLarge)
} }
} }
} }
@ -603,7 +640,7 @@ impl TryFrom<ShadowS3MediaBackend> for S3MediaBackend {
path: value.path, path: value.path,
directory_structure: value.directory_structure, directory_structure: value.directory_structure,
}), }),
Err(_) => Err(Error::bad_config("Invalid S3 config")), Err(_) => Err(Error::S3),
} }
} }
} }
@ -618,39 +655,14 @@ pub struct S3MediaBackend {
pub directory_structure: DirectoryStructure, pub directory_structure: DirectoryStructure,
} }
const DEPRECATED_KEYS: &[&str] = &[ #[cfg(any(feature = "sqlite", feature = "rocksdb"))]
"cache_capacity", impl std::fmt::Display for Config {
"turn_username", fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
"turn_password",
"turn_uris",
"turn_secret",
"turn_ttl",
];
impl Config {
pub fn warn_deprecated(&self) {
let mut was_deprecated = false;
for key in self
.catchall
.keys()
.filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key))
{
warn!("Config parameter {} is deprecated", key);
was_deprecated = true;
}
if was_deprecated {
warn!("Read conduit documentation and check your configuration if any new configuration parameters should be adjusted");
}
}
}
impl fmt::Display for Config {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// Prepare a list of config values to show // Prepare a list of config values to show
// TODO: Replace this with something more fit-for-purpose, especially with tables in mind.
let lines = [ let lines = [
("Server name", self.server_name.host()), ("Server name", self.server_name.host()),
("Database backend", &self.database_backend), ("Database backend", &self.database_backend.to_string()),
("Database path", &self.database_path), ("Database path", &self.database_path),
( (
"Database cache capacity (MB)", "Database cache capacity (MB)",

35
conduit/src/config/proxy.rs → conduit-config/src/proxy.rs

@ -1,7 +1,9 @@
use std::{fmt, str::FromStr};
use reqwest::{Proxy, Url}; use reqwest::{Proxy, Url};
use serde::Deserialize; use serde::Deserialize;
use crate::Result; use crate::error::Result;
/// ## Examples: /// ## Examples:
/// - No proxy (default): /// - No proxy (default):
@ -34,7 +36,6 @@ pub enum ProxyConfig {
#[default] #[default]
None, None,
Global { Global {
#[serde(deserialize_with = "crate::utils::deserialize_from_str")]
url: Url, url: Url,
}, },
ByDomain(Vec<PartialProxyConfig>), ByDomain(Vec<PartialProxyConfig>),
@ -53,7 +54,6 @@ impl ProxyConfig {
#[derive(Clone, Debug, Deserialize)] #[derive(Clone, Debug, Deserialize)]
pub struct PartialProxyConfig { pub struct PartialProxyConfig {
#[serde(deserialize_with = "crate::utils::deserialize_from_str")]
url: Url, url: Url,
#[serde(default)] #[serde(default)]
include: Vec<WildCardedDomain>, include: Vec<WildCardedDomain>,
@ -120,7 +120,8 @@ impl WildCardedDomain {
} }
} }
} }
impl std::str::FromStr for WildCardedDomain {
impl FromStr for WildCardedDomain {
type Err = std::convert::Infallible; type Err = std::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> { fn from_str(s: &str) -> Result<Self, Self::Err> {
// maybe do some domain validation? // maybe do some domain validation?
@ -138,6 +139,30 @@ impl<'de> Deserialize<'de> for WildCardedDomain {
where where
D: serde::de::Deserializer<'de>, D: serde::de::Deserializer<'de>,
{ {
crate::utils::deserialize_from_str(deserializer) deserialize_from_str(deserializer)
}
}
fn deserialize_from_str<
'de,
D: serde::de::Deserializer<'de>,
T: FromStr<Err = E>,
E: fmt::Display,
>(
deserializer: D,
) -> Result<T, D::Error> {
struct Visitor<T: FromStr<Err = E>, E>(std::marker::PhantomData<T>);
impl<T: FromStr<Err = Err>, Err: fmt::Display> serde::de::Visitor<'_> for Visitor<T, Err> {
type Value = T;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "a parsable string")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
v.parse().map_err(serde::de::Error::custom)
}
} }
deserializer.deserialize_str(Visitor(std::marker::PhantomData))
} }

29
conduit/Cargo.toml

@ -16,6 +16,9 @@ version = "0.11.0-alpha"
workspace = true workspace = true
[dependencies] [dependencies]
# For the configuration
conduit-config.workspace = true
# Web framework # Web framework
axum = { version = "0.8", default-features = false, features = [ axum = { version = "0.8", default-features = false, features = [
"form", "form",
@ -42,8 +45,6 @@ tokio = { version = "1", features = ["fs", "macros", "signal", "sync"] }
# Used for the http request / response body type for Ruma endpoints used with reqwest # Used for the http request / response body type for Ruma endpoints used with reqwest
bytes = "1" bytes = "1"
http = "1" http = "1"
# Used to find data directory for default db path
directories = "6"
# Used for ruma wrapper # Used for ruma wrapper
serde_json = { version = "1", features = ["raw_value"] } serde_json = { version = "1", features = ["raw_value"] }
# Used for appservice registration files # Used for appservice registration files
@ -62,12 +63,9 @@ hyper-util = { version = "0.1", features = [
"http1", "http1",
"http2", "http2",
] } ] }
reqwest = { version = "0.12", default-features = false, features = [ reqwest = { workspace = true, features = ["rustls-tls-native-roots", "socks"] }
"rustls-tls-native-roots",
"socks",
] }
# Used for conduit::Error type # Used for conduit::Error type
thiserror = "2" #TODO: 2 thiserror.workspace = true #TODO: 2
# Used to generate thumbnails for images # Used to generate thumbnails for images
image = { version = "0.25", default-features = false, features = [ image = { version = "0.25", default-features = false, features = [
"gif", "gif",
@ -78,9 +76,9 @@ image = { version = "0.25", default-features = false, features = [
# Used for creating media filenames # Used for creating media filenames
hex = "0.4" hex = "0.4"
sha2 = "0.10" sha2 = "0.10"
# Used for parsing media retention policies from the config
bytesize = { version = "2", features = ["serde"] } # Used for parsing admin commands and purging media files for space limitations
humantime-serde = "1" bytesize.workspace = true
# Used to encode server public key # Used to encode server public key
base64 = "0.22" base64 = "0.22"
# Used when hashing the state # Used when hashing the state
@ -132,9 +130,6 @@ futures-util = { version = "0.3", default-features = false }
# Used for reading the configuration from conduit.toml & environment variables # Used for reading the configuration from conduit.toml & environment variables
figment = { version = "0.10", features = ["env", "toml"] } figment = { version = "0.10", features = ["env", "toml"] }
# Validating urls in config
url = { version = "2", features = ["serde"] }
async-trait = "0.1" async-trait = "0.1"
tikv-jemallocator = { version = "0.6", features = [ tikv-jemallocator = { version = "0.6", features = [
"unprefixed_malloc_on_supported_platforms", "unprefixed_malloc_on_supported_platforms",
@ -144,7 +139,7 @@ sd-notify = { version = "0.4", optional = true }
# Used for inspecting request errors # Used for inspecting request errors
http-body-util = "0.1.3" http-body-util = "0.1.3"
# Used for S3 media backend # Used for S3 media backend
rusty-s3 = "0.8.1" rusty-s3.workspace = true
# Used for matrix spec type definitions and helpers # Used for matrix spec type definitions and helpers
[dependencies.ruma] [dependencies.ruma]
@ -169,7 +164,7 @@ features = [
"unstable-msc4186", "unstable-msc4186",
"unstable-msc4311", "unstable-msc4311",
] ]
git = "https://github.com/ruma/ruma.git" workspace = true
[dependencies.rocksdb] [dependencies.rocksdb]
features = ["lz4", "multi-threaded-cf", "zstd"] features = ["lz4", "multi-threaded-cf", "zstd"]
@ -181,8 +176,8 @@ version = "0.43"
nix = { version = "0.30", features = ["resource"] } nix = { version = "0.30", features = ["resource"] }
[features] [features]
backend_rocksdb = ["rocksdb"] backend_rocksdb = ["conduit-config/rocksdb", "rocksdb"]
backend_sqlite = ["sqlite"] backend_sqlite = ["conduit-config/sqlite", "sqlite"]
conduit_bin = ["axum"] conduit_bin = ["axum"]
default = ["backend_rocksdb", "backend_sqlite", "conduit_bin", "systemd"] default = ["backend_rocksdb", "backend_sqlite", "conduit_bin", "systemd"]
jemalloc = ["tikv-jemallocator"] jemalloc = ["tikv-jemallocator"]

3
conduit/src/api/client_server/voip.rs

@ -1,5 +1,6 @@
use crate::{config::TurnAuth, services, Error, Result, Ruma}; use crate::{services, Error, Result, Ruma};
use base64::{engine::general_purpose, Engine as _}; use base64::{engine::general_purpose, Engine as _};
use conduit_config::TurnAuth;
use hmac::{Hmac, Mac}; use hmac::{Hmac, Mac};
use ruma::{ use ruma::{
api::client::{error::ErrorKind, voip::get_turn_server_info}, api::client::{error::ErrorKind, voip::get_turn_server_info},

4
conduit/src/api/server_server.rs

@ -130,11 +130,11 @@ where
T: OutgoingRequest + Debug, T: OutgoingRequest + Debug,
{ {
if !services().globals.allow_federation() { if !services().globals.allow_federation() {
return Err(Error::bad_config("Federation is disabled.")); return Err(Error::BadServerResponse("Federation is disabled."));
} }
if destination == services().globals.server_name() { if destination == services().globals.server_name() {
return Err(Error::bad_config( return Err(Error::BadServerResponse(
"Won't send federation request to ourselves", "Won't send federation request to ourselves",
)); ));
} }

3
conduit/src/database/abstraction.rs

@ -1,6 +1,7 @@
use super::Config;
use crate::Result; use crate::Result;
use conduit_config::Config;
use std::{future::Future, pin::Pin, sync::Arc}; use std::{future::Future, pin::Pin, sync::Arc};
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]

3
conduit/src/database/abstraction/rocksdb.rs

@ -1,5 +1,6 @@
use super::{super::Config, watchers::Watchers, KeyValueDatabaseEngine, KvTree}; use super::{watchers::Watchers, KeyValueDatabaseEngine, KvTree};
use crate::{utils, Result}; use crate::{utils, Result};
use conduit_config::Config;
use std::{ use std::{
future::Future, future::Future,
pin::Pin, pin::Pin,

3
conduit/src/database/abstraction/sqlite.rs

@ -1,5 +1,6 @@
use super::{watchers::Watchers, KeyValueDatabaseEngine, KvTree}; use super::{watchers::Watchers, KeyValueDatabaseEngine, KvTree};
use crate::{database::Config, Result}; use crate::Result;
use conduit_config::Config;
use parking_lot::{Mutex, MutexGuard}; use parking_lot::{Mutex, MutexGuard};
use rusqlite::{Connection, DatabaseName::Main, OptionalExtension}; use rusqlite::{Connection, DatabaseName::Main, OptionalExtension};
use std::{ use std::{

2
conduit/src/database/key_value/media.rs

@ -1,12 +1,12 @@
use std::{collections::BTreeMap, ops::Range, slice::Split}; use std::{collections::BTreeMap, ops::Range, slice::Split};
use bytesize::ByteSize; use bytesize::ByteSize;
use conduit_config::{MediaRetentionConfig, MediaRetentionScope};
use ruma::{api::client::error::ErrorKind, OwnedServerName, ServerName, UserId}; use ruma::{api::client::error::ErrorKind, OwnedServerName, ServerName, UserId};
use sha2::{digest::Output, Sha256}; use sha2::{digest::Output, Sha256};
use tracing::error; use tracing::error;
use crate::{ use crate::{
config::{MediaRetentionConfig, MediaRetentionScope},
database::KeyValueDatabase, database::KeyValueDatabase,
service::{ service::{
self, self,

71
conduit/src/database/mod.rs

@ -7,7 +7,7 @@ use crate::{
}; };
use abstraction::{KeyValueDatabaseEngine, KvTree}; use abstraction::{KeyValueDatabaseEngine, KvTree};
use base64::{engine::general_purpose, Engine}; use base64::{engine::general_purpose, Engine};
use directories::ProjectDirs; use conduit_config::DatabaseBackend;
use key_value::media::FilehashMetadata; use key_value::media::FilehashMetadata;
use lru_cache::LruCache; use lru_cache::LruCache;
@ -25,7 +25,7 @@ use serde::Deserialize;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use std::{ use std::{
collections::{BTreeMap, HashMap, HashSet}, collections::{BTreeMap, HashMap, HashSet},
fs::{self, remove_dir_all}, fs,
io::Write, io::Write,
mem::size_of, mem::size_of,
path::{Path, PathBuf}, path::{Path, PathBuf},
@ -214,18 +214,6 @@ pub struct KeyValueDatabase {
} }
impl KeyValueDatabase { impl KeyValueDatabase {
/// Tries to remove the old database but ignores all errors.
pub fn try_remove(server_name: &str) -> Result<()> {
let mut path = ProjectDirs::from("xyz", "koesters", "conduit")
.ok_or_else(|| Error::bad_config("The OS didn't return a valid home directory path."))?
.data_dir()
.to_path_buf();
path.push(server_name);
let _ = remove_dir_all(path);
Ok(())
}
fn check_db_setup(config: &Config) -> Result<()> { fn check_db_setup(config: &Config) -> Result<()> {
let path = Path::new(&config.database_path); let path = Path::new(&config.database_path);
@ -252,22 +240,32 @@ impl KeyValueDatabase {
return Ok(()); return Ok(());
} }
if sled_exists && config.database_backend != "sled" { if sled_exists {
return Err(Error::bad_config( return Err(Error::Initialization(
"Found sled at database_path, but is not specified in config.", "Found sled at database_path, but is not specified in config.",
)); ));
} }
if sqlite_exists && config.database_backend != "sqlite" { // Only works as these are the only two active backends currently. If there ever were
return Err(Error::bad_config( // to be more than 2, this could get complicated due to there not being attributes
"Found sqlite at database_path, but is not specified in config.", // available on expressions yet: https://github.com/rust-lang/rust/issues/15701
)); match config.database_backend {
} #[cfg(feature = "rocksdb")]
DatabaseBackend::RocksDB => {
if rocksdb_exists && config.database_backend != "rocksdb" { if sqlite_exists {
return Err(Error::bad_config( return Err(Error::Initialization(
"Found rocksdb at database_path, but is not specified in config.", "Found sqlite at database_path, but is not specified in config.",
)); ));
}
}
#[cfg(feature = "sqlite")]
DatabaseBackend::SQLite => {
if rocksdb_exists {
return Err(Error::Initialization(
"Found rocksdb at database_path, but is not specified in config.",
));
}
}
} }
Ok(()) Ok(())
@ -279,23 +277,18 @@ impl KeyValueDatabase {
if !Path::new(&config.database_path).exists() { if !Path::new(&config.database_path).exists() {
fs::create_dir_all(&config.database_path) fs::create_dir_all(&config.database_path)
.map_err(|_| Error::BadConfig("Database folder doesn't exists and couldn't be created (e.g. due to missing permissions). Please create the database folder yourself."))?; .map_err(|_| Error::Initialization("Database folder doesn't exists and couldn't be created (e.g. due to missing permissions). Please create the database folder yourself."))?;
} }
let builder: Arc<dyn KeyValueDatabaseEngine> = match &*config.database_backend { let builder: Arc<dyn KeyValueDatabaseEngine> = match &config.database_backend {
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
"sqlite" => Arc::new(Arc::<abstraction::sqlite::Engine>::open(&config)?), DatabaseBackend::SQLite => Arc::new(Arc::<abstraction::sqlite::Engine>::open(&config)?),
#[cfg(feature = "rocksdb")] #[cfg(feature = "rocksdb")]
"rocksdb" => Arc::new(Arc::<abstraction::rocksdb::Engine>::open(&config)?), DatabaseBackend::RocksDB => {
_ => { Arc::new(Arc::<abstraction::rocksdb::Engine>::open(&config)?)
return Err(Error::BadConfig("Database backend not found."));
} }
}; };
if config.registration_token == Some(String::new()) {
return Err(Error::bad_config("Registration token is empty"));
}
if config.max_request_size < 1024 { if config.max_request_size < 1024 {
error!(?config.max_request_size, "Max request size is less than 1KB. Please increase it."); error!(?config.max_request_size, "Max request size is less than 1KB. Please increase it.");
} }
@ -1056,9 +1049,9 @@ impl KeyValueDatabase {
} }
if services().globals.database_version()? < 18 { if services().globals.database_version()? < 18 {
if let crate::config::MediaBackendConfig::FileSystem { if let conduit_config::MediaBackendConfig::FileSystem {
path, path,
directory_structure: crate::config::DirectoryStructure::Deep { length, depth }, directory_structure: conduit_config::DirectoryStructure::Deep { length, depth },
} = &services().globals.config.media.backend } = &services().globals.config.media.backend
{ {
for file in fs::read_dir(path) for file in fs::read_dir(path)
@ -1074,7 +1067,7 @@ impl KeyValueDatabase {
file.path(), file.path(),
services().globals.get_media_path( services().globals.get_media_path(
path.as_str(), path.as_str(),
&crate::config::DirectoryStructure::Deep { &conduit_config::DirectoryStructure::Deep {
length: *length, length: *length,
depth: *depth, depth: *depth,
}, },

3
conduit/src/lib.rs

@ -1,6 +1,5 @@
pub mod api; pub mod api;
pub mod clap; pub mod clap;
mod config;
mod database; mod database;
mod service; mod service;
mod utils; mod utils;
@ -14,7 +13,7 @@ use std::{
}; };
pub use api::ruma_wrapper::{Ruma, RumaResponse}; pub use api::ruma_wrapper::{Ruma, RumaResponse};
pub use config::Config; pub use conduit_config::Config;
pub use database::KeyValueDatabase; pub use database::KeyValueDatabase;
use ruma::api::{MatrixVersion, SupportedVersions}; use ruma::api::{MatrixVersion, SupportedVersions};
pub use service::{pdu::PduEvent, Services}; pub use service::{pdu::PduEvent, Services};

25
conduit/src/main.rs

@ -52,6 +52,15 @@ static SUB_TABLES: [&str; 3] = ["well_known", "tls", "media"]; // Not doing `pro
// this is what we have to deal with. Also see: https://github.com/SergioBenitez/Figment/issues/12#issuecomment-801449465 // this is what we have to deal with. Also see: https://github.com/SergioBenitez/Figment/issues/12#issuecomment-801449465
static SUB_SUB_TABLES: [&str; 2] = ["directory_structure", "retention"]; static SUB_SUB_TABLES: [&str; 2] = ["directory_structure", "retention"];
const DEPRECATED_KEYS: &[&str] = &[
"cache_capacity",
"turn_username",
"turn_password",
"turn_uris",
"turn_secret",
"turn_ttl",
];
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
clap::parse(); clap::parse();
@ -104,7 +113,19 @@ async fn main() {
} }
}; };
config.warn_deprecated(); let mut was_deprecated = false;
for key in config
.catchall
.keys()
.filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key))
{
warn!("Config parameter {} is deprecated", key);
was_deprecated = true;
}
if was_deprecated {
warn!("Read conduit documentation and check your configuration if any new configuration parameters should be adjusted");
}
let jaeger = if config.allow_jaeger { let jaeger = if config.allow_jaeger {
opentelemetry::global::set_text_map_propagator( opentelemetry::global::set_text_map_propagator(
@ -544,7 +565,7 @@ async fn shutdown_signal(handle: ServerHandle) {
} }
async fn federation_disabled(_: Uri) -> impl IntoResponse { async fn federation_disabled(_: Uri) -> impl IntoResponse {
Error::bad_config("Federation is disabled.") Error::BadServerResponse("Federation is disabled.")
} }
async fn not_found(uri: Uri) -> impl IntoResponse { async fn not_found(uri: Uri) -> impl IntoResponse {

12
conduit/src/service/globals/mod.rs

@ -7,10 +7,8 @@ use ruma::{
use crate::api::server_server::DestinationResponse; use crate::api::server_server::DestinationResponse;
use crate::{ use crate::{services, Config, Error, Result};
config::{DirectoryStructure, MediaBackendConfig, TurnConfig}, use conduit_config::{DirectoryStructure, MediaBackendConfig, TurnConfig};
services, Config, Error, Result,
};
use futures_util::FutureExt; use futures_util::FutureExt;
use hickory_resolver::TokioResolver; use hickory_resolver::TokioResolver;
use hyper_util::client::legacy::connect::dns::{GaiResolver, Name as HyperName}; use hyper_util::client::legacy::connect::dns::{GaiResolver, Name as HyperName};
@ -207,7 +205,7 @@ impl Service {
"Failed to set up trust dns resolver with system config: {}", "Failed to set up trust dns resolver with system config: {}",
e e
); );
Error::bad_config("Failed to set up trust dns resolver with system config.") Error::Initialization("Failed to set up trust dns resolver with system config.")
})? })?
.build(), .build(),
actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())), actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())),
@ -239,8 +237,8 @@ impl Service {
.supported_room_versions() .supported_room_versions()
.contains(&s.config.default_room_version) .contains(&s.config.default_room_version)
{ {
error!(config=?s.config.default_room_version, fallback=?crate::config::default_default_room_version(), "Room version in config isn't supported, falling back to default version"); error!(config=?s.config.default_room_version, fallback=?conduit_config::default_default_room_version(), "Room version in config isn't supported, falling back to default version");
s.config.default_room_version = crate::config::default_default_room_version(); s.config.default_room_version = conduit_config::default_default_room_version();
}; };
Ok(s) Ok(s)

3
conduit/src/service/media/data.rs

@ -1,7 +1,8 @@
use conduit_config::MediaRetentionConfig;
use ruma::{OwnedServerName, ServerName, UserId}; use ruma::{OwnedServerName, ServerName, UserId};
use sha2::{digest::Output, Sha256}; use sha2::{digest::Output, Sha256};
use crate::{config::MediaRetentionConfig, Error, Result}; use crate::{Error, Result};
use super::{ use super::{
BlockedMediaInfo, DbFileMeta, MediaListItem, MediaQuery, MediaType, ServerNameOrUserId, BlockedMediaInfo, DbFileMeta, MediaListItem, MediaQuery, MediaType, ServerNameOrUserId,

15
conduit/src/service/media/mod.rs

@ -1,6 +1,7 @@
mod data; mod data;
use std::{io::Cursor, sync::Arc}; use std::{io::Cursor, sync::Arc};
use conduit_config::{DirectoryStructure, MediaBackendConfig, S3MediaBackend};
pub use data::Data; pub use data::Data;
use http::StatusCode; use http::StatusCode;
use ruma::{ use ruma::{
@ -15,10 +16,7 @@ use rusty_s3::{
use sha2::{digest::Output, Digest, Sha256}; use sha2::{digest::Output, Digest, Sha256};
use tracing::{error, info, warn}; use tracing::{error, info, warn};
use crate::{ use crate::{services, utils, Error, Result};
config::{DirectoryStructure, MediaBackendConfig, S3MediaBackend},
services, utils, Error, Result,
};
use image::imageops::FilterType; use image::imageops::FilterType;
pub struct DbFileMeta { pub struct DbFileMeta {
@ -31,6 +29,7 @@ pub struct DbFileMeta {
use tokio::{ use tokio::{
fs::{self, File}, fs::{self, File},
io::{AsyncReadExt, AsyncWriteExt}, io::{AsyncReadExt, AsyncWriteExt},
time::interval,
}; };
pub struct MediaQuery { pub struct MediaQuery {
@ -126,7 +125,13 @@ pub struct BlockedMediaInfo {
impl Service { impl Service {
pub fn start_time_retention_checker(self: &Arc<Self>) { pub fn start_time_retention_checker(self: &Arc<Self>) {
let self2 = Arc::clone(self); let self2 = Arc::clone(self);
if let Some(cleanup_interval) = services().globals.config.media.retention.cleanup_interval() if let Some(cleanup_interval) = services()
.globals
.config
.media
.retention
.cleanup_interval()
.map(interval)
{ {
tokio::spawn(async move { tokio::spawn(async move {
let mut i = cleanup_interval; let mut i = cleanup_interval;

6
conduit/src/service/rooms/alias/mod.rs

@ -2,6 +2,7 @@ mod data;
pub use data::Data; pub use data::Data;
use rand::seq::SliceRandom; use rand::seq::SliceRandom;
use tracing::warn;
use crate::{services, Error, Result}; use crate::{services, Error, Result};
use ruma::{ use ruma::{
@ -131,7 +132,10 @@ impl Service {
{ {
room_id = room_id =
Some(self.resolve_local_alias(&room_alias)?.ok_or_else(|| { Some(self.resolve_local_alias(&room_alias)?.ok_or_else(|| {
Error::bad_config("Appservice lied to us. Room does not exist.") warn!(id = appservice.registration.id, alias = room_alias.to_string(), "Appservice claimed to create room for alias, but the alias could not be resolved.");
Error::BadServerResponse(
"Appservice lied to us. Room does not exist.",
)
})?); })?);
break; break;
} }

12
conduit/src/utils/error.rs

@ -57,8 +57,13 @@ pub enum Error {
CannotDeleteS3File(String), // This is only needed when an S3 deletion fails CannotDeleteS3File(String), // This is only needed when an S3 deletion fails
#[error("{0}")] #[error("{0}")]
BadServerResponse(&'static str), BadServerResponse(&'static str),
#[error("{source}")]
BadConfig {
#[from]
source: conduit_config::Error,
},
#[error("{0}")] #[error("{0}")]
BadConfig(&'static str), Initialization(&'static str),
#[error("{0}")] #[error("{0}")]
/// Don't create this directly. Use Error::bad_database instead. /// Don't create this directly. Use Error::bad_database instead.
BadDatabase(&'static str), BadDatabase(&'static str),
@ -90,11 +95,6 @@ impl Error {
error!("BadDatabase: {}", message); error!("BadDatabase: {}", message);
Self::BadDatabase(message) Self::BadDatabase(message)
} }
pub fn bad_config(message: &'static str) -> Self {
error!("BadConfig: {}", message);
Self::BadConfig(message)
}
} }
impl Error { impl Error {

25
conduit/src/utils/mod.rs

@ -18,7 +18,6 @@ use std::{
cmp, cmp,
collections::BTreeMap, collections::BTreeMap,
fmt, fmt,
str::FromStr,
time::{SystemTime, UNIX_EPOCH}, time::{SystemTime, UNIX_EPOCH},
}; };
use tokio::sync::RwLock; use tokio::sync::RwLock;
@ -140,30 +139,6 @@ pub fn to_canonical_object<T: serde::Serialize>(
} }
} }
pub fn deserialize_from_str<
'de,
D: serde::de::Deserializer<'de>,
T: FromStr<Err = E>,
E: fmt::Display,
>(
deserializer: D,
) -> Result<T, D::Error> {
struct Visitor<T: FromStr<Err = E>, E>(std::marker::PhantomData<T>);
impl<T: FromStr<Err = Err>, Err: fmt::Display> serde::de::Visitor<'_> for Visitor<T, Err> {
type Value = T;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "a parsable string")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
v.parse().map_err(serde::de::Error::custom)
}
}
deserializer.deserialize_str(Visitor(std::marker::PhantomData))
}
// Copied from librustdoc: // Copied from librustdoc:
// https://github.com/rust-lang/rust/blob/cbaeec14f90b59a91a6b0f17fc046c66fa811892/src/librustdoc/html/escape.rs // https://github.com/rust-lang/rust/blob/cbaeec14f90b59a91a6b0f17fc046c66fa811892/src/librustdoc/html/escape.rs

2
flake.nix

@ -26,6 +26,8 @@
craneLib = craneLib =
(inputs.crane.mkLib pkgs).overrideToolchain (_: self.toolchain); (inputs.crane.mkLib pkgs).overrideToolchain (_: self.toolchain);
workspaceMembers = (pkgs.lib.importTOML ./Cargo.toml).workspace.members;
default = self.callPackage ./nix/pkgs/default {}; default = self.callPackage ./nix/pkgs/default {};
inherit inputs; inherit inputs;

5
nix/pkgs/default/default.nix

@ -6,6 +6,7 @@
, rocksdb , rocksdb
, rust , rust
, stdenv , stdenv
, workspaceMembers
# Options (keep sorted) # Options (keep sorted)
, default-features ? true , default-features ? true
@ -56,8 +57,8 @@ let
".cargo" ".cargo"
"Cargo.lock" "Cargo.lock"
"Cargo.toml" "Cargo.toml"
"conduit" ]
]; ++ workspaceMembers;
}; };
nativeBuildInputs = [ nativeBuildInputs = [

Loading…
Cancel
Save