Browse Source
In addition, metadata about the file, such as creation time, last access, and file size, are stored in the databasemerge-requests/762/head
Matthias Ahouansou
1 year ago
14 changed files with 830 additions and 276 deletions
@ -1,71 +1,199 @@
|
||||
use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition}; |
||||
use ruma::{api::client::error::ErrorKind, ServerName}; |
||||
use sha2::{digest::Output, Sha256}; |
||||
use tracing::error; |
||||
|
||||
use crate::{database::KeyValueDatabase, service, utils, Error, Result}; |
||||
use crate::{ |
||||
database::KeyValueDatabase, |
||||
service::{self, media::DbFileMeta}, |
||||
utils, Error, Result, |
||||
}; |
||||
|
||||
impl service::media::Data for KeyValueDatabase { |
||||
fn create_file_metadata( |
||||
&self, |
||||
mxc: String, |
||||
sha256_digest: Output<Sha256>, |
||||
file_size: u64, |
||||
servername: &ServerName, |
||||
media_id: &str, |
||||
filename: Option<&str>, |
||||
content_type: Option<&str>, |
||||
) -> Result<()> { |
||||
let metadata = FilehashMetadata::new(file_size); |
||||
|
||||
self.filehash_metadata |
||||
.insert(&sha256_digest, metadata.value())?; |
||||
|
||||
let mut key = sha256_digest.to_vec(); |
||||
key.extend_from_slice(servername.as_bytes()); |
||||
key.push(0xff); |
||||
key.extend_from_slice(media_id.as_bytes()); |
||||
|
||||
self.filehash_servername_mediaid.insert(&key, &[])?; |
||||
|
||||
let mut key = servername.as_bytes().to_vec(); |
||||
key.push(0xff); |
||||
key.extend_from_slice(media_id.as_bytes()); |
||||
|
||||
let mut value = sha256_digest.to_vec(); |
||||
value.extend_from_slice(filename.map(|f| f.as_bytes()).unwrap_or_default()); |
||||
value.push(0xff); |
||||
value.extend_from_slice(content_type.map(|f| f.as_bytes()).unwrap_or_default()); |
||||
|
||||
self.servernamemediaid_metadata.insert(&key, &value) |
||||
} |
||||
|
||||
fn search_file_metadata(&self, servername: &ServerName, media_id: &str) -> Result<DbFileMeta> { |
||||
let mut key = servername.as_bytes().to_vec(); |
||||
key.push(0xff); |
||||
key.extend_from_slice(media_id.as_bytes()); |
||||
|
||||
let value = self |
||||
.servernamemediaid_metadata |
||||
.get(&key)? |
||||
.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found."))?; |
||||
|
||||
let metadata = parse_metadata(&value).inspect_err(|e| { |
||||
error!("Error parsing metadata for \"mxc://{servername}/{media_id}\" from servernamemediaid_metadata: {e}"); |
||||
})?; |
||||
|
||||
// Only assume file is available if there is metadata about the filehash itself
|
||||
self.filehash_metadata |
||||
.get(&metadata.sha256_digest)? |
||||
.map(|_| metadata) |
||||
.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found.")) |
||||
} |
||||
|
||||
fn create_thumbnail_metadata( |
||||
&self, |
||||
sha256_digest: Output<Sha256>, |
||||
file_size: u64, |
||||
servername: &ServerName, |
||||
media_id: &str, |
||||
width: u32, |
||||
height: u32, |
||||
content_disposition: &ContentDisposition, |
||||
filename: Option<&str>, |
||||
content_type: Option<&str>, |
||||
) -> Result<Vec<u8>> { |
||||
let mut key = mxc.as_bytes().to_vec(); |
||||
) -> Result<()> { |
||||
let metadata = FilehashMetadata::new(file_size); |
||||
|
||||
self.filehash_metadata |
||||
.insert(&sha256_digest, metadata.value())?; |
||||
|
||||
let mut key = sha256_digest.to_vec(); |
||||
key.extend_from_slice(servername.as_bytes()); |
||||
key.push(0xff); |
||||
key.extend_from_slice(media_id.as_bytes()); |
||||
key.push(0xff); |
||||
key.extend_from_slice(&width.to_be_bytes()); |
||||
key.extend_from_slice(&height.to_be_bytes()); |
||||
|
||||
self.filehash_thumbnailid.insert(&key, &[])?; |
||||
|
||||
let mut key = servername.as_bytes().to_vec(); |
||||
key.push(0xff); |
||||
key.extend_from_slice(content_disposition.to_string().as_bytes()); |
||||
key.extend_from_slice(media_id.as_bytes()); |
||||
key.push(0xff); |
||||
key.extend_from_slice( |
||||
content_type |
||||
.as_ref() |
||||
.map(|c| c.as_bytes()) |
||||
.unwrap_or_default(), |
||||
); |
||||
key.extend_from_slice(&width.to_be_bytes()); |
||||
key.extend_from_slice(&height.to_be_bytes()); |
||||
|
||||
self.mediaid_file.insert(&key, &[])?; |
||||
let mut value = sha256_digest.to_vec(); |
||||
value.extend_from_slice(filename.map(|f| f.as_bytes()).unwrap_or_default()); |
||||
value.push(0xff); |
||||
value.extend_from_slice(content_type.map(|f| f.as_bytes()).unwrap_or_default()); |
||||
|
||||
Ok(key) |
||||
self.thumbnailid_metadata.insert(&key, &value) |
||||
} |
||||
|
||||
fn search_file_metadata( |
||||
fn search_thumbnail_metadata( |
||||
&self, |
||||
mxc: String, |
||||
servername: &ServerName, |
||||
media_id: &str, |
||||
width: u32, |
||||
height: u32, |
||||
) -> Result<(ContentDisposition, Option<String>, Vec<u8>)> { |
||||
let mut prefix = mxc.as_bytes().to_vec(); |
||||
prefix.push(0xff); |
||||
prefix.extend_from_slice(&width.to_be_bytes()); |
||||
prefix.extend_from_slice(&height.to_be_bytes()); |
||||
prefix.push(0xff); |
||||
|
||||
let (key, _) = self |
||||
.mediaid_file |
||||
.scan_prefix(prefix) |
||||
.next() |
||||
.ok_or(Error::BadRequest(ErrorKind::NotFound, "Media not found"))?; |
||||
|
||||
let mut parts = key.rsplit(|&b| b == 0xff); |
||||
|
||||
let content_type = parts |
||||
.next() |
||||
.map(|bytes| { |
||||
utils::string_from_bytes(bytes).map_err(|_| { |
||||
Error::bad_database("Content type in mediaid_file is invalid unicode.") |
||||
}) |
||||
) -> Result<DbFileMeta> { |
||||
let mut key = servername.as_bytes().to_vec(); |
||||
key.push(0xff); |
||||
key.extend_from_slice(media_id.as_bytes()); |
||||
key.push(0xff); |
||||
key.extend_from_slice(&width.to_be_bytes()); |
||||
key.extend_from_slice(&height.to_be_bytes()); |
||||
|
||||
let value = self |
||||
.thumbnailid_metadata |
||||
.get(&key)? |
||||
.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found."))?; |
||||
|
||||
let metadata = parse_metadata(&value).inspect_err(|e| { |
||||
error!("Error parsing metadata for thumbnail \"mxc://{servername}/{media_id}\" with dimensions {width}x{height} from thumbnailid_metadata: {e}"); |
||||
})?; |
||||
|
||||
// Only assume file is available if there is metadata about the filehash itself
|
||||
self.filehash_metadata |
||||
.get(&metadata.sha256_digest)? |
||||
.map(|_| metadata) |
||||
.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found.")) |
||||
} |
||||
} |
||||
|
||||
fn parse_metadata(value: &[u8]) -> Result<DbFileMeta> { |
||||
let (sha256_digest, mut parts) = value |
||||
.split_at_checked(32) |
||||
.map(|(digest, value)| (digest.to_vec(), value.split(|&b| b == 0xff))) |
||||
.ok_or_else(|| Error::BadDatabase("Invalid format for media metadata"))?; |
||||
|
||||
let filename = parts |
||||
.next() |
||||
.map(|bytes| { |
||||
utils::string_from_bytes(bytes) |
||||
.map_err(|_| Error::BadDatabase("filename in media metadata is invalid unicode")) |
||||
}) |
||||
.transpose()? |
||||
.and_then(|s| (!s.is_empty()).then_some(s)); |
||||
|
||||
let content_type = parts |
||||
.next() |
||||
.map(|bytes| { |
||||
utils::string_from_bytes(bytes).map_err(|_| { |
||||
Error::BadDatabase("content type in media metadata is invalid unicode") |
||||
}) |
||||
.transpose()?; |
||||
}) |
||||
.transpose()? |
||||
.and_then(|s| (!s.is_empty()).then_some(s)); |
||||
|
||||
let unauthenticated_access_permitted = parts.next().is_some_and(|v| v.is_empty()); |
||||
|
||||
Ok(DbFileMeta { |
||||
sha256_digest, |
||||
filename, |
||||
content_type, |
||||
unauthenticated_access_permitted, |
||||
}) |
||||
} |
||||
|
||||
pub struct FilehashMetadata { |
||||
value: Vec<u8>, |
||||
} |
||||
|
||||
let content_disposition_bytes = parts |
||||
.next() |
||||
.ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?; |
||||
impl FilehashMetadata { |
||||
pub fn new_with_times(size: u64, creation: u64, last_access: u64) -> Self { |
||||
let mut value = size.to_be_bytes().to_vec(); |
||||
value.extend_from_slice(&creation.to_be_bytes()); |
||||
value.extend_from_slice(&last_access.to_be_bytes()); |
||||
|
||||
Self { value } |
||||
} |
||||
|
||||
pub fn new(size: u64) -> Self { |
||||
let now = utils::secs_since_unix_epoch(); |
||||
|
||||
let mut value = size.to_be_bytes().to_vec(); |
||||
value.extend_from_slice(&now.to_be_bytes()); |
||||
value.extend_from_slice(&now.to_be_bytes()); |
||||
|
||||
Self { value } |
||||
} |
||||
|
||||
let content_disposition = content_disposition_bytes.try_into().unwrap_or_else(|_| { |
||||
ContentDisposition::new(ruma::http_headers::ContentDispositionType::Inline) |
||||
}); |
||||
Ok((content_disposition, content_type, key)) |
||||
pub fn value(&self) -> &[u8] { |
||||
&self.value |
||||
} |
||||
} |
||||
|
||||
@ -1,22 +1,43 @@
|
||||
use ruma::http_headers::ContentDisposition; |
||||
use ruma::ServerName; |
||||
use sha2::{digest::Output, Sha256}; |
||||
|
||||
use crate::Result; |
||||
|
||||
use super::DbFileMeta; |
||||
|
||||
pub trait Data: Send + Sync { |
||||
fn create_file_metadata( |
||||
&self, |
||||
mxc: String, |
||||
sha256_digest: Output<Sha256>, |
||||
file_size: u64, |
||||
servername: &ServerName, |
||||
media_id: &str, |
||||
filename: Option<&str>, |
||||
content_type: Option<&str>, |
||||
) -> Result<()>; |
||||
|
||||
fn search_file_metadata(&self, servername: &ServerName, media_id: &str) -> Result<DbFileMeta>; |
||||
|
||||
#[allow(clippy::too_many_arguments)] |
||||
fn create_thumbnail_metadata( |
||||
&self, |
||||
sha256_digest: Output<Sha256>, |
||||
file_size: u64, |
||||
servername: &ServerName, |
||||
media_id: &str, |
||||
width: u32, |
||||
height: u32, |
||||
content_disposition: &ContentDisposition, |
||||
filename: Option<&str>, |
||||
content_type: Option<&str>, |
||||
) -> Result<Vec<u8>>; |
||||
) -> Result<()>; |
||||
|
||||
/// Returns content_disposition, content_type and the metadata key.
|
||||
fn search_file_metadata( |
||||
// Returns the sha256 hash, filename and content_type and whether the media should be accessible via
|
||||
/// unauthenticated endpoints.
|
||||
fn search_thumbnail_metadata( |
||||
&self, |
||||
mxc: String, |
||||
servername: &ServerName, |
||||
media_id: &str, |
||||
width: u32, |
||||
height: u32, |
||||
) -> Result<(ContentDisposition, Option<String>, Vec<u8>)>; |
||||
) -> Result<DbFileMeta>; |
||||
} |
||||
|
||||
Loading…
Reference in new issue