From bcf16cc7c37abfe01432caeef2f7e8e3363cee62 Mon Sep 17 00:00:00 2001 From: lennoxlotl Date: Wed, 23 Oct 2024 17:32:37 +0200 Subject: [PATCH] feat(api): add `Drive` storage driver --- README.md | 2 +- api/src/endpoint/fairing/storage.rs | 16 ++++++- api/src/endpoint/v1/error.rs | 2 +- api/src/storage/drive.rs | 68 +++++++++++++++++++++++++++++ api/src/storage/driver.rs | 20 ++++++++- api/src/storage/mod.rs | 1 + docker/production/Rocket.toml | 5 ++- 7 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 api/src/storage/drive.rs diff --git a/README.md b/README.md index 63fa5c2..403c498 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ This roadmap is constantly updated with new ideas and features that are planned #### API - [ ] Multiple auth-keys stored in Postgres - [ ] Simple CLI application for creating auth-keys and administrating the service -- [ ] Support for multiple storage "drivers" (e.g. local file system) +- [x] Support for multiple storage "drivers" (e.g. local file system) #### General - [ ] Issue and Pull Request templates to make contribution easier diff --git a/api/src/endpoint/fairing/storage.rs b/api/src/endpoint/fairing/storage.rs index 73aec63..5b7c4ec 100644 --- a/api/src/endpoint/fairing/storage.rs +++ b/api/src/endpoint/fairing/storage.rs @@ -1,4 +1,4 @@ -use std::ops::Deref; +use std::{ops::Deref, path::Path}; use aws_credential_types::Credentials; use rocket::{ @@ -13,6 +13,7 @@ use crate::{ storage::driver::StorageDriver, }; +// Provides access to the selected storage driver pub struct StorageDriverGuard(pub StorageDriver); pub struct StorageDriverFairing; @@ -21,6 +22,7 @@ pub struct StorageDriverFairing; #[serde(rename_all = "snake_case")] pub enum StorageDriverType { ObjectStorage, + Drive, } #[derive(Debug, Clone, Deserialize)] @@ -37,6 +39,11 @@ pub struct ObjectStorageConfig { region: Option, } +#[derive(Debug, Clone, Deserialize)] +pub struct DriveStorageConfig { + path: String, +} + impl Deref for StorageDriverGuard { type Target = StorageDriver; @@ -82,6 +89,13 @@ impl Fairing for StorageDriverFairing { ), )) } + StorageDriverType::Drive => { + let config: DriveStorageConfig = + rocket.figment().focus("storage.drive").extract().expect( + "Unable to load drive storage config, is it defined in Rocket.toml?", + ); + StorageDriver::drive(Path::new(config.path.as_str()).to_path_buf()) + } }; Ok(rocket.manage(driver)) } diff --git a/api/src/endpoint/v1/error.rs b/api/src/endpoint/v1/error.rs index c0e61d9..80fc797 100644 --- a/api/src/endpoint/v1/error.rs +++ b/api/src/endpoint/v1/error.rs @@ -14,7 +14,7 @@ pub struct ErrorAttributes { pub status_code: u16, } -#[derive(Debug, Clone, thiserror::Error, UploaderError)] +#[derive(Debug, thiserror::Error, UploaderError)] pub enum Error { #[error("Invalid auth key")] #[uploader(status_code = 403)] diff --git a/api/src/storage/drive.rs b/api/src/storage/drive.rs new file mode 100644 index 0000000..76ddeba --- /dev/null +++ b/api/src/storage/drive.rs @@ -0,0 +1,68 @@ +use std::{ + fs::File, + io::{Read, Write}, + path::PathBuf, +}; + +use super::driver::{StorageError, StorageResult}; + +macro_rules! try_write { + ($f:expr, $i:expr) => { + $f.write($i).map_err(|err| StorageError::from(err))? + }; +} + +/// Implements save_file for the Drive type +pub(crate) async fn save_file( + root: &PathBuf, + id: &str, + content_type: &str, + bytes: Vec, +) -> StorageResult<()> { + let mut file_path = root.clone(); + file_path.push(id); + + std::fs::create_dir_all(root).map_err(|err| StorageError::from(err))?; + let mut file = File::create(file_path).map_err(|err| StorageError::from(err))?; + try_write!(file, &[content_type.len() as u8]); + try_write!(file, content_type.as_bytes()); + try_write!(file, &bytes); + Ok(()) +} + +/// Implements get_file for the Drive type +pub(crate) async fn get_file(root: &PathBuf, id: &str) -> StorageResult<(Vec, String)> { + let mut file_path = root.clone(); + file_path.push(id); + + let mut file = File::open(file_path).map_err(|_| StorageError::DriveLoadError)?; + let content_type = read_content_type(&mut file)?; + let bytes = read_file_bytes(&mut file)?; + Ok((bytes, content_type)) +} + +/// Implements delete_file for the Drive type +pub(crate) async fn delete_file(root: &PathBuf, id: &str) -> StorageResult<()> { + let mut file_path = root.clone(); + file_path.push(id); + std::fs::remove_file(file_path).map_err(|_| StorageError::DriveDeleteError) +} + +/// Reads the content type stored in the file +fn read_content_type(file: &mut File) -> StorageResult { + let mut ct_len = [0 as u8]; + file.read_exact(&mut ct_len) + .map_err(|_| StorageError::DriveLoadError)?; + let mut ct = vec![0 as u8; ct_len[0] as usize]; + file.read_exact(&mut ct) + .map_err(|_| StorageError::DriveLoadError)?; + String::from_utf8(ct).map_err(|_| StorageError::DriveLoadError) +} + +/// Reads the file bytes stored in the file +fn read_file_bytes(file: &mut File) -> StorageResult> { + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes) + .map_err(|_| StorageError::DriveLoadError)?; + Ok(bytes) +} diff --git a/api/src/storage/driver.rs b/api/src/storage/driver.rs index 60cd91a..6384fa6 100644 --- a/api/src/storage/driver.rs +++ b/api/src/storage/driver.rs @@ -1,17 +1,20 @@ +use std::path::PathBuf; + use thiserror::Error; use crate::s3::bucket::Bucket; -use super::object_storage; +use super::{drive, object_storage}; pub type StorageResult = std::result::Result; #[derive(Debug, Clone)] pub enum StorageDriver { ObjectStorage { bucket: Bucket }, + Drive { path: PathBuf }, } -#[derive(Debug, Clone, Error)] +#[derive(Debug, Error)] pub enum StorageError { #[error("Failed to save file in object storage bucket")] BucketSaveError, @@ -19,6 +22,12 @@ pub enum StorageError { BucketLoadError, #[error("Failed to delete file from object storage bucket")] BucketDeleteError, + #[error("Failed to write file to drive ({0})")] + DriveWriteError(#[from] std::io::Error), + #[error("Failed to load file from drive")] + DriveLoadError, + #[error("Failed to delete file from drive")] + DriveDeleteError, } impl StorageDriver { @@ -26,6 +35,10 @@ impl StorageDriver { Self::ObjectStorage { bucket } } + pub fn drive(path: PathBuf) -> Self { + Self::Drive { path } + } + /// Saves a file in the storage driver /// /// # Arguments @@ -43,6 +56,7 @@ impl StorageDriver { Self::ObjectStorage { bucket } => { object_storage::save_file(bucket, id, content_type, bytes).await } + Self::Drive { path } => drive::save_file(path, id, content_type, bytes).await, } } @@ -58,6 +72,7 @@ impl StorageDriver { pub async fn get_file(&self, id: &str) -> StorageResult<(Vec, String)> { match self { Self::ObjectStorage { bucket } => object_storage::get_file(bucket, id).await, + Self::Drive { path } => drive::get_file(path, id).await, } } @@ -69,6 +84,7 @@ impl StorageDriver { pub async fn delete_file(&self, id: &str) -> StorageResult<()> { match self { Self::ObjectStorage { bucket } => object_storage::delete_file(bucket, id).await, + Self::Drive { path } => drive::delete_file(path, id).await, } } } diff --git a/api/src/storage/mod.rs b/api/src/storage/mod.rs index b62a489..2b6a0bd 100644 --- a/api/src/storage/mod.rs +++ b/api/src/storage/mod.rs @@ -1,2 +1,3 @@ +pub mod drive; pub mod driver; pub mod object_storage; diff --git a/docker/production/Rocket.toml b/docker/production/Rocket.toml index a69af17..f4dcabc 100644 --- a/docker/production/Rocket.toml +++ b/docker/production/Rocket.toml @@ -9,7 +9,10 @@ auth_key = "hi" data-form = "16MiB" file = "16MiB" -[default.bucket] +[default.storage] +storage_type = "object" + +[default.storage.object] access_key = "" access_key_secret = "" name = "files"