feat(api): rewrite storage system for easy expandability

api/migrations/3_rename_to_storage_id.sql

@@ -0,0 +1 @@
+ALTER TABLE files RENAME COLUMN bucket_id TO storage_id;

api/src/database/file.rs

@@ -5,7 +5,7 @@ use sqlx::Row;
 #[derive(Debug, Clone, PostgresRow)]
 pub struct FileEntity {
     pub id: String,
-    pub bucket_id: String,
+    pub storage_id: String,
     pub secret: String,
     pub uploaded_at: i64,
     pub size: i64,

api/src/database/query/file.rs

@@ -23,7 +23,7 @@ pub async fn save_file(
     size: &i64,
 ) -> DbResult<()> {
     sqlx::query(
-        r"INSERT INTO files (id, bucket_id, secret, uploaded_at, size) VALUES ($1, $2, $3, $4, $5)",
+        r"INSERT INTO files (id, storage_id, secret, uploaded_at, size) VALUES ($1, $2, $3, $4, $5)",
     )
     .bind(&id)
     .bind(&bucket_id)

api/src/endpoint/fairing/mod.rs

@@ -1,2 +1,2 @@
-pub mod bucket;
 pub mod database;
+pub mod storage;

api/src/endpoint/fairing/storage.rs

@@ -0,0 +1,104 @@
+use std::ops::Deref;
+
+use aws_credential_types::Credentials;
+use rocket::{
+    fairing::{self, Fairing, Info, Kind},
+    request::{FromRequest, Outcome},
+    Build, Request, Rocket,
+};
+use serde::Deserialize;
+
+use crate::{
+    s3::{bucket::Bucket, credentials::BucketCredentials},
+    storage::driver::StorageDriver,
+};
+
+pub struct StorageDriverGuard(pub StorageDriver);
+
+pub struct StorageDriverFairing;
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum StorageDriverType {
+    ObjectStorage,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct StorageDriverFairingConfig {
+    storage_type: StorageDriverType,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct ObjectStorageConfig {
+    url: String,
+    name: String,
+    access_key: String,
+    access_key_secret: String,
+    region: Option<String>,
+}
+
+impl Deref for StorageDriverGuard {
+    type Target = StorageDriver;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl StorageDriverFairing {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+#[rocket::async_trait]
+impl Fairing for StorageDriverFairing {
+    fn info(&self) -> Info {
+        Info {
+            name: "Storage Driver Fairing",
+            kind: Kind::Ignite,
+        }
+    }
+
+    async fn on_ignite(&self, rocket: Rocket<Build>) -> fairing::Result {
+        let driver = match rocket
+            .figment()
+            .focus("storage")
+            .extract::<StorageDriverFairingConfig>()
+            .expect("Unable to load storage config, is it defined in Rocket.toml?")
+            .storage_type
+        {
+            StorageDriverType::ObjectStorage => {
+                let config: ObjectStorageConfig =
+                    rocket.figment().focus("storage.object").extract().expect(
+                        "Unable to load object storage config, is it defined in Rocket.toml?",
+                    );
+                StorageDriver::object(Bucket::new(
+                    config.name,
+                    config.url,
+                    BucketCredentials::new(
+                        Credentials::from_keys(config.access_key, config.access_key_secret, None),
+                        config.region,
+                    ),
+                ))
+            }
+        };
+        Ok(rocket.manage(driver))
+    }
+}
+
+#[rocket::async_trait]
+impl<'r> FromRequest<'r> for StorageDriverGuard {
+    type Error = crate::endpoint::v1::error::Error;
+
+    async fn from_request(request: &'r Request<'_>) -> Outcome<Self, Self::Error> {
+        if let Some(driver) = request.rocket().state::<StorageDriver>() {
+            Outcome::Success(StorageDriverGuard(driver.clone()))
+        } else {
+            Outcome::Error((
+                rocket::http::Status::InternalServerError,
+                Self::Error::StorageUnavailableError,
+            ))
+        }
+    }
+}

api/src/endpoint/index.rs

@@ -1,8 +1,8 @@
 use super::{
-    fairing::{bucket::BucketGuard, database::PostgresDb},
+    fairing::{database::PostgresDb, storage::StorageDriverGuard},
     v1::{error::Error, UploaderResult},
 };
-use crate::{database::query::file::find_file_by_id, s3::bucket::BucketOperations, GlobalConfig};
+use crate::{database::query::file::find_file_by_id, GlobalConfig};
 use rocket::{
     get,
     http::ContentType,
@@ -55,24 +55,17 @@ pub async fn index() -> &'static str {
 pub async fn show_file(
     id: &str,
     database: PostgresDb,
-    bucket: BucketGuard,
+    storage: StorageDriverGuard,
     config: &State<GlobalConfig>,
 ) -> UploaderResult<FileShowResponse> {
     let mut transaction = database.begin().await.map_err(|_| Error::DatabaseError)?;
     let file = find_file_by_id(&mut transaction, &id.to_string())
         .await
         .map_err(|_| Error::FileNotFoundError)?;
-    let data = bucket.get(&file.bucket_id).await.unwrap();
-    let file_type = &data.content_type.ok_or(Error::FileConvertError)?;
-    let file_bytes = data
-        .body
-        .collect()
-        .await
-        .map_err(|_| Error::FileConvertError)?
-        .to_vec();
+    let (data, content_type) = storage.get_file(&file.storage_id).await.unwrap();
     Ok(FileShowResponse::new(
-        file_bytes,
-        file_type.to_string(),
+        data,
+        content_type,
         config.cache_length.unwrap_or(0),
     ))
 }

api/src/endpoint/v1/error.rs

@@ -7,6 +7,8 @@ use rocket::{
 };
 use serde::Serialize;
 
+use crate::storage::driver::StorageError;
+
 /// Stores attributes about an error
 pub struct ErrorAttributes {
     pub status_code: u16,
@@ -23,12 +25,12 @@ pub enum Error {
     #[error("The file does not exist")]
     #[uploader(status_code = 404)]
     FileNotFoundError,
-    #[error("Failed to delete file from bucket, try again later")]
+    #[error("Storage driver is not available")]
     #[uploader(status_code = 500)]
-    BucketDeleteError,
-    #[error("Failed to upload file to storage bucket")]
+    StorageUnavailableError,
+    #[error("Failed to access storage driver ({0})")]
     #[uploader(status_code = 500)]
-    BucketConnectionError,
+    InternalStorageError(#[from] StorageError),
     #[error("Failed to convert file byte stream")]
     #[uploader(status_code = 500)]
     FileConvertError,

api/src/endpoint/v1/file/delete.rs

@@ -1,36 +1,47 @@
 use crate::{
     database::query::file::delete_file_by_secret,
     endpoint::{
-        fairing::{bucket::BucketGuard, database::PostgresDb},
+        fairing::{database::PostgresDb, storage::StorageDriverGuard},
         v1::{error::Error, UploaderResult},
     },
-    s3::bucket::BucketOperations,
 };
 use rocket::{delete, get};
 
 // Also offer deletion using GET requests as some screenshotting / uploading tools do that unfortunately
 #[get("/file/delete/<id>")]
-pub async fn delete_get(id: &str, database: PostgresDb, bucket: BucketGuard) -> UploaderResult<()> {
-    inner_delete(id, database, bucket).await
+pub async fn delete_get(
+    id: &str,
+    database: PostgresDb,
+    storage: StorageDriverGuard,
+) -> UploaderResult<()> {
+    inner_delete(id, database, storage).await
 }
 
 #[delete("/file/delete/<id>")]
-pub async fn delete(id: &str, database: PostgresDb, bucket: BucketGuard) -> UploaderResult<()> {
-    inner_delete(id, database, bucket).await
+pub async fn delete(
+    id: &str,
+    database: PostgresDb,
+    storage: StorageDriverGuard,
+) -> UploaderResult<()> {
+    inner_delete(id, database, storage).await
 }
 
 /// Deletes a file by its secret id, this prevents unauthorized third parties to
 /// delete random file ids
-async fn inner_delete(id: &str, database: PostgresDb, bucket: BucketGuard) -> UploaderResult<()> {
+async fn inner_delete(
+    id: &str,
+    database: PostgresDb,
+    storage: StorageDriverGuard,
+) -> UploaderResult<()> {
     let mut transaction = database.begin().await.map_err(|_| Error::DatabaseError)?;
 
     let file = delete_file_by_secret(&mut transaction, &id.to_string())
         .await
         .map_err(|_| Error::FileNotFoundError)?;
-    bucket
-        .delete(&file.bucket_id.as_str())
+    storage
+        .delete_file(&file.storage_id.as_str())
         .await
-        .map_err(|_| Error::BucketDeleteError)?;
+        .map_err(|err| Error::from(err))?;
 
     transaction
         .commit()

api/src/endpoint/v1/file/upload.rs

@@ -13,11 +13,10 @@ use serde::Serialize;
 use uuid::Uuid;
 
 use crate::database::query::file::save_file;
-use crate::endpoint::fairing::bucket::BucketGuard;
 use crate::endpoint::fairing::database::PostgresDb;
+use crate::endpoint::fairing::storage::StorageDriverGuard;
 use crate::endpoint::v1::error::Error;
-use crate::endpoint::v1::{convert_to_byte_stream, UploaderResult};
-use crate::s3::bucket::BucketOperations;
+use crate::endpoint::v1::{convert_to_bytes, UploaderResult};
 use crate::GlobalConfig;
 
 #[derive(FromForm)]
@@ -63,7 +62,7 @@ impl<'r> FromRequest<'r> for AuthToken {
 #[post("/file/upload", data = "<file_data>")]
 pub async fn upload(
     file_data: Form<FileData<'_>>,
-    bucket: BucketGuard,
+    storage: StorageDriverGuard,
     database: PostgresDb,
     config: &State<GlobalConfig>,
     token: AuthToken,
@@ -89,27 +88,25 @@ pub async fn upload(
     )
     .await
     .map_err(|_| Error::DatabaseError)?;
-    bucket
-        .put(
+    storage
+        .save_file(
             &bucket_id,
-            convert_to_byte_stream(
+            &file_data
+                .file
+                .content_type()
+                .unwrap_or(&ContentType::default())
+                .to_string(),
+            convert_to_bytes(
                 &mut file_data
                     .file
                     .open()
                     .await
                     .map_err(|_| Error::FileConvertError)?,
             )
             .await?,
-            Some(
-                &file_data
-                    .file
-                    .content_type()
-                    .unwrap_or(&ContentType::default())
-                    .to_string(),
-            ),
         )
         .await
-        .map_err(|_| Error::BucketConnectionError)?;
+        .map_err(|err| Error::from(err))?;
 
     transaction
         .commit()