From e44bc9b32de13e17c3d459cd283d8bfb045d1a18 Mon Sep 17 00:00:00 2001 From: Narayan Bhat Date: Sun, 18 Aug 2024 16:45:07 +0530 Subject: [PATCH 1/9] feat: add a check in the store file function to check whether the upload dir is full --- Cargo.toml | 1 + src/config.rs | 2 ++ src/paste.rs | 23 +++++++++++++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 63e51c8a..3047d2a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } uts2ts = "0.4.1" path-clean = "1.0.1" +fs_extra = "1.3.0" [dependencies.config] version = "0.14.0" diff --git a/src/config.rs b/src/config.rs index a7aac7e2..a995f808 100644 --- a/src/config.rs +++ b/src/config.rs @@ -43,6 +43,8 @@ pub struct ServerConfig { pub max_content_length: Byte, /// Storage path. pub upload_path: PathBuf, + /// Maximum upload directory size. + pub max_upload_dir_size: Option, /// Request timeout. #[serde(default, with = "humantime_serde")] pub timeout: Option, diff --git a/src/paste.rs b/src/paste.rs index 0b6261d3..7b4bf744 100644 --- a/src/paste.rs +++ b/src/paste.rs @@ -3,13 +3,16 @@ use crate::file::Directory; use crate::header::ContentDisposition; use crate::util; use actix_web::{error, Error}; -use awc::Client; -use std::convert::{TryFrom, TryInto}; +use awc::{body::MessageBody, Client}; use std::fs::{self, File}; use std::io::{Error as IoError, ErrorKind as IoErrorKind, Result as IoResult, Write}; use std::path::{Path, PathBuf}; use std::str; use std::sync::RwLock; +use std::{ + convert::{TryFrom, TryInto}, + ops::Add, +}; use url::Url; /// Type of the data to store. @@ -109,6 +112,22 @@ impl Paste { } } } + + if let Some(max_dir_size) = config.server.max_upload_dir_size { + let file_size = u64::try_from(self.data.len()).unwrap(); + let upload_dir = self.type_.get_path(&config.server.upload_path)?; + let current_size_of_upload_dir = fs_extra::dir::get_size(upload_dir) + .map_err(|_| error::ErrorInternalServerError("Internal server error occured"))?; + + let expected_size_of_upload_dir = current_size_of_upload_dir.add(file_size); + + if expected_size_of_upload_dir > max_dir_size { + return Err(error::ErrorInsufficientStorage( + "upload directory size limit exceeded", + )); + } + } + let mut file_name = match PathBuf::from(file_name) .file_name() .and_then(|v| v.to_str()) From 7430197d07974fcbf6ed5da23bda9a14d3d961ce Mon Sep 17 00:00:00 2001 From: Narayan Bhat Date: Sun, 18 Aug 2024 16:51:23 +0530 Subject: [PATCH 2/9] chore: cargo clippy --- src/paste.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/paste.rs b/src/paste.rs index 7b4bf744..11b3eb11 100644 --- a/src/paste.rs +++ b/src/paste.rs @@ -3,7 +3,7 @@ use crate::file::Directory; use crate::header::ContentDisposition; use crate::util; use actix_web::{error, Error}; -use awc::{body::MessageBody, Client}; +use awc::Client; use std::fs::{self, File}; use std::io::{Error as IoError, ErrorKind as IoErrorKind, Result as IoResult, Write}; use std::path::{Path, PathBuf}; @@ -114,7 +114,8 @@ impl Paste { } if let Some(max_dir_size) = config.server.max_upload_dir_size { - let file_size = u64::try_from(self.data.len()).unwrap(); + // The unwrap here should be fine as the max value of u64 will be within the limits + let file_size = u64::try_from(self.data.len()).unwrap_or_default(); let upload_dir = self.type_.get_path(&config.server.upload_path)?; let current_size_of_upload_dir = fs_extra::dir::get_size(upload_dir) .map_err(|_| error::ErrorInternalServerError("Internal server error occured"))?; From d229895c01d123507681fed1d28c0b486db7b2fa Mon Sep 17 00:00:00 2001 From: Narayan Bhat Date: Sun, 18 Aug 2024 17:13:46 +0530 Subject: [PATCH 3/9] feat: add test cases --- .../test-server-upload-dir-limit/config.toml | 9 +++++++++ fixtures/test-server-upload-dir-limit/test.sh | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 fixtures/test-server-upload-dir-limit/config.toml create mode 100755 fixtures/test-server-upload-dir-limit/test.sh diff --git a/fixtures/test-server-upload-dir-limit/config.toml b/fixtures/test-server-upload-dir-limit/config.toml new file mode 100644 index 00000000..53bd752d --- /dev/null +++ b/fixtures/test-server-upload-dir-limit/config.toml @@ -0,0 +1,9 @@ +[server] +address = "127.0.0.1:8000" +max_content_length = "10KB" +max_upload_dir_size = "20KB" +upload_path = "./upload" + +[paste] +default_extension = "txt" +duplicate_files = true diff --git a/fixtures/test-server-upload-dir-limit/test.sh b/fixtures/test-server-upload-dir-limit/test.sh new file mode 100755 index 00000000..6fafabe3 --- /dev/null +++ b/fixtures/test-server-upload-dir-limit/test.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +setup() { + truncate -s 9KB bigfile1 bigfile2 bigfile3 +} + +run_test() { + result=$(curl -s -F "file=@bigfile1" localhost:8000) + result=$(curl -s -F "file=@bigfile2" localhost:8000) + + result=$(curl -s -F "file=@bigfile3" localhost:8000) + test "upload directory size limit exceeded" = "$result" +} + +teardown() { + rm bigfile* + rm -r upload +} From 23284736b85c2fef00299f56a7aeb15171d713aa Mon Sep 17 00:00:00 2001 From: Narayan Bhat Date: Sun, 18 Aug 2024 17:18:33 +0530 Subject: [PATCH 4/9] refactor: rename config variable --- fixtures/test-server-upload-dir-limit/config.toml | 2 +- src/config.rs | 2 +- src/paste.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fixtures/test-server-upload-dir-limit/config.toml b/fixtures/test-server-upload-dir-limit/config.toml index 53bd752d..2757a0b3 100644 --- a/fixtures/test-server-upload-dir-limit/config.toml +++ b/fixtures/test-server-upload-dir-limit/config.toml @@ -1,7 +1,7 @@ [server] address = "127.0.0.1:8000" max_content_length = "10KB" -max_upload_dir_size = "20KB" +max_uploads = "20KB" upload_path = "./upload" [paste] diff --git a/src/config.rs b/src/config.rs index a995f808..d3af418a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -44,7 +44,7 @@ pub struct ServerConfig { /// Storage path. pub upload_path: PathBuf, /// Maximum upload directory size. - pub max_upload_dir_size: Option, + pub max_uploads: Option, /// Request timeout. #[serde(default, with = "humantime_serde")] pub timeout: Option, diff --git a/src/paste.rs b/src/paste.rs index 11b3eb11..6a79f13f 100644 --- a/src/paste.rs +++ b/src/paste.rs @@ -113,7 +113,7 @@ impl Paste { } } - if let Some(max_dir_size) = config.server.max_upload_dir_size { + if let Some(max_dir_size) = config.server.max_uploads { // The unwrap here should be fine as the max value of u64 will be within the limits let file_size = u64::try_from(self.data.len()).unwrap_or_default(); let upload_dir = self.type_.get_path(&config.server.upload_path)?; From d1ff9405ae10d07e1f5c81621b19ac7bdab90d77 Mon Sep 17 00:00:00 2001 From: Narayan Bhat Date: Mon, 19 Aug 2024 14:12:21 +0530 Subject: [PATCH 5/9] Revert "refactor: rename config variable" This reverts commit 23284736b85c2fef00299f56a7aeb15171d713aa. --- fixtures/test-server-upload-dir-limit/config.toml | 2 +- src/config.rs | 2 +- src/paste.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fixtures/test-server-upload-dir-limit/config.toml b/fixtures/test-server-upload-dir-limit/config.toml index 2757a0b3..53bd752d 100644 --- a/fixtures/test-server-upload-dir-limit/config.toml +++ b/fixtures/test-server-upload-dir-limit/config.toml @@ -1,7 +1,7 @@ [server] address = "127.0.0.1:8000" max_content_length = "10KB" -max_uploads = "20KB" +max_upload_dir_size = "20KB" upload_path = "./upload" [paste] diff --git a/src/config.rs b/src/config.rs index d3af418a..a995f808 100644 --- a/src/config.rs +++ b/src/config.rs @@ -44,7 +44,7 @@ pub struct ServerConfig { /// Storage path. pub upload_path: PathBuf, /// Maximum upload directory size. - pub max_uploads: Option, + pub max_upload_dir_size: Option, /// Request timeout. #[serde(default, with = "humantime_serde")] pub timeout: Option, diff --git a/src/paste.rs b/src/paste.rs index 6a79f13f..11b3eb11 100644 --- a/src/paste.rs +++ b/src/paste.rs @@ -113,7 +113,7 @@ impl Paste { } } - if let Some(max_dir_size) = config.server.max_uploads { + if let Some(max_dir_size) = config.server.max_upload_dir_size { // The unwrap here should be fine as the max value of u64 will be within the limits let file_size = u64::try_from(self.data.len()).unwrap_or_default(); let upload_dir = self.type_.get_path(&config.server.upload_path)?; From a5c3740ff6060eaf3d37a22d6afd17bdc073ec9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhun=20Parmaks=C4=B1z?= Date: Wed, 11 Dec 2024 14:18:20 +0300 Subject: [PATCH 6/9] test(fixtures): update fixture for max upload size --- fixtures/test-server-upload-dir-limit/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/fixtures/test-server-upload-dir-limit/test.sh b/fixtures/test-server-upload-dir-limit/test.sh index 6fafabe3..a730b5db 100755 --- a/fixtures/test-server-upload-dir-limit/test.sh +++ b/fixtures/test-server-upload-dir-limit/test.sh @@ -7,6 +7,7 @@ setup() { run_test() { result=$(curl -s -F "file=@bigfile1" localhost:8000) result=$(curl -s -F "file=@bigfile2" localhost:8000) + curl -s "$result" result=$(curl -s -F "file=@bigfile3" localhost:8000) test "upload directory size limit exceeded" = "$result" From 84305fb37df2879c8ba713ffc6b48c1b7dfffef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhun=20Parmaks=C4=B1z?= Date: Wed, 11 Dec 2024 14:19:37 +0300 Subject: [PATCH 7/9] refactor(server): vendor get_size function from fx_extra --- Cargo.lock | 2 +- Cargo.toml | 1 - src/paste.rs | 5 +---- src/util.rs | 25 +++++++++++++++++++++++++ 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c266a71..e1afd6e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "actix-codec" diff --git a/Cargo.toml b/Cargo.toml index 3047d2a7..63e51c8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,6 @@ tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } uts2ts = "0.4.1" path-clean = "1.0.1" -fs_extra = "1.3.0" [dependencies.config] version = "0.14.0" diff --git a/src/paste.rs b/src/paste.rs index 11b3eb11..e09f95b9 100644 --- a/src/paste.rs +++ b/src/paste.rs @@ -114,14 +114,11 @@ impl Paste { } if let Some(max_dir_size) = config.server.max_upload_dir_size { - // The unwrap here should be fine as the max value of u64 will be within the limits let file_size = u64::try_from(self.data.len()).unwrap_or_default(); let upload_dir = self.type_.get_path(&config.server.upload_path)?; - let current_size_of_upload_dir = fs_extra::dir::get_size(upload_dir) + let current_size_of_upload_dir = util::get_dir_size(&upload_dir) .map_err(|_| error::ErrorInternalServerError("Internal server error occured"))?; - let expected_size_of_upload_dir = current_size_of_upload_dir.add(file_size); - if expected_size_of_upload_dir > max_dir_size { return Err(error::ErrorInsufficientStorage( "upload directory size limit exceeded", diff --git a/src/util.rs b/src/util.rs index 11bac079..af6e45a6 100644 --- a/src/util.rs +++ b/src/util.rs @@ -131,6 +131,31 @@ pub fn safe_path_join, P: AsRef>(base: B, part: P) -> IoRes Ok(new_path) } +/// Returns the size of the directory at the given path. +/// +/// This function is recursive, and will calculate the size of all files and directories. +/// If a symlink is encountered, the size of the symlink itself is counted, not its target. +/// +/// Adopted from +pub fn get_dir_size(path: &Path) -> IoResult { + let path_metadata = path.symlink_metadata()?; + let mut size_in_bytes = 0; + if path_metadata.is_dir() { + for entry in std::fs::read_dir(&path)? { + let entry = entry?; + let entry_metadata = entry.metadata()?; + if entry_metadata.is_dir() { + size_in_bytes += get_dir_size(&entry.path())?; + } else { + size_in_bytes += entry_metadata.len(); + } + } + } else { + size_in_bytes = path_metadata.len(); + } + Ok(size_in_bytes) +} + #[cfg(test)] mod tests { use super::*; From c4040f3cfc65234215ce2a706b05e359aea31a97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhun=20Parmaks=C4=B1z?= Date: Wed, 11 Dec 2024 14:22:00 +0300 Subject: [PATCH 8/9] refactor(paste): update error message --- src/paste.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/paste.rs b/src/paste.rs index e09f95b9..32a32128 100644 --- a/src/paste.rs +++ b/src/paste.rs @@ -116,8 +116,9 @@ impl Paste { if let Some(max_dir_size) = config.server.max_upload_dir_size { let file_size = u64::try_from(self.data.len()).unwrap_or_default(); let upload_dir = self.type_.get_path(&config.server.upload_path)?; - let current_size_of_upload_dir = util::get_dir_size(&upload_dir) - .map_err(|_| error::ErrorInternalServerError("Internal server error occured"))?; + let current_size_of_upload_dir = util::get_dir_size(&upload_dir).map_err(|e| { + error::ErrorInternalServerError(format!("could not get directory size: {e}")) + })?; let expected_size_of_upload_dir = current_size_of_upload_dir.add(file_size); if expected_size_of_upload_dir > max_dir_size { return Err(error::ErrorInsufficientStorage( From aad5f77b8b479e63cc928f2c1b038ad97b126913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhun=20Parmaks=C4=B1z?= Date: Wed, 11 Dec 2024 14:27:56 +0300 Subject: [PATCH 9/9] fix(lints): apply clippy suggestions --- src/util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util.rs b/src/util.rs index af6e45a6..91a43139 100644 --- a/src/util.rs +++ b/src/util.rs @@ -141,7 +141,7 @@ pub fn get_dir_size(path: &Path) -> IoResult { let path_metadata = path.symlink_metadata()?; let mut size_in_bytes = 0; if path_metadata.is_dir() { - for entry in std::fs::read_dir(&path)? { + for entry in std::fs::read_dir(path)? { let entry = entry?; let entry_metadata = entry.metadata()?; if entry_metadata.is_dir() {