Separated odf-storage-s3 crate, gated with feature

kamu-data · Jan 9, 2025 · ba5cdac · ba5cdac
1 parent 398acbc
commit ba5cdac
Show file tree

Hide file tree

Showing 29 changed files with 245 additions and 60 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -31,6 +31,7 @@ members = [
     "src/odf/data-utils",
     "src/odf/dataset-impl",
     "src/odf/storage-impl",
+    "src/odf/storage-s3",
     # Domain
     "src/domain/accounts/domain",
     "src/domain/auth-rebac/domain",
@@ -142,6 +143,7 @@ odf-storage = { version = "0.217.0", path = "src/odf/storage", default-features
 odf-data-utils = { version = "0.217.0", path = "src/odf/data-utils", default-features = false, package="opendatafabric-data-utils" }
 odf-dataset-impl = { version = "0.217.0", path = "src/odf/dataset-impl", default-features = false, package="opendatafabric-dataset-impl" }
 odf-storage-impl = { version = "0.217.0", path = "src/odf/storage-impl", default-features = false, package="opendatafabric-storage-impl" }
+odf-storage-s3 = { version = "0.217.0", path = "src/odf/storage-s3", default-features = false, package="opendatafabric-storage-s3" }
 
 # Domain service layer
 kamu-accounts-services = { version = "0.217.0", path = "src/domain/accounts/services", default-features = false }

diff --git a/src/infra/core/Cargo.toml b/src/infra/core/Cargo.toml
@@ -155,7 +155,7 @@ kamu-accounts-inmem = { workspace = true }
 kamu-accounts-services = { workspace = true }
 kamu-datasets-services = { workspace = true }
 kamu-datasets-inmem = { workspace = true }
-odf = { workspace = true, features = ["testing"] }
+odf = { workspace = true, features = ["s3", "testing"] }
 test-utils = { workspace = true }
 
 criterion = { version = "0.5", features = ["async_tokio"] }

diff --git a/src/infra/core/src/services/storage/dataset_storage_unit_s3.rs b/src/infra/core/src/services/storage/dataset_storage_unit_s3.rs
@@ -16,13 +16,12 @@ use dill::*;
 use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal};
 use kamu_accounts::{CurrentAccountSubject, DEFAULT_ACCOUNT_NAME_STR};
 use kamu_core::TenancyConfig;
+use odf::storage::s3::{NamedObjectRepositoryS3, ObjectRepositoryS3Sha3};
 use odf_dataset_impl::{DatasetImpl, MetadataChainImpl};
 use odf_storage_impl::{
     MetadataBlockRepositoryCachingInMem,
     MetadataBlockRepositoryImpl,
-    NamedObjectRepositoryS3,
     ObjectRepositoryCachingLocalFs,
-    ObjectRepositoryS3Sha3,
     ReferenceRepositoryImpl,
 };
 use s3_utils::S3Context;

diff --git a/src/odf/dataset-impl/Cargo.toml b/src/odf/dataset-impl/Cargo.toml
@@ -28,7 +28,7 @@ testing = ["dep:mockall"]
 [dependencies]
 internal-error = { workspace = true }
 file-utils = { workspace = true }
-odf = { workspace = true }
+odf = { workspace = true, features = ["s3"] }
 odf-storage-impl = { workspace = true }
 s3-utils = { workspace = true }
 

diff --git a/src/odf/dataset-impl/src/services/dataset_factory_impl.rs b/src/odf/dataset-impl/src/services/dataset_factory_impl.rs
@@ -12,6 +12,7 @@ use std::sync::Arc;
 use dill::*;
 use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal};
 use odf::dataset::*;
+use odf::storage::s3::*;
 use odf_storage_impl::*;
 use s3_utils::S3Context;
 use url::Url;

diff --git a/src/odf/odf/Cargo.toml b/src/odf/odf/Cargo.toml
@@ -24,10 +24,11 @@ doctest = false
 [features]
 default = []
 arrow = ["odf-metadata/arrow"]
+s3 = ["dep:odf-storage-s3"]
 sqlx-mysql = ["odf-metadata/sqlx-mysql"]
 sqlx-postgres = ["odf-metadata/sqlx-postgres"]
 sqlx-sqlite = ["odf-metadata/sqlx-sqlite"]
-testing = ["odf-dataset/testing", "odf-metadata/testing", "odf-data-utils/testing"]
+testing = ["odf-dataset/testing", "odf-metadata/testing", "odf-storage/testing", "odf-data-utils/testing"]
 utoipa = ["odf-metadata/utoipa"]
 
 
@@ -36,3 +37,5 @@ odf-dataset = { workspace = true }
 odf-metadata = { workspace = true }
 odf-storage = { workspace = true }
 odf-data-utils = { workspace = true }
+
+odf-storage-s3 = { optional = true, workspace = true }
diff --git a/src/odf/odf/src/lib.rs b/src/odf/odf/src/lib.rs
@@ -53,6 +53,8 @@ pub mod dataset {
 
 pub mod storage {
     pub use odf_storage::*;
+    #[cfg(feature = "s3")]
+    pub use odf_storage_s3 as s3;
 }
 
 pub mod serde {

diff --git a/src/odf/storage-impl/Cargo.toml b/src/odf/storage-impl/Cargo.toml
@@ -48,6 +48,8 @@ url = { version = "2", default-features = false }
 
 
 [dev-dependencies]
+odf = { workspace = true, features = ["testing"]}
+
 mockall = { version = "0.13", default-features = false }
 pretty_assertions = { version = "1" }
 tempfile = "3"

diff --git a/src/odf/storage-impl/src/repos/mod.rs b/src/odf/storage-impl/src/repos/mod.rs
@@ -13,12 +13,10 @@ mod named_object_repository_http;
 mod named_object_repository_in_memory;
 mod named_object_repository_ipfs_http;
 mod named_object_repository_local_fs;
-mod named_object_repository_s3;
 mod object_repository_caching_local_fs;
 mod object_repository_http;
 mod object_repository_in_memory;
 mod object_repository_local_fs;
-mod object_repository_s3;
 mod reference_repository_impl;
 mod repo_helpers;
 
@@ -28,11 +26,9 @@ pub use named_object_repository_http::*;
 pub use named_object_repository_in_memory::*;
 pub use named_object_repository_ipfs_http::*;
 pub use named_object_repository_local_fs::*;
-pub use named_object_repository_s3::*;
 pub use object_repository_caching_local_fs::*;
 pub use object_repository_http::*;
 pub use object_repository_in_memory::*;
 pub use object_repository_local_fs::*;
-pub use object_repository_s3::*;
 pub use reference_repository_impl::*;
 pub use repo_helpers::*;
diff --git a/src/odf/storage-impl/tests/repos/mod.rs b/src/odf/storage-impl/tests/repos/mod.rs
@@ -14,6 +14,4 @@ mod test_named_object_repository;
 mod test_object_repository_http;
 mod test_object_repository_in_memory;
 mod test_object_repository_local_fs;
-mod test_object_repository_s3;
-mod test_object_repository_shared;
 mod test_reference_repository_impl;
diff --git a/src/odf/storage-impl/tests/repos/test_named_object_repository.rs b/src/odf/storage-impl/tests/repos/test_named_object_repository.rs
@@ -9,31 +9,17 @@
 
 use std::assert_matches::assert_matches;
 
+use odf::storage::testing::test_named_object_repository_shared;
 use odf::storage::*;
 use opendatafabric_storage_impl::*;
-use test_utils::{HttpFileServer, LocalS3Server};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-async fn test_named_repository_operations(repo: &dyn NamedObjectRepository) {
-    assert_matches!(repo.get("head").await, Err(GetNamedError::NotFound(_)));
-
-    repo.set("head", b"foo").await.unwrap();
-    assert_eq!(&repo.get("head").await.unwrap()[..], b"foo");
-
-    repo.set("head", b"bar").await.unwrap();
-    assert_eq!(&repo.get("head").await.unwrap()[..], b"bar");
-
-    repo.delete("head").await.unwrap();
-    assert_matches!(repo.get("head").await, Err(GetNamedError::NotFound(_)));
-}
+use test_utils::HttpFileServer;
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #[tokio::test]
 async fn test_basics_in_memory() {
     let repo = NamedObjectRepositoryInMemory::new();
-    test_named_repository_operations(&repo).await;
+    test_named_object_repository_shared::test_named_repository_operations(&repo).await;
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -43,19 +29,7 @@ async fn test_basics_local_fs() {
     let tmp_dir = tempfile::tempdir().unwrap();
     let repo = NamedObjectRepositoryLocalFS::new(tmp_dir.path());
 
-    test_named_repository_operations(&repo).await;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-#[test_group::group(containerized)]
-#[tokio::test]
-async fn test_basics_s3() {
-    let s3 = LocalS3Server::new().await;
-    let s3_context = s3_utils::S3Context::from_url(&s3.url).await;
-    let repo = NamedObjectRepositoryS3::new(s3_context);
-
-    test_named_repository_operations(&repo).await;
+    test_named_object_repository_shared::test_named_repository_operations(&repo).await;
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

diff --git a/src/odf/storage-impl/tests/repos/test_object_repository_http.rs b/src/odf/storage-impl/tests/repos/test_object_repository_http.rs
@@ -10,14 +10,13 @@
 use std::assert_matches::assert_matches;
 
 use odf::metadata::*;
+use odf::storage::testing::test_object_repository_shared;
+use odf::storage::testing::test_object_repository_shared::ExternalUrlTestOptions;
 use odf::storage::*;
 use opendatafabric_storage_impl::*;
 use test_utils::HttpFileServer;
 use url::Url;
 
-use super::test_object_repository_shared;
-use super::test_object_repository_shared::ExternalUrlTestOptions;
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #[test_log::test(tokio::test)]

diff --git a/src/odf/storage-impl/tests/repos/test_object_repository_in_memory.rs b/src/odf/storage-impl/tests/repos/test_object_repository_in_memory.rs
@@ -9,11 +9,10 @@
 
 use std::assert_matches::assert_matches;
 
+use odf::storage::testing::test_object_repository_shared;
 use odf::storage::*;
 use opendatafabric_storage_impl::*;
 
-use super::test_object_repository_shared;
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #[tokio::test]

diff --git a/src/odf/storage-impl/tests/repos/test_object_repository_local_fs.rs b/src/odf/storage-impl/tests/repos/test_object_repository_local_fs.rs
@@ -10,11 +10,10 @@
 use std::assert_matches::assert_matches;
 
 use odf::metadata::*;
+use odf::storage::testing::test_object_repository_shared;
 use odf::storage::*;
 use opendatafabric_storage_impl::*;
 
-use super::test_object_repository_shared;
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #[tokio::test]

diff --git a/src/odf/storage-s3/Cargo.toml b/src/odf/storage-s3/Cargo.toml
@@ -0,0 +1,62 @@
+[package]
+name = "opendatafabric-storage-s3"
+description = "AWS S3 based implementation of storage facilities for Open Data Fabric based datasets"
+version = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+authors = { workspace = true }
+readme = { workspace = true }
+license-file = { workspace = true }
+keywords = { workspace = true }
+include = { workspace = true }
+edition = { workspace = true }
+publish = { workspace = true }
+
+
+[lints]
+workspace = true
+
+
+[lib]
+doctest = false
+
+
+[dependencies]
+async-utils = { workspace = true }
+internal-error = { workspace = true }
+odf-metadata = { workspace = true }
+odf-storage = { workspace = true }
+#random-names = { workspace = true }
+s3-utils = { workspace = true }
+
+#async-stream = "0.3"
+async-trait = "0.1"
+aws-sdk-s3 = { version = "1" }
+bytes = { version = "1", default-features = false }
+chrono = { version = "0.4", default-features = false}
+#dashmap = { version = "6", default-features = false }
+digest = "0.10"
+http = "1"
+#futures = "0.3"
+
+#rand = "0.8"
+#reqwest = { version = "0.12", default-features = false }
+sha3 = "0.10"
+tokio = { version = "1", default-features = false }
+#tokio-util = { version = "0.7", default-features = false }
+tracing = "0.1"
+url = { version = "2", default-features = false }
+
+
+[dev-dependencies]
+odf-storage = { workspace = true, features = ["testing"]}
+
+#mockall = { version = "0.13", default-features = false }
+#pretty_assertions = { version = "1" }
+rand = "0.8"
+#tempfile = "3"
+#thiserror = { version = "2", default-features = false }
+test-log = { version = "0.2", features = ["trace"] }
+test-utils = { workspace = true }
+test-group = { version = "1" }
+#tokio-stream = { version = "0.1", default-features = false }
diff --git a/src/odf/storage-s3/src/lib.rs b/src/odf/storage-s3/src/lib.rs
@@ -0,0 +1,11 @@
+// Copyright Kamu Data, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+mod repos;
+pub use repos::*;
diff --git a/src/odf/storage-s3/src/repos/mod.rs b/src/odf/storage-s3/src/repos/mod.rs
@@ -0,0 +1,14 @@
+// Copyright Kamu Data, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+mod named_object_repository_s3;
+mod object_repository_s3;
+
+pub use named_object_repository_s3::*;
+pub use object_repository_s3::*;
diff --git a/...l/src/repos/named_object_repository_s3.rs → ...3/src/repos/named_object_repository_s3.rs b/...l/src/repos/named_object_repository_s3.rs → ...3/src/repos/named_object_repository_s3.rs
@@ -11,7 +11,7 @@ use async_trait::async_trait;
 use aws_sdk_s3::operation::get_object::GetObjectError;
 use bytes::Bytes;
 use internal_error::{ErrorIntoInternal, ResultIntoInternal};
-use odf::storage::*;
+use odf_storage::*;
 use s3_utils::S3Context;
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

diff --git a/...ge-impl/src/repos/object_repository_s3.rs → ...rage-s3/src/repos/object_repository_s3.rs b/...ge-impl/src/repos/object_repository_s3.rs → ...rage-s3/src/repos/object_repository_s3.rs
@@ -18,8 +18,8 @@ use aws_sdk_s3::operation::head_object::HeadObjectError;
 use aws_sdk_s3::presigning::PresigningConfig;
 use bytes::Bytes;
 use internal_error::{ErrorIntoInternal, ResultIntoInternal};
-use odf::metadata::*;
-use odf::storage::*;
+use odf_metadata::*;
+use odf_storage::*;
 use s3_utils::S3Context;
 use url::Url;
 

diff --git a/src/odf/storage-s3/tests/mod.rs b/src/odf/storage-s3/tests/mod.rs
@@ -0,0 +1,12 @@
+// Copyright Kamu Data, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+#![feature(assert_matches)]
+
+mod repos;