diff --git a/Cargo.lock b/Cargo.lock index 77ab775c8b..21f9cad0e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5624,6 +5624,7 @@ dependencies = [ "messaging-outbox", "mockall", "odf-dataset", + "odf-dataset-impl", "odf-storage", "odf-storage-impl", "opendatafabric", @@ -5879,6 +5880,7 @@ dependencies = [ "num-format", "observability", "odf-dataset", + "odf-dataset-impl", "odf-storage", "odf-storage-impl", "opendatafabric", @@ -6627,6 +6629,7 @@ dependencies = [ "mockall", "observability", "odf-dataset", + "odf-dataset-impl", "serde_json", "tempfile", "test-log", @@ -7577,6 +7580,7 @@ dependencies = [ "async-trait", "async-utils", "chrono", + "dill", "file-utils", "futures", "internal-error", @@ -7596,7 +7600,10 @@ dependencies = [ "async-stream", "async-trait", "chrono", + "dill", "file-utils", + "hickory-resolver", + "http 1.2.0", "internal-error", "kamu-data-utils", "mockall", @@ -7604,6 +7611,9 @@ dependencies = [ "odf-storage", "odf-storage-impl", "opendatafabric", + "regex", + "reqwest", + "s3-utils", "serde_yaml", "tempfile", "test-log", diff --git a/deny.toml b/deny.toml index 2fb06c86b2..016b1dd71a 100644 --- a/deny.toml +++ b/deny.toml @@ -99,8 +99,5 @@ ignore = [ "RUSTSEC-2024-0370", # Unmaintained (instant) # https://rustsec.org/advisories/RUSTSEC-2024-0384.html - "RUSTSEC-2024-0384", - # Security: requires update in hickory-resolver, which is not available yet - # https://github.com/rust-lang/crates.io-index - "RUSTSEC-2024-0421" + "RUSTSEC-2024-0384" ] diff --git a/src/adapter/http/Cargo.toml b/src/adapter/http/Cargo.toml index 4135515741..f4df000c77 100644 --- a/src/adapter/http/Cargo.toml +++ b/src/adapter/http/Cargo.toml @@ -114,7 +114,8 @@ kamu-datasets-services = { workspace = true } kamu-datasets-inmem = { workspace = true } kamu-ingest-datafusion = { workspace = true } messaging-outbox = { workspace = true } -odf-storage-impl = { workspace = true, features = ["testing"]} +odf-dataset-impl = { workspace = true, features = ["testing"] } +odf-storage-impl = { workspace = true, features = ["testing"] } test-utils = { workspace = true } fs_extra = "1.3" # Recursive folder copy diff --git a/src/adapter/http/src/smart_protocol/ws_tungstenite_client.rs b/src/adapter/http/src/smart_protocol/ws_tungstenite_client.rs index 6a505613dd..4ca1e6bbef 100644 --- a/src/adapter/http/src/smart_protocol/ws_tungstenite_client.rs +++ b/src/adapter/http/src/smart_protocol/ws_tungstenite_client.rs @@ -17,7 +17,14 @@ use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu::utils::smart_transfer_protocol::{SmartTransferProtocolClient, TransferOptions}; use kamu_core::*; use odf::AsTypedBlock; -use odf_dataset::{BlockRef, Dataset, DatasetVisibility, InvalidIntervalError, RefCASError}; +use odf_dataset::{ + BlockRef, + Dataset, + DatasetVisibility, + InvalidIntervalError, + OdfServerAccessTokenResolver, + RefCASError, +}; use odf_storage::GetRefError; use opendatafabric as odf; use serde::de::DeserializeOwned; @@ -39,7 +46,7 @@ use crate::OdfSmtpVersion; pub struct WsSmartTransferProtocolClient { catalog: Catalog, - dataset_credential_resolver: Arc, + dataset_credential_resolver: Arc, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -49,7 +56,7 @@ pub struct WsSmartTransferProtocolClient { impl WsSmartTransferProtocolClient { pub fn new( catalog: Catalog, - dataset_credential_resolver: Arc, + dataset_credential_resolver: Arc, ) -> Self { Self { catalog, diff --git a/src/adapter/http/tests/harness/client_side_harness.rs b/src/adapter/http/tests/harness/client_side_harness.rs index f54d6c2735..1526997410 100644 --- a/src/adapter/http/tests/harness/client_side_harness.rs +++ b/src/adapter/http/tests/harness/client_side_harness.rs @@ -11,7 +11,6 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; -use auth::OdfServerAccessTokenResolver; use container_runtime::ContainerRuntime; use database_common::NoOpDatabasePlugin; use dill::Component; @@ -25,7 +24,8 @@ use kamu_adapter_http::{OdfSmtpVersion, SmartTransferProtocolClientWs}; use kamu_datasets_inmem::InMemoryDatasetDependencyRepository; use kamu_datasets_services::{DatasetKeyValueServiceSysEnv, DependencyGraphServiceImpl}; use messaging_outbox::DummyOutboxImpl; -use odf_dataset::DatasetVisibility; +use odf_dataset::{DatasetVisibility, OdfServerAccessTokenResolver}; +use odf_dataset_impl::{DatasetFactoryImpl, DatasetLayout, IpfsGateway}; use opendatafabric::{ AccountID, AccountName, @@ -94,10 +94,10 @@ impl ClientSideHarness { b.add::(); if options.authenticated_remotely { - b.add::(); + b.add::(); } else { - b.add_value(kamu::testing::MockOdfServerAccessTokenResolver::empty()); - b.bind::(); + b.add_value(odf_dataset_impl::testing::MockOdfServerAccessTokenResolver::empty()); + b.bind::(); } b.add::(); diff --git a/src/adapter/http/tests/harness/common_harness.rs b/src/adapter/http/tests/harness/common_harness.rs index cbf2e413c4..3ad4d37aa7 100644 --- a/src/adapter/http/tests/harness/common_harness.rs +++ b/src/adapter/http/tests/harness/common_harness.rs @@ -16,8 +16,8 @@ use datafusion::arrow::array::{Array, RecordBatch, UInt64Array}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use kamu::domain::*; use kamu::testing::ParquetWriterHelper; -use kamu::DatasetLayout; use odf_dataset::{CommitOpts, CommitResult}; +use odf_dataset_impl::DatasetLayout; use odf_storage::{InsertOpts, ObjectRepository}; use odf_storage_impl::testing::{AddDataBuilder, MetadataFactory}; use odf_storage_impl::ObjectRepositoryLocalFSSha3; diff --git a/src/adapter/http/tests/harness/server_side_harness.rs b/src/adapter/http/tests/harness/server_side_harness.rs index 0bec16e831..5661786957 100644 --- a/src/adapter/http/tests/harness/server_side_harness.rs +++ b/src/adapter/http/tests/harness/server_side_harness.rs @@ -24,10 +24,10 @@ use kamu::domain::{ CreateDatasetUseCase, }; use kamu::testing::MockDatasetActionAuthorizer; -use kamu::DatasetLayout; use kamu_accounts::testing::MockAuthenticationService; use kamu_accounts::{Account, AccountType, CurrentAccountSubject, PROVIDER_PASSWORD}; use kamu_core::{CompactionExecutor, CompactionPlanner, DatasetRegistry, TenancyConfig}; +use odf_dataset_impl::DatasetLayout; use opendatafabric::{AccountID, AccountName, DatasetAlias, DatasetHandle}; use reqwest::Url; use time_source::SystemTimeSourceStub; @@ -84,7 +84,7 @@ pub(crate) struct ServerSideHarnessOptions { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub(crate) fn server_authentication_mock(account: &Account) -> MockAuthenticationService { - MockAuthenticationService::resolving_token(kamu_accounts::DUMMY_ACCESS_TOKEN, account.clone()) + MockAuthenticationService::resolving_token(odf_dataset::DUMMY_ODF_ACCESS_TOKEN, account.clone()) } pub(crate) fn make_server_account() -> Account { diff --git a/src/adapter/http/tests/harness/server_side_local_fs_harness.rs b/src/adapter/http/tests/harness/server_side_local_fs_harness.rs index cdb620211d..2fcdf05639 100644 --- a/src/adapter/http/tests/harness/server_side_local_fs_harness.rs +++ b/src/adapter/http/tests/harness/server_side_local_fs_harness.rs @@ -30,7 +30,6 @@ use kamu::{ CompactionPlannerImpl, CreateDatasetFromSnapshotUseCaseImpl, CreateDatasetUseCaseImpl, - DatasetLayout, DatasetRegistryRepoBridge, DatasetRepositoryLocalFs, DatasetRepositoryWriter, @@ -44,6 +43,7 @@ use kamu_core::{CompactionExecutor, CompactionPlanner, DatasetRegistry, TenancyC use kamu_datasets_inmem::InMemoryDatasetDependencyRepository; use kamu_datasets_services::DependencyGraphServiceImpl; use messaging_outbox::DummyOutboxImpl; +use odf_dataset_impl::DatasetLayout; use opendatafabric::{AccountName, DatasetAlias, DatasetHandle}; use tempfile::TempDir; use time_source::{SystemTimeSource, SystemTimeSourceStub}; diff --git a/src/adapter/http/tests/harness/server_side_s3_harness.rs b/src/adapter/http/tests/harness/server_side_s3_harness.rs index 238d1a7979..e61c763442 100644 --- a/src/adapter/http/tests/harness/server_side_s3_harness.rs +++ b/src/adapter/http/tests/harness/server_side_s3_harness.rs @@ -32,7 +32,6 @@ use kamu::{ CompactionPlannerImpl, CreateDatasetFromSnapshotUseCaseImpl, CreateDatasetUseCaseImpl, - DatasetLayout, DatasetRegistryRepoBridge, DatasetRepositoryS3, DatasetRepositoryWriter, @@ -46,6 +45,7 @@ use kamu_core::{DatasetRegistry, TenancyConfig}; use kamu_datasets_inmem::InMemoryDatasetDependencyRepository; use kamu_datasets_services::DependencyGraphServiceImpl; use messaging_outbox::DummyOutboxImpl; +use odf_dataset_impl::DatasetLayout; use opendatafabric::{AccountName, DatasetAlias, DatasetHandle}; use s3_utils::S3Context; use test_utils::LocalS3Server; diff --git a/src/adapter/http/tests/tests/test_account_info.rs b/src/adapter/http/tests/tests/test_account_info.rs index 96e12943e1..639d11c454 100644 --- a/src/adapter/http/tests/tests/test_account_info.rs +++ b/src/adapter/http/tests/tests/test_account_info.rs @@ -7,7 +7,6 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu_accounts::DUMMY_ACCESS_TOKEN; use kamu_core::{RunInfoDir, TenancyConfig}; use serde_json::json; @@ -50,7 +49,10 @@ async fn test_get_account_info() { let res = cl .get(&format!("{}accounts/me", harness.root_url)) - .header("Authorization", format!("Bearer {DUMMY_ACCESS_TOKEN}")) + .header( + "Authorization", + format!("Bearer {}", odf_dataset::DUMMY_ODF_ACCESS_TOKEN), + ) .send() .await .unwrap(); diff --git a/src/adapter/http/tests/tests/test_data_ingest.rs b/src/adapter/http/tests/tests/test_data_ingest.rs index 9e9c827550..3502a12b0b 100644 --- a/src/adapter/http/tests/tests/test_data_ingest.rs +++ b/src/adapter/http/tests/tests/test_data_ingest.rs @@ -12,7 +12,6 @@ use indoc::indoc; use kamu::domain::*; use kamu::testing::DatasetDataHelper; use kamu::*; -use kamu_accounts::DUMMY_ACCESS_TOKEN; use kamu_adapter_http::{ FileUploadLimitConfig, UploadServiceLocal, @@ -318,7 +317,7 @@ async fn test_data_push_ingest_upload_token_no_initial_source() { let res = cl .execute( cl.post(&ingest_url) - .bearer_auth(DUMMY_ACCESS_TOKEN) + .bearer_auth(odf_dataset::DUMMY_ODF_ACCESS_TOKEN) .build() .unwrap(), ) @@ -406,7 +405,7 @@ async fn test_data_push_ingest_upload_token_with_initial_source() { let res = cl .execute( cl.post(&ingest_url) - .bearer_auth(DUMMY_ACCESS_TOKEN) + .bearer_auth(odf_dataset::DUMMY_ODF_ACCESS_TOKEN) .build() .unwrap(), ) @@ -485,7 +484,7 @@ async fn test_data_push_ingest_upload_content_type_not_specified() { cl.execute( cl.post(&ingest_url) - .bearer_auth(DUMMY_ACCESS_TOKEN) + .bearer_auth(odf_dataset::DUMMY_ODF_ACCESS_TOKEN) .build() .unwrap(), ) @@ -573,7 +572,7 @@ async fn test_data_push_ingest_upload_token_actual_file_different_size() { let res = cl .execute( cl.post(&ingest_url) - .bearer_auth(DUMMY_ACCESS_TOKEN) + .bearer_auth(odf_dataset::DUMMY_ODF_ACCESS_TOKEN) .build() .unwrap(), ) diff --git a/src/adapter/http/tests/tests/test_dataset_info.rs b/src/adapter/http/tests/tests/test_dataset_info.rs index e9a204c848..ce949d558c 100644 --- a/src/adapter/http/tests/tests/test_dataset_info.rs +++ b/src/adapter/http/tests/tests/test_dataset_info.rs @@ -8,7 +8,6 @@ // by the Apache License, Version 2.0. use chrono::Utc; -use kamu_accounts::DUMMY_ACCESS_TOKEN; use kamu_core::TenancyConfig; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::{DatasetAlias, DatasetID, DatasetKind, DatasetName}; @@ -44,7 +43,10 @@ async fn test_get_dataset_info_by_id() { "{}datasets/{}", harness.root_url, create_result.dataset_handle.id )) - .header("Authorization", format!("Bearer {DUMMY_ACCESS_TOKEN}")) + .header( + "Authorization", + format!("Bearer {}", odf_dataset::DUMMY_ODF_ACCESS_TOKEN), + ) .send() .await .unwrap() @@ -75,7 +77,10 @@ async fn test_get_dataset_info_by_id_not_found_err() { let res = cl .get(&format!("{}datasets/{dataset_id}", harness.root_url)) - .header("Authorization", format!("Bearer {DUMMY_ACCESS_TOKEN}")) + .header( + "Authorization", + format!("Bearer {}", odf_dataset::DUMMY_ODF_ACCESS_TOKEN), + ) .send() .await .unwrap(); diff --git a/src/adapter/http/tests/tests/test_routing.rs b/src/adapter/http/tests/tests/test_routing.rs index 4aaf1ad780..3ce834d3ab 100644 --- a/src/adapter/http/tests/tests/test_routing.rs +++ b/src/adapter/http/tests/tests/test_routing.rs @@ -20,7 +20,8 @@ use kamu_accounts::CurrentAccountSubject; use kamu_datasets_inmem::InMemoryDatasetDependencyRepository; use kamu_datasets_services::DependencyGraphServiceImpl; use messaging_outbox::DummyOutboxImpl; -use odf_dataset::BlockRef; +use odf_dataset::{BlockRef, DatasetFactory, DummyOdfServerAccessTokenResolver}; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; use time_source::SystemTimeSourceDefault; @@ -131,7 +132,7 @@ where async fn setup_client(dataset_url: url::Url, head_expected: Multihash) { let catalog = dill::CatalogBuilder::new() - .add::() + .add::() .build(); let dataset = DatasetFactoryImpl::new(IpfsGateway::default(), catalog.get_one().unwrap()) diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_existing_evolved_dataset_reread_succeeds.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_existing_evolved_dataset_reread_succeeds.rs index 5d8b6fde09..ad4b28662c 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_existing_evolved_dataset_reread_succeeds.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_existing_evolved_dataset_reread_succeeds.rs @@ -8,8 +8,8 @@ // by the Apache License, Version 2.0. use kamu::domain::*; -use kamu::DatasetLayout; use odf_dataset::{CommitOpts, CommitResult}; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_new_reread_succeeds.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_new_reread_succeeds.rs index 7d5b018671..cf550a5508 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_new_reread_succeeds.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_new_reread_succeeds.rs @@ -7,8 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::DatasetLayout; use odf_dataset::CommitResult; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_diverged_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_diverged_dataset.rs index 18a7e1fcb8..9b7eb50779 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_diverged_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_diverged_dataset.rs @@ -8,8 +8,8 @@ // by the Apache License, Version 2.0. use kamu::domain::*; -use kamu::DatasetLayout; use odf_dataset::CommitResult; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_evolved_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_evolved_dataset.rs index 2e2944f80b..14c38508ba 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_evolved_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_evolved_dataset.rs @@ -8,8 +8,8 @@ // by the Apache License, Version 2.0. use kamu::domain::*; -use kamu::DatasetLayout; use odf_dataset::{CommitOpts, CommitResult}; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_up_to_date_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_up_to_date_dataset.rs index 50ba940617..6dab8fb47d 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_up_to_date_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_up_to_date_dataset.rs @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::DatasetLayout; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_dataset.rs index 458c506ab2..aa6097aaf0 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_dataset.rs @@ -7,8 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::DatasetLayout; use odf_dataset::CommitResult; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_empty_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_empty_dataset.rs index f049fd09ae..ca8dfbfcb8 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_empty_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_empty_dataset.rs @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0. use kamu::domain::*; -use kamu::DatasetLayout; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_new_rewrite_succeeds.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_new_rewrite_succeeds.rs index d6c7b4df88..95e75dbb67 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_new_rewrite_succeeds.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_new_rewrite_succeeds.rs @@ -7,8 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::DatasetLayout; use odf_dataset::CommitResult; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_updated_rewrite_succeeds.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_updated_rewrite_succeeds.rs index af5744f3ca..c835226e10 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_updated_rewrite_succeeds.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_updated_rewrite_succeeds.rs @@ -8,8 +8,8 @@ // by the Apache License, Version 2.0. use kamu::domain::*; -use kamu::DatasetLayout; use odf_dataset::{CommitOpts, CommitResult}; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_diverged_dataset.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_diverged_dataset.rs index d0ccc0654f..5e1ea5acb0 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_diverged_dataset.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_diverged_dataset.rs @@ -8,8 +8,8 @@ // by the Apache License, Version 2.0. use kamu::domain::*; -use kamu::DatasetLayout; use odf_dataset::CommitResult; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_evolved_dataset.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_evolved_dataset.rs index 7c1e41ad38..a6830cd208 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_evolved_dataset.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_evolved_dataset.rs @@ -8,8 +8,8 @@ // by the Apache License, Version 2.0. use kamu::domain::*; -use kamu::DatasetLayout; use odf_dataset::{CommitOpts, CommitResult}; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_up_to_date_dataset.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_up_to_date_dataset.rs index 59a54de46b..b61fe8063c 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_up_to_date_dataset.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_up_to_date_dataset.rs @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::DatasetLayout; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset.rs index 9669de0859..4f0494e733 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset.rs @@ -7,8 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::DatasetLayout; use odf_dataset::CommitResult; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset_via_repo_ref.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset_via_repo_ref.rs index 2987bb6794..6c8e4fdf6d 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset_via_repo_ref.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset_via_repo_ref.rs @@ -7,8 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::DatasetLayout; use odf_dataset::CommitResult; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_empty_dataset.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_empty_dataset.rs index ebc2e6b590..102bf2a4be 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_empty_dataset.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_empty_dataset.rs @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0. use kamu::domain::*; -use kamu::DatasetLayout; +use odf_dataset_impl::DatasetLayout; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; diff --git a/src/app/cli/Cargo.toml b/src/app/cli/Cargo.toml index cfc87ae186..450164db86 100644 --- a/src/app/cli/Cargo.toml +++ b/src/app/cli/Cargo.toml @@ -65,6 +65,7 @@ kamu-adapter-oauth = { workspace = true } kamu-adapter-odata = { workspace = true } kamu-datafusion-cli = { workspace = true } odf-dataset = { workspace = true } +odf-dataset-impl = { workspace = true } odf-storage = { workspace = true } opendatafabric = { workspace = true } random-names = { workspace = true } diff --git a/src/app/cli/src/app.rs b/src/app/cli/src/app.rs index c2eb03f370..5a6aec89e6 100644 --- a/src/app/cli/src/app.rs +++ b/src/app/cli/src/app.rs @@ -32,6 +32,7 @@ use kamu_flow_system_services::{ }; use kamu_task_system_inmem::domain::{TaskProgressMessage, MESSAGE_PRODUCER_KAMU_TASK_AGENT}; use messaging_outbox::{register_message_dispatcher, Outbox, OutboxDispatchingImpl}; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use time_source::{SystemTimeSource, SystemTimeSourceDefault, SystemTimeSourceStub}; use tracing::{warn, Instrument}; diff --git a/src/app/cli/src/explore/sql_shell_impl.rs b/src/app/cli/src/explore/sql_shell_impl.rs index db6e254671..4ec411a691 100644 --- a/src/app/cli/src/explore/sql_shell_impl.rs +++ b/src/app/cli/src/explore/sql_shell_impl.rs @@ -16,7 +16,7 @@ use std::time::Duration; use container_runtime::*; use internal_error::*; -use kamu::*; +use odf_dataset_impl::DatasetLayout; use random_names::get_random_name; use crate::error::{CommandRunError, SubprocessError}; diff --git a/src/app/cli/src/services/odf_server/access_token_registry_service.rs b/src/app/cli/src/services/odf_server/access_token_registry_service.rs index fa62f3d555..59e1172f73 100644 --- a/src/app/cli/src/services/odf_server/access_token_registry_service.rs +++ b/src/app/cli/src/services/odf_server/access_token_registry_service.rs @@ -32,7 +32,7 @@ pub struct AccessTokenRegistryService { } #[component(pub)] -#[interface(dyn kamu::domain::auth::OdfServerAccessTokenResolver)] +#[interface(dyn odf_dataset::OdfServerAccessTokenResolver)] impl AccessTokenRegistryService { pub fn new( storage: Arc, @@ -256,7 +256,7 @@ impl AccessTokenRegistryService { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl kamu::domain::auth::OdfServerAccessTokenResolver for AccessTokenRegistryService { +impl odf_dataset::OdfServerAccessTokenResolver for AccessTokenRegistryService { fn resolve_odf_dataset_access_token(&self, odf_dataset_http_url: &Url) -> Option { assert!(!odf_dataset_http_url.scheme().starts_with("odf+")); diff --git a/src/app/cli/tests/tests/test_access_token_registry_svc.rs b/src/app/cli/tests/tests/test_access_token_registry_svc.rs index 11511856ff..b3fee79a71 100644 --- a/src/app/cli/tests/tests/test_access_token_registry_svc.rs +++ b/src/app/cli/tests/tests/test_access_token_registry_svc.rs @@ -12,9 +12,9 @@ use std::assert_matches::assert_matches; use std::sync::Arc; -use kamu::domain::auth::OdfServerAccessTokenResolver; use kamu_accounts::CurrentAccountSubject; use kamu_cli::odf_server::*; +use odf_dataset::OdfServerAccessTokenResolver; use url::Url; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/app/cli/tests/tests/test_workspace_svc.rs b/src/app/cli/tests/tests/test_workspace_svc.rs index de6a8ad7b4..9bb006e27b 100644 --- a/src/app/cli/tests/tests/test_workspace_svc.rs +++ b/src/app/cli/tests/tests/test_workspace_svc.rs @@ -17,6 +17,7 @@ use kamu_cli::*; use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; use odf_dataset::{AddDataParams, CheckpointRef, CommitOpts, Dataset, GetSummaryOpts}; +use odf_dataset_impl::{DatasetFactoryImpl, DatasetLayout}; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::serde::yaml::Manifest; use opendatafabric::*; diff --git a/src/domain/core/src/auth/mod.rs b/src/domain/core/src/auth/mod.rs index ea2ca70886..3360513b88 100644 --- a/src/domain/core/src/auth/mod.rs +++ b/src/domain/core/src/auth/mod.rs @@ -9,6 +9,3 @@ mod dataset_action_authorizer; pub use dataset_action_authorizer::*; - -mod odf_server_access_token_resolver; -pub use odf_server_access_token_resolver::*; diff --git a/src/domain/core/src/repos/mod.rs b/src/domain/core/src/repos/mod.rs index e7a580061e..49fcf42a24 100644 --- a/src/domain/core/src/repos/mod.rs +++ b/src/domain/core/src/repos/mod.rs @@ -7,8 +7,6 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -pub mod dataset_factory; pub mod dataset_repository; -pub use dataset_factory::*; pub use dataset_repository::*; diff --git a/src/domain/core/src/services/remote_repository_registry.rs b/src/domain/core/src/services/remote_repository_registry.rs index 7845490275..7525d016ab 100644 --- a/src/domain/core/src/services/remote_repository_registry.rs +++ b/src/domain/core/src/services/remote_repository_registry.rs @@ -11,12 +11,11 @@ use ::serde::{Deserialize, Serialize}; use ::serde_with::skip_serializing_none; use async_trait::async_trait; use internal_error::InternalError; +use odf_dataset::UnsupportedProtocolError; use opendatafabric::*; use thiserror::Error; use url::Url; -use crate::*; - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[skip_serializing_none] diff --git a/src/domain/core/src/services/search_service.rs b/src/domain/core/src/services/search_service.rs index ca4e3592e5..61df1b8cf9 100644 --- a/src/domain/core/src/services/search_service.rs +++ b/src/domain/core/src/services/search_service.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. use internal_error::{BoxedError, InternalError}; +use odf_dataset::UnsupportedProtocolError; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/sync_service.rs b/src/domain/core/src/services/sync_service.rs index daaddd635e..0503288d95 100644 --- a/src/domain/core/src/services/sync_service.rs +++ b/src/domain/core/src/services/sync_service.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use internal_error::{BoxedError, InternalError}; -use odf_dataset::{Dataset, DatasetVisibility}; +use odf_dataset::{BuildDatasetError, Dataset, DatasetVisibility, UnsupportedProtocolError}; use opendatafabric::*; use thiserror::Error; use url::Url; diff --git a/src/domain/odf/dataset/Cargo.toml b/src/domain/odf/dataset/Cargo.toml index 04df277921..303e620345 100644 --- a/src/domain/odf/dataset/Cargo.toml +++ b/src/domain/odf/dataset/Cargo.toml @@ -30,6 +30,7 @@ odf-storage = { workspace = true } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", features = ["serde"] } +dill = { version = "0.10", default-features = false } futures = { version = "0.3", default-features = false } pin-project = { version = "1", default-features = false } serde = { version = "1", default-features = false, features = ["derive"] } diff --git a/src/domain/core/src/repos/dataset_factory.rs b/src/domain/odf/dataset/src/services/dataset_factory.rs similarity index 98% rename from src/domain/core/src/repos/dataset_factory.rs rename to src/domain/odf/dataset/src/services/dataset_factory.rs index e41df5813e..4c1a5b7b46 100644 --- a/src/domain/core/src/repos/dataset_factory.rs +++ b/src/domain/odf/dataset/src/services/dataset_factory.rs @@ -11,9 +11,10 @@ use std::sync::Arc; use async_trait::async_trait; use internal_error::InternalError; -use odf_dataset::Dataset; use url::Url; +use crate::Dataset; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait] diff --git a/src/domain/odf/dataset/src/services/mod.rs b/src/domain/odf/dataset/src/services/mod.rs index 24e2a29658..9357b81c63 100644 --- a/src/domain/odf/dataset/src/services/mod.rs +++ b/src/domain/odf/dataset/src/services/mod.rs @@ -7,8 +7,12 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +mod dataset_factory; mod metadata_chain_visitor; mod metadata_chain_visitors; +mod odf_server_access_token_resolver; +pub use dataset_factory::*; pub use metadata_chain_visitor::*; pub use metadata_chain_visitors::*; +pub use odf_server_access_token_resolver::*; diff --git a/src/domain/core/src/auth/odf_server_access_token_resolver.rs b/src/domain/odf/dataset/src/services/odf_server_access_token_resolver.rs similarity index 85% rename from src/domain/core/src/auth/odf_server_access_token_resolver.rs rename to src/domain/odf/dataset/src/services/odf_server_access_token_resolver.rs index 7d17face95..a755e85fe8 100644 --- a/src/domain/core/src/auth/odf_server_access_token_resolver.rs +++ b/src/domain/odf/dataset/src/services/odf_server_access_token_resolver.rs @@ -8,11 +8,14 @@ // by the Apache License, Version 2.0. use dill::*; -use kamu_accounts::DUMMY_ACCESS_TOKEN; use url::Url; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub const DUMMY_ODF_ACCESS_TOKEN: &str = "some-odf-token"; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub trait OdfServerAccessTokenResolver: Send + Sync { fn resolve_odf_dataset_access_token(&self, odf_dataset_http_url: &Url) -> Option; } @@ -32,7 +35,7 @@ impl DummyOdfServerAccessTokenResolver { #[async_trait::async_trait] impl OdfServerAccessTokenResolver for DummyOdfServerAccessTokenResolver { fn resolve_odf_dataset_access_token(&self, _odf_dataset_http_url: &Url) -> Option { - Some(DUMMY_ACCESS_TOKEN.to_string()) + Some(DUMMY_ODF_ACCESS_TOKEN.to_string()) } } diff --git a/src/domain/task-system/services/Cargo.toml b/src/domain/task-system/services/Cargo.toml index 862bc1b641..42fdffd10a 100644 --- a/src/domain/task-system/services/Cargo.toml +++ b/src/domain/task-system/services/Cargo.toml @@ -47,6 +47,7 @@ kamu-accounts = { workspace = true } kamu-datasets-services = { workspace = true } kamu-datasets-inmem = { workspace = true } kamu-task-system-inmem = { workspace = true } +odf-dataset-impl = { workspace = true } chrono = { version = "0.4", default-features = false } mockall = "0.13" diff --git a/src/domain/task-system/services/tests/tests/test_task_agent_impl.rs b/src/domain/task-system/services/tests/tests/test_task_agent_impl.rs index 63b1d52a9a..c035c64fb5 100644 --- a/src/domain/task-system/services/tests/tests/test_task_agent_impl.rs +++ b/src/domain/task-system/services/tests/tests/test_task_agent_impl.rs @@ -15,7 +15,6 @@ use dill::{Catalog, CatalogBuilder, Component}; use kamu::utils::ipfs_wrapper::IpfsClient; use kamu::*; use kamu_accounts::CurrentAccountSubject; -use kamu_core::auth::DummyOdfServerAccessTokenResolver; use kamu_core::{DatasetRepository, TenancyConfig}; use kamu_datasets::DatasetEnvVarsConfig; use kamu_datasets_inmem::InMemoryDatasetEnvVarRepository; @@ -25,6 +24,8 @@ use kamu_task_system_inmem::InMemoryTaskEventStore; use kamu_task_system_services::*; use messaging_outbox::{MockOutbox, Outbox}; use mockall::predicate::{eq, function}; +use odf_dataset::DummyOdfServerAccessTokenResolver; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use tempfile::TempDir; use time_source::SystemTimeSourceDefault; diff --git a/src/infra/core/src/lib.rs b/src/infra/core/src/lib.rs index 0bd4173e17..f4c71cacb7 100644 --- a/src/infra/core/src/lib.rs +++ b/src/infra/core/src/lib.rs @@ -33,7 +33,5 @@ pub use services::*; pub use use_cases::*; mod dataset_config; -mod dataset_layout; pub use dataset_config::*; -pub use dataset_layout::*; diff --git a/src/infra/core/src/repos/dataset_repository_local_fs.rs b/src/infra/core/src/repos/dataset_repository_local_fs.rs index 74676b197f..44eea67cf6 100644 --- a/src/infra/core/src/repos/dataset_repository_local_fs.rs +++ b/src/infra/core/src/repos/dataset_repository_local_fs.rs @@ -24,7 +24,7 @@ use odf_dataset::{ GetSummaryError, GetSummaryOpts, }; -use odf_dataset_impl::{DatasetImpl, MetadataChainImpl}; +use odf_dataset_impl::{DatasetImpl, DatasetLayout, MetadataChainImpl}; use odf_storage::{GetNamedError, GetRefError}; use odf_storage_impl::{ MetadataBlockRepositoryCachingInMem, diff --git a/src/infra/core/src/repos/mod.rs b/src/infra/core/src/repos/mod.rs index d31c96608e..d7f87a27a3 100644 --- a/src/infra/core/src/repos/mod.rs +++ b/src/infra/core/src/repos/mod.rs @@ -7,13 +7,11 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod dataset_factory_impl; mod dataset_repository_helpers; mod dataset_repository_local_fs; mod dataset_repository_s3; mod dataset_repository_writer; -pub use dataset_factory_impl::*; pub use dataset_repository_helpers::*; pub use dataset_repository_local_fs::*; pub use dataset_repository_s3::*; diff --git a/src/infra/core/src/services/remote/remote_alias_resolver_impl.rs b/src/infra/core/src/services/remote/remote_alias_resolver_impl.rs index 9a8edb2c3d..80867e863b 100644 --- a/src/infra/core/src/services/remote/remote_alias_resolver_impl.rs +++ b/src/infra/core/src/services/remote/remote_alias_resolver_impl.rs @@ -10,10 +10,10 @@ use std::str::FromStr; use std::sync::Arc; -use auth::OdfServerAccessTokenResolver; use dill::*; use internal_error::{InternalError, ResultIntoInternal}; use kamu_core::*; +use odf_dataset::OdfServerAccessTokenResolver; use opendatafabric::{self as odf, DatasetHandle}; use url::Url; diff --git a/src/infra/core/src/services/remote/remote_status_service_impl.rs b/src/infra/core/src/services/remote/remote_status_service_impl.rs index 4e39a394f2..895ef41314 100644 --- a/src/infra/core/src/services/remote/remote_status_service_impl.rs +++ b/src/infra/core/src/services/remote/remote_status_service_impl.rs @@ -18,7 +18,6 @@ use kamu_core::utils::metadata_chain_comparator::{ NullCompareChainsListener, }; use kamu_core::{ - DatasetFactory, DatasetPushStatuses, DatasetRegistry, PushStatus, @@ -27,7 +26,7 @@ use kamu_core::{ RemoteStatusService, StatusCheckError, }; -use odf_dataset::{BlockRef, MetadataChain}; +use odf_dataset::{BlockRef, DatasetFactory, MetadataChain}; use odf_storage::GetRefError; use opendatafabric as odf; diff --git a/src/infra/core/src/services/remote/search_service_impl.rs b/src/infra/core/src/services/remote/search_service_impl.rs index 27aec8c46a..cdf972ca6f 100644 --- a/src/infra/core/src/services/remote/search_service_impl.rs +++ b/src/infra/core/src/services/remote/search_service_impl.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use dill::*; use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; +use odf_dataset::UnsupportedProtocolError; use opendatafabric::*; use s3_utils::S3Context; use serde_json::json; diff --git a/src/infra/core/src/services/sync/sync_request_builder.rs b/src/infra/core/src/services/sync/sync_request_builder.rs index 2cf593eb6e..7c5b7d24f0 100644 --- a/src/infra/core/src/services/sync/sync_request_builder.rs +++ b/src/infra/core/src/services/sync/sync_request_builder.rs @@ -13,7 +13,6 @@ use dill::component; use internal_error::ErrorIntoInternal; use kamu_core::services::DatasetNotFoundError; use kamu_core::{ - DatasetFactory, DatasetRegistry, DatasetRegistryExt, GetDatasetError, @@ -23,7 +22,7 @@ use kamu_core::{ SyncRefRemote, SyncRequest, }; -use odf_dataset::{BlockRef, Dataset}; +use odf_dataset::{BlockRef, Dataset, DatasetFactory}; use odf_storage::GetRefError; use opendatafabric as odf; use url::Url; diff --git a/src/infra/core/src/services/sync/sync_service_impl.rs b/src/infra/core/src/services/sync/sync_service_impl.rs index bc1201d0be..19606c7e66 100644 --- a/src/infra/core/src/services/sync/sync_service_impl.rs +++ b/src/infra/core/src/services/sync/sync_service_impl.rs @@ -15,7 +15,14 @@ use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_core::services::sync_service::DatasetNotFoundError; use kamu_core::utils::metadata_chain_comparator::*; use kamu_core::*; -use odf_dataset::{AppendValidation, BlockRef, Dataset, IterBlocksError}; +use odf_dataset::{ + AppendValidation, + BlockRef, + Dataset, + DatasetFactory, + IterBlocksError, + UnsupportedProtocolError, +}; use odf_storage::GetRefError; use opendatafabric as odf; use url::Url; diff --git a/src/infra/core/src/testing/dataset_test_helper.rs b/src/infra/core/src/testing/dataset_test_helper.rs index a4a000c5d7..1afb4e0156 100644 --- a/src/infra/core/src/testing/dataset_test_helper.rs +++ b/src/infra/core/src/testing/dataset_test_helper.rs @@ -12,10 +12,10 @@ use std::path::{Path, PathBuf}; use file_utils::OwnedFile; use kamu_core::*; use odf_dataset::{AddDataParams, CheckpointRef, CommitOpts, MetadataChainExt, TryStreamExtExt}; +use odf_dataset_impl::DatasetLayout; use opendatafabric::*; use super::ParquetWriterHelper; -use crate::DatasetLayout; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/testing/mod.rs b/src/infra/core/src/testing/mod.rs index 84af0a3fdd..c454127811 100644 --- a/src/infra/core/src/testing/mod.rs +++ b/src/infra/core/src/testing/mod.rs @@ -13,7 +13,6 @@ mod dataset_test_helper; mod dummy_smart_transfer_protocol_client; mod mock_dataset_action_authorizer; mod mock_dataset_changes_service; -mod mock_odf_server_access_token_resolver; mod mock_polling_source_service; mod mock_sync_service; mod mock_transform_elaboration_service; @@ -28,7 +27,6 @@ pub use dataset_test_helper::*; pub use dummy_smart_transfer_protocol_client::*; pub use mock_dataset_action_authorizer::*; pub use mock_dataset_changes_service::*; -pub use mock_odf_server_access_token_resolver::*; pub use mock_polling_source_service::*; pub use mock_sync_service::*; pub use mock_transform_elaboration_service::*; diff --git a/src/infra/core/tests/benches/parallel_simple_transfer_protocol.rs b/src/infra/core/tests/benches/parallel_simple_transfer_protocol.rs index 0899565cbf..85be4236d2 100644 --- a/src/infra/core/tests/benches/parallel_simple_transfer_protocol.rs +++ b/src/infra/core/tests/benches/parallel_simple_transfer_protocol.rs @@ -18,11 +18,9 @@ use kamu::testing::{DatasetTestHelper, DummySmartTransferProtocolClient}; use kamu::utils::ipfs_wrapper::IpfsClient; use kamu::utils::simple_transfer_protocol::ENV_VAR_SIMPLE_PROTOCOL_MAX_PARALLEL_TRANSFERS; use kamu::{ - DatasetFactoryImpl, DatasetRegistryRepoBridge, DatasetRepositoryLocalFs, DatasetRepositoryWriter, - IpfsGateway, RemoteReposDir, RemoteRepositoryRegistryImpl, SyncRequestBuilder, @@ -31,6 +29,8 @@ use kamu::{ use kamu_accounts::CurrentAccountSubject; use kamu_datasets_inmem::InMemoryDatasetDependencyRepository; use kamu_datasets_services::DependencyGraphServiceImpl; +use odf_dataset::DummyOdfServerAccessTokenResolver; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; use test_utils::HttpFileServer; @@ -74,7 +74,7 @@ async fn setup_dataset( .add::() .add_value(RemoteReposDir::new(repos_dir)) .add::() - .add::() + .add::() .add::() .add::() .add::() diff --git a/src/infra/core/tests/tests/repos/mod.rs b/src/infra/core/tests/tests/repos/mod.rs index 89d0575290..f71372d5e0 100644 --- a/src/infra/core/tests/tests/repos/mod.rs +++ b/src/infra/core/tests/tests/repos/mod.rs @@ -7,7 +7,6 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_dataset_impl; mod test_dataset_repository_local_fs; mod test_dataset_repository_s3; mod test_dataset_repository_shared; diff --git a/src/infra/core/tests/tests/test_pull_request_planner_impl.rs b/src/infra/core/tests/tests/test_pull_request_planner_impl.rs index 8ca98f1d35..1a2218a1be 100644 --- a/src/infra/core/tests/tests/test_pull_request_planner_impl.rs +++ b/src/infra/core/tests/tests/test_pull_request_planner_impl.rs @@ -19,6 +19,8 @@ use kamu::utils::simple_transfer_protocol::SimpleTransferProtocol; use kamu::*; use kamu_accounts::CurrentAccountSubject; use messaging_outbox::DummyOutboxImpl; +use odf_dataset::DummyOdfServerAccessTokenResolver; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; use time_source::SystemTimeSourceDefault; @@ -678,7 +680,7 @@ impl PullTestHarness { .add::() .add::() .add::() - .add::() + .add::() .add::() .add::() .add::() diff --git a/src/infra/core/tests/tests/test_push_request_planner_impl.rs b/src/infra/core/tests/tests/test_push_request_planner_impl.rs index af5b044f4a..4d8feaaf74 100644 --- a/src/infra/core/tests/tests/test_push_request_planner_impl.rs +++ b/src/infra/core/tests/tests/test_push_request_planner_impl.rs @@ -10,10 +10,10 @@ use std::assert_matches::assert_matches; use std::sync::Arc; -use auth::DummyOdfServerAccessTokenResolver; use kamu::testing::BaseRepoHarness; use kamu::*; use kamu_core::*; +use odf_dataset::DummyOdfServerAccessTokenResolver; use opendatafabric::{ DatasetAlias, DatasetAliasRemote, diff --git a/src/infra/core/tests/tests/test_remote_status_service.rs b/src/infra/core/tests/tests/test_remote_status_service.rs index 4cdb33e740..f7a0ad4bca 100644 --- a/src/infra/core/tests/tests/test_remote_status_service.rs +++ b/src/infra/core/tests/tests/test_remote_status_service.rs @@ -18,8 +18,6 @@ use kamu::testing::{BaseRepoHarness, DummySmartTransferProtocolClient}; use kamu::utils::ipfs_wrapper::IpfsClient; use kamu::utils::simple_transfer_protocol::SimpleTransferProtocol; use kamu::{ - DatasetFactoryImpl, - IpfsGateway, RemoteAliasesRegistryImpl, RemoteReposDir, RemoteRepositoryRegistryImpl, @@ -27,10 +25,16 @@ use kamu::{ SyncRequestBuilder, SyncServiceImpl, }; -use kamu_core::auth::DummyOdfServerAccessTokenResolver; use kamu_core::utils::metadata_chain_comparator::CompareChainsResult; use kamu_core::*; -use odf_dataset::{AppendOpts, BlockRef, DatasetVisibility, SetRefOpts}; +use odf_dataset::{ + AppendOpts, + BlockRef, + DatasetVisibility, + DummyOdfServerAccessTokenResolver, + SetRefOpts, +}; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; use url::Url; diff --git a/src/infra/core/tests/tests/test_search_service_impl.rs b/src/infra/core/tests/tests/test_search_service_impl.rs index 695912e6c4..a997e0ee61 100644 --- a/src/infra/core/tests/tests/test_search_service_impl.rs +++ b/src/infra/core/tests/tests/test_search_service_impl.rs @@ -16,6 +16,8 @@ use kamu::utils::simple_transfer_protocol::SimpleTransferProtocol; use kamu::*; use kamu_accounts::CurrentAccountSubject; use messaging_outbox::DummyOutboxImpl; +use odf_dataset::DummyOdfServerAccessTokenResolver; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; use test_utils::LocalS3Server; @@ -45,7 +47,7 @@ async fn do_test_search(tmp_workspace_dir: &Path, repo_url: Url) { .bind::() .add_value(IpfsGateway::default()) .add_value(kamu::utils::ipfs_wrapper::IpfsClient::default()) - .add::() + .add::() .add::() .add::() .add::() diff --git a/src/infra/core/tests/tests/test_sync_service_impl.rs b/src/infra/core/tests/tests/test_sync_service_impl.rs index 8622fb53de..8355488397 100644 --- a/src/infra/core/tests/tests/test_sync_service_impl.rs +++ b/src/infra/core/tests/tests/test_sync_service_impl.rs @@ -19,6 +19,8 @@ use kamu::utils::simple_transfer_protocol::SimpleTransferProtocol; use kamu::*; use kamu_accounts::CurrentAccountSubject; use messaging_outbox::DummyOutboxImpl; +use odf_dataset::DummyOdfServerAccessTokenResolver; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*; use test_utils::{HttpFileServer, LocalS3Server}; @@ -83,7 +85,7 @@ async fn do_test_sync( .add::() .add_value(RemoteReposDir::new(tmp_workspace_dir_foo.join("repos"))) .add::() - .add::() + .add::() .add::() .add::() .add::() @@ -105,7 +107,7 @@ async fn do_test_sync( .add::() .add_value(RemoteReposDir::new(tmp_workspace_dir_bar.join("repos"))) .add::() - .add::() + .add::() .add::() .add::() .add::() diff --git a/src/infra/core/tests/tests/use_cases/test_pull_dataset_use_case.rs b/src/infra/core/tests/tests/use_cases/test_pull_dataset_use_case.rs index 8af02e3d05..141f917c92 100644 --- a/src/infra/core/tests/tests/use_cases/test_pull_dataset_use_case.rs +++ b/src/infra/core/tests/tests/use_cases/test_pull_dataset_use_case.rs @@ -13,8 +13,10 @@ use std::sync::Arc; use kamu::testing::*; use kamu::*; -use kamu_core::auth::{DatasetAction, DummyOdfServerAccessTokenResolver}; +use kamu_core::auth::DatasetAction; use kamu_core::*; +use odf_dataset::DummyOdfServerAccessTokenResolver; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use opendatafabric::*; use tempfile::TempDir; use url::Url; diff --git a/src/infra/core/tests/tests/use_cases/test_push_dataset_use_case.rs b/src/infra/core/tests/tests/use_cases/test_push_dataset_use_case.rs index 987fe37f73..90b521357d 100644 --- a/src/infra/core/tests/tests/use_cases/test_push_dataset_use_case.rs +++ b/src/infra/core/tests/tests/use_cases/test_push_dataset_use_case.rs @@ -15,8 +15,10 @@ use kamu::testing::{DummySmartTransferProtocolClient, MockDatasetActionAuthorize use kamu::utils::ipfs_wrapper::IpfsClient; use kamu::utils::simple_transfer_protocol::SimpleTransferProtocol; use kamu::*; -use kamu_core::auth::{DatasetAction, DummyOdfServerAccessTokenResolver}; +use kamu_core::auth::DatasetAction; use kamu_core::*; +use odf_dataset::DummyOdfServerAccessTokenResolver; +use odf_dataset_impl::{DatasetFactoryImpl, IpfsGateway}; use opendatafabric::*; use tempfile::TempDir; use url::Url; diff --git a/src/infra/odf/dataset/Cargo.toml b/src/infra/odf/dataset/Cargo.toml index cb9e75c53b..ca3484715c 100644 --- a/src/infra/odf/dataset/Cargo.toml +++ b/src/infra/odf/dataset/Cargo.toml @@ -21,17 +21,30 @@ workspace = true doctest = false +[features] +testing = ["dep:mockall"] + + [dependencies] internal-error = { workspace = true } file-utils = { workspace = true } kamu-data-utils = { workspace = true } odf-dataset = { workspace = true } odf-storage = { workspace = true } +odf-storage-impl = { workspace = true } opendatafabric = { workspace = true } +s3-utils = { workspace = true } async-stream = "0.3" async-trait = "0.1" chrono = { version = "0.4", default-features = false} +dill = { version = "0.10", default-features = false } +http = { version = "1", default-features = false } +hickory-resolver = "0.24" +mockall = { optional = true, version = "0.13", default-features = false } +regex = { version = "1", default-features = false } +reqwest = { version = "0.12", default-features = false } +thiserror = { version = "2", default-features = false } tokio = { version = "1", default-features = false } tokio-stream = { version = "0.1", default-features = false } tracing = "0.1" diff --git a/src/infra/core/src/dataset_layout.rs b/src/infra/odf/dataset/src/entities/dataset_layout.rs similarity index 100% rename from src/infra/core/src/dataset_layout.rs rename to src/infra/odf/dataset/src/entities/dataset_layout.rs diff --git a/src/infra/odf/dataset/src/entities/mod.rs b/src/infra/odf/dataset/src/entities/mod.rs index 60e982c20b..2818f5b08a 100644 --- a/src/infra/odf/dataset/src/entities/mod.rs +++ b/src/infra/odf/dataset/src/entities/mod.rs @@ -8,9 +8,11 @@ // by the Apache License, Version 2.0. mod dataset_impl; +mod dataset_layout; mod metadata_chain_impl; mod metadata_chain_validators; pub use dataset_impl::*; +pub use dataset_layout::*; pub use metadata_chain_impl::*; pub use metadata_chain_validators::*; diff --git a/src/infra/odf/dataset/src/lib.rs b/src/infra/odf/dataset/src/lib.rs index 11e91f7b4f..b95a6a70f8 100644 --- a/src/infra/odf/dataset/src/lib.rs +++ b/src/infra/odf/dataset/src/lib.rs @@ -10,5 +10,10 @@ #![feature(let_chains)] mod entities; +mod services; pub use entities::*; +pub use services::*; + +#[cfg(any(feature = "testing", test))] +pub mod testing; diff --git a/src/infra/core/src/repos/dataset_factory_impl.rs b/src/infra/odf/dataset/src/services/dataset_factory_impl.rs similarity index 96% rename from src/infra/core/src/repos/dataset_factory_impl.rs rename to src/infra/odf/dataset/src/services/dataset_factory_impl.rs index 266fa9e6de..4ad4607f78 100644 --- a/src/infra/core/src/repos/dataset_factory_impl.rs +++ b/src/infra/odf/dataset/src/services/dataset_factory_impl.rs @@ -11,20 +11,24 @@ use std::sync::Arc; use dill::*; use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; -use kamu_core::*; -use odf_dataset::Dataset; -use odf_dataset_impl::{DatasetImpl, MetadataChainImpl}; +use odf_dataset::{ + BuildDatasetError, + Dataset, + DatasetFactory, + OdfServerAccessTokenResolver, + UnsupportedProtocolError, +}; use odf_storage_impl::*; use s3_utils::S3Context; use url::Url; -use crate::*; +use crate::{DatasetImpl, DatasetLayout, MetadataChainImpl}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct DatasetFactoryImpl { ipfs_gateway: IpfsGateway, - access_token_resolver: Arc, + access_token_resolver: Arc, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -49,7 +53,7 @@ type DatasetImplLocalFS = DatasetImpl< impl DatasetFactoryImpl { pub fn new( ipfs_gateway: IpfsGateway, - access_token_resolver: Arc, + access_token_resolver: Arc, ) -> Self { Self { ipfs_gateway, @@ -75,7 +79,9 @@ impl DatasetFactoryImpl { fn get_http(base_url: &Url, header_map: http::HeaderMap) -> impl Dataset { // When joining url without trailing '/', last path part is being dropped let mut base_url = base_url.clone(); - base_url.ensure_trailing_slash(); + if !base_url.path().ends_with('/') { + base_url.set_path(&format!("{}/", base_url.path())); + } let client = reqwest::Client::new(); diff --git a/src/infra/odf/dataset/src/services/mod.rs b/src/infra/odf/dataset/src/services/mod.rs new file mode 100644 index 0000000000..70fa1bad38 --- /dev/null +++ b/src/infra/odf/dataset/src/services/mod.rs @@ -0,0 +1,12 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod dataset_factory_impl; + +pub use dataset_factory_impl::*; diff --git a/src/infra/core/src/testing/mock_odf_server_access_token_resolver.rs b/src/infra/odf/dataset/src/testing/mock_odf_server_access_token_resolver.rs similarity index 96% rename from src/infra/core/src/testing/mock_odf_server_access_token_resolver.rs rename to src/infra/odf/dataset/src/testing/mock_odf_server_access_token_resolver.rs index 28458af742..99d764b358 100644 --- a/src/infra/core/src/testing/mock_odf_server_access_token_resolver.rs +++ b/src/infra/odf/dataset/src/testing/mock_odf_server_access_token_resolver.rs @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu_core::auth::OdfServerAccessTokenResolver; +use odf_dataset::OdfServerAccessTokenResolver; use url::Url; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/odf/dataset/src/testing/mod.rs b/src/infra/odf/dataset/src/testing/mod.rs new file mode 100644 index 0000000000..f309862ab2 --- /dev/null +++ b/src/infra/odf/dataset/src/testing/mod.rs @@ -0,0 +1,12 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod mock_odf_server_access_token_resolver; + +pub use mock_odf_server_access_token_resolver::*; diff --git a/src/infra/odf/dataset/tests/entities/mod.rs b/src/infra/odf/dataset/tests/entities/mod.rs index c54657c636..4f71d0ef6e 100644 --- a/src/infra/odf/dataset/tests/entities/mod.rs +++ b/src/infra/odf/dataset/tests/entities/mod.rs @@ -7,4 +7,5 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +mod test_dataset_impl; mod test_metadata_chain_impl; diff --git a/src/infra/core/tests/tests/repos/test_dataset_impl.rs b/src/infra/odf/dataset/tests/entities/test_dataset_impl.rs similarity index 98% rename from src/infra/core/tests/tests/repos/test_dataset_impl.rs rename to src/infra/odf/dataset/tests/entities/test_dataset_impl.rs index 0d3bd950f7..3a4f5fc88a 100644 --- a/src/infra/core/tests/tests/repos/test_dataset_impl.rs +++ b/src/infra/odf/dataset/tests/entities/test_dataset_impl.rs @@ -9,8 +9,8 @@ use std::assert_matches::assert_matches; -use kamu::*; use odf_dataset::{AppendOpts, Dataset, DatasetSummary, GetSummaryError, GetSummaryOpts}; +use odf_dataset_impl::{DatasetFactoryImpl, DatasetLayout}; use odf_storage_impl::testing::MetadataFactory; use opendatafabric::*;