diff --git a/.github/workflows/release-images.yaml b/.github/workflows/release-images.yaml index 8b13cb6b..78564762 100644 --- a/.github/workflows/release-images.yaml +++ b/.github/workflows/release-images.yaml @@ -35,9 +35,11 @@ jobs: run: | cd images/ make api-server-with-data + make api-server-with-data-mt - name: Publish images run: | cd images make api-server-push make api-server-with-data-push + make api-server-with-data-mt-push diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..8e1bf7a3 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,8 @@ +# This avoids cross picking up custom linker from ~/.cargo/config.toml +# See: https://github.com/cross-rs/cross/issues/621 + +[build.env] +passthrough = [ + "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc", + "RUSTFLAGS", +] diff --git a/images/Makefile b/images/Makefile index 51867cf5..18e50a34 100644 --- a/images/Makefile +++ b/images/Makefile @@ -3,6 +3,8 @@ KAMU_VERSION = $(shell cargo metadata --format-version 1 | jq -r '.packages[] | API_SERVER_VERSION = $(shell cargo metadata --format-version 1 | jq -r '.packages[] | select( .name == "kamu-api-server") | .version') +######################################################################################### + .PHONY: api-server api-server: docker build \ @@ -19,6 +21,8 @@ api-server-push: docker push $(IMAGE_REPO)/kamu-api-server:latest +######################################################################################### + .PHONY: api-server-with-data api-server-with-data: docker build \ @@ -31,3 +35,19 @@ api-server-with-data: .PHONY: api-server-with-data-push api-server-with-data-push: docker push $(IMAGE_REPO)/kamu-api-server:latest-with-data + + +######################################################################################### + +.PHONY: api-server-with-data-mt +api-server-with-data-mt: + docker build \ + --build-arg KAMU_VERSION=latest-with-data-mt \ + --build-arg API_SERVER_VERSION=$(API_SERVER_VERSION) \ + -t $(IMAGE_REPO)/kamu-api-server:latest-with-data-mt \ + api-server-with-data-mt/ + + +.PHONY: api-server-with-data-mt-push +api-server-with-data-mt-push: + docker push $(IMAGE_REPO)/kamu-api-server:latest-with-data-mt diff --git a/images/api-server-with-data-mt/Dockerfile b/images/api-server-with-data-mt/Dockerfile new file mode 100644 index 00000000..a32fb742 --- /dev/null +++ b/images/api-server-with-data-mt/Dockerfile @@ -0,0 +1,19 @@ +ARG KAMU_VERSION +FROM ghcr.io/kamu-data/kamu-base:${KAMU_VERSION} +ARG API_SERVER_VERSION + +WORKDIR /opt/kamu + +RUN wget -q https://github.com/kamu-data/kamu-platform/releases/download/v${API_SERVER_VERSION}/kamu-api-server-x86_64-unknown-linux-gnu.tar.gz && \ + tar -xf kamu-api-server-x86_64-unknown-linux-gnu.tar.gz && \ + chmod +x kamu-api-server-x86_64-unknown-linux-gnu/kamu-api-server && \ + mv kamu-api-server-x86_64-unknown-linux-gnu/kamu-api-server /opt/kamu/ && \ + rm -rf kamu-api-server-x86_64-unknown-linux-gnu* + +COPY config.yaml /opt/kamu/config.yaml + +ENTRYPOINT ["/usr/bin/tini", "--"] + +CMD [ "/opt/kamu/kamu-api-server", "--config=/opt/kamu/config.yaml", "--repo-url=file:///opt/kamu/workspace/.kamu/datasets", "--multi-tenant", "run", "--address=0.0.0.0", "--http-port=8080" ] + +EXPOSE 8080/tcp diff --git a/images/api-server-with-data-mt/config.yaml b/images/api-server-with-data-mt/config.yaml new file mode 100644 index 00000000..08479c8f --- /dev/null +++ b/images/api-server-with-data-mt/config.yaml @@ -0,0 +1,81 @@ +auth: + providers: + # Dummy accounts for owners of example datasets + - kind: dummy + accounts: + - accountId: kamu + accountName: kamu + accountType: User + avatarUrl: https://avatars.githubusercontent.com/u/50896974?s=200&v=4 + displayName: kamu + - accountId: sh101-bowen + accountName: sh101-bowen + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png + displayName: sh101-bowen + - accountId: sh102-gambier + accountName: sh102-gambier + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png + displayName: sh102-gambier + - accountId: sh103-howe + accountName: sh103-howe + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png + displayName: sh103-howe + - accountId: sh104-granville + accountName: sh104-granville + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png + displayName: sh104-granville + - accountId: sh105-keats + accountName: sh105-keats + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png + displayName: sh105-keats + - accountId: sh106-seymour + accountName: sh106-seymour + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png + displayName: sh106-seymour + - accountId: mim.dk + accountName: mim.dk + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png + displayName: mim.dk + - accountId: mmo.gov.uk + accountName: mmo.gov.uk + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png + displayName: mmo.gov.uk + - accountId: moa.gov.nl + accountName: moa.gov.nl + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png + displayName: moa.gov.nl + - accountId: ofb.gouv.fr + accountName: ofb.gouv.fr + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png + displayName: ofb.gouv.fr + - accountId: protectedseas.net + accountName: protectedseas.net + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/3530/3530558.png + displayName: protectedseas.net + - accountId: acme.fishing.co + accountName: acme.fishing.co + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/1090/1090630.png + displayName: ACME Fishing Company + - accountId: globalfishingwatch.org + accountName: globalfishingwatch.org + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/512/744/744480.png + displayName: globalfishingwatch.org + - accountId: bmis-bycatch.org + accountName: bmis-bycatch.org + accountType: User + avatarUrl: https://cdn-icons-png.flaticon.com/128/10197/10197245.png + displayName: Bycatch Management Information System + - kind: github diff --git a/images/api-server/Dockerfile b/images/api-server/Dockerfile index 8eefff80..c65c8e5d 100644 --- a/images/api-server/Dockerfile +++ b/images/api-server/Dockerfile @@ -1,11 +1,9 @@ ARG KAMU_VERSION FROM ghcr.io/kamu-data/kamu-base:${KAMU_VERSION} - ARG API_SERVER_VERSION WORKDIR /opt/kamu -# API Server RUN wget -q https://github.com/kamu-data/kamu-platform/releases/download/v${API_SERVER_VERSION}/kamu-api-server-x86_64-unknown-linux-gnu.tar.gz && \ tar -xf kamu-api-server-x86_64-unknown-linux-gnu.tar.gz && \ chmod +x kamu-api-server-x86_64-unknown-linux-gnu/kamu-api-server && \ @@ -15,7 +13,6 @@ RUN wget -q https://github.com/kamu-data/kamu-platform/releases/download/v${API_ ENTRYPOINT ["/usr/bin/tini", "--"] -# Local repository: -CMD ["/opt/kamu/kamu-api-server --local-repo /opt/kamu/workspace run --address 0.0.0.0 --http-port 80"] +CMD ["/opt/kamu/kamu-api-server", "--repo-url=file:///opt/kamu/workspace/.kamu/datasets", "run", "--address=0.0.0.0", "--http-port=8080"] -EXPOSE 80/tcp +EXPOSE 8080/tcp diff --git a/src/app/api-server/src/app.rs b/src/app/api-server/src/app.rs index b877fac3..3aa967b7 100644 --- a/src/app/api-server/src/app.rs +++ b/src/app/api-server/src/app.rs @@ -46,6 +46,8 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> { Url::from_directory_path(workspace_dir.join("datasets")).unwrap() }; + let multi_tenant = matches.get_flag("multi-tenant") || repo_url.scheme() != "file"; + let local_dir = tempfile::tempdir().unwrap(); info!( @@ -53,6 +55,7 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> { args = ?std::env::args().collect::>(), repo_url = %repo_url, local_dir = %local_dir.path().display(), + config = ?config, "Initializing {}", BINARY_NAME ); @@ -63,10 +66,11 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> { true, ), &repo_url, + multi_tenant, ) .await; - let catalog = init_dependencies(config, &repo_url, local_dir.path()) + let catalog = init_dependencies(config, &repo_url, multi_tenant, local_dir.path()) .await .add_value(dependencies_graph_repository) .bind::() @@ -104,7 +108,7 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> { address, sub.get_one("http-port").map(|p| *p), catalog.clone(), - should_use_multi_tenancy(&repo_url), + multi_tenant, ); let flightsql_server = crate::flightsql_server::FlightSqlServer::new( @@ -210,9 +214,11 @@ pub fn load_config(path: Option<&PathBuf>) -> Result kamu::DependencyGraphRepositoryInMemory { // Construct a special catalog just to create 1 object, but with a repository // bound to API server command line user. It also should be authorized to access @@ -220,29 +226,7 @@ pub async fn prepare_dependencies_graph_repository( let mut b = CatalogBuilder::new(); - match repo_url.scheme() { - "file" => { - let datasets_dir = repo_url.to_file_path().unwrap(); - - b.add_builder( - kamu::DatasetRepositoryLocalFs::builder() - .with_root(datasets_dir.clone()) - .with_multi_tenant(false), - ); - b.bind::(); - } - "s3" | "s3+http" | "s3+https" => { - let s3_context = kamu::utils::s3_context::S3Context::from_url(&repo_url).await; - - b.add_builder( - kamu::DatasetRepositoryS3::builder() - .with_s3_context(s3_context.clone()) - .with_multi_tenant(true), - ) - .bind::(); - } - _ => panic!("Unsupported repository scheme: {}", repo_url.scheme()), - } + configure_repository(&mut b, repo_url, multi_tenant).await; let special_catlaog = b .add::() @@ -260,6 +244,7 @@ pub async fn prepare_dependencies_graph_repository( pub async fn init_dependencies( config: ApiServerConfig, repo_url: &Url, + multi_tenant: bool, local_dir: &Path, ) -> CatalogBuilder { let mut b = dill::CatalogBuilder::new(); @@ -348,6 +333,34 @@ pub async fn init_dependencies( b.add::(); b.add::(); + configure_repository(&mut b, repo_url, multi_tenant).await; + + if !multi_tenant { + b.add_value(DummyAuthProvider::new_with_default_account()); + b.bind::(); + } else { + // Default to GitHub auth + if config.auth.providers.is_empty() { + b.add::(); + } + + for provider in config.auth.providers { + match provider { + AuthProviderConfig::Github(_) => { + b.add::(); + } + AuthProviderConfig::Dummy(prov) => { + b.add_value(DummyAuthProvider::new(prov.accounts)); + b.bind::(); + } + } + } + } + + b +} + +async fn configure_repository(b: &mut CatalogBuilder, repo_url: &Url, multi_tenant: bool) { match repo_url.scheme() { "file" => { let datasets_dir = repo_url.to_file_path().unwrap(); @@ -355,13 +368,11 @@ pub async fn init_dependencies( b.add_builder( kamu::DatasetRepositoryLocalFs::builder() .with_root(datasets_dir.clone()) - .with_multi_tenant(false), + .with_multi_tenant(multi_tenant), ); b.bind::(); b.add::(); - b.add_value(DummyAuthProvider::new_with_default_account()); - b.bind::(); } "s3" | "s3+http" | "s3+https" => { let s3_context = kamu::utils::s3_context::S3Context::from_url(&repo_url).await; @@ -376,38 +387,9 @@ pub async fn init_dependencies( let allow_http = repo_url.scheme() == "s3+http"; b.add_value(kamu::ObjectStoreBuilderS3::new(s3_context, allow_http)) .bind::(); - - // Default to GitHub auth - if config.auth.providers.is_empty() { - b.add::(); - } - - for provider in config.auth.providers { - match provider { - AuthProviderConfig::Github(_) => { - b.add::(); - } - AuthProviderConfig::Dummy(prov) => { - b.add_value(DummyAuthProvider::new(prov.accounts)); - b.bind::(); - } - } - } } _ => panic!("Unsupported repository scheme: {}", repo_url.scheme()), } - - b -} - -///////////////////////////////////////////////////////////////////////////////////////// - -fn should_use_multi_tenancy(repo_url: &Url) -> bool { - match repo_url.scheme() { - "file" => false, - "s3" | "s3+http" | "s3+https" => true, - _ => panic!("Unsupported repository scheme: {}", repo_url.scheme()), - } } ///////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/app/api-server/src/cli_parser.rs b/src/app/api-server/src/cli_parser.rs index 9051da6d..da4a5389 100644 --- a/src/app/api-server/src/cli_parser.rs +++ b/src/app/api-server/src/cli_parser.rs @@ -33,6 +33,12 @@ pub fn cli() -> Command { .long("repo-url") .value_parser(value_parse_repo_url) .help("URL of the remote dataset repository"), + // TODO: This is temporary and will be removed soon + // See: https://github.com/kamu-data/kamu-cli/issues/342 + Arg::new("multi-tenant") + .long("multi-tenant") + .action(ArgAction::SetTrue) + .help("Indicates that target repo is multi-tenant (for file:// only)"), ]) .subcommands([ Command::new("run").about("Run the server").args([ diff --git a/src/app/api-server/tests/tests/test_di_graph.rs b/src/app/api-server/tests/tests/test_di_graph.rs index 8b8d4728..4fdbdf3d 100644 --- a/src/app/api-server/tests/tests/test_di_graph.rs +++ b/src/app/api-server/tests/tests/test_di_graph.rs @@ -15,6 +15,7 @@ async fn test_di_graph_validates_local() { let mut catalog_builder = kamu_api_server::init_dependencies( kamu_api_server::config::ApiServerConfig::default(), &url::Url::from_directory_path(tempdir.path()).unwrap(), + false, tempdir.path(), ) .await; @@ -61,6 +62,7 @@ async fn test_di_graph_validates_remote() { let mut catalog_builder = kamu_api_server::init_dependencies( kamu_api_server::config::ApiServerConfig::default(), &repo_url, + true, tmp_repo_dir.path(), ) .await;