diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e4d85be3..f384ef1fb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ Recommendation: for ease of reading, use the following order: - Fixed --> +## [0.206.3] - 2024-10-28 +### Fixed +- Improved telemetry for dataset entry indexing process +- Corrected recent migration related to outbox consumptions of old dataset events + ## [0.206.2] - 2024-10-26 ### Changed - GraphQL: Removed deprecated `JSON_LD` in favor of `ND_JSON` in `DataBatchFormat` diff --git a/Cargo.lock b/Cargo.lock index 74cc341f99..ba92ec5aa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1306,7 +1306,7 @@ dependencies = [ [[package]] name = "async-utils" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", ] @@ -2455,7 +2455,7 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "container-runtime" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "cfg-if", @@ -2870,7 +2870,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "database-common" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "aws-config", @@ -2894,7 +2894,7 @@ dependencies = [ [[package]] name = "database-common-macros" -version = "0.206.2" +version = "0.206.3" dependencies = [ "quote", "syn 2.0.79", @@ -3738,7 +3738,7 @@ dependencies = [ [[package]] name = "enum-variants" -version = "0.206.2" +version = "0.206.3" [[package]] name = "env_filter" @@ -3807,7 +3807,7 @@ dependencies = [ [[package]] name = "event-sourcing" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -3823,7 +3823,7 @@ dependencies = [ [[package]] name = "event-sourcing-macros" -version = "0.206.2" +version = "0.206.3" dependencies = [ "quote", "syn 2.0.79", @@ -4503,7 +4503,7 @@ dependencies = [ [[package]] name = "http-common" -version = "0.206.2" +version = "0.206.3" dependencies = [ "axum", "http 1.1.0", @@ -4779,7 +4779,7 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "init-on-startup" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "database-common", @@ -4822,7 +4822,7 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "internal-error" -version = "0.206.2" +version = "0.206.3" dependencies = [ "thiserror", ] @@ -4970,7 +4970,7 @@ dependencies = [ [[package]] name = "kamu" -version = "0.206.2" +version = "0.206.3" dependencies = [ "alloy", "async-recursion", @@ -5059,7 +5059,7 @@ dependencies = [ [[package]] name = "kamu-accounts" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "base32", @@ -5085,7 +5085,7 @@ dependencies = [ [[package]] name = "kamu-accounts-inmem" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5106,7 +5106,7 @@ dependencies = [ [[package]] name = "kamu-accounts-mysql" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5127,7 +5127,7 @@ dependencies = [ [[package]] name = "kamu-accounts-postgres" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5148,7 +5148,7 @@ dependencies = [ [[package]] name = "kamu-accounts-repo-tests" -version = "0.206.2" +version = "0.206.3" dependencies = [ "argon2", "chrono", @@ -5164,7 +5164,7 @@ dependencies = [ [[package]] name = "kamu-accounts-services" -version = "0.206.2" +version = "0.206.3" dependencies = [ "argon2", "async-trait", @@ -5191,7 +5191,7 @@ dependencies = [ [[package]] name = "kamu-accounts-sqlite" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5212,7 +5212,7 @@ dependencies = [ [[package]] name = "kamu-adapter-auth-oso" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "dill", @@ -5234,7 +5234,7 @@ dependencies = [ [[package]] name = "kamu-adapter-flight-sql" -version = "0.206.2" +version = "0.206.3" dependencies = [ "arrow-flight", "async-trait", @@ -5257,7 +5257,7 @@ dependencies = [ [[package]] name = "kamu-adapter-graphql" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-graphql", "async-trait", @@ -5308,7 +5308,7 @@ dependencies = [ [[package]] name = "kamu-adapter-http" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "aws-sdk-s3", @@ -5377,7 +5377,7 @@ dependencies = [ [[package]] name = "kamu-adapter-oauth" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5396,7 +5396,7 @@ dependencies = [ [[package]] name = "kamu-adapter-odata" -version = "0.206.2" +version = "0.206.3" dependencies = [ "axum", "chrono", @@ -5434,7 +5434,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "internal-error", @@ -5446,7 +5446,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-inmem" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "database-common-macros", @@ -5460,7 +5460,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-postgres" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "database-common", @@ -5477,7 +5477,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-repo-tests" -version = "0.206.2" +version = "0.206.3" dependencies = [ "dill", "kamu-auth-rebac", @@ -5486,7 +5486,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-services" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "dill", @@ -5505,7 +5505,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-sqlite" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "database-common", @@ -5522,7 +5522,7 @@ dependencies = [ [[package]] name = "kamu-cli" -version = "0.206.2" +version = "0.206.3" dependencies = [ "arrow-flight", "async-graphql", @@ -5646,7 +5646,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5668,7 +5668,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common-macros" -version = "0.206.2" +version = "0.206.3" dependencies = [ "quote", "syn 2.0.79", @@ -5676,7 +5676,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-inmem" -version = "0.206.2" +version = "0.206.3" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5689,7 +5689,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-mysql" -version = "0.206.2" +version = "0.206.3" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5703,7 +5703,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-postgres" -version = "0.206.2" +version = "0.206.3" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5717,7 +5717,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-repo-tests" -version = "0.206.2" +version = "0.206.3" dependencies = [ "chrono", "indoc 2.0.5", @@ -5737,7 +5737,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-sqlite" -version = "0.206.2" +version = "0.206.3" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5751,7 +5751,7 @@ dependencies = [ [[package]] name = "kamu-cli-puppet" -version = "0.206.2" +version = "0.206.3" dependencies = [ "assert_cmd", "async-trait", @@ -5770,7 +5770,7 @@ dependencies = [ [[package]] name = "kamu-core" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -5801,7 +5801,7 @@ dependencies = [ [[package]] name = "kamu-data-utils" -version = "0.206.2" +version = "0.206.3" dependencies = [ "arrow", "arrow-digest", @@ -5826,7 +5826,7 @@ dependencies = [ [[package]] name = "kamu-datafusion-cli" -version = "0.206.2" +version = "0.206.3" dependencies = [ "arrow", "async-trait", @@ -5850,7 +5850,7 @@ dependencies = [ [[package]] name = "kamu-datasets" -version = "0.206.2" +version = "0.206.3" dependencies = [ "aes-gcm", "async-trait", @@ -5870,7 +5870,7 @@ dependencies = [ [[package]] name = "kamu-datasets-inmem" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5893,7 +5893,7 @@ dependencies = [ [[package]] name = "kamu-datasets-postgres" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5916,7 +5916,7 @@ dependencies = [ [[package]] name = "kamu-datasets-repo-tests" -version = "0.206.2" +version = "0.206.3" dependencies = [ "chrono", "database-common", @@ -5930,7 +5930,7 @@ dependencies = [ [[package]] name = "kamu-datasets-services" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5961,7 +5961,7 @@ dependencies = [ [[package]] name = "kamu-datasets-sqlite" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -5984,7 +5984,7 @@ dependencies = [ [[package]] name = "kamu-flow-system" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -6013,7 +6013,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-inmem" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6043,7 +6043,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-postgres" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6068,7 +6068,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-repo-tests" -version = "0.206.2" +version = "0.206.3" dependencies = [ "chrono", "database-common", @@ -6081,7 +6081,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-services" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6125,7 +6125,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-sqlite" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6150,7 +6150,7 @@ dependencies = [ [[package]] name = "kamu-ingest-datafusion" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -6186,7 +6186,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-inmem" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -6205,7 +6205,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-postgres" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6228,7 +6228,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-repo-tests" -version = "0.206.2" +version = "0.206.3" dependencies = [ "chrono", "database-common", @@ -6242,7 +6242,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-sqlite" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6264,7 +6264,7 @@ dependencies = [ [[package]] name = "kamu-repo-tools" -version = "0.206.2" +version = "0.206.3" dependencies = [ "chrono", "clap", @@ -6279,7 +6279,7 @@ dependencies = [ [[package]] name = "kamu-task-system" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -6297,7 +6297,7 @@ dependencies = [ [[package]] name = "kamu-task-system-inmem" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -6316,7 +6316,7 @@ dependencies = [ [[package]] name = "kamu-task-system-postgres" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6339,7 +6339,7 @@ dependencies = [ [[package]] name = "kamu-task-system-repo-tests" -version = "0.206.2" +version = "0.206.3" dependencies = [ "chrono", "database-common", @@ -6351,7 +6351,7 @@ dependencies = [ [[package]] name = "kamu-task-system-services" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6379,7 +6379,7 @@ dependencies = [ [[package]] name = "kamu-task-system-sqlite" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-stream", "async-trait", @@ -6752,7 +6752,7 @@ dependencies = [ [[package]] name = "messaging-outbox" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -6881,7 +6881,7 @@ dependencies = [ [[package]] name = "multiformats" -version = "0.206.2" +version = "0.206.3" dependencies = [ "base64 0.22.1", "bs58", @@ -7206,7 +7206,7 @@ dependencies = [ [[package]] name = "observability" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "axum", @@ -7264,7 +7264,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendatafabric" -version = "0.206.2" +version = "0.206.3" dependencies = [ "arrow", "base64 0.22.1", @@ -8224,7 +8224,7 @@ dependencies = [ [[package]] name = "random-names" -version = "0.206.2" +version = "0.206.3" dependencies = [ "rand", ] @@ -9193,7 +9193,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.79", @@ -9777,7 +9777,7 @@ dependencies = [ [[package]] name = "time-source" -version = "0.206.2" +version = "0.206.3" dependencies = [ "async-trait", "chrono", @@ -10191,7 +10191,7 @@ dependencies = [ [[package]] name = "tracing-perfetto" -version = "0.206.2" +version = "0.206.3" dependencies = [ "conv", "serde", diff --git a/Cargo.toml b/Cargo.toml index 662e1916f8..458f10c0ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,95 +92,95 @@ resolver = "2" [workspace.dependencies] # Apps -kamu-cli = { version = "0.206.2", path = "src/app/cli", default-features = false } +kamu-cli = { version = "0.206.3", path = "src/app/cli", default-features = false } # Utils -async-utils = { version = "0.206.2", path = "src/utils/async-utils", default-features = false } -container-runtime = { version = "0.206.2", path = "src/utils/container-runtime", default-features = false } -database-common = { version = "0.206.2", path = "src/utils/database-common", default-features = false } -database-common-macros = { version = "0.206.2", path = "src/utils/database-common-macros", default-features = false } -enum-variants = { version = "0.206.2", path = "src/utils/enum-variants", default-features = false } -event-sourcing = { version = "0.206.2", path = "src/utils/event-sourcing", default-features = false } -event-sourcing-macros = { version = "0.206.2", path = "src/utils/event-sourcing-macros", default-features = false } -http-common = { version = "0.206.2", path = "src/utils/http-common", default-features = false } -init-on-startup = { version = "0.206.2", path = "src/utils/init-on-startup", default-features = false } -internal-error = { version = "0.206.2", path = "src/utils/internal-error", default-features = false } -kamu-cli-puppet = { version = "0.206.2", path = "src/utils/kamu-cli-puppet", default-features = false } -kamu-data-utils = { version = "0.206.2", path = "src/utils/data-utils", default-features = false } -kamu-datafusion-cli = { version = "0.206.2", path = "src/utils/datafusion-cli", default-features = false } -messaging-outbox = { version = "0.206.2", path = "src/utils/messaging-outbox", default-features = false } -multiformats = { version = "0.206.2", path = "src/utils/multiformats", default-features = false } -observability = { version = "0.206.2", path = "src/utils/observability", default-features = false } -random-names = { version = "0.206.2", path = "src/utils/random-names", default-features = false } -time-source = { version = "0.206.2", path = "src/utils/time-source", default-features = false } -tracing-perfetto = { version = "0.206.2", path = "src/utils/tracing-perfetto", default-features = false } +async-utils = { version = "0.206.3", path = "src/utils/async-utils", default-features = false } +container-runtime = { version = "0.206.3", path = "src/utils/container-runtime", default-features = false } +database-common = { version = "0.206.3", path = "src/utils/database-common", default-features = false } +database-common-macros = { version = "0.206.3", path = "src/utils/database-common-macros", default-features = false } +enum-variants = { version = "0.206.3", path = "src/utils/enum-variants", default-features = false } +event-sourcing = { version = "0.206.3", path = "src/utils/event-sourcing", default-features = false } +event-sourcing-macros = { version = "0.206.3", path = "src/utils/event-sourcing-macros", default-features = false } +http-common = { version = "0.206.3", path = "src/utils/http-common", default-features = false } +init-on-startup = { version = "0.206.3", path = "src/utils/init-on-startup", default-features = false } +internal-error = { version = "0.206.3", path = "src/utils/internal-error", default-features = false } +kamu-cli-puppet = { version = "0.206.3", path = "src/utils/kamu-cli-puppet", default-features = false } +kamu-data-utils = { version = "0.206.3", path = "src/utils/data-utils", default-features = false } +kamu-datafusion-cli = { version = "0.206.3", path = "src/utils/datafusion-cli", default-features = false } +messaging-outbox = { version = "0.206.3", path = "src/utils/messaging-outbox", default-features = false } +multiformats = { version = "0.206.3", path = "src/utils/multiformats", default-features = false } +observability = { version = "0.206.3", path = "src/utils/observability", default-features = false } +random-names = { version = "0.206.3", path = "src/utils/random-names", default-features = false } +time-source = { version = "0.206.3", path = "src/utils/time-source", default-features = false } +tracing-perfetto = { version = "0.206.3", path = "src/utils/tracing-perfetto", default-features = false } # Domain -kamu-accounts = { version = "0.206.2", path = "src/domain/accounts/domain", default-features = false } -kamu-auth-rebac = { version = "0.206.2", path = "src/domain/auth-rebac/domain", default-features = false } -kamu-core = { version = "0.206.2", path = "src/domain/core", default-features = false } -kamu-datasets = { version = "0.206.2", path = "src/domain/datasets/domain", default-features = false } -kamu-flow-system = { version = "0.206.2", path = "src/domain/flow-system/domain", default-features = false } -kamu-task-system = { version = "0.206.2", path = "src/domain/task-system/domain", default-features = false } -opendatafabric = { version = "0.206.2", path = "src/domain/opendatafabric", default-features = false } +kamu-accounts = { version = "0.206.3", path = "src/domain/accounts/domain", default-features = false } +kamu-auth-rebac = { version = "0.206.3", path = "src/domain/auth-rebac/domain", default-features = false } +kamu-core = { version = "0.206.3", path = "src/domain/core", default-features = false } +kamu-datasets = { version = "0.206.3", path = "src/domain/datasets/domain", default-features = false } +kamu-flow-system = { version = "0.206.3", path = "src/domain/flow-system/domain", default-features = false } +kamu-task-system = { version = "0.206.3", path = "src/domain/task-system/domain", default-features = false } +opendatafabric = { version = "0.206.3", path = "src/domain/opendatafabric", default-features = false } # Domain service layer -kamu-accounts-services = { version = "0.206.2", path = "src/domain/accounts/services", default-features = false } -kamu-auth-rebac-services = { version = "0.206.2", path = "src/domain/auth-rebac/services", default-features = false } -kamu-datasets-services = { version = "0.206.2", path = "src/domain/datasets/services", default-features = false } -kamu-flow-system-services = { version = "0.206.2", path = "src/domain/flow-system/services", default-features = false } -kamu-task-system-services = { version = "0.206.2", path = "src/domain/task-system/services", default-features = false } +kamu-accounts-services = { version = "0.206.3", path = "src/domain/accounts/services", default-features = false } +kamu-auth-rebac-services = { version = "0.206.3", path = "src/domain/auth-rebac/services", default-features = false } +kamu-datasets-services = { version = "0.206.3", path = "src/domain/datasets/services", default-features = false } +kamu-flow-system-services = { version = "0.206.3", path = "src/domain/flow-system/services", default-features = false } +kamu-task-system-services = { version = "0.206.3", path = "src/domain/task-system/services", default-features = false } # Infra -kamu = { version = "0.206.2", path = "src/infra/core", default-features = false } -kamu-ingest-datafusion = { version = "0.206.2", path = "src/infra/ingest-datafusion", default-features = false } +kamu = { version = "0.206.3", path = "src/infra/core", default-features = false } +kamu-ingest-datafusion = { version = "0.206.3", path = "src/infra/ingest-datafusion", default-features = false } ## Flow System -kamu-flow-system-repo-tests = { version = "0.206.2", path = "src/infra/flow-system/repo-tests", default-features = false } -kamu-flow-system-inmem = { version = "0.206.2", path = "src/infra/flow-system/inmem", default-features = false } -kamu-flow-system-postgres = { version = "0.206.2", path = "src/infra/flow-system/postgres", default-features = false } -kamu-flow-system-sqlite = { version = "0.206.2", path = "src/infra/flow-system/sqlite", default-features = false } +kamu-flow-system-repo-tests = { version = "0.206.3", path = "src/infra/flow-system/repo-tests", default-features = false } +kamu-flow-system-inmem = { version = "0.206.3", path = "src/infra/flow-system/inmem", default-features = false } +kamu-flow-system-postgres = { version = "0.206.3", path = "src/infra/flow-system/postgres", default-features = false } +kamu-flow-system-sqlite = { version = "0.206.3", path = "src/infra/flow-system/sqlite", default-features = false } ## Accounts -kamu-accounts-inmem = { version = "0.206.2", path = "src/infra/accounts/inmem", default-features = false } -kamu-accounts-mysql = { version = "0.206.2", path = "src/infra/accounts/mysql", default-features = false } -kamu-accounts-postgres = { version = "0.206.2", path = "src/infra/accounts/postgres", default-features = false } -kamu-accounts-sqlite = { version = "0.206.2", path = "src/infra/accounts/sqlite", default-features = false } -kamu-accounts-repo-tests = { version = "0.206.2", path = "src/infra/accounts/repo-tests", default-features = false } +kamu-accounts-inmem = { version = "0.206.3", path = "src/infra/accounts/inmem", default-features = false } +kamu-accounts-mysql = { version = "0.206.3", path = "src/infra/accounts/mysql", default-features = false } +kamu-accounts-postgres = { version = "0.206.3", path = "src/infra/accounts/postgres", default-features = false } +kamu-accounts-sqlite = { version = "0.206.3", path = "src/infra/accounts/sqlite", default-features = false } +kamu-accounts-repo-tests = { version = "0.206.3", path = "src/infra/accounts/repo-tests", default-features = false } ## Datasets -kamu-datasets-inmem = { version = "0.206.2", path = "src/infra/datasets/inmem", default-features = false } -kamu-datasets-postgres = { version = "0.206.2", path = "src/infra/datasets/postgres", default-features = false } -kamu-datasets-sqlite = { version = "0.206.2", path = "src/infra/datasets/sqlite", default-features = false } -kamu-datasets-repo-tests = { version = "0.206.2", path = "src/infra/datasets/repo-tests", default-features = false } +kamu-datasets-inmem = { version = "0.206.3", path = "src/infra/datasets/inmem", default-features = false } +kamu-datasets-postgres = { version = "0.206.3", path = "src/infra/datasets/postgres", default-features = false } +kamu-datasets-sqlite = { version = "0.206.3", path = "src/infra/datasets/sqlite", default-features = false } +kamu-datasets-repo-tests = { version = "0.206.3", path = "src/infra/datasets/repo-tests", default-features = false } ## Task System -kamu-task-system-inmem = { version = "0.206.2", path = "src/infra/task-system/inmem", default-features = false } -kamu-task-system-postgres = { version = "0.206.2", path = "src/infra/task-system/postgres", default-features = false } -kamu-task-system-sqlite = { version = "0.206.2", path = "src/infra/task-system/sqlite", default-features = false } -kamu-task-system-repo-tests = { version = "0.206.2", path = "src/infra/task-system/repo-tests", default-features = false } +kamu-task-system-inmem = { version = "0.206.3", path = "src/infra/task-system/inmem", default-features = false } +kamu-task-system-postgres = { version = "0.206.3", path = "src/infra/task-system/postgres", default-features = false } +kamu-task-system-sqlite = { version = "0.206.3", path = "src/infra/task-system/sqlite", default-features = false } +kamu-task-system-repo-tests = { version = "0.206.3", path = "src/infra/task-system/repo-tests", default-features = false } ## ReBAC -kamu-auth-rebac-inmem = { version = "0.206.2", path = "src/infra/auth-rebac/inmem", default-features = false } -kamu-auth-rebac-repo-tests = { version = "0.206.2", path = "src/infra/auth-rebac/repo-tests", default-features = false } -kamu-auth-rebac-postgres = { version = "0.206.2", path = "src/infra/auth-rebac/postgres", default-features = false } -kamu-auth-rebac-sqlite = { version = "0.206.2", path = "src/infra/auth-rebac/sqlite", default-features = false } +kamu-auth-rebac-inmem = { version = "0.206.3", path = "src/infra/auth-rebac/inmem", default-features = false } +kamu-auth-rebac-repo-tests = { version = "0.206.3", path = "src/infra/auth-rebac/repo-tests", default-features = false } +kamu-auth-rebac-postgres = { version = "0.206.3", path = "src/infra/auth-rebac/postgres", default-features = false } +kamu-auth-rebac-sqlite = { version = "0.206.3", path = "src/infra/auth-rebac/sqlite", default-features = false } ## Outbox -kamu-messaging-outbox-inmem = { version = "0.206.2", path = "src/infra/messaging-outbox/inmem", default-features = false } -kamu-messaging-outbox-postgres = { version = "0.206.2", path = "src/infra/messaging-outbox/postgres", default-features = false } -kamu-messaging-outbox-sqlite = { version = "0.206.2", path = "src/infra/messaging-outbox/sqlite", default-features = false } -kamu-messaging-outbox-repo-tests = { version = "0.206.2", path = "src/infra/messaging-outbox/repo-tests", default-features = false } +kamu-messaging-outbox-inmem = { version = "0.206.3", path = "src/infra/messaging-outbox/inmem", default-features = false } +kamu-messaging-outbox-postgres = { version = "0.206.3", path = "src/infra/messaging-outbox/postgres", default-features = false } +kamu-messaging-outbox-sqlite = { version = "0.206.3", path = "src/infra/messaging-outbox/sqlite", default-features = false } +kamu-messaging-outbox-repo-tests = { version = "0.206.3", path = "src/infra/messaging-outbox/repo-tests", default-features = false } # Adapters -kamu-adapter-auth-oso = { version = "0.206.2", path = "src/adapter/auth-oso", default-features = false } -kamu-adapter-flight-sql = { version = "0.206.2", path = "src/adapter/flight-sql", default-features = false } -kamu-adapter-graphql = { version = "0.206.2", path = "src/adapter/graphql", default-features = false } -kamu-adapter-http = { version = "0.206.2", path = "src/adapter/http", default-features = false } -kamu-adapter-odata = { version = "0.206.2", path = "src/adapter/odata", default-features = false } -kamu-adapter-oauth = { version = "0.206.2", path = "src/adapter/oauth", default-features = false } +kamu-adapter-auth-oso = { version = "0.206.3", path = "src/adapter/auth-oso", default-features = false } +kamu-adapter-flight-sql = { version = "0.206.3", path = "src/adapter/flight-sql", default-features = false } +kamu-adapter-graphql = { version = "0.206.3", path = "src/adapter/graphql", default-features = false } +kamu-adapter-http = { version = "0.206.3", path = "src/adapter/http", default-features = false } +kamu-adapter-odata = { version = "0.206.3", path = "src/adapter/odata", default-features = false } +kamu-adapter-oauth = { version = "0.206.3", path = "src/adapter/oauth", default-features = false } # E2E -kamu-cli-e2e-common = { version = "0.206.2", path = "src/e2e/app/cli/common", default-features = false } -kamu-cli-e2e-common-macros = { version = "0.206.2", path = "src/e2e/app/cli/common-macros", default-features = false } -kamu-cli-e2e-repo-tests = { version = "0.206.2", path = "src/e2e/app/cli/repo-tests", default-features = false } +kamu-cli-e2e-common = { version = "0.206.3", path = "src/e2e/app/cli/common", default-features = false } +kamu-cli-e2e-common-macros = { version = "0.206.3", path = "src/e2e/app/cli/common-macros", default-features = false } +kamu-cli-e2e-repo-tests = { version = "0.206.3", path = "src/e2e/app/cli/repo-tests", default-features = false } [workspace.package] -version = "0.206.2" +version = "0.206.3" edition = "2021" homepage = "https://github.com/kamu-data/kamu-cli" repository = "https://github.com/kamu-data/kamu-cli" diff --git a/LICENSE.txt b/LICENSE.txt index a902378f6f..14ccc5e26b 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -11,7 +11,7 @@ Business Source License 1.1 Licensor: Kamu Data, Inc. -Licensed Work: Kamu CLI Version 0.206.2 +Licensed Work: Kamu CLI Version 0.206.3 The Licensed Work is © 2023 Kamu Data, Inc. Additional Use Grant: You may use the Licensed Work for any purpose, diff --git a/resources/openapi-mt.json b/resources/openapi-mt.json index e52b5c20b7..d1ebedb947 100644 --- a/resources/openapi-mt.json +++ b/resources/openapi-mt.json @@ -840,7 +840,7 @@ "name": "" }, "title": "kamu-cli", - "version": "0.206.2" + "version": "0.206.3" }, "openapi": "3.1.0", "paths": { diff --git a/resources/openapi.json b/resources/openapi.json index bdd4f8e7f4..19098761b4 100644 --- a/resources/openapi.json +++ b/resources/openapi.json @@ -840,7 +840,7 @@ "name": "" }, "title": "kamu-cli", - "version": "0.206.2" + "version": "0.206.3" }, "openapi": "3.1.0", "paths": { diff --git a/src/domain/datasets/services/src/dataset_entry_indexer.rs b/src/domain/datasets/services/src/dataset_entry_indexer.rs index 3b29a1336d..8e53d53de6 100644 --- a/src/domain/datasets/services/src/dataset_entry_indexer.rs +++ b/src/domain/datasets/services/src/dataset_entry_indexer.rs @@ -15,6 +15,7 @@ use init_on_startup::{InitOnStartup, InitOnStartupMeta}; use internal_error::{InternalError, ResultIntoInternal}; use kamu_accounts::{ AccountRepository, + GetAccountByNameError, DEFAULT_ACCOUNT_ID, JOB_KAMU_ACCOUNTS_PREDEFINED_ACCOUNTS_REGISTRATOR, }; @@ -71,50 +72,44 @@ impl DatasetEntryIndexer { Ok(stored_dataset_entries_count > 0) } + #[tracing::instrument( + level = "debug", + skip_all, + name = "DatasetEntryIndexer::index_datasets" + )] async fn index_datasets(&self) -> Result<(), InternalError> { use futures::TryStreamExt; let dataset_handles: Vec<_> = self.dataset_repo.get_all_datasets().try_collect().await?; - self.concurrent_dataset_handles_processing(dataset_handles) - .await?; - - Ok(()) - } - - async fn concurrent_dataset_handles_processing( - &self, - dataset_handles: Vec, - ) -> Result<(), InternalError> { - let mut join_set = tokio::task::JoinSet::new(); - let now = self.time_source.now(); let account_name_id_mapping = self.build_account_name_id_mapping(&dataset_handles).await?; for dataset_handle in dataset_handles { - let task_owner_account_id = - account_name_id_mapping[&dataset_handle.alias.account_name].clone(); - let task_now = now; - let task_dataset_entry_repo = self.dataset_entry_repo.clone(); - - join_set.spawn(async move { - let dataset_entry = DatasetEntry::new( - dataset_handle.id, - task_owner_account_id, - dataset_handle.alias.dataset_name, - task_now, - ); - - task_dataset_entry_repo - .save_dataset_entry(&dataset_entry) - .await - .int_err() - }); - } - - while let Some(join_result) = join_set.join_next().await { - let task_res = join_result.int_err()?; - - task_res?; + let Some(owner_account_id) = account_name_id_mapping + .get(&dataset_handle.alias.account_name) + .cloned() + else { + tracing::debug!(dataset_handle=%dataset_handle, "Skipped indexing dataset due to unresolved owner"); + continue; + }; + + let dataset_entry = DatasetEntry::new( + dataset_handle.id, + owner_account_id, + dataset_handle.alias.dataset_name, + self.time_source.now(), + ); + + use tracing::Instrument; + self.dataset_entry_repo + .save_dataset_entry(&dataset_entry) + .instrument(tracing::debug_span!( + "Saving indexed dataset entry", + dataset_entry = ?dataset_entry, + owner_account_name = ?dataset_handle.alias.account_name, + )) + .await + .int_err()?; } Ok(()) @@ -133,9 +128,15 @@ impl DatasetEntryIndexer { continue; } - let owner_account_id = self.get_dataset_owner_id(maybe_owner_name).await?; - - map.insert(maybe_owner_name.clone(), owner_account_id); + match self.get_dataset_owner_id(maybe_owner_name).await { + Ok(owner_account_id) => { + map.insert(maybe_owner_name.clone(), owner_account_id); + } + Err(e) => { + // Log error, but don't crash + tracing::error!(error=?e, account_name = ?maybe_owner_name, "Account unresolved by dataset indexing job"); + } + } } Ok(map) @@ -144,15 +145,15 @@ impl DatasetEntryIndexer { async fn get_dataset_owner_id( &self, maybe_owner_name: &Option, - ) -> Result { + ) -> Result { match &maybe_owner_name { Some(account_name) => { let account = self .account_repository .get_account_by_name(account_name) - .await - .int_err()?; + .await?; + tracing::debug!(account_id=%account.id, account_name=%account_name, "Account resolved by dataset indexing job"); Ok(account.id) } None => Ok(DEFAULT_ACCOUNT_ID.clone()), diff --git a/src/infra/core/Cargo.toml b/src/infra/core/Cargo.toml index fba33387ac..553f277fd0 100644 --- a/src/infra/core/Cargo.toml +++ b/src/infra/core/Cargo.toml @@ -70,7 +70,7 @@ secrecy = "0.10" zip = "2" # Data -datafusion = { version = "42", default-features = false } +datafusion = { version = "42.0", default-features = false } digest = "0.10" object_store = { version = "0.11", features = ["aws"] } parking_lot = { version = "0.12" } @@ -159,7 +159,7 @@ kamu-data-utils = { workspace = true, features = ["testing"] } kamu-datasets-services = { workspace = true } criterion = { version = "0.5", features = ["async_tokio"] } -datafusion = { version = "42", default-features = false, features = [ +datafusion = { version = "42.0", default-features = false, features = [ "parquet", ] } filetime = "0.2"