From b884acd717ea4f2d52b6482000ae549e83ee6c07 Mon Sep 17 00:00:00 2001 From: Sergii Mikhtoniuk Date: Thu, 26 Oct 2023 15:31:07 -0700 Subject: [PATCH] FlightSQL endpoint --- CHANGELOG.md | 5 +- Cargo.lock | 465 ++++++++++++--------- Cargo.toml | 30 +- LICENSE.txt | 2 +- README.md | 6 +- deny.toml | 4 - src/app/api-server/Cargo.toml | 9 +- src/app/api-server/src/app.rs | 22 +- src/app/api-server/src/cli_parser.rs | 4 + src/app/api-server/src/flightsql_server.rs | 100 +++++ src/app/api-server/src/http_server.rs | 11 +- src/app/api-server/src/lib.rs | 1 + 12 files changed, 417 insertions(+), 242 deletions(-) create mode 100644 src/app/api-server/src/flightsql_server.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e7691e..16a054dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.0] - 2023-10-26 +### Added +- FlightSQL endpoint + ## [0.8.0] - 2023-10-13 ### Added - Integrated multi-tenancy support: authentication & authorization for public datasets - ## [0.7.0] - 2023-07-27 ### Added - Keeping a CHANGELOG diff --git a/Cargo.lock b/Cargo.lock index 04e7a1ae..a02c4b17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,26 +40,16 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - -[[package]] -name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", "const-random", "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -188,7 +178,7 @@ version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04a8801ebb147ad240b2d978d3ab9f73c9ccd4557ba6a03e7800496770ed10e0" dependencies = [ - "ahash 0.8.3", + "ahash", "arrow-arith", "arrow-array", "arrow-buffer", @@ -225,14 +215,14 @@ version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "226fdc6c3a4ae154a74c24091d36a90b514f0ed7112f5b8322c1d8f354d8e20d" dependencies = [ - "ahash 0.8.3", + "ahash", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", "chrono-tz", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "num", ] @@ -306,6 +296,33 @@ dependencies = [ "digest", ] +[[package]] +name = "arrow-flight" +version = "46.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40372d37a860a742f248d4d7c137950cd793f1b46f2b99a5116c55efbe2699f" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "base64 0.21.5", + "bytes", + "futures", + "once_cell", + "paste", + "prost", + "tokio", + "tonic", +] + [[package]] name = "arrow-ipc" version = "46.0.0" @@ -361,13 +378,13 @@ version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e32afc1329f7b372463b21c6ca502b07cf237e1ed420d87706c1770bb0ebd38" dependencies = [ - "ahash 0.8.3", + "ahash", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -441,9 +458,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "6.0.7" +version = "6.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1addb0b551c59640e15de99e7566a4e3a1186cf42269e160c485ba6d8b43fe30" +checksum = "117113a7ff4a98f2a864fa7a5274033b0907fce65dc8464993c75033f8074f90" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -476,9 +493,9 @@ dependencies = [ [[package]] name = "async-graphql-axum" -version = "6.0.7" +version = "6.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c21af134ab9419aae6658298f819a28e4737ac81f96cde8008f9d49db1802662" +checksum = "0ddd7767f83c3273a6d26a36cbdd562c6009aae87c6fa2c0f1eebb76f60c4151" dependencies = [ "async-graphql", "async-trait", @@ -494,9 +511,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "6.0.7" +version = "6.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e1121ff0be2feea705c24f6940162c4f14a077e50a217b16e091e6534a8c08a" +checksum = "6e4bb7b7b2344d24af860776b7fe4e4ee4a67cd965f076048d023f555703b854" dependencies = [ "Inflector", "async-graphql-parser", @@ -504,16 +521,16 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "strum 0.25.0", + "strum", "syn 2.0.38", "thiserror", ] [[package]] name = "async-graphql-parser" -version = "6.0.7" +version = "6.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b6713fd4ffd610b8b6f6e911bf31277cbb84b7c2a9cdeeb39d1b3eed3b88e4" +checksum = "c47e1c1ff6cb7cae62c9cd768d76475cc68f156d8234b024fd2499ad0e91da21" dependencies = [ "async-graphql-value", "pest", @@ -523,9 +540,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "6.0.7" +version = "6.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d74240f9daa8c1e8f73e9cfcc338d20a88d00bbeb83ded49ce8e5b4dcec0f5" +checksum = "2270df3a642efce860ed06fbcf61fc6db10f83c2ecb5613127fb453c82e012a4" dependencies = [ "bytes", "indexmap 2.0.2", @@ -952,7 +969,7 @@ checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", - "base64 0.21.4", + "base64 0.21.5", "bitflags 1.3.2", "bytes", "futures-util", @@ -1042,9 +1059,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "base64-simd" @@ -1267,18 +1284,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstream", "anstyle", @@ -1288,9 +1305,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -1300,12 +1317,12 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "comfy-table" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "strum 0.24.1", - "strum_macros 0.24.3", + "strum", + "strum_macros 0.25.3", "unicode-width", ] @@ -1317,23 +1334,21 @@ checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "const-random" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +checksum = "11df32a13d7892ec42d51d3d175faba5211ffe13ed25d4fb348ac9e9ce835593" dependencies = [ "const-random-macro", - "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ "getrandom", "once_cell", - "proc-macro-hack", "tiny-keccak", ] @@ -1351,8 +1366,8 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "container-runtime" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-trait", "cfg-if", @@ -1385,9 +1400,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -1536,7 +1551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "lock_api", "once_cell", "parking_lot_core", @@ -1554,7 +1569,7 @@ version = "31.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a4e4fc25698a14c90b34dda647ba10a5a966dc04b036d22e77fb1048663375d" dependencies = [ - "ahash 0.8.3", + "ahash", "arrow", "arrow-array", "arrow-schema", @@ -1574,7 +1589,7 @@ dependencies = [ "futures", "glob", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "indexmap 2.0.2", "itertools 0.11.0", "log", @@ -1630,7 +1645,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "log", "object_store", "parking_lot", @@ -1645,11 +1660,11 @@ version = "31.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6ea9844395f537730a145e5d87f61fecd37c2bc9d54e1dc89b35590d867345d" dependencies = [ - "ahash 0.8.3", + "ahash", "arrow", "datafusion-common", "sqlparser", - "strum 0.25.0", + "strum", "strum_macros 0.25.3", ] @@ -1665,7 +1680,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "itertools 0.11.0", "log", "regex-syntax 0.7.5", @@ -1677,19 +1692,19 @@ version = "31.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "766c567082c9bbdcb784feec8fe40c7049cedaeb3a18d54f563f75fe0dc1932c" dependencies = [ - "ahash 0.8.3", + "ahash", "arrow", "arrow-array", "arrow-buffer", "arrow-schema", - "base64 0.21.4", + "base64 0.21.5", "blake2", "blake3", "chrono", "datafusion-common", "datafusion-expr", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "hex", "indexmap 2.0.2", "itertools 0.11.0", @@ -1889,8 +1904,8 @@ dependencies = [ [[package]] name = "enum-variants" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" [[package]] name = "env_logger" @@ -1923,8 +1938,8 @@ dependencies = [ [[package]] name = "event-sourcing" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-trait", "chrono", @@ -1937,8 +1952,8 @@ dependencies = [ [[package]] name = "event-sourcing-macros" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "quote", "syn 2.0.38", @@ -2044,9 +2059,9 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -2059,9 +2074,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -2069,15 +2084,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -2086,15 +2101,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", @@ -2103,21 +2118,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -2247,17 +2262,14 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.6", -] [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ - "ahash 0.8.3", + "ahash", "allocator-api2", ] @@ -2267,7 +2279,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "bytes", "headers-core", "http", @@ -2386,7 +2398,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -2410,14 +2422,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.7", + "rustls 0.21.8", "tokio", "tokio-rustls 0.24.1", ] @@ -2502,7 +2514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "serde", ] @@ -2544,8 +2556,8 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "internal-error" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "thiserror", ] @@ -2556,7 +2568,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.4", + "socket2 0.5.5", "widestring", "windows-sys", "winreg", @@ -2564,9 +2576,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" @@ -2627,7 +2639,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "pem", "ring 0.16.20", "serde", @@ -2637,8 +2649,8 @@ dependencies = [ [[package]] name = "kamu" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-recursion", "async-stream", @@ -2683,7 +2695,7 @@ dependencies = [ "serde_with", "serde_yaml", "sha3", - "strum 0.25.0", + "strum", "strum_macros 0.25.3", "tar", "tempfile", @@ -2700,8 +2712,8 @@ dependencies = [ [[package]] name = "kamu-adapter-auth-oso" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-trait", "dill", @@ -2711,10 +2723,28 @@ dependencies = [ "oso-derive", ] +[[package]] +name = "kamu-adapter-flight-sql" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" +dependencies = [ + "arrow-flight", + "async-trait", + "base64 0.21.5", + "dashmap", + "datafusion", + "futures", + "like", + "prost", + "tonic", + "tracing", + "uuid", +] + [[package]] name = "kamu-adapter-graphql" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-graphql", "async-trait", @@ -2737,8 +2767,8 @@ dependencies = [ [[package]] name = "kamu-adapter-http" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-trait", "axum", @@ -2768,8 +2798,8 @@ dependencies = [ [[package]] name = "kamu-adapter-oauth" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-trait", "dill", @@ -2784,8 +2814,9 @@ dependencies = [ [[package]] name = "kamu-api-server" -version = "0.8.3" +version = "0.9.0" dependencies = [ + "arrow-flight", "async-graphql", "async-graphql-axum", "async-trait", @@ -2794,6 +2825,7 @@ dependencies = [ "chrono", "clap", "container-runtime", + "datafusion", "dill", "env_logger", "futures", @@ -2803,6 +2835,7 @@ dependencies = [ "internal-error", "kamu", "kamu-adapter-auth-oso", + "kamu-adapter-flight-sql", "kamu-adapter-graphql", "kamu-adapter-http", "kamu-adapter-oauth", @@ -2816,7 +2849,9 @@ dependencies = [ "test-log", "thiserror", "tokio", + "tokio-stream", "tokio-util", + "tonic", "tower", "tower-http", "tracing", @@ -2829,8 +2864,8 @@ dependencies = [ [[package]] name = "kamu-core" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-stream", "async-trait", @@ -2857,8 +2892,8 @@ dependencies = [ [[package]] name = "kamu-data-utils" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "arrow-digest", "async-trait", @@ -2874,8 +2909,8 @@ dependencies = [ [[package]] name = "kamu-ingest-datafusion" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-trait", "chrono", @@ -2904,7 +2939,7 @@ dependencies = [ [[package]] name = "kamu-repo-tools" -version = "0.8.3" +version = "0.9.0" dependencies = [ "chrono", "clap", @@ -2918,8 +2953,8 @@ dependencies = [ [[package]] name = "kamu-task-system" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-trait", "chrono", @@ -2938,8 +2973,8 @@ dependencies = [ [[package]] name = "kamu-task-system-inmem" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "async-stream", "async-trait", @@ -3076,6 +3111,12 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "like" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7281e4b2b1a1fae03463a7c49dd21464de50251a450f6da9715c40c7b21a70" + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -3204,9 +3245,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi", @@ -3260,9 +3301,9 @@ dependencies = [ [[package]] name = "multimap" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70db9248a93dc36a36d9a47898caa007a32755c7ad140ec64eeeb50d5a730631" +checksum = "e1a5d38b9b352dbd913288736af36af41c48d61b1a8cd34bcecd727561b7d511" dependencies = [ "serde", ] @@ -3392,7 +3433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.5", "bytes", "chrono", "futures", @@ -3422,8 +3463,8 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opendatafabric" -version = "0.145.3" -source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.3#84f685eae1e79e1f4e174eefb5c9d4abd1aa4918" +version = "0.145.5" +source = "git+https://github.com/kamu-data/kamu-cli?tag=v0.145.5#de67ad14000128a13073c18aebf4a363ff3ab7b1" dependencies = [ "bs58", "byteorder", @@ -3529,7 +3570,7 @@ version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad2cba786ae07da4d73371a88b9e0f9d3ffac1a9badc83922e0e15814f5c5fa" dependencies = [ - "ahash 0.8.3", + "ahash", "arrow-array", "arrow-buffer", "arrow-cast", @@ -3537,13 +3578,13 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.4", + "base64 0.21.5", "brotli", "bytes", "chrono", "flate2", "futures", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "lz4", "num", "num-bigint", @@ -3618,9 +3659,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c022f1e7b65d6a24c0dbbd5fb344c66881bc01f3e5ae74a1c8100f2f985d98a4" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" dependencies = [ "memchr", "thiserror", @@ -3629,9 +3670,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35513f630d46400a977c4cb58f78e1bfbe01434316e60c37d27b9ad6139c66d8" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" dependencies = [ "pest", "pest_generator", @@ -3639,9 +3680,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9fc1b9e7057baba189b5c626e2d6f40681ae5b6eb064dc7c7834101ec8123a" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" dependencies = [ "pest", "pest_meta", @@ -3652,9 +3693,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" dependencies = [ "once_cell", "pest", @@ -3856,12 +3897,6 @@ dependencies = [ "toml_edit", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.69" @@ -4044,7 +4079,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "async-compression", - "base64 0.21.4", + "base64 0.21.5", "bytes", "encoding_rs", "futures-core", @@ -4053,7 +4088,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.1", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -4061,7 +4096,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls 0.21.8", "rustls-pemfile", "serde", "serde_json", @@ -4107,9 +4142,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.4" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce3045ffa7c981a6ee93f640b538952e155f1ae3a1a02b84547fc7a56b7059a" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" dependencies = [ "cc", "getrandom", @@ -4145,9 +4180,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.19" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ "bitflags 2.4.1", "errno", @@ -4170,12 +4205,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" dependencies = [ "log", - "ring 0.16.20", + "ring 0.17.5", "rustls-webpki", "sct", ] @@ -4198,17 +4233,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -4249,12 +4284,12 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -4294,18 +4329,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", @@ -4335,9 +4370,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" dependencies = [ "serde", ] @@ -4360,7 +4395,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "chrono", "hex", "indexmap 1.9.3", @@ -4385,9 +4420,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" dependencies = [ "indexmap 2.0.2", "itoa", @@ -4526,9 +4561,9 @@ checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -4536,9 +4571,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys", @@ -4612,12 +4647,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strum" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" - [[package]] name = "strum" version = "0.25.0" @@ -4890,7 +4919,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.5", "tokio-macros", "windows-sys", ] @@ -4933,7 +4962,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls 0.21.8", "tokio", ] @@ -4962,16 +4991,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", "futures-util", - "hashbrown 0.12.3", + "hashbrown 0.14.2", "pin-project-lite", "tokio", "tracing", @@ -4991,9 +5020,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] @@ -5019,7 +5048,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.4", + "base64 0.21.5", "bytes", "futures-core", "futures-util", @@ -5092,9 +5121,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", "pin-project-lite", @@ -5130,7 +5159,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5c266b9ac83dedf0e0385ad78514949e6d89491269e7065bee51d2bb8ec7373" dependencies = [ - "ahash 0.8.3", + "ahash", "gethostname", "log", "serde", @@ -5154,12 +5183,12 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] @@ -5181,9 +5210,9 @@ dependencies = [ [[package]] name = "trust-dns-proto" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "559ac980345f7f5020883dd3bcacf176355225e01916f8c2efecad7534f682c6" +checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" dependencies = [ "async-trait", "cfg-if", @@ -5206,9 +5235,9 @@ dependencies = [ [[package]] name = "trust-dns-resolver" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c723b0e608b24ad04c73b2607e0241b2c98fd79795a95e98b068b6966138a29d" +checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" dependencies = [ "cfg-if", "futures-util", @@ -5512,7 +5541,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.4", + "ring 0.17.5", "untrusted 0.9.0", ] @@ -5683,6 +5712,26 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "zerocopy" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81ba595b9f2772fbee2312de30eeb80ec773b4cb2f1e8098db024afadda6c06f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "772666c41fb6dceaf520b564b962d738a8e1a83b41bd48945f50837aed78bb1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", +] + [[package]] name = "zeroize" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 9dedf4dd..dd75fb9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,21 +10,23 @@ resolver = "2" [workspace.dependencies] # Utils -container-runtime = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } -internal-error = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } +container-runtime = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } +internal-error = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } # Domain -opendatafabric = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } -kamu-task-system = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } +opendatafabric = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } +kamu-task-system = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } # Infra -kamu = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } -kamu-task-system-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } -kamu-adapter-graphql = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } -kamu-adapter-http = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } -kamu-adapter-oauth = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } -kamu-adapter-auth-oso = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.3", version = "0.145.3", default-features = false } +kamu = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } +kamu-task-system-inmem = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } +kamu-adapter-graphql = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } +kamu-adapter-http = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } +kamu-adapter-oauth = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } +kamu-adapter-auth-oso = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } +kamu-adapter-flight-sql = { git = "https://github.com/kamu-data/kamu-cli", tag = "v0.145.5", version = "0.145.5", default-features = false } + [workspace.package] -version = "0.8.3" +version = "0.9.0" edition = "2021" homepage = "https://github.com/kamu-data/kamu-platform" repository = "https://github.com/kamu-data/kamu-platform" @@ -53,13 +55,17 @@ debug = 2 # We don't emit info for dependencies as this significantly increases binary size. # See: https://doc.rust-lang.org/cargo/reference/profiles.html#debug [profile.release.package] +container-runtime = { debug = 1 } +internal-error = { debug = 1 } opendatafabric = { debug = 1 } +kamu-task-system = { debug = 1 } kamu = { debug = 1 } +kamu-task-system-inmem = { debug = 1 } kamu-adapter-graphql = { debug = 1 } kamu-adapter-http = { debug = 1 } kamu-adapter-oauth = { debug = 1 } kamu-adapter-auth-oso = { debug = 1 } -kamu-api-server = { debug = 1 } +kamu-adapter-flight-sql = { debug = 1 } [profile.ci] diff --git a/LICENSE.txt b/LICENSE.txt index 08e92a0b..10231d4f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -24,7 +24,7 @@ Additional Use Grant: You may use the Licensed Work for any purpose, Licensed Work where data or transformations are controlled by such third parties. -Change Date: 2027-07-27 +Change Date: 2027-10-26 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index ef67eb20..41bc32b4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Prerequisites: To run API server using local `kamu` workspace: ```bash -cargo run -- --repo-url workspace/.kamu/datasets run | bunyan +cargo run --bin kamu-api-server -- --repo-url workspace/.kamu/datasets run | bunyan ``` To control log verbosity use the standard `RUST_LOG` env var: @@ -34,7 +34,7 @@ To explore GQL schema run server and open http://127.0.0.1:8080/playground. To test GQL queries from the CLI: ```bash -cargo run -- gql query '{ apiVersion }' | jq +cargo run --bin kamu-api-server -- gql query '{ apiVersion }' | jq ``` @@ -43,7 +43,7 @@ cargo run -- gql query '{ apiVersion }' | jq To use it: ```bash -cargo run -- --repo-url s3://example.com/kamu_repo run | bunyan +cargo run --bin kamu-api-server -- --repo-url s3://example.com/kamu_repo run | bunyan ``` diff --git a/deny.toml b/deny.toml index b6ff1870..9d4f54f3 100644 --- a/deny.toml +++ b/deny.toml @@ -72,9 +72,5 @@ unsound = "warn" yanked = "deny" notice = "warn" ignore = [ - # https://rustsec.org/advisories/RUSTSEC-2020-0071 - # Ignoring it until `chrono` updates to new version of `time` crate. - # We are not affected by this voulnerability as `chrono` doesn't call the affected code. - "RUSTSEC-2020-0071", ] diff --git a/src/app/api-server/Cargo.toml b/src/app/api-server/Cargo.toml index e28c8be7..fe8475cc 100644 --- a/src/app/api-server/Cargo.toml +++ b/src/app/api-server/Cargo.toml @@ -24,12 +24,12 @@ kamu-adapter-graphql = { workspace = true } kamu-adapter-http = { workspace = true } kamu-adapter-oauth = { workspace = true } kamu-adapter-auth-oso = { workspace = true } +kamu-adapter-flight-sql = { workspace = true } # APIs http = "0.2" hyper = "0.14" -tower = { version = "0.4" } -tower-http = { version = "0.4", features = ["trace", "cors"] } +arrow-flight = { version = "46", features = ["flight-sql-experimental"] } axum = { version = "0.6", features = ["ws"] } axum-extra = { version = "0.8", features = ["async-read-body"] } async-graphql = { version = "6", features = [ @@ -38,6 +38,9 @@ async-graphql = { version = "6", features = [ "apollo_tracing", ] } async-graphql-axum = "6" +tonic = { version = "0.9", default-features = false } +tower = { version = "0.4" } +tower-http = { version = "0.4", features = ["trace", "cors"] } # Tracing / logging / telemetry tracing = "0.1" @@ -55,12 +58,14 @@ tracing-bunyan-formatter = "0.3" async-trait = { version = "0.1", default-features = false } chrono = "0.4" clap = "4" +datafusion = "31" futures = "0.3" indoc = "2" serde = "1" serde_json = "1" tempfile = "3" tokio = { version = "1", default-features = false, features = ["macros"] } +tokio-stream = { version = "0.1", default-features = false, features = ["net"] } tokio-util = { version = "0.7", default-features = false, features = ["rt"] } thiserror = { version = "=1.0.44", default-features = false } # TODO: Pinned until rustc upgrade url = "2" diff --git a/src/app/api-server/src/app.rs b/src/app/api-server/src/app.rs index 83d2a513..561cb62a 100644 --- a/src/app/api-server/src/app.rs +++ b/src/app/api-server/src/app.rs @@ -72,15 +72,28 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> { _ => unimplemented!(), }, Some(("run", sub)) => { - let server = crate::http_server::build_server( - sub.get_one("address").map(|a| *a), + let address = sub + .get_one::("address") + .map(|a| *a) + .unwrap_or(std::net::Ipv4Addr::new(127, 0, 0, 1).into()); + + let http_server = crate::http_server::build_server( + address, sub.get_one("http-port").map(|p| *p), catalog.clone(), should_use_multi_tenancy(&repo_url), ); + let flightsql_server = crate::flightsql_server::FlightSqlServer::new( + address, + sub.get_one("flightsql-port").map(|p| *p), + catalog.clone(), + ) + .await; + tracing::info!( - http_endpoint = format!("http://{}", server.local_addr()), + http_endpoint = format!("http://{}", http_server.local_addr()), + flightsql_endpoint = format!("flightsql://{}", flightsql_server.local_addr()), "Serving traffic" ); @@ -89,7 +102,8 @@ pub async fn run(matches: clap::ArgMatches) -> Result<(), InternalError> { .unwrap(); tokio::select! { - res = server => { res.int_err() }, + res = http_server => { res.int_err() }, + res = flightsql_server.run() => { res.int_err() }, res = task_executor.run() => { res.int_err() }, } } diff --git a/src/app/api-server/src/cli_parser.rs b/src/app/api-server/src/cli_parser.rs index eafaaba1..97f217e0 100644 --- a/src/app/api-server/src/cli_parser.rs +++ b/src/app/api-server/src/cli_parser.rs @@ -38,6 +38,10 @@ pub fn cli() -> Command { .long("http-port") .value_parser(value_parser!(u16)) .help("Expose HTTP server on specific port"), + Arg::new("flightsql-port") + .long("flightsql-port") + .value_parser(value_parser!(u16)) + .help("Expose Flight SQL server on specific port"), ]), Command::new("gql") .about("GraphQL related command group") diff --git a/src/app/api-server/src/flightsql_server.rs b/src/app/api-server/src/flightsql_server.rs new file mode 100644 index 00000000..660ca002 --- /dev/null +++ b/src/app/api-server/src/flightsql_server.rs @@ -0,0 +1,100 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +///////////////////////////////////////////////////////////////////////////////////////// + +use std::net::SocketAddr; +use std::sync::Arc; + +use arrow_flight::flight_service_server::FlightServiceServer; +use datafusion::prelude::SessionContext; +use futures::Future; +use kamu::domain::CurrentAccountSubject; +use kamu_adapter_flight_sql::{KamuFlightSqlService, SessionFactory, Token}; +use tokio::net::TcpListener; +use tonic::transport::Server; +use tonic::Status; + +///////////////////////////////////////////////////////////////////////////////////////// + +pub(crate) struct FlightSqlServer { + service: KamuFlightSqlService, + listener: TcpListener, +} + +///////////////////////////////////////////////////////////////////////////////////////// + +impl FlightSqlServer { + pub async fn new( + address: std::net::IpAddr, + port: Option, + base_catalog: dill::Catalog, + ) -> Self { + let service = kamu_adapter_flight_sql::KamuFlightSqlService::builder() + .with_server_name(crate::BINARY_NAME, crate::VERSION) + .with_session_factory(Arc::new(SessionFactoryImpl { base_catalog })) + .build(); + + let listener = TcpListener::bind((address, port.unwrap_or_default())) + .await + .unwrap(); + + Self { service, listener } + } + + pub fn local_addr(&self) -> SocketAddr { + self.listener.local_addr().unwrap() + } + + pub fn run(self) -> impl Future> { + Server::builder() + .add_service(FlightServiceServer::new(self.service)) + .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new( + self.listener, + )) + } +} + +///////////////////////////////////////////////////////////////////////////////////////// + +struct SessionFactoryImpl { + base_catalog: dill::Catalog, +} + +#[async_trait::async_trait] +impl SessionFactory for SessionFactoryImpl { + #[tracing::instrument(level = "debug", skip_all, fields(username))] + async fn authenticate(&self, username: &str, password: &str) -> Result { + // TODO: SEC: Real auth via app token + if username == "kamu" && password == "kamu" { + Ok(String::new()) + } else { + Err(Status::unauthenticated("Invalid credentials!")) + } + } + + #[tracing::instrument(level = "debug", skip_all)] + async fn get_context(&self, _token: &Token) -> Result, Status> { + let subject = CurrentAccountSubject::Anonymous( + kamu::domain::AnonymousAccountReason::NoAuthenticationProvided, + ); + + let session_catalog = dill::CatalogBuilder::new_chained(&self.base_catalog) + .add_value(subject) + .build(); + + let query_svc = session_catalog + .get_one::() + .unwrap(); + + let ctx = Arc::new(query_svc.create_session().await.unwrap()); + + Ok(ctx) + } +} diff --git a/src/app/api-server/src/http_server.rs b/src/app/api-server/src/http_server.rs index fab247c8..c5ea82fc 100644 --- a/src/app/api-server/src/http_server.rs +++ b/src/app/api-server/src/http_server.rs @@ -9,14 +9,14 @@ ///////////////////////////////////////////////////////////////////////////////////////// +use std::net::SocketAddr; + pub(crate) fn build_server( - address: Option, + address: std::net::IpAddr, http_port: Option, catalog: dill::Catalog, multi_tenant_workspace: bool, ) -> axum::Server> { - use std::net::{IpAddr, Ipv4Addr, SocketAddr}; - let gql_schema = kamu_adapter_graphql::schema(); let app = axum::Router::new() @@ -54,10 +54,7 @@ pub(crate) fn build_server( .layer(kamu_adapter_http::AuthenticationLayer::new()), ); - let addr = SocketAddr::from(( - address.unwrap_or(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))), - http_port.unwrap_or(0), - )); + let addr = SocketAddr::from((address, http_port.unwrap_or(0))); axum::Server::bind(&addr).serve(app.into_make_service()) } diff --git a/src/app/api-server/src/lib.rs b/src/app/api-server/src/lib.rs index 62b8d132..2f2381c2 100644 --- a/src/app/api-server/src/lib.rs +++ b/src/app/api-server/src/lib.rs @@ -10,6 +10,7 @@ pub(crate) mod app; pub(crate) mod builtin_authentication_provider; pub(crate) mod cli_parser; +pub(crate) mod flightsql_server; pub(crate) mod gql_server; pub(crate) mod http_server;