diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1189a4dc800e..ff73a39ef0a0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,7 +89,6 @@ jobs: hydroflow hydroflow_lang hydroflow_macro hydroflow_plus hydroflow_datalog hydroflow_datalog_core hydro_deploy hydro_cli hydroflow_deploy_integration - hydroflow_plus_deploy stageleft stageleft_macro stageleft_tool multiplatform_test env: diff --git a/Cargo.lock b/Cargo.lock index 28c15037ea84..dbd0f1aa4a07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -778,6 +778,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +dependencies = [ + "quote", + "syn 2.0.75", +] + [[package]] name = "ctrlc" version = "3.4.5" @@ -1433,28 +1443,19 @@ dependencies = [ name = "hydroflow_plus" version = "0.9.0" dependencies = [ + "async-ssh2-lite", "bincode", + "ctor", + "hydro_deploy", "hydroflow", "hydroflow_lang", "insta", + "nameof", + "prettyplease", "proc-macro-crate", "proc-macro2", "quote", "serde", - "stageleft", - "stageleft_tool", - "syn 2.0.75", -] - -[[package]] -name = "hydroflow_plus_deploy" -version = "0.9.0" -dependencies = [ - "hydro_deploy", - "hydroflow_plus", - "nameof", - "prettyplease", - "serde", "sha2", "stageleft", "stageleft_tool", @@ -1471,7 +1472,6 @@ dependencies = [ "futures", "hydro_deploy", "hydroflow_plus", - "hydroflow_plus_deploy", "insta", "rand", "serde", diff --git a/Cargo.toml b/Cargo.toml index 4c0713b657a9..668519191c00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,6 @@ members = [ "hydro_deploy/hydro_cli", "hydro_deploy/hydro_cli_examples", "hydro_deploy/hydroflow_deploy_integration", - "hydro_deploy/hydroflow_plus_deploy", "hydroflow", "hydroflow_datalog", "hydroflow_datalog_core", diff --git a/docs/docs/hydroflow_plus/quickstart/clusters.mdx b/docs/docs/hydroflow_plus/quickstart/clusters.mdx index 454c636a2883..240b8fd5e153 100644 --- a/docs/docs/hydroflow_plus/quickstart/clusters.mdx +++ b/docs/docs/hydroflow_plus/quickstart/clusters.mdx @@ -59,7 +59,7 @@ To deploy this application, we must set up the Hydro Deploy configuration as bef use std::cell::RefCell; use hydro_deploy::{Deployment, HydroflowCrate}; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; #[tokio::main] async fn main() { diff --git a/hydro_deploy/core/Cargo.toml b/hydro_deploy/core/Cargo.toml index 0c754f3c9049..f74fc73fa066 100644 --- a/hydro_deploy/core/Cargo.toml +++ b/hydro_deploy/core/Cargo.toml @@ -37,4 +37,3 @@ tempfile = "3.0.0" tokio = { version = "1.29.0", features = [ "full" ] } tokio-stream = { version = "0.1.3", default-features = false } tokio-util = { version = "0.7.5", features = [ "compat", "io-util" ] } - diff --git a/hydro_deploy/hydroflow_plus_deploy/CHANGELOG.md b/hydro_deploy/hydroflow_plus_deploy/CHANGELOG.md deleted file mode 100644 index 615125a01968..000000000000 --- a/hydro_deploy/hydroflow_plus_deploy/CHANGELOG.md +++ /dev/null @@ -1,358 +0,0 @@ - - -## 0.9.0 (2024-08-30) - - - -### New Features - - - use trybuild to compile subgraph binaries - - Add end-to-end flamegraph generation for macos and linux localhost, fix #1351 - -### Refactor (BREAKING) - - - disentangle instantiated nodes from locations - - simplify process/cluster specs - --- - [//]: # (BEGIN SAPLING FOOTER) - Stack created with [Sapling](https://sapling-scm.com). Best reviewed - with - [ReviewStack](https://reviewstack.dev/hydro-project/hydroflow/pull/1394). - * #1395 - * __->__ #1394 - - defer network instantiation until after finalizing IR - --- - [//]: # (BEGIN SAPLING FOOTER) - Stack created with [Sapling](https://sapling-scm.com). Best reviewed - with - [ReviewStack](https://reviewstack.dev/hydro-project/hydroflow/pull/1377). - * #1395 - * #1394 - * __->__ #1377 - -### Style (BREAKING) - - - rename some `CLI`->`Deploy`, decapitalize acronym names - -### Refactor (BREAKING) - - - rename integration crates to drop CLI references - -### Style - - - use `name_of!` macro - -### Other - - - update `RELEASING.md` notes, prep for release, wip - -### Bug Fixes - - - remove `FlowProps` - -### Chore - - - manually set versions for crates renamed in #1413 - - lower min dependency versions where possible, update `Cargo.lock` - Moved from #1418 - - --------- - - simplify process/cluster specs - --- - [//]: # (BEGIN SAPLING FOOTER) - Stack created with [Sapling](https://sapling-scm.com). Best reviewed - with - [ReviewStack](https://reviewstack.dev/hydro-project/hydroflow/pull/1394). - * #1395 - * __->__ #1394 - - defer network instantiation until after finalizing IR - --- - [//]: # (BEGIN SAPLING FOOTER) - Stack created with [Sapling](https://sapling-scm.com). Best reviewed - with - [ReviewStack](https://reviewstack.dev/hydro-project/hydroflow/pull/1377). - * #1395 - * #1394 - * __->__ #1377 - - defer network instantiation until after finalizing IR - --- - [//]: # (BEGIN SAPLING FOOTER) - Stack created with [Sapling](https://sapling-scm.com). Best reviewed - with - [ReviewStack](https://reviewstack.dev/hydro-project/hydroflow/pull/1377). - * #1395 - * #1394 - * __->__ #1377 - -### Pre-Move Commit Statistics - - - - - 4 commits contributed to the release over the course of 7 calendar days. - - 30 days passed between releases. - - 4 commits were understood as [conventional](https://www.conventionalcommits.org). - - 4 unique issues were worked on: [#1377](https://github.com/hydro-project/hydroflow/issues/1377), [#1394](https://github.com/hydro-project/hydroflow/issues/1394), [#1395](https://github.com/hydro-project/hydroflow/issues/1395), [#1398](https://github.com/hydro-project/hydroflow/issues/1398) - -### Pre-Move Commit Details - - - -
view details - - * **[#1377](https://github.com/hydro-project/hydroflow/issues/1377)** - - Defer network instantiation until after finalizing IR ([`0eba702`](https://github.com/hydro-project/hydroflow/commit/0eba702f62e7a6816cf931b01a2ea5643bd7321d)) - * **[#1394](https://github.com/hydro-project/hydroflow/issues/1394)** - - Simplify process/cluster specs ([`128aaec`](https://github.com/hydro-project/hydroflow/commit/128aaecd40edce57dc254afdcd61ecd5b9948d71)) - * **[#1395](https://github.com/hydro-project/hydroflow/issues/1395)** - - Disentangle instantiated nodes from locations ([`5f2789a`](https://github.com/hydro-project/hydroflow/commit/5f2789a13d1602f170e678fe9bbc59caf69db4b5)) - * **[#1398](https://github.com/hydro-project/hydroflow/issues/1398)** - - Use trybuild to compile subgraph binaries ([`46a8a2c`](https://github.com/hydro-project/hydroflow/commit/46a8a2cb08732bb21096e824bc4542d208c68fb2)) -
- -### Commit Statistics - - - - - 8 commits contributed to the release. - - 8 commits were understood as [conventional](https://www.conventionalcommits.org). - - 6 unique issues were worked on: [#1396](https://github.com/hydro-project/hydroflow/issues/1396), [#1399](https://github.com/hydro-project/hydroflow/issues/1399), [#1413](https://github.com/hydro-project/hydroflow/issues/1413), [#1420](https://github.com/hydro-project/hydroflow/issues/1420), [#1423](https://github.com/hydro-project/hydroflow/issues/1423), [#1429](https://github.com/hydro-project/hydroflow/issues/1429) - -### Commit Details - - - -
view details - - * **[#1396](https://github.com/hydro-project/hydroflow/issues/1396)** - - Add end-to-end flamegraph generation for macos and linux localhost, fix #1351 ([`6568263`](https://github.com/hydro-project/hydroflow/commit/6568263e03899d4e96837690e6e59284c194d7ff)) - * **[#1399](https://github.com/hydro-project/hydroflow/issues/1399)** - - Rename some `CLI`->`Deploy`, decapitalize acronym names ([`fa41720`](https://github.com/hydro-project/hydroflow/commit/fa417205569d8c49c85b0c2324118e0f9b1c8407)) - * **[#1413](https://github.com/hydro-project/hydroflow/issues/1413)** - - Rename integration crates to drop CLI references ([`0a465e5`](https://github.com/hydro-project/hydroflow/commit/0a465e55dd39c76bc1aefb020460a639d792fe87)) - * **[#1420](https://github.com/hydro-project/hydroflow/issues/1420)** - - Remove `FlowProps` ([`22c7218`](https://github.com/hydro-project/hydroflow/commit/22c72189bb76412955d29b03c5d99894c558a07c)) - * **[#1423](https://github.com/hydro-project/hydroflow/issues/1423)** - - Lower min dependency versions where possible, update `Cargo.lock` ([`11af328`](https://github.com/hydro-project/hydroflow/commit/11af32828bab6e4a4264d2635ff71a12bb0bb778)) - * **[#1429](https://github.com/hydro-project/hydroflow/issues/1429)** - - Use `name_of!` macro ([`3fde68d`](https://github.com/hydro-project/hydroflow/commit/3fde68d0db0414017cfb771a218b14b8f57d1686)) - * **Uncategorized** - - Manually set versions for crates renamed in #1413 ([`a2ec110`](https://github.com/hydro-project/hydroflow/commit/a2ec110ccadb97e293b19d83a155d98d94224bba)) - - Update `RELEASING.md` notes, prep for release, wip ([`c41787f`](https://github.com/hydro-project/hydroflow/commit/c41787f527859cb9d704736ecdea5ca7bc641460)) -
- -## v0.8.0 (2024-07-23) - - - - - - -### Refactor (BREAKING) - - - make `Host` trait use `&self` interior mutability to remove `RwLock` wrappings #430 - Depends on #1346 - - make `HydroflowSource`, `HydroflowSink` traits use `&self` interior mutability to remove `RwLock` wrappings #430 - Depends on #1339 - - replace `async-channel` with `tokio::sync::mpsc::unbounded_channel` - Depends on #1339 - - We could make the publicly facing `stdout`, `stderr` APIs return `impl Stream` in the future, maybe - - replace some uses of `tokio::sync::RwLock` with `std::sync::Mutex` #430 (3/3) - -### Pre-Move Commit Statistics - - - - - 5 commits contributed to the release over the course of 4 calendar days. - - 59 days passed between releases. - - 4 commits were understood as [conventional](https://www.conventionalcommits.org). - - 4 unique issues were worked on: [#1339](https://github.com/hydro-project/hydroflow/issues/1339), [#1346](https://github.com/hydro-project/hydroflow/issues/1346), [#1347](https://github.com/hydro-project/hydroflow/issues/1347), [#1356](https://github.com/hydro-project/hydroflow/issues/1356) - -### Pre-Move Commit Details - - - -
view details - - * **[#1339](https://github.com/hydro-project/hydroflow/issues/1339)** - - Replace some uses of `tokio::sync::RwLock` with `std::sync::Mutex` #430 (3/3) ([`141eae1`](https://github.com/hydro-project/hydroflow/commit/141eae1c3a1869fa42756250618a21ea2a2c7e34)) - * **[#1346](https://github.com/hydro-project/hydroflow/issues/1346)** - - Make `HydroflowSource`, `HydroflowSink` traits use `&self` interior mutability to remove `RwLock` wrappings #430 ([`057a0a5`](https://github.com/hydro-project/hydroflow/commit/057a0a510568cf81932368c8c65e056f91af7202)) - * **[#1347](https://github.com/hydro-project/hydroflow/issues/1347)** - - Make `Host` trait use `&self` interior mutability to remove `RwLock` wrappings #430 ([`c5a8de2`](https://github.com/hydro-project/hydroflow/commit/c5a8de28e7844b3c29d58116d8340967f2e6bcc4)) - * **[#1356](https://github.com/hydro-project/hydroflow/issues/1356)** - - Replace `async-channel` with `tokio::sync::mpsc::unbounded_channel` ([`6039078`](https://github.com/hydro-project/hydroflow/commit/60390782dd7dcec18d193c800af716843a944dba)) - * **Uncategorized** - - Release hydroflow_lang v0.8.0, hydroflow_datalog_core v0.8.0, hydroflow_datalog v0.8.0, hydroflow_macro v0.8.0, lattices_macro v0.5.5, lattices v0.5.6, variadics v0.0.5, pusherator v0.0.7, hydroflow v0.8.0, hydroflow_plus v0.8.0, hydro_deploy v0.8.0, hydro_cli v0.8.0, hydroflow_plus_cli_integration v0.8.0, safety bump 7 crates ([`ca6c16b`](https://github.com/hydro-project/hydroflow/commit/ca6c16b4a7ce35e155fe7fc6c7d1676c37c9e4de)) -
- -## v0.7.0 (2024-05-24) - - - -### Chore - - - use workaround for `cargo smart-release` not properly ordering `dev-`/`build-dependencies` - -### New Features - - - add API to get the cluster ID of the current node - feat(hydroflow_plus): add API to get the cluster ID of the current node - -### Pre-Move Commit Statistics - - - - - 3 commits contributed to the release over the course of 9 calendar days. - - 44 days passed between releases. - - 2 commits were understood as [conventional](https://www.conventionalcommits.org). - - 2 unique issues were worked on: [#1194](https://github.com/hydro-project/hydroflow/issues/1194), [#1238](https://github.com/hydro-project/hydroflow/issues/1238) - -### Pre-Move Commit Details - - - -
view details - - * **[#1194](https://github.com/hydro-project/hydroflow/issues/1194)** - - Add API to get the cluster ID of the current node ([`6e57172`](https://github.com/hydro-project/hydroflow/commit/6e571726ff40818fbe9bbe9923511877c20fb243)) - * **[#1238](https://github.com/hydro-project/hydroflow/issues/1238)** - - Use workaround for `cargo smart-release` not properly ordering `dev-`/`build-dependencies` ([`c9dfddc`](https://github.com/hydro-project/hydroflow/commit/c9dfddc680e0ce5415539d7b77bc5beb97ab59d9)) - * **Uncategorized** - - Release hydroflow_lang v0.7.0, hydroflow_datalog_core v0.7.0, hydroflow_datalog v0.7.0, hydroflow_macro v0.7.0, lattices v0.5.5, multiplatform_test v0.1.0, pusherator v0.0.6, hydroflow v0.7.0, stageleft_macro v0.2.0, stageleft v0.3.0, stageleft_tool v0.2.0, hydroflow_plus v0.7.0, hydro_deploy v0.7.0, hydro_cli v0.7.0, hydroflow_plus_cli_integration v0.7.0, safety bump 8 crates ([`2852147`](https://github.com/hydro-project/hydroflow/commit/285214740627685e911781793e05d234ab2ad2bd)) -
- -## v0.6.1 (2024-04-09) - -### New Features - - - simplify Location trait to remove lifetimes - - simplify lifetime bounds for processes and clusters - feat(hydroflow_plus): simplify lifetime bounds for processes and - clusters - - This allows `extract` to move the flow builder, which is a prerequisite - for having developers run the optimizer during deployment as well in - case it changes the network topology. - -### Pre-Move Commit Statistics - - - - - 5 commits contributed to the release over the course of 18 calendar days. - - 38 days passed between releases. - - 2 commits were understood as [conventional](https://www.conventionalcommits.org). - - 2 unique issues were worked on: [#1100](https://github.com/hydro-project/hydroflow/issues/1100), [#1101](https://github.com/hydro-project/hydroflow/issues/1101) - -### Pre-Move Commit Details - - - -
view details - - * **[#1100](https://github.com/hydro-project/hydroflow/issues/1100)** - - Simplify lifetime bounds for processes and clusters ([`77f3e5a`](https://github.com/hydro-project/hydroflow/commit/77f3e5afb9e276d1d6c643574ebac75ed0003939)) - * **[#1101](https://github.com/hydro-project/hydroflow/issues/1101)** - - Simplify Location trait to remove lifetimes ([`7f68ebf`](https://github.com/hydro-project/hydroflow/commit/7f68ebf2a23e8e73719229a6f0408bffc7fbe7af)) - * **Uncategorized** - - Release hydroflow_plus v0.6.1, hydro_deploy v0.6.1, hydro_cli v0.6.1, hydroflow_plus_cli_integration v0.6.1 ([`c385c13`](https://github.com/hydro-project/hydroflow/commit/c385c132c9733d1bace82156aa14216b8e7fef9f)) - - Release hydroflow_lang v0.6.2, hydroflow v0.6.2, hydroflow_plus v0.6.1, hydro_deploy v0.6.1, hydro_cli v0.6.1, hydroflow_plus_cli_integration v0.6.1, stageleft_tool v0.1.1 ([`23cfe08`](https://github.com/hydro-project/hydroflow/commit/23cfe0839079aa17d042bbd3976f6d188689d290)) - - Release hydroflow_cli_integration v0.5.2, hydroflow_lang v0.6.1, hydroflow_datalog_core v0.6.1, lattices v0.5.4, hydroflow v0.6.1, stageleft_macro v0.1.1, stageleft v0.2.1, hydroflow_plus v0.6.1, hydro_deploy v0.6.1, hydro_cli v0.6.1, hydroflow_plus_cli_integration v0.6.1, stageleft_tool v0.1.1 ([`cd63f22`](https://github.com/hydro-project/hydroflow/commit/cd63f2258c961a40f0e5dbef20ac329a2d570ad0)) -
- -## v0.6.0 (2024-03-02) - -### New Features - - - unify send/demux/tagged APIs - feat(hydroflow_plus): unify send/demux/tagged APIs - - use an IR before lowering to Hydroflow - Makes it possible to write custom optimization passes. - -### Pre-Move Commit Statistics - - - - - 3 commits contributed to the release over the course of 2 calendar days. - - 28 days passed between releases. - - 2 commits were understood as [conventional](https://www.conventionalcommits.org). - - 2 unique issues were worked on: [#1070](https://github.com/hydro-project/hydroflow/issues/1070), [#1080](https://github.com/hydro-project/hydroflow/issues/1080) - -### Pre-Move Commit Details - - - -
view details - - * **[#1070](https://github.com/hydro-project/hydroflow/issues/1070)** - - Use an IR before lowering to Hydroflow ([`eb34ccd`](https://github.com/hydro-project/hydroflow/commit/eb34ccd13f56e1d07cbae35ead79daeb3b9bad20)) - * **[#1080](https://github.com/hydro-project/hydroflow/issues/1080)** - - Unify send/demux/tagged APIs ([`c1d1b51`](https://github.com/hydro-project/hydroflow/commit/c1d1b51ee26cc9946af59ac02c040e0a33d15fde)) - * **Uncategorized** - - Release hydroflow_lang v0.6.0, hydroflow_datalog_core v0.6.0, hydroflow_datalog v0.6.0, hydroflow_macro v0.6.0, lattices v0.5.3, variadics v0.0.4, pusherator v0.0.5, hydroflow v0.6.0, stageleft v0.2.0, hydroflow_plus v0.6.0, hydro_deploy v0.6.0, hydro_cli v0.6.0, hydroflow_plus_cli_integration v0.6.0, safety bump 7 crates ([`09ea65f`](https://github.com/hydro-project/hydroflow/commit/09ea65fe9cd45c357c43bffca30e60243fa45cc8)) -
- -## v0.5.1 (2024-02-03) - - - - - -### Chore - - - prep for initial release - > it contains the logic linking hydroflow+ to deploy, it should be published that’s a bug - - manually set lockstep-versioned crates (and `lattices`) to version `0.5.1` - Setting manually since - https://github.com/frewsxcv/rust-crates-index/issues/159 is messing with - smart-release - -### Chore - - - fix/add releasing for `hydroflow_plus_cli_integration`, `stageleft[_macro/_tool]` - -### New Features - - - add APIs for declaring external ports on clusters - - improve API naming and polish docs - - pass subgraph ID through deploy metadata - - improve API naming and eliminate wire API for builders - - improve Rust API for defining services - - split Rust core from Python bindings - -### Pre-Move Commit Statistics - - - - - 11 commits contributed to the release over the course of 43 calendar days. - - 9 commits were understood as [conventional](https://www.conventionalcommits.org). - - 6 unique issues were worked on: [#1013](https://github.com/hydro-project/hydroflow/issues/1013), [#1056](https://github.com/hydro-project/hydroflow/issues/1056), [#986](https://github.com/hydro-project/hydroflow/issues/986), [#987](https://github.com/hydro-project/hydroflow/issues/987), [#995](https://github.com/hydro-project/hydroflow/issues/995), [#996](https://github.com/hydro-project/hydroflow/issues/996) - -### Pre-Move Commit Details - - - -
view details - - * **[#1013](https://github.com/hydro-project/hydroflow/issues/1013)** - - Improve API naming and polish docs ([`6eeb9be`](https://github.com/hydro-project/hydroflow/commit/6eeb9be9bc4136041a2855f650ae640c478b7fc9)) - * **[#1056](https://github.com/hydro-project/hydroflow/issues/1056)** - - Prep for initial release ([`e9c7ced`](https://github.com/hydro-project/hydroflow/commit/e9c7ced8760f88e3215a4b1b4e23f8b9db159a84)) - * **[#986](https://github.com/hydro-project/hydroflow/issues/986)** - - Split Rust core from Python bindings ([`c50ca12`](https://github.com/hydro-project/hydroflow/commit/c50ca121b6d5e30dc07843f82caa135b68626301)) - * **[#987](https://github.com/hydro-project/hydroflow/issues/987)** - - Improve Rust API for defining services ([`53d7aee`](https://github.com/hydro-project/hydroflow/commit/53d7aee8dcc574d47864ec89bfea30a82eab0ee7)) - * **[#995](https://github.com/hydro-project/hydroflow/issues/995)** - - Improve API naming and eliminate wire API for builders ([`b7aafd3`](https://github.com/hydro-project/hydroflow/commit/b7aafd3c97897db4bff62c4ab0b7480ef9a799e0)) - * **[#996](https://github.com/hydro-project/hydroflow/issues/996)** - - Pass subgraph ID through deploy metadata ([`46d87fa`](https://github.com/hydro-project/hydroflow/commit/46d87fa364d3fe01422cf3c404fbc8a1d5e9fb88)) - * **Uncategorized** - - Release hydroflow_plus_cli_integration v0.5.1 ([`8202feb`](https://github.com/hydro-project/hydroflow/commit/8202febcd8ed160c23d08c37d7c13773831c97da)) - - Fix/add releasing for `hydroflow_plus_cli_integration`, `stageleft[_macro/_tool]` ([`03ce73d`](https://github.com/hydro-project/hydroflow/commit/03ce73d2eb1377179f5ebd6f80e082c2889588a7)) - - Release hydroflow_lang v0.5.2, hydroflow_datalog_core v0.5.2, hydroflow_macro v0.5.2, lattices v0.5.2, hydroflow v0.5.2, hydro_cli v0.5.1, hydroflow_plus_cli_integration v0.5.1 ([`6ac8720`](https://github.com/hydro-project/hydroflow/commit/6ac872081753548ebb8ec95549b4d820dc050d3e)) - - Manually set lockstep-versioned crates (and `lattices`) to version `0.5.1` ([`1b555e5`](https://github.com/hydro-project/hydroflow/commit/1b555e57c8c812bed4d6495d2960cbf77fb0b3ef)) - - Add APIs for declaring external ports on clusters ([`7d930a2`](https://github.com/hydro-project/hydroflow/commit/7d930a2ccf656d3d6bc5db3e22eb63c5fd6d37d1)) -
- diff --git a/hydro_deploy/hydroflow_plus_deploy/Cargo.toml b/hydro_deploy/hydroflow_plus_deploy/Cargo.toml deleted file mode 100644 index f17e98929809..000000000000 --- a/hydro_deploy/hydroflow_plus_deploy/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -name = "hydroflow_plus_deploy" -publish = true -version = "0.9.0" -edition = "2021" -license = "Apache-2.0" -documentation = "https://docs.rs/hydroflow_plus_deploy/" -description = "Library for working with hydro_deploy and hydroflow_plus" - -[lints] -workspace = true - -[features] -deploy = [ "hydro_deploy", "trybuild-internals-api", "toml", "prettyplease" ] -stageleft_devel = [] - -[dependencies] -hydroflow_plus = { path = "../../hydroflow_plus", version = "^0.9.0", features = [ "deploy_integration" ] } -nameof = "1.0.0" -serde = { version = "1.0.197", features = [ "derive" ] } -sha2 = "0.10.0" -stageleft = { path = "../../stageleft", version = "^0.4.0" } -syn = { version = "2.0.46", features = [ "parsing", "extra-traits" ] } -tokio = { version = "1.29.0", features = [ "full" ] } - -hydro_deploy = { path = "../core", version = "^0.9.0", optional = true } -prettyplease = { version = "0.2.0", features = [ "verbatim" ], optional = true } -toml = { version = "0.8.0", optional = true } -trybuild-internals-api = { version = "1.0.99", optional = true } - -# added to workaround `cargo smart-release` https://github.com/Byron/cargo-smart-release/issues/16 -stageleft_tool = { path = "../../stageleft_tool", version = "^0.3.0", optional = true } - -[build-dependencies] -stageleft_tool = { path = "../../stageleft_tool", version = "^0.3.0" } diff --git a/hydro_deploy/hydroflow_plus_deploy/build.rs b/hydro_deploy/hydroflow_plus_deploy/build.rs deleted file mode 100644 index 99775c3c7daa..000000000000 --- a/hydro_deploy/hydroflow_plus_deploy/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - stageleft_tool::gen_final!(); -} diff --git a/hydro_deploy/hydroflow_plus_deploy/src/lib.rs b/hydro_deploy/hydroflow_plus_deploy/src/lib.rs deleted file mode 100644 index 9625f38f8db7..000000000000 --- a/hydro_deploy/hydroflow_plus_deploy/src/lib.rs +++ /dev/null @@ -1,26 +0,0 @@ -stageleft::stageleft_no_entry_crate!(); - -mod runtime; -use std::collections::HashMap; - -#[cfg(feature = "deploy")] -pub(crate) mod trybuild; - -pub use runtime::*; - -#[allow(clippy::allow_attributes, unused, reason = "stageleft")] -pub(crate) mod deploy_runtime; - -#[cfg(feature = "deploy")] -mod deploy; - -#[cfg(feature = "deploy")] -pub use deploy::*; -use serde::{Deserialize, Serialize}; - -#[derive(Default, Serialize, Deserialize)] -pub struct HydroflowPlusMeta { - pub clusters: HashMap>, - pub cluster_id: Option, - pub subgraph_id: usize, -} diff --git a/hydroflow_plus/Cargo.toml b/hydroflow_plus/Cargo.toml index 8eee4219ba8c..c2ddf2a000c8 100644 --- a/hydroflow_plus/Cargo.toml +++ b/hydroflow_plus/Cargo.toml @@ -14,9 +14,10 @@ workspace = true path = "src/lib.rs" [features] +default = ["deploy"] diagnostics = [ "hydroflow_lang/diagnostics" ] -deploy_integration = [ "hydroflow/deploy_integration" ] stageleft_devel = [] +deploy = [ "hydroflow/deploy_integration", "dep:hydro_deploy", "dep:trybuild-internals-api", "dep:toml", "dep:prettyplease" ] [dependencies] quote = "1.0.35" @@ -25,15 +26,24 @@ proc-macro2 = "1.0.74" proc-macro-crate = "1.0.0" hydroflow = { path = "../hydroflow", version = "^0.9.0", default-features = false } hydroflow_lang = { path = "../hydroflow_lang", version = "^0.9.0" } -serde = "1.0.197" +serde = { version = "1.0.197", features = [ "derive" ] } bincode = "1.3.1" +tokio = { version = "1.29.0", features = [ "full" ] } stageleft = { path = "../stageleft", version = "^0.4.0" } -# added to workaround `cargo smart-release` https://github.com/Byron/cargo-smart-release/issues/16 -stageleft_tool = { path = "../stageleft_tool", version = "^0.3.0", optional = true } +nameof = "1.0.0" +sha2 = "0.10.0" +stageleft_tool = { path = "../stageleft_tool", version = "^0.3.0" } +hydro_deploy = { path = "../hydro_deploy/core", version = "^0.9.0", optional = true } +prettyplease = { version = "0.2.0", features = [ "verbatim" ], optional = true } +toml = { version = "0.8.0", optional = true } +trybuild-internals-api = { version = "1.0.99", optional = true } [build-dependencies] stageleft_tool = { path = "../stageleft_tool", version = "^0.3.0" } [dev-dependencies] insta = "1.39" +hydro_deploy = { path = "../hydro_deploy/core", version = "^0.9.0" } +async-ssh2-lite = { version = "0.5.0", features = ["vendored-openssl"] } +ctor = "0.2.8" diff --git a/hydro_deploy/hydroflow_plus_deploy/src/deploy.rs b/hydroflow_plus/src/deploy/deploy_graph.rs similarity index 87% rename from hydro_deploy/hydroflow_plus_deploy/src/deploy.rs rename to hydroflow_plus/src/deploy/deploy_graph.rs index 266a37bec892..e6b41f6e62f1 100644 --- a/hydro_deploy/hydroflow_plus_deploy/src/deploy.rs +++ b/hydroflow_plus/src/deploy/deploy_graph.rs @@ -13,19 +13,27 @@ use hydro_deploy::hydroflow_crate::ports::{ use hydro_deploy::hydroflow_crate::tracing_options::TracingOptions; use hydro_deploy::hydroflow_crate::HydroflowCrateService; use hydro_deploy::{CustomService, Deployment, Host, HydroflowCrate}; -use hydroflow_plus::deploy::{ClusterSpec, Deploy, ExternalSpec, Node, ProcessSpec, RegisterPort}; -use hydroflow_plus::futures::SinkExt; -use hydroflow_plus::lang::graph::HydroflowGraph; -use hydroflow_plus::util::deploy::ConnectedSink; use nameof::name_of; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use stageleft::{Quoted, RuntimeData}; +use syn::visit_mut::VisitMut; use tokio::sync::RwLock; - -use super::HydroflowPlusMeta; -use crate::deploy_runtime::*; -use crate::trybuild::{compile_graph_trybuild, create_trybuild}; +use trybuild_internals_api::path; + +use super::deploy_runtime::*; +use super::trybuild::{compile_graph_trybuild, create_trybuild}; +use super::{ClusterSpec, Deploy, ExternalSpec, Node, ProcessSpec, RegisterPort}; +use crate::futures::SinkExt; +use crate::lang::graph::HydroflowGraph; +use crate::util::deploy::ConnectedSink; + +#[derive(Default, Serialize, Deserialize)] +pub struct HydroflowPlusMeta { + pub clusters: HashMap>, + pub cluster_id: Option, + pub subgraph_id: usize, +} pub struct HydroDeploy {} @@ -413,37 +421,24 @@ impl<'a> RegisterPort<'a, HydroDeploy> for DeployExternal { fn as_bytes_sink( &self, key: usize, - ) -> impl Future< - Output = Pin< - Box>, - >, - > + 'a { + ) -> impl Future>>> + 'a + { let port = self.raw_port(key); async move { let sink = port.connect().await.into_sink(); - Box::pin(sink) - as Pin< - Box< - dyn hydroflow_plus::futures::Sink< - hydroflow_plus::bytes::Bytes, - Error = Error, - >, - >, - > + Box::pin(sink) as Pin>> } } fn as_bincode_sink( &self, key: usize, - ) -> impl Future>>> + 'a - { + ) -> impl Future>>> + 'a { let port = self.raw_port(key); async move { let sink = port.connect().await.into_sink(); - Box::pin(sink.with(|item| async move { - Ok(hydroflow_plus::bincode::serialize(&item).unwrap().into()) - })) as Pin>> + Box::pin(sink.with(|item| async move { Ok(bincode::serialize(&item).unwrap().into()) })) + as Pin>> } } } @@ -732,6 +727,44 @@ fn clean_name_hint(name_hint: &str) -> String { .replace(")", "") } +// TODO(shadaj): has to be public due to stageleft limitations +#[doc(hidden)] +pub struct ReplaceCrateNameWithStaged { + pub crate_name: String, +} + +impl VisitMut for ReplaceCrateNameWithStaged { + fn visit_type_path_mut(&mut self, i: &mut syn::TypePath) { + if let Some(first) = i.path.segments.first() { + if first.ident == self.crate_name { + let tail = i.path.segments.iter().skip(1).collect::>(); + *i = syn::parse_quote!(crate::__staged #(::#tail)*); + } + } + + syn::visit_mut::visit_type_path_mut(self, i); + } +} + +// TODO(shadaj): has to be public due to stageleft limitations +#[doc(hidden)] +pub struct ReplaceCrateWithOrig { + pub crate_name: String, +} + +impl VisitMut for ReplaceCrateWithOrig { + fn visit_item_use_mut(&mut self, i: &mut syn::ItemUse) { + if let syn::UseTree::Path(p) = &mut i.tree { + if p.ident == "crate" { + p.ident = syn::Ident::new(&self.crate_name, p.ident.span()); + i.leading_colon = Some(Default::default()); + } + } + + syn::visit_mut::visit_item_use_mut(self, i); + } +} + fn create_graph_trybuild( graph: HydroflowGraph, extra_stmts: Vec, @@ -740,15 +773,44 @@ fn create_graph_trybuild( String, (std::path::PathBuf, std::path::PathBuf, Option>), ) { - let source_ast = compile_graph_trybuild(graph, extra_stmts); - let source_dir = trybuild_internals_api::cargo::manifest_dir().unwrap(); let source_manifest = trybuild_internals_api::dependencies::get_manifest(&source_dir).unwrap(); let crate_name = &source_manifest.package.name.to_string().replace("-", "_"); - let source = prettyplease::unparse(&source_ast) - .to_string() - .replace(crate_name, &format!("{crate_name}::__staged")) - .replace("crate::__staged", &format!("{crate_name}::__staged")); + + let is_test = super::trybuild::IS_TEST.load(std::sync::atomic::Ordering::Relaxed); + + let mut generated_code = compile_graph_trybuild(graph, extra_stmts); + + ReplaceCrateNameWithStaged { + crate_name: crate_name.clone(), + } + .visit_file_mut(&mut generated_code); + + let mut inlined_staged = stageleft_tool::gen_staged_trybuild( + &path!(source_dir / "src" / "lib.rs"), + crate_name.clone(), + is_test, + ); + + ReplaceCrateWithOrig { + crate_name: crate_name.clone(), + } + .visit_file_mut(&mut inlined_staged); + + let source = prettyplease::unparse(&syn::parse_quote! { + #generated_code + + #[allow( + unused, + ambiguous_glob_reexports, + clippy::suspicious_else_formatting, + unexpected_cfgs, + reason = "generated code" + )] + pub mod __staged { + #inlined_staged + } + }); let mut hasher = Sha256::new(); hasher.update(&source); @@ -763,7 +825,7 @@ fn create_graph_trybuild( hash }; - let trybuild_created = create_trybuild(&source, &bin_name).unwrap(); + let trybuild_created = create_trybuild(&source, &bin_name, is_test).unwrap(); (bin_name, trybuild_created) } diff --git a/hydro_deploy/hydroflow_plus_deploy/src/deploy_runtime.rs b/hydroflow_plus/src/deploy/deploy_runtime.rs similarity index 98% rename from hydro_deploy/hydroflow_plus_deploy/src/deploy_runtime.rs rename to hydroflow_plus/src/deploy/deploy_runtime.rs index 60c5ddb15c39..eb3cdf40c171 100644 --- a/hydro_deploy/hydroflow_plus_deploy/src/deploy_runtime.rs +++ b/hydroflow_plus/src/deploy/deploy_runtime.rs @@ -1,9 +1,9 @@ -use hydroflow_plus::util::deploy::{ - ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, DeployPorts, -}; use stageleft::{q, Quoted, RuntimeData}; -use crate::HydroflowPlusMeta; +use super::HydroflowPlusMeta; +use crate::util::deploy::{ + ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, DeployPorts, +}; pub fn cluster_members( cli: RuntimeData<&DeployPorts>, diff --git a/hydroflow_plus/src/deploy/graphs.rs b/hydroflow_plus/src/deploy/in_memory_graph.rs similarity index 100% rename from hydroflow_plus/src/deploy/graphs.rs rename to hydroflow_plus/src/deploy/in_memory_graph.rs diff --git a/hydro_deploy/hydroflow_plus_deploy/src/runtime.rs b/hydroflow_plus/src/deploy/macro_runtime.rs similarity index 87% rename from hydro_deploy/hydroflow_plus_deploy/src/runtime.rs rename to hydroflow_plus/src/deploy/macro_runtime.rs index ba761ef3e1d3..a30f25d9699a 100644 --- a/hydro_deploy/hydroflow_plus_deploy/src/runtime.rs +++ b/hydroflow_plus/src/deploy/macro_runtime.rs @@ -2,12 +2,12 @@ use std::cell::RefCell; use std::pin::Pin; use std::rc::Rc; -use hydroflow_plus::deploy::{ClusterSpec, Deploy, ExternalSpec, Node, ProcessSpec, RegisterPort}; -use hydroflow_plus::lang::graph::HydroflowGraph; -use hydroflow_plus::util::deploy::DeployPorts; use stageleft::{Quoted, RuntimeData}; use super::HydroflowPlusMeta; +use crate::deploy::{ClusterSpec, Deploy, ExternalSpec, Node, ProcessSpec, RegisterPort}; +use crate::lang::graph::HydroflowGraph; +use crate::util::deploy::DeployPorts; pub struct DeployRuntime {} @@ -57,7 +57,7 @@ impl<'a> Deploy<'a> for DeployRuntime { _p2: &Self::Process, p2_port: &Self::Port, ) -> (syn::Expr, syn::Expr) { - crate::deploy_runtime::deploy_o2o(*env, p1_port.as_str(), p2_port.as_str()) + super::deploy_runtime::deploy_o2o(*env, p1_port.as_str(), p2_port.as_str()) } fn o2o_connect( @@ -76,7 +76,7 @@ impl<'a> Deploy<'a> for DeployRuntime { _c2: &Self::Cluster, c2_port: &Self::Port, ) -> (syn::Expr, syn::Expr) { - crate::deploy_runtime::deploy_o2m(*env, p1_port.as_str(), c2_port.as_str()) + super::deploy_runtime::deploy_o2m(*env, p1_port.as_str(), c2_port.as_str()) } fn o2m_connect( @@ -95,7 +95,7 @@ impl<'a> Deploy<'a> for DeployRuntime { _p2: &Self::Process, p2_port: &Self::Port, ) -> (syn::Expr, syn::Expr) { - crate::deploy_runtime::deploy_m2o(*env, c1_port.as_str(), p2_port.as_str()) + super::deploy_runtime::deploy_m2o(*env, c1_port.as_str(), p2_port.as_str()) } fn m2o_connect( @@ -114,7 +114,7 @@ impl<'a> Deploy<'a> for DeployRuntime { _c2: &Self::Cluster, c2_port: &Self::Port, ) -> (syn::Expr, syn::Expr) { - crate::deploy_runtime::deploy_m2m(*env, c1_port.as_str(), c2_port.as_str()) + super::deploy_runtime::deploy_m2m(*env, c1_port.as_str(), c2_port.as_str()) } fn m2m_connect( @@ -149,11 +149,11 @@ impl<'a> Deploy<'a> for DeployRuntime { env: &Self::CompileEnv, of_cluster: usize, ) -> impl Quoted<'a, &'a Vec> + Copy + 'a { - crate::deploy_runtime::cluster_members(*env, of_cluster) + super::deploy_runtime::cluster_members(*env, of_cluster) } fn cluster_self_id(env: &Self::CompileEnv) -> impl Quoted<'a, u32> + Copy + 'a { - crate::deploy_runtime::cluster_self_id(*env) + super::deploy_runtime::cluster_self_id(*env) } } @@ -179,14 +179,7 @@ impl<'a> RegisterPort<'a, DeployRuntime> for DeployRuntimeNode { &self, _key: usize, ) -> impl std::future::Future< - Output = Pin< - Box< - dyn hydroflow_plus::futures::Sink< - hydroflow_plus::bytes::Bytes, - Error = std::io::Error, - >, - >, - >, + Output = Pin>>, > + 'a { async { panic!() } } @@ -199,7 +192,7 @@ impl<'a> RegisterPort<'a, DeployRuntime> for DeployRuntimeNode { &self, _key: usize, ) -> impl std::future::Future< - Output = Pin>>, + Output = Pin>>, > + 'a { async { panic!() } } diff --git a/hydroflow_plus/src/deploy/mod.rs b/hydroflow_plus/src/deploy/mod.rs index 56d8317bbd7d..2e82b9c6fc93 100644 --- a/hydroflow_plus/src/deploy/mod.rs +++ b/hydroflow_plus/src/deploy/mod.rs @@ -8,8 +8,26 @@ use hydroflow_lang::graph::HydroflowGraph; use serde::Serialize; use stageleft::Quoted; -pub mod graphs; -pub use graphs::*; +pub mod macro_runtime; + +#[cfg(feature = "deploy")] +pub(crate) mod trybuild; + +pub use macro_runtime::*; +#[cfg(feature = "deploy")] +pub use trybuild::init_test; + +#[allow(clippy::allow_attributes, unused, reason = "stageleft")] +pub(crate) mod deploy_runtime; + +#[cfg(feature = "deploy")] +pub mod deploy_graph; + +#[cfg(feature = "deploy")] +pub use deploy_graph::*; + +pub mod in_memory_graph; +pub use in_memory_graph::*; pub trait LocalDeploy<'a> { type Process: Node; diff --git a/hydro_deploy/hydroflow_plus_deploy/src/trybuild.rs b/hydroflow_plus/src/deploy/trybuild.rs similarity index 82% rename from hydro_deploy/hydroflow_plus_deploy/src/trybuild.rs rename to hydroflow_plus/src/deploy/trybuild.rs index c71f91a9d277..c9ed72535ff8 100644 --- a/hydro_deploy/hydroflow_plus_deploy/src/trybuild.rs +++ b/hydroflow_plus/src/deploy/trybuild.rs @@ -1,13 +1,20 @@ use std::fs; use std::path::PathBuf; -use hydroflow_plus::lang::graph::{partition_graph, HydroflowGraph}; use stageleft::internal::quote; use trybuild_internals_api::cargo::{self, Metadata}; use trybuild_internals_api::env::Update; use trybuild_internals_api::run::{PathDependency, Project}; use trybuild_internals_api::{dependencies, features, path, Runner}; +use crate::lang::graph::{partition_graph, HydroflowGraph}; + +pub static IS_TEST: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +pub fn init_test() { + IS_TEST.store(true, std::sync::atomic::Ordering::Relaxed); +} + pub fn compile_graph_trybuild(graph: HydroflowGraph, extra_stmts: Vec) -> syn::File { let partitioned_graph = partition_graph(graph).expect("Failed to partition (cycle detected)."); @@ -16,11 +23,12 @@ pub fn compile_graph_trybuild(graph: HydroflowGraph, extra_stmts: Vec partitioned_graph.as_code("e! { hydroflow_plus }, true, quote!(), &mut diagnostics); let source_ast: syn::File = syn::parse_quote! { + #![feature(box_patterns)] #![allow(unused_crate_dependencies, missing_docs)] use hydroflow_plus::*; #[allow(unused)] - fn __hfplus_runtime<'a>(__hydroflow_plus_trybuild_cli: &'a hydroflow_plus::util::deploy::DeployPorts) -> hydroflow_plus::Hydroflow<'a> { + fn __hfplus_runtime<'a>(__hydroflow_plus_trybuild_cli: &'a hydroflow_plus::util::deploy::DeployPorts) -> hydroflow_plus::Hydroflow<'a> { #(#extra_stmts)* #tokens } @@ -39,6 +47,7 @@ pub fn compile_graph_trybuild(graph: HydroflowGraph, extra_stmts: Vec pub fn create_trybuild( source: &str, bin: &str, + is_test: bool, ) -> Result<(PathBuf, PathBuf, Option>), trybuild_internals_api::error::Error> { let Metadata { target_directory: target_dir, @@ -48,7 +57,10 @@ pub fn create_trybuild( let source_dir = cargo::manifest_dir()?; let mut source_manifest = dependencies::get_manifest(&source_dir)?; - source_manifest.dev_dependencies.clear(); + + if !is_test { + source_manifest.dev_dependencies.clear(); + } let mut features = features::find(); @@ -74,7 +86,7 @@ pub fn create_trybuild( fs::create_dir_all(&project_dir)?; let project_name = format!("{}-hfplus-trybuild", crate_name); - let manifest = Runner::make_manifest( + let mut manifest = Runner::make_manifest( &workspace, &project_name, &source_dir, @@ -83,12 +95,19 @@ pub fn create_trybuild( source_manifest, )?; + manifest.features.remove("stageleft_devel"); + if let Some(enabled_features) = &mut features { - enabled_features.retain(|feature| { - manifest.features.contains_key(feature) - && feature != "default" - && feature != "stageleft_devel" - }); + enabled_features + .retain(|feature| manifest.features.contains_key(feature) || feature == "default"); + + manifest + .features + .get_mut("default") + .iter_mut() + .for_each(|v| { + v.retain(|f| f != "stageleft_devel"); + }); } let project = Project { diff --git a/hydroflow_plus/src/lib.rs b/hydroflow_plus/src/lib.rs index ffbdf2325cc2..e214ec5ac6c6 100644 --- a/hydroflow_plus/src/lib.rs +++ b/hydroflow_plus/src/lib.rs @@ -165,3 +165,12 @@ impl<'a> FreeVariable> for HfBuiltWithId<'a> { (None, Some(self.tokens)) } } + +#[stageleft::runtime] +#[cfg(test)] +mod tests { + #[ctor::ctor] + fn init() { + crate::deploy::init_test(); + } +} diff --git a/hydroflow_plus/src/profiler.rs b/hydroflow_plus/src/profiler.rs index 26ab364229ce..85409f7ec4ee 100644 --- a/hydroflow_plus/src/profiler.rs +++ b/hydroflow_plus/src/profiler.rs @@ -3,9 +3,8 @@ use std::cell::RefCell; use hydroflow::futures::channel::mpsc::UnboundedSender; use stageleft::*; -use crate as hydroflow_plus; use crate::ir::*; -use crate::RuntimeContext; +use crate::{profiler as myself, RuntimeContext}; pub fn increment_counter(count: &mut u64) { *count += 1; @@ -43,9 +42,7 @@ fn add_profiling_node<'a>( .unwrap(); counters.borrow_mut()[my_id as usize] = 0; move |_| { - hydroflow_plus::profiler::increment_counter( - &mut counters.borrow_mut()[my_id as usize], - ); + myself::increment_counter(&mut counters.borrow_mut()[my_id as usize]); } })) .splice_untyped() diff --git a/hydroflow_plus/src/snapshots/hydroflow_plus__profiler__tests__profiler_wrapping_all_operators-2.snap b/hydroflow_plus/src/snapshots/hydroflow_plus__profiler__tests__profiler_wrapping_all_operators-2.snap index e090e82de5e0..b2a4cfe8e7a0 100644 --- a/hydroflow_plus/src/snapshots/hydroflow_plus__profiler__tests__profiler_wrapping_all_operators-2.snap +++ b/hydroflow_plus/src/snapshots/hydroflow_plus__profiler__tests__profiler_wrapping_all_operators-2.snap @@ -6,11 +6,11 @@ expression: "&pushed_down.ir()" ForEach { f: stageleft :: runtime_support :: fn1_type_hint :: < i32 , () > ({ use crate :: __staged :: profiler :: tests :: * ; | n | println ! ("{}" , n) }), input: Inspect { - f: { use crate :: __staged :: profiler :: * ; let counter_queue = Fake ; let counters = Fake ; let my_id = 0u32 ; { counter_queue . borrow () . unbounded_send ((my_id as usize , counters . borrow () [my_id as usize])) . unwrap () ; counters . borrow_mut () [my_id as usize] = 0 ; move | _ | { hydroflow_plus :: profiler :: increment_counter (& mut counters . borrow_mut () [my_id as usize] ,) ; } } }, + f: { use crate :: __staged :: profiler :: * ; let counter_queue = Fake ; let counters = Fake ; let my_id = 0u32 ; { counter_queue . borrow () . unbounded_send ((my_id as usize , counters . borrow () [my_id as usize])) . unwrap () ; counters . borrow_mut () [my_id as usize] = 0 ; move | _ | { myself :: increment_counter (& mut counters . borrow_mut () [my_id as usize]) ; } } }, input: Map { f: stageleft :: runtime_support :: fn1_type_hint :: < i32 , i32 > ({ use crate :: __staged :: profiler :: tests :: * ; | v | v + 1 }), input: Inspect { - f: { use crate :: __staged :: profiler :: * ; let counter_queue = Fake ; let counters = Fake ; let my_id = 1u32 ; { counter_queue . borrow () . unbounded_send ((my_id as usize , counters . borrow () [my_id as usize])) . unwrap () ; counters . borrow_mut () [my_id as usize] = 0 ; move | _ | { hydroflow_plus :: profiler :: increment_counter (& mut counters . borrow_mut () [my_id as usize] ,) ; } } }, + f: { use crate :: __staged :: profiler :: * ; let counter_queue = Fake ; let counters = Fake ; let my_id = 1u32 ; { counter_queue . borrow () . unbounded_send ((my_id as usize , counters . borrow () [my_id as usize])) . unwrap () ; counters . borrow_mut () [my_id as usize] = 0 ; move | _ | { myself :: increment_counter (& mut counters . borrow_mut () [my_id as usize]) ; } } }, input: Source { source: Iter( { use crate :: __staged :: profiler :: tests :: * ; 0 .. 10 }, diff --git a/hydroflow_plus/src/stream.rs b/hydroflow_plus/src/stream.rs index 4884c52b379d..2a33ebc0fccf 100644 --- a/hydroflow_plus/src/stream.rs +++ b/hydroflow_plus/src/stream.rs @@ -475,7 +475,7 @@ impl<'a, T, N: Location> Stream<'a, T, Unbounded, NoTick, N> { ) -> Stream<'a, T, Unbounded, NoTick, N> { let interval = duration.splice_typed(); - let samples = Stream::<'a, hydroflow::tokio::time::Instant, Bounded, Tick, N>::new( + let samples = Stream::<'a, tokio::time::Instant, Bounded, Tick, N>::new( self.location_kind, self.ir_leaves.clone(), HfPlusNode::Source { @@ -835,3 +835,52 @@ impl<'a, T, W, N: Location> Stream<'a, T, W, NoTick, N> { self.broadcast_bytes(other).map(q!(|(_, b)| b)) } } + +#[cfg(test)] +mod tests { + use hydro_deploy::Deployment; + use serde::{Deserialize, Serialize}; + use stageleft::q; + + use crate::deploy::{DeployCrateWrapper, TrybuildHost}; + use crate::FlowBuilder; + + struct P1 {} + struct P2 {} + + #[derive(Serialize, Deserialize, Debug)] + struct SendOverNetwork { + n: u32, + } + + #[tokio::test] + async fn first_ten_distributed() { + let mut deployment = Deployment::new(); + + let flow = FlowBuilder::new(); + let first_node = flow.process::(); + let second_node = flow.process::(); + + let numbers = flow.source_iter(&first_node, q!(0..10)); + numbers + .map(q!(|n| SendOverNetwork { n })) + .send_bincode(&second_node) + .for_each(q!(|n| println!("{}", n.n))); + + let nodes = flow + .with_default_optimize() + .with_process(&first_node, TrybuildHost::new(deployment.Localhost())) + .with_process(&second_node, TrybuildHost::new(deployment.Localhost())) + .deploy(&mut deployment); + + deployment.deploy().await.unwrap(); + + let mut second_node_stdout = nodes.get_process(&second_node).stdout().await; + + deployment.start().await.unwrap(); + + for i in 0..10 { + assert_eq!(second_node_stdout.recv().await.unwrap(), i.to_string()); + } + } +} diff --git a/hydroflow_plus_test/Cargo.toml b/hydroflow_plus_test/Cargo.toml index bc6589360878..ce6502dbcce2 100644 --- a/hydroflow_plus_test/Cargo.toml +++ b/hydroflow_plus_test/Cargo.toml @@ -15,7 +15,6 @@ stageleft_devel = [] hydroflow_plus = { path = "../hydroflow_plus", version = "^0.9.0" } tokio = { version = "1.29.0", features = [ "full" ] } stageleft = { path = "../stageleft", version = "^0.4.0" } -hydroflow_plus_deploy = { path = "../hydro_deploy/hydroflow_plus_deploy", version = "^0.9.0" } rand = "0.8.0" serde = { version = "1.0.197", features = [ "derive" ] } @@ -25,5 +24,5 @@ stageleft_tool = { path = "../stageleft_tool", version = "^0.3.0" } [dev-dependencies] insta = "1.39" hydro_deploy = { path = "../hydro_deploy/core", version = "^0.9.0" } -hydroflow_plus_deploy = { path = "../hydro_deploy/hydroflow_plus_deploy", version = "^0.9.0", features = [ "deploy" ] } +hydroflow_plus = { path = "../hydroflow_plus", version = "^0.9.0", features = [ "deploy" ] } futures = "0.3.0" diff --git a/hydroflow_plus_test/examples/compute_pi.rs b/hydroflow_plus_test/examples/compute_pi.rs index 4ff6e008744e..37c02da439f3 100644 --- a/hydroflow_plus_test/examples/compute_pi.rs +++ b/hydroflow_plus_test/examples/compute_pi.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/examples/first_ten_distributed.rs b/hydroflow_plus_test/examples/first_ten_distributed.rs index f55badf07484..6716c6fa52b4 100644 --- a/hydroflow_plus_test/examples/first_ten_distributed.rs +++ b/hydroflow_plus_test/examples/first_ten_distributed.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use futures::SinkExt; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/examples/map_reduce.rs b/hydroflow_plus_test/examples/map_reduce.rs index ce9a39c3419f..c18d9c59bb52 100644 --- a/hydroflow_plus_test/examples/map_reduce.rs +++ b/hydroflow_plus_test/examples/map_reduce.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/examples/paxos.rs b/hydroflow_plus_test/examples/paxos.rs index cfd326b6b7de..e1ebab02c91a 100644 --- a/hydroflow_plus_test/examples/paxos.rs +++ b/hydroflow_plus_test/examples/paxos.rs @@ -1,5 +1,5 @@ use hydro_deploy::Deployment; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; #[tokio::main] async fn main() { diff --git a/hydroflow_plus_test/examples/perf_compute_pi.rs b/hydroflow_plus_test/examples/perf_compute_pi.rs index ebdf36db01dd..56d8db5a9746 100644 --- a/hydroflow_plus_test/examples/perf_compute_pi.rs +++ b/hydroflow_plus_test/examples/perf_compute_pi.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::hydroflow_crate::tracing_options::TracingOptions; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/examples/simple_cluster.rs b/hydroflow_plus_test/examples/simple_cluster.rs index 64ebad93641f..e1c22b702f45 100644 --- a/hydroflow_plus_test/examples/simple_cluster.rs +++ b/hydroflow_plus_test/examples/simple_cluster.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::{Deployment, Host}; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; use tokio::sync::RwLock; type HostCreator = Box Arc>; diff --git a/hydroflow_plus_test/src/cluster/compute_pi.rs b/hydroflow_plus_test/src/cluster/compute_pi.rs index 6aac46b86d77..0be1e74b64f9 100644 --- a/hydroflow_plus_test/src/cluster/compute_pi.rs +++ b/hydroflow_plus_test/src/cluster/compute_pi.rs @@ -46,7 +46,7 @@ pub fn compute_pi(flow: &FlowBuilder, batch_size: usize) -> (Cluster, Pr #[cfg(test)] mod tests { - use hydroflow_plus_deploy::DeployRuntime; + use hydroflow_plus::deploy::DeployRuntime; use stageleft::RuntimeData; #[test] diff --git a/hydroflow_plus_test/src/cluster/many_to_many.rs b/hydroflow_plus_test/src/cluster/many_to_many.rs index ba9d865d6e3c..c3e35b85b5cb 100644 --- a/hydroflow_plus_test/src/cluster/many_to_many.rs +++ b/hydroflow_plus_test/src/cluster/many_to_many.rs @@ -13,7 +13,7 @@ pub fn many_to_many(flow: &FlowBuilder) -> Cluster<()> { #[cfg(test)] mod tests { use hydro_deploy::Deployment; - use hydroflow_plus_deploy::{DeployCrateWrapper, TrybuildHost}; + use hydroflow_plus::deploy::{DeployCrateWrapper, TrybuildHost}; #[tokio::test] async fn many_to_many() { diff --git a/hydroflow_plus_test/src/cluster/map_reduce.rs b/hydroflow_plus_test/src/cluster/map_reduce.rs index 5c4c2829f10f..3cdfce85da74 100644 --- a/hydroflow_plus_test/src/cluster/map_reduce.rs +++ b/hydroflow_plus_test/src/cluster/map_reduce.rs @@ -41,7 +41,7 @@ pub fn map_reduce(flow: &FlowBuilder) -> (Process, Cluster) { #[cfg(test)] mod tests { - use hydroflow_plus_deploy::DeployRuntime; + use hydroflow_plus::deploy::DeployRuntime; use stageleft::RuntimeData; #[test] diff --git a/hydroflow_plus_test/src/cluster/paxos_bench.rs b/hydroflow_plus_test/src/cluster/paxos_bench.rs index c53938d9406d..31275f79eeba 100644 --- a/hydroflow_plus_test/src/cluster/paxos_bench.rs +++ b/hydroflow_plus_test/src/cluster/paxos_bench.rs @@ -454,7 +454,7 @@ fn bench_client<'a, B: LeaderElected + std::fmt::Debug>( #[cfg(test)] mod tests { - use hydroflow_plus_deploy::DeployRuntime; + use hydroflow_plus::deploy::DeployRuntime; use stageleft::RuntimeData; #[test] diff --git a/hydroflow_plus_test/src/cluster/simple_cluster.rs b/hydroflow_plus_test/src/cluster/simple_cluster.rs index e50965f52908..9f0c9003203c 100644 --- a/hydroflow_plus_test/src/cluster/simple_cluster.rs +++ b/hydroflow_plus_test/src/cluster/simple_cluster.rs @@ -30,7 +30,7 @@ pub fn simple_cluster(flow: &FlowBuilder) -> (Process<()>, Cluster<()>) { #[cfg(test)] mod tests { use hydro_deploy::Deployment; - use hydroflow_plus_deploy::{DeployCrateWrapper, TrybuildHost}; + use hydroflow_plus::deploy::{DeployCrateWrapper, TrybuildHost}; #[tokio::test] async fn simple_cluster() { diff --git a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_0.snap b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_0.snap index 110524ff4791..234bb8f616bb 100644 --- a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_0.snap +++ b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_0.snap @@ -9,7 +9,7 @@ expression: ir.surface_syntax_string() 5v1 = map (stageleft :: runtime_support :: fn1_type_hint :: < (f64 , f64) , bool > ({ use crate :: __staged :: cluster :: compute_pi :: * ; | (x , y) | x * x + y * y < 1.0 })); 6v1 = fold :: < 'tick > (stageleft :: runtime_support :: fn0_type_hint :: < (u64 , u64) > ({ use crate :: __staged :: cluster :: compute_pi :: * ; | | (0u64 , 0u64) }) , stageleft :: runtime_support :: fn2_borrow_mut_type_hint :: < (u64 , u64) , bool , () > ({ use crate :: __staged :: cluster :: compute_pi :: * ; | (inside , total) , sample_inside | { if sample_inside { * inside += 1 ; } * total += 1 ; } })); 7v1 = map (| data | { hydroflow_plus :: runtime_support :: bincode :: serialize :: < (u64 , u64) > (& data) . unwrap () . into () }); -8v1 = dest_sink ({ use hydroflow_plus_deploy :: __staged :: deploy_runtime :: * ; let c1_port = "port_0" ; let env = FAKE ; { env . port (c1_port) . connect_local_blocking :: < ConnectedDirect > () . into_sink () } }); +8v1 = dest_sink ({ use hydroflow_plus :: __staged :: deploy :: deploy_runtime :: * ; let c1_port = "port_0" ; let env = FAKE ; { env . port (c1_port) . connect_local_blocking :: < ConnectedDirect > () . into_sink () } }); 1v1 -> 2v1; 2v1 -> 3v1; diff --git a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_1.snap b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_1.snap index 97e5b2013726..ca6d96f060dc 100644 --- a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_1.snap +++ b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__compute_pi__tests__compute_pi_ir@surface_graph_1.snap @@ -2,7 +2,7 @@ source: hydroflow_plus_test/src/cluster/compute_pi.rs expression: ir.surface_syntax_string() --- -1v1 = source_stream ({ use hydroflow_plus_deploy :: __staged :: deploy_runtime :: * ; let env = FAKE ; let p2_port = "port_0" ; { env . port (p2_port) . connect_local_blocking :: < ConnectedTagged < ConnectedDirect > > () . into_source () } }); +1v1 = source_stream ({ use hydroflow_plus :: __staged :: deploy :: deploy_runtime :: * ; let env = FAKE ; let p2_port = "port_0" ; { env . port (p2_port) . connect_local_blocking :: < ConnectedTagged < ConnectedDirect > > () . into_source () } }); 2v1 = map (| res | { let (id , b) = res . unwrap () ; (id , hydroflow_plus :: runtime_support :: bincode :: deserialize :: < (u64 , u64) > (& b) . unwrap ()) }); 3v1 = map (stageleft :: runtime_support :: fn1_type_hint :: < (u32 , (u64 , u64)) , (u64 , u64) > ({ use hydroflow_plus :: __staged :: stream :: * ; | (_ , b) | b })); 4v1 = reduce :: < 'static > (stageleft :: runtime_support :: fn2_borrow_mut_type_hint :: < (u64 , u64) , (u64 , u64) , () > ({ use crate :: __staged :: cluster :: compute_pi :: * ; | (inside , total) , (inside_batch , total_batch) | { * inside += inside_batch ; * total += total_batch ; } })); diff --git a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_0.snap b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_0.snap index 6bbd7ffaaf93..de23c7f7e01c 100644 --- a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_0.snap +++ b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_0.snap @@ -7,8 +7,8 @@ expression: ir.surface_syntax_string() 3v1 = enumerate (); 4v1 = map (stageleft :: runtime_support :: fn1_type_hint :: < (usize , std :: string :: String) , (u32 , std :: string :: String) > ({ use crate :: __staged :: cluster :: map_reduce :: * ; let all_ids_vec = __hydroflow_plus_cluster_ids_1 ; | (i , w) | ((i % all_ids_vec . len ()) as u32 , w) })); 5v1 = map (| (id , data) | { (id , hydroflow_plus :: runtime_support :: bincode :: serialize :: < std :: string :: String > (& data) . unwrap () . into ()) }); -6v1 = dest_sink ({ use hydroflow_plus_deploy :: __staged :: deploy_runtime :: * ; let env = FAKE ; let p1_port = "port_0" ; { env . port (p1_port) . connect_local_blocking :: < ConnectedDemux < ConnectedDirect > > () . into_sink () } }); -7v1 = source_stream ({ use hydroflow_plus_deploy :: __staged :: deploy_runtime :: * ; let env = FAKE ; let p2_port = "port_1" ; { env . port (p2_port) . connect_local_blocking :: < ConnectedTagged < ConnectedDirect > > () . into_source () } }); +6v1 = dest_sink ({ use hydroflow_plus :: __staged :: deploy :: deploy_runtime :: * ; let env = FAKE ; let p1_port = "port_0" ; { env . port (p1_port) . connect_local_blocking :: < ConnectedDemux < ConnectedDirect > > () . into_sink () } }); +7v1 = source_stream ({ use hydroflow_plus :: __staged :: deploy :: deploy_runtime :: * ; let env = FAKE ; let p2_port = "port_1" ; { env . port (p2_port) . connect_local_blocking :: < ConnectedTagged < ConnectedDirect > > () . into_source () } }); 8v1 = map (| res | { let (id , b) = res . unwrap () ; (id , hydroflow_plus :: runtime_support :: bincode :: deserialize :: < (std :: string :: String , i32) > (& b) . unwrap ()) }); 9v1 = map (stageleft :: runtime_support :: fn1_type_hint :: < (u32 , (std :: string :: String , i32)) , (std :: string :: String , i32) > ({ use hydroflow_plus :: __staged :: stream :: * ; | (_ , b) | b })); 10v1 = reduce_keyed :: < 'static > (stageleft :: runtime_support :: fn2_borrow_mut_type_hint :: < i32 , i32 , () > ({ use crate :: __staged :: cluster :: map_reduce :: * ; | total , count | * total += count })); diff --git a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_1.snap b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_1.snap index 2449f07876e9..2c79806257e9 100644 --- a/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_1.snap +++ b/hydroflow_plus_test/src/cluster/snapshots/hydroflow_plus_test__cluster__map_reduce__tests__map_reduce_ir@surface_graph_1.snap @@ -2,13 +2,13 @@ source: hydroflow_plus_test/src/cluster/map_reduce.rs expression: ir.surface_syntax_string() --- -1v1 = source_stream ({ use hydroflow_plus_deploy :: __staged :: deploy_runtime :: * ; let c2_port = "port_0" ; let env = FAKE ; { env . port (c2_port) . connect_local_blocking :: < ConnectedDirect > () . into_source () } }); +1v1 = source_stream ({ use hydroflow_plus :: __staged :: deploy :: deploy_runtime :: * ; let c2_port = "port_0" ; let env = FAKE ; { env . port (c2_port) . connect_local_blocking :: < ConnectedDirect > () . into_source () } }); 2v1 = map (| res | { hydroflow_plus :: runtime_support :: bincode :: deserialize :: < std :: string :: String > (& res . unwrap ()) . unwrap () }); 3v1 = map (stageleft :: runtime_support :: fn1_type_hint :: < std :: string :: String , (std :: string :: String , ()) > ({ use crate :: __staged :: cluster :: map_reduce :: * ; | string | (string , ()) })); 4v1 = fold_keyed :: < 'tick > (stageleft :: runtime_support :: fn0_type_hint :: < i32 > ({ use crate :: __staged :: cluster :: map_reduce :: * ; | | 0 }) , stageleft :: runtime_support :: fn2_borrow_mut_type_hint :: < i32 , () , () > ({ use crate :: __staged :: cluster :: map_reduce :: * ; | count , _ | * count += 1 })); 5v1 = inspect (stageleft :: runtime_support :: fn1_borrow_type_hint :: < (std :: string :: String , i32) , () > ({ use crate :: __staged :: cluster :: map_reduce :: * ; | (string , count) | println ! ("partition count: {} - {}" , string , count) })); 6v1 = map (| data | { hydroflow_plus :: runtime_support :: bincode :: serialize :: < (std :: string :: String , i32) > (& data) . unwrap () . into () }); -7v1 = dest_sink ({ use hydroflow_plus_deploy :: __staged :: deploy_runtime :: * ; let c1_port = "port_1" ; let env = FAKE ; { env . port (c1_port) . connect_local_blocking :: < ConnectedDirect > () . into_sink () } }); +7v1 = dest_sink ({ use hydroflow_plus :: __staged :: deploy :: deploy_runtime :: * ; let c1_port = "port_1" ; let env = FAKE ; { env . port (c1_port) . connect_local_blocking :: < ConnectedDirect > () . into_sink () } }); 1v1 -> 2v1; 2v1 -> 3v1; diff --git a/hydroflow_plus_test/src/distributed/first_ten.rs b/hydroflow_plus_test/src/distributed/first_ten.rs index ea10a880b729..9372b174d717 100644 --- a/hydroflow_plus_test/src/distributed/first_ten.rs +++ b/hydroflow_plus_test/src/distributed/first_ten.rs @@ -31,7 +31,7 @@ pub fn first_ten_distributed( numbers .map(q!(|n| SendOverNetwork { n })) .send_bincode(&second_process) - .for_each(q!(|n: SendOverNetwork| println!("{}", n.n))); // TODO(shadaj): why is the explicit type required here? + .for_each(q!(|n| println!("{}", n.n))); ( external_process, @@ -47,7 +47,7 @@ mod tests { use futures::SinkExt; use hydro_deploy::{Deployment, Host}; - use hydroflow_plus_deploy::{DeployCrateWrapper, TrybuildHost}; + use hydroflow_plus::deploy::{DeployCrateWrapper, TrybuildHost}; #[tokio::test] async fn first_ten_distributed() { @@ -61,7 +61,6 @@ mod tests { insta::assert_debug_snapshot!(built.ir()); - // if we drop this, we drop the references to the deployment nodes let nodes = built .with_process(&first_node, TrybuildHost::new(deployment.Localhost())) .with_process(&second_node, TrybuildHost::new(deployment.Localhost())) diff --git a/hydroflow_plus_test/src/distributed/snapshots/hydroflow_plus_test__distributed__first_ten__tests__first_ten_distributed.snap b/hydroflow_plus_test/src/distributed/snapshots/hydroflow_plus_test__distributed__first_ten__tests__first_ten_distributed.snap index fca0ec10245f..fb73641059a3 100644 --- a/hydroflow_plus_test/src/distributed/snapshots/hydroflow_plus_test__distributed__first_ten__tests__first_ten_distributed.snap +++ b/hydroflow_plus_test/src/distributed/snapshots/hydroflow_plus_test__distributed__first_ten__tests__first_ten_distributed.snap @@ -37,7 +37,7 @@ expression: built.ir() }, }, ForEach { - f: stageleft :: runtime_support :: fn1_type_hint :: < hydroflow_plus_test :: distributed :: first_ten :: SendOverNetwork , () > ({ use crate :: __staged :: distributed :: first_ten :: * ; | n : SendOverNetwork | println ! ("{}" , n . n) }), + f: stageleft :: runtime_support :: fn1_type_hint :: < hydroflow_plus_test :: distributed :: first_ten :: SendOverNetwork , () > ({ use crate :: __staged :: distributed :: first_ten :: * ; | n | println ! ("{}" , n . n) }), input: Network { from_location: Process( 1, diff --git a/hydroflow_plus_test_local/src/local/negation.rs b/hydroflow_plus_test_local/src/local/negation.rs index 8e4180d5db5d..c5517c47cfb7 100644 --- a/hydroflow_plus_test_local/src/local/negation.rs +++ b/hydroflow_plus_test_local/src/local/negation.rs @@ -52,13 +52,9 @@ pub fn test_anti_join<'a>( source2 = source2.persist(); } - // TODO(shadaj): inference fails without a for_each type annotation here - source - .anti_join(source2) - .all_ticks() - .for_each(q!(|v: (u32, u32)| { - output.send(v.0).unwrap(); - })); + source.anti_join(source2).all_ticks().for_each(q!(|v| { + output.send(v.0).unwrap(); + })); flow.with_default_optimize() .compile_no_network::() diff --git a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_static@graphvis_dot.snap b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_static@graphvis_dot.snap index 6834ac3f3c6f..a42f15c8200d 100644 --- a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_static@graphvis_dot.snap +++ b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_static@graphvis_dot.snap @@ -10,7 +10,7 @@ digraph { n3v1 [label="(n3v1) persist::<'static>()", shape=invhouse, fillcolor="#88aaff"] n4v1 [label="(n4v1) source_iter({\l use crate::__staged::local::negation::*;\l 3..6\l})\l", shape=invhouse, fillcolor="#88aaff"] n5v1 [label="(n5v1) anti_join_multiset::<'tick, 'static>()", shape=invhouse, fillcolor="#88aaff"] - n6v1 [label="(n6v1) for_each(\l stageleft::runtime_support::fn1_type_hint::<\l (u32, u32),\l (),\l >({\l use crate::__staged::local::negation::*;\l let output = output;\l |v: (u32, u32)| {\l output.send(v.0).unwrap();\l }\l }),\l)\l", shape=house, fillcolor="#ffff88"] + n6v1 [label="(n6v1) for_each(\l stageleft::runtime_support::fn1_type_hint::<\l (u32, u32),\l (),\l >({\l use crate::__staged::local::negation::*;\l let output = output;\l |v| {\l output.send(v.0).unwrap();\l }\l }),\l)\l", shape=house, fillcolor="#ffff88"] n7v1 [label="(n7v1) handoff", shape=parallelogram, fillcolor="#ddddff"] n1v1 -> n2v1 n2v1 -> n3v1 diff --git a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_static@graphvis_mermaid.snap b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_static@graphvis_mermaid.snap index b07490f8d474..a551e93b7771 100644 --- a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_static@graphvis_mermaid.snap +++ b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_static@graphvis_mermaid.snap @@ -13,7 +13,7 @@ linkStyle default stroke:#aaa 3v1[\"(3v1) persist::<'static>()"/]:::pullClass 4v1[\"
(4v1)
source_iter({
use crate::__staged::local::negation::*;
3..6
})"/]:::pullClass 5v1[\"(5v1) anti_join_multiset::<'tick, 'static>()"/]:::pullClass -6v1[/"
(6v1)
for_each(
stageleft::runtime_support::fn1_type_hint::<
(u32, u32),
(),
>({
use crate::__staged::local::negation::*;
let output = output;
|v: (u32, u32)| {
output.send(v.0).unwrap();
}
}),
)"\]:::pushClass +6v1[/"
(6v1)
for_each(
stageleft::runtime_support::fn1_type_hint::<
(u32, u32),
(),
>({
use crate::__staged::local::negation::*;
let output = output;
|v| {
output.send(v.0).unwrap();
}
}),
)"\]:::pushClass 7v1["(7v1) handoff"]:::otherClass 1v1-->2v1 2v1-->3v1 diff --git a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_tick@graphvis_dot.snap b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_tick@graphvis_dot.snap index 490eb91c910e..064e8c2ad217 100644 --- a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_tick@graphvis_dot.snap +++ b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_tick@graphvis_dot.snap @@ -10,7 +10,7 @@ digraph { n3v1 [label="(n3v1) persist::<'static>()", shape=invhouse, fillcolor="#88aaff"] n4v1 [label="(n4v1) source_iter({\l use crate::__staged::local::negation::*;\l 3..6\l})\l", shape=invhouse, fillcolor="#88aaff"] n5v1 [label="(n5v1) anti_join_multiset::<'tick, 'tick>()", shape=invhouse, fillcolor="#88aaff"] - n6v1 [label="(n6v1) for_each(\l stageleft::runtime_support::fn1_type_hint::<\l (u32, u32),\l (),\l >({\l use crate::__staged::local::negation::*;\l let output = output;\l |v: (u32, u32)| {\l output.send(v.0).unwrap();\l }\l }),\l)\l", shape=house, fillcolor="#ffff88"] + n6v1 [label="(n6v1) for_each(\l stageleft::runtime_support::fn1_type_hint::<\l (u32, u32),\l (),\l >({\l use crate::__staged::local::negation::*;\l let output = output;\l |v| {\l output.send(v.0).unwrap();\l }\l }),\l)\l", shape=house, fillcolor="#ffff88"] n7v1 [label="(n7v1) handoff", shape=parallelogram, fillcolor="#ddddff"] n1v1 -> n2v1 n2v1 -> n3v1 diff --git a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_tick@graphvis_mermaid.snap b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_tick@graphvis_mermaid.snap index 7900177a4d72..72e7e8e12cc0 100644 --- a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_tick@graphvis_mermaid.snap +++ b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_static_tick@graphvis_mermaid.snap @@ -13,7 +13,7 @@ linkStyle default stroke:#aaa 3v1[\"(3v1) persist::<'static>()"/]:::pullClass 4v1[\"
(4v1)
source_iter({
use crate::__staged::local::negation::*;
3..6
})"/]:::pullClass 5v1[\"(5v1) anti_join_multiset::<'tick, 'tick>()"/]:::pullClass -6v1[/"
(6v1)
for_each(
stageleft::runtime_support::fn1_type_hint::<
(u32, u32),
(),
>({
use crate::__staged::local::negation::*;
let output = output;
|v: (u32, u32)| {
output.send(v.0).unwrap();
}
}),
)"\]:::pushClass +6v1[/"
(6v1)
for_each(
stageleft::runtime_support::fn1_type_hint::<
(u32, u32),
(),
>({
use crate::__staged::local::negation::*;
let output = output;
|v| {
output.send(v.0).unwrap();
}
}),
)"\]:::pushClass 7v1["(7v1) handoff"]:::otherClass 1v1-->2v1 2v1-->3v1 diff --git a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_static@graphvis_dot.snap b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_static@graphvis_dot.snap index 89df6b6109ec..e08580edcfca 100644 --- a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_static@graphvis_dot.snap +++ b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_static@graphvis_dot.snap @@ -9,7 +9,7 @@ digraph { n2v1 [label="(n2v1) map(\l stageleft::runtime_support::fn1_type_hint::<\l u32,\l (u32, u32),\l >({\l use crate::__staged::local::negation::*;\l |v| (v, v)\l }),\l)\l", shape=invhouse, fillcolor="#88aaff"] n3v1 [label="(n3v1) source_iter({\l use crate::__staged::local::negation::*;\l 3..6\l})\l", shape=invhouse, fillcolor="#88aaff"] n4v1 [label="(n4v1) anti_join_multiset::<'tick, 'static>()", shape=invhouse, fillcolor="#88aaff"] - n5v1 [label="(n5v1) for_each(\l stageleft::runtime_support::fn1_type_hint::<\l (u32, u32),\l (),\l >({\l use crate::__staged::local::negation::*;\l let output = output;\l |v: (u32, u32)| {\l output.send(v.0).unwrap();\l }\l }),\l)\l", shape=house, fillcolor="#ffff88"] + n5v1 [label="(n5v1) for_each(\l stageleft::runtime_support::fn1_type_hint::<\l (u32, u32),\l (),\l >({\l use crate::__staged::local::negation::*;\l let output = output;\l |v| {\l output.send(v.0).unwrap();\l }\l }),\l)\l", shape=house, fillcolor="#ffff88"] n6v1 [label="(n6v1) handoff", shape=parallelogram, fillcolor="#ddddff"] n1v1 -> n2v1 n2v1 -> n4v1 [label="pos"] diff --git a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_static@graphvis_mermaid.snap b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_static@graphvis_mermaid.snap index b100ec877195..bd3ed201ed8b 100644 --- a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_static@graphvis_mermaid.snap +++ b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_static@graphvis_mermaid.snap @@ -12,7 +12,7 @@ linkStyle default stroke:#aaa 2v1[\"
(2v1)
map(
stageleft::runtime_support::fn1_type_hint::<
u32,
(u32, u32),
>({
use crate::__staged::local::negation::*;
|v| (v, v)
}),
)"/]:::pullClass 3v1[\"
(3v1)
source_iter({
use crate::__staged::local::negation::*;
3..6
})"/]:::pullClass 4v1[\"(4v1) anti_join_multiset::<'tick, 'static>()"/]:::pullClass -5v1[/"
(5v1)
for_each(
stageleft::runtime_support::fn1_type_hint::<
(u32, u32),
(),
>({
use crate::__staged::local::negation::*;
let output = output;
|v: (u32, u32)| {
output.send(v.0).unwrap();
}
}),
)"\]:::pushClass +5v1[/"
(5v1)
for_each(
stageleft::runtime_support::fn1_type_hint::<
(u32, u32),
(),
>({
use crate::__staged::local::negation::*;
let output = output;
|v| {
output.send(v.0).unwrap();
}
}),
)"\]:::pushClass 6v1["(6v1) handoff"]:::otherClass 1v1-->2v1 2v1-->|pos|4v1 diff --git a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_tick@graphvis_dot.snap b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_tick@graphvis_dot.snap index 69a5615e8b16..19c6f15739b8 100644 --- a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_tick@graphvis_dot.snap +++ b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_tick@graphvis_dot.snap @@ -9,7 +9,7 @@ digraph { n2v1 [label="(n2v1) map(\l stageleft::runtime_support::fn1_type_hint::<\l u32,\l (u32, u32),\l >({\l use crate::__staged::local::negation::*;\l |v| (v, v)\l }),\l)\l", shape=invhouse, fillcolor="#88aaff"] n3v1 [label="(n3v1) source_iter({\l use crate::__staged::local::negation::*;\l 3..6\l})\l", shape=invhouse, fillcolor="#88aaff"] n4v1 [label="(n4v1) anti_join_multiset::<'tick, 'tick>()", shape=invhouse, fillcolor="#88aaff"] - n5v1 [label="(n5v1) for_each(\l stageleft::runtime_support::fn1_type_hint::<\l (u32, u32),\l (),\l >({\l use crate::__staged::local::negation::*;\l let output = output;\l |v: (u32, u32)| {\l output.send(v.0).unwrap();\l }\l }),\l)\l", shape=house, fillcolor="#ffff88"] + n5v1 [label="(n5v1) for_each(\l stageleft::runtime_support::fn1_type_hint::<\l (u32, u32),\l (),\l >({\l use crate::__staged::local::negation::*;\l let output = output;\l |v| {\l output.send(v.0).unwrap();\l }\l }),\l)\l", shape=house, fillcolor="#ffff88"] n6v1 [label="(n6v1) handoff", shape=parallelogram, fillcolor="#ddddff"] n1v1 -> n2v1 n2v1 -> n4v1 [label="pos"] diff --git a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_tick@graphvis_mermaid.snap b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_tick@graphvis_mermaid.snap index f2d7fcdd8ff9..f43ea369d190 100644 --- a/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_tick@graphvis_mermaid.snap +++ b/hydroflow_plus_test_local/src/local/snapshots/hydroflow_plus_test_local__local__negation__tests__anti_join_tick_tick@graphvis_mermaid.snap @@ -12,7 +12,7 @@ linkStyle default stroke:#aaa 2v1[\"
(2v1)
map(
stageleft::runtime_support::fn1_type_hint::<
u32,
(u32, u32),
>({
use crate::__staged::local::negation::*;
|v| (v, v)
}),
)"/]:::pullClass 3v1[\"
(3v1)
source_iter({
use crate::__staged::local::negation::*;
3..6
})"/]:::pullClass 4v1[\"(4v1) anti_join_multiset::<'tick, 'tick>()"/]:::pullClass -5v1[/"
(5v1)
for_each(
stageleft::runtime_support::fn1_type_hint::<
(u32, u32),
(),
>({
use crate::__staged::local::negation::*;
let output = output;
|v: (u32, u32)| {
output.send(v.0).unwrap();
}
}),
)"\]:::pushClass +5v1[/"
(5v1)
for_each(
stageleft::runtime_support::fn1_type_hint::<
(u32, u32),
(),
>({
use crate::__staged::local::negation::*;
let output = output;
|v| {
output.send(v.0).unwrap();
}
}),
)"\]:::pushClass 6v1["(6v1) handoff"]:::otherClass 1v1-->2v1 2v1-->|pos|4v1 diff --git a/scripts/build_dist_release.sh b/scripts/build_dist_release.sh index 43ea621bf0f2..64b0cb9dc321 100755 --- a/scripts/build_dist_release.sh +++ b/scripts/build_dist_release.sh @@ -52,4 +52,4 @@ fi # The CARGO_NET_GIT_FETCH_WITH_CLI="true" environment variable is a Workaround to an issue similar # to the one encountered by pytorch in https://github.com/pytorch/pytorch/issues/82174 -CARGO_NET_GIT_FETCH_WITH_CLI="true" cargo build --release --all-targets --workspace --exclude hydro_deploy --exclude hydro_cli --exclude hydroflow_plus_deploy --exclude hydroflow_plus_test --exclude hydroflow_plus_test_macro --target ${RUST_TARGET} +CARGO_NET_GIT_FETCH_WITH_CLI="true" cargo build --release --all-targets --workspace --exclude hydro_deploy --exclude hydro_cli --exclude hydroflow_plus_test --exclude hydroflow_plus_test_macro --target ${RUST_TARGET} diff --git a/stageleft_macro/src/lib.rs b/stageleft_macro/src/lib.rs index f608a32c3199..c6972c694c23 100644 --- a/stageleft_macro/src/lib.rs +++ b/stageleft_macro/src/lib.rs @@ -67,7 +67,7 @@ fn gen_use_paths( #[allow(non_upper_case_globals, non_snake_case)] let #name_ident_unspanned = #root::runtime_support::create_import( #full_path, - env!("STAGELEFT_FINAL_CRATE_NAME"), + option_env!("STAGELEFT_FINAL_CRATE_NAME").unwrap_or(env!("CARGO_PKG_NAME")), { let __quse_local = (); { @@ -366,7 +366,7 @@ pub fn entry( #(#param_parsing)* let macro_crate_name = env!("CARGO_PKG_NAME"); - let final_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + let final_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME").unwrap_or(macro_crate_name); #root::runtime_support::set_macro_to_crate(macro_crate_name, final_crate_name); let output_core = { diff --git a/stageleft_macro/src/quote_impl/mod.rs b/stageleft_macro/src/quote_impl/mod.rs index 928ae91cbb76..f26b533be7d3 100644 --- a/stageleft_macro/src/quote_impl/mod.rs +++ b/stageleft_macro/src/quote_impl/mod.rs @@ -42,7 +42,7 @@ pub fn q_impl(root: TokenStream, expr: syn::Expr) -> TokenStream { #(#unitialized_free_variables;)* *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME").unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = #root::internal::quote! { #expr_without_spans }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local@macro_tokens.snap index 5385e11b6c04..7a48e3cdfdef 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -22,7 +22,8 @@ fn main() { _out }; *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { (x + 2) + (x + 2) }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local_block@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local_block@macro_tokens.snap index 96d63583b5b0..b5f826004f69 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local_block@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local_block@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -22,7 +22,8 @@ fn main() { _out }; *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { { let _ = x + 2; let _ = x + 2; } }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local_block_let@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local_block_let@macro_tokens.snap index c5288baaf7cd..84a8c67dea1d 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local_block_let@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_copy_local_block_let@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -22,7 +22,8 @@ fn main() { _out }; *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { { let x = x + 2; let _ = x + 2; } }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_in_macro@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_in_macro@macro_tokens.snap index 46dff03f98f0..13d47903d640 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_in_macro@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_in_macro@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -22,7 +22,8 @@ fn main() { _out }; *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { dbg!(x) }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_local@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_local@macro_tokens.snap index 6e78d455407a..1ba7491cf55f 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_local@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_local@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -22,7 +22,8 @@ fn main() { _out }; *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { x + 2 }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_local_mut@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_local_mut@macro_tokens.snap index 06a002e03e90..70bb2d59ba99 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_local_mut@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__capture_local_mut@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -22,7 +22,8 @@ fn main() { _out }; *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { x += 2 }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_enum_creation@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_enum_creation@macro_tokens.snap index ad8caabe4841..819cbbd28c3c 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_enum_creation@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_enum_creation@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -12,7 +12,8 @@ fn main() { run: bool| { *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { Foo::Bar { x : 1 } }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_local@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_local@macro_tokens.snap index 99e8d91bff17..7169ceb62557 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_local@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_local@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -12,7 +12,8 @@ fn main() { run: bool| { *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { { let x = 1; x + 2 } }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_struct_creation@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_struct_creation@macro_tokens.snap index 53b8b24eb4ab..e72911de50bc 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_struct_creation@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__non_capture_struct_creation@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -12,7 +12,8 @@ fn main() { run: bool| { *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { Foo { x : 1 } }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-2.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-2.snap index eefa385bc688..236653f1143e 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-2.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-2.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -12,7 +12,8 @@ fn main() { run: bool| { *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { None }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-3.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-3.snap index b0c1d999fcfe..a7c8bf5a87af 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-3.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-3.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -12,7 +12,8 @@ fn main() { run: bool| { *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { Ok(1) }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-4.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-4.snap index 5cf03ca9f084..3a7639ba687a 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-4.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens-4.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -12,7 +12,8 @@ fn main() { run: bool| { *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { Err(1) }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens.snap index 9f63c497ecfd..06b3866d73bd 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__prelude_enum_variants@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -12,7 +12,8 @@ fn main() { run: bool| { *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { Some(1) }; diff --git a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__simple@macro_tokens.snap b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__simple@macro_tokens.snap index f6e626405c16..d9143ddb3d98 100644 --- a/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__simple@macro_tokens.snap +++ b/stageleft_macro/src/quote_impl/snapshots/stageleft_macro__quote_impl__tests__simple@macro_tokens.snap @@ -1,6 +1,6 @@ --- source: stageleft_macro/src/quote_impl/mod.rs -expression: "prettyplease::unparse(&wrapped)" +expression: "prettyplease :: unparse(& wrapped)" --- fn main() { { @@ -12,7 +12,8 @@ fn main() { run: bool| { *set_mod = module_path!().to_string(); - *set_crate_name = env!("STAGELEFT_FINAL_CRATE_NAME"); + *set_crate_name = option_env!("STAGELEFT_FINAL_CRATE_NAME") + .unwrap_or(env!("CARGO_PKG_NAME")); *set_tokens = stageleft::internal::quote! { 1 + 2 }; diff --git a/stageleft_tool/src/lib.rs b/stageleft_tool/src/lib.rs index 217b155e4ced..35bbff0f0f06 100644 --- a/stageleft_tool/src/lib.rs +++ b/stageleft_tool/src/lib.rs @@ -116,6 +116,7 @@ impl VisitMut for InlineTopLevelMod { struct GenFinalPubVistor { current_mod: Option, + test_mode: bool, } impl VisitMut for GenFinalPubVistor { @@ -142,15 +143,30 @@ impl VisitMut for GenFinalPubVistor { fn visit_item_mod_mut(&mut self, i: &mut syn::ItemMod) { let is_runtime_or_test = i.attrs.iter().any(|a| { a.path().to_token_stream().to_string() == "stageleft :: runtime" - || a.to_token_stream().to_string() == "# [cfg (test)]" + || a.to_token_stream().to_string() == "# [test]" + || a.to_token_stream().to_string() == "# [tokio::test]" }); + let is_test_mod = i + .attrs + .iter() + .any(|a| a.to_token_stream().to_string() == "# [cfg (test)]"); + if is_runtime_or_test { *i = parse_quote! { #[cfg(stageleft_macro)] #i }; } else { + if is_test_mod { + i.attrs + .retain(|a| a.to_token_stream().to_string() != "# [cfg (test)]"); + + if !self.test_mode { + i.attrs.push(parse_quote!(#[cfg(stageleft_macro)])); + } + } + let old_mod = self.current_mod.clone(); let i_ident = &i.ident; self.current_mod = self @@ -233,6 +249,22 @@ impl VisitMut for GenFinalPubVistor { } } +pub fn gen_staged_trybuild(lib_path: &Path, orig_crate_name: String, test_mode: bool) -> syn::File { + let mut orig_flow_lib = syn_inline_mod::parse_and_inline_modules(lib_path); + InlineTopLevelMod {}.visit_file_mut(&mut orig_flow_lib); + + let mut flow_lib_pub = syn_inline_mod::parse_and_inline_modules(lib_path); + + let orig_crate_ident = syn::Ident::new(&orig_crate_name, Span::call_site()); + let mut final_pub_visitor = GenFinalPubVistor { + current_mod: Some(parse_quote!(#orig_crate_ident)), + test_mode, + }; + final_pub_visitor.visit_file_mut(&mut flow_lib_pub); + + flow_lib_pub +} + pub fn gen_final_helper() { let out_dir = env::var_os("OUT_DIR").unwrap(); @@ -243,6 +275,7 @@ pub fn gen_final_helper() { let mut final_pub_visitor = GenFinalPubVistor { current_mod: Some(parse_quote!(crate)), + test_mode: false, }; final_pub_visitor.visit_file_mut(&mut flow_lib_pub); @@ -260,11 +293,6 @@ pub fn gen_final_helper() { #[macro_export] macro_rules! gen_final { () => { - println!( - "cargo::rustc-env=STAGELEFT_FINAL_CRATE_NAME={}", - env!("CARGO_PKG_NAME") - ); - #[cfg(not(feature = "stageleft_devel"))] $crate::gen_final_helper() }; diff --git a/template/hydroflow_plus/Cargo.toml b/template/hydroflow_plus/Cargo.toml index b9aea292adf0..2c521b6155d6 100644 --- a/template/hydroflow_plus/Cargo.toml +++ b/template/hydroflow_plus/Cargo.toml @@ -10,7 +10,6 @@ stageleft_devel = [] [dependencies] hydroflow_plus = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } -hydroflow_plus_deploy = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } stageleft = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } tokio = { version = "1.29.0", features = [ "full" ] } @@ -20,7 +19,7 @@ stageleft_tool = { git = "{{ hydroflow_git | default: 'https://github.com/hydro- [dev-dependencies] async-ssh2-lite = { version = "0.5.0", features = ["vendored-openssl"] } hydro_deploy = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}" } -hydroflow_plus_deploy = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}", features = [ +hydroflow_plus = { git = "{{ hydroflow_git | default: 'https://github.com/hydro-project/hydroflow.git' }}", branch = "{{ hydroflow_branch | default: 'main' }}", features = [ "deploy", ] } tokio-stream = { version = "0.1.3", default-features = false } diff --git a/template/hydroflow_plus/examples/first_ten_distributed.rs b/template/hydroflow_plus/examples/first_ten_distributed.rs index bdef569b0ba1..6cae922a757d 100644 --- a/template/hydroflow_plus/examples/first_ten_distributed.rs +++ b/template/hydroflow_plus/examples/first_ten_distributed.rs @@ -1,5 +1,5 @@ use hydro_deploy::Deployment; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; #[tokio::main] async fn main() { diff --git a/template/hydroflow_plus/examples/first_ten_distributed_gcp.rs b/template/hydroflow_plus/examples/first_ten_distributed_gcp.rs index 9ce50f956fa3..3caf775ce50d 100644 --- a/template/hydroflow_plus/examples/first_ten_distributed_gcp.rs +++ b/template/hydroflow_plus/examples/first_ten_distributed_gcp.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use hydro_deploy::gcp::GcpNetwork; use hydro_deploy::Deployment; -use hydroflow_plus_deploy::TrybuildHost; +use hydroflow_plus::deploy::TrybuildHost; use tokio::sync::RwLock; static RELEASE_RUSTFLAGS: &str = diff --git a/template/hydroflow_plus/src/first_ten_distributed.rs b/template/hydroflow_plus/src/first_ten_distributed.rs index 4afaf7d1d0c1..7b0e966bf7da 100644 --- a/template/hydroflow_plus/src/first_ten_distributed.rs +++ b/template/hydroflow_plus/src/first_ten_distributed.rs @@ -19,8 +19,8 @@ pub fn first_ten_distributed(flow: &FlowBuilder) -> (Process, Process) { #[cfg(test)] mod tests { use hydro_deploy::Deployment; + use hydroflow_plus::deploy::{DeployCrateWrapper, TrybuildHost}; use hydroflow_plus::futures::StreamExt; - use hydroflow_plus_deploy::{DeployCrateWrapper, TrybuildHost}; use tokio_stream::wrappers::UnboundedReceiverStream; #[tokio::test]