feat(hydro_deploy): split Rust core from Python bindings (#986)

hydro-project · Dec 20, 2023 · c50ca12 · c50ca12
1 parent 174607d
commit c50ca12
Show file tree

Hide file tree

Showing 76 changed files with 112 additions and 165 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -210,7 +210,7 @@ jobs:
 
       - name: Build CLI
         run: |
-          cd hydro_cli
+          cd hydro_deploy/hydro_cli
           python -m venv .venv
           source .venv/bin/activate
           pip install maturin
@@ -219,7 +219,7 @@ jobs:
       - name: Run Python tests
         run: |
           ulimit -c unlimited
-          cd hydro_cli
+          cd hydro_deploy/hydro_cli
           source .venv/bin/activate
           cd python_tests
           pip install -r requirements.txt

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -2,16 +2,16 @@
 
 members = [
     "benches",
-    "hydro_cli",
-    "hydro_cli_examples",
+    "hydro_deploy/hydro_cli",
+    "hydro_deploy/hydro_cli_examples",
+    "hydro_deploy/hydroflow_cli_integration",
+    "hydro_deploy/hydroflow_plus_cli_integration",
     "hydroflow",
-    "hydroflow_cli_integration",
     "hydroflow_datalog",
     "hydroflow_datalog_core",
     "hydroflow_lang",
     "hydroflow_macro",
     "hydroflow_plus",
-    "hydroflow_plus_cli_integration",
     "hydroflow_plus_test",
     "hydroflow_plus_test_macro",
     "lattices",

diff --git a/docs/docs/hydroflow_plus/distributed.mdx b/docs/docs/hydroflow_plus/distributed.mdx
@@ -81,7 +81,7 @@ async fn main() {
 Finally, we need to deploy our dataflow with the appropriate network topology. We achieve this by using [Hydro Deploy](../deploy/index.md). Hydroflow+ integrates with Hydro Deploy to automatically construct the topology based on the flow graph. We can create a new file `examples/first_ten_distributed.rs` with the following contents:
 
 ```rust
-use hydro_cli::core::Deployment;
+use hydro_deploy::Deployment;
 use hydroflow_plus_cli_integration::CLIDeployNodeBuilder;
 
 #[tokio::main]

diff --git a/hydro_cli/echo.hydro.py b/hydro_cli/echo.hydro.py
diff --git a/hydro_cli/.gitignore → hydro_deploy/core/.gitignore b/hydro_cli/.gitignore → hydro_deploy/core/.gitignore
diff --git a/hydro_cli/Cargo.toml → hydro_deploy/core/Cargo.toml b/hydro_cli/Cargo.toml → hydro_deploy/core/Cargo.toml
@@ -1,26 +1,17 @@
 [package]
-name = "hydro_cli"
+name = "hydro_deploy"
 publish = true
 version = "0.5.0"
 edition = "2021"
 license = "Apache-2.0"
-documentation = "https://docs.rs/hydro_cli/"
-description = "Hydro Deploy Command Line Interface"
-
-[lib]
-name = "hydro_cli"
-# "cdylib" is necessary to produce a shared library for Python to import from.
-crate-type = ["lib", "cdylib"]
+documentation = "https://docs.rs/hydro_deploy/"
+description = "Hydro Deploy"
 
 [dependencies]
 tokio = { version = "1.16", features = [ "full" ] }
 tokio-util = { version = "0.7.7", features=[ "compat" ] }
 once_cell = "1.17"
 anyhow = { version = "1.0.69", features = [ "backtrace" ] }
-clap = { version = "4.1.8", features = ["derive"] }
-pyo3 = { version = "0.18", features = ["abi3-py37"] }
-pyo3-asyncio = { version = "0.18", features = ["attributes", "tokio-runtime"] }
-pythonize = "0.18"
 async-trait = "0.1.64"
 async-process = "1.6.0"
 async-recursion = "1"
@@ -35,11 +26,8 @@ shell-escape = "0.1.5"
 dyn-clone = "1"
 bytes = "1.1.0"
 nanoid = "0.4.0"
-ctrlc = "3.2.5"
 nix = "0.26.2"
 hydroflow_cli_integration = { path = "../hydroflow_cli_integration", version = "^0.3.0" }
 indicatif = "0.17.6"
 cargo_metadata = "0.15.4"
 async-once-cell = "0.5.3"
-
-[dev-dependencies]
diff --git a/hydro_cli/src/core/custom_service.rs → hydro_deploy/core/src/custom_service.rs b/hydro_cli/src/core/custom_service.rs → hydro_deploy/core/src/custom_service.rs
diff --git a/hydro_cli/src/core/deployment.rs → hydro_deploy/core/src/deployment.rs b/hydro_cli/src/core/deployment.rs → hydro_deploy/core/src/deployment.rs
@@ -42,7 +42,7 @@ impl Deployment {
         user: Option<String>,
     ) -> Arc<RwLock<GCPComputeEngineHost>> {
         self.add_host(|id| {
-            crate::core::GCPComputeEngineHost::new(
+            super::GCPComputeEngineHost::new(
                 id,
                 project,
                 machine_type,
@@ -77,7 +77,7 @@ impl Deployment {
         external_ports: Vec<u16>,
     ) -> Arc<RwLock<HydroflowCrate>> {
         self.add_service(|id| {
-            crate::core::HydroflowCrate::new(
+            super::HydroflowCrate::new(
                 id,
                 src.into(),
                 on,

diff --git a/hydro_cli/src/core/gcp.rs → hydro_deploy/core/src/gcp.rs b/hydro_cli/src/core/gcp.rs → hydro_deploy/core/src/gcp.rs
diff --git a/hydro_cli/src/core/hydroflow_crate/build.rs → ..._deploy/core/src/hydroflow_crate/build.rs b/hydro_cli/src/core/hydroflow_crate/build.rs → ..._deploy/core/src/hydroflow_crate/build.rs
@@ -8,8 +8,8 @@ use nanoid::nanoid;
 use once_cell::sync::Lazy;
 use tokio::sync::OnceCell;
 
-use crate::core::progress::ProgressTracker;
-use crate::core::HostTargetType;
+use crate::progress::ProgressTracker;
+use crate::HostTargetType;
 
 #[derive(PartialEq, Eq, Hash)]
 struct CacheKey {

diff --git a/hydro_cli/src/core/hydroflow_crate/mod.rs → hydro_deploy/core/src/hydroflow_crate/mod.rs b/hydro_cli/src/core/hydroflow_crate/mod.rs → hydro_deploy/core/src/hydroflow_crate/mod.rs
@@ -7,7 +7,7 @@ use anyhow::{anyhow, bail, Result};
 use async_channel::Receiver;
 use async_trait::async_trait;
 use futures_core::Future;
-use hydroflow_cli_integration::{ServerBindConfig, ServerPort};
+use hydroflow_cli_integration::{InitConfig, ServerPort};
 use tokio::sync::RwLock;
 
 use self::ports::{HydroflowPortConfig, HydroflowSink, SourcePath};
@@ -205,8 +205,6 @@ impl HydroflowCrate {
     }
 }
 
-type InitConfig = (HashMap<String, ServerBindConfig>, Option<String>);
-
 #[async_trait]
 impl Service for HydroflowCrate {
     fn collect_resources(&mut self, _resource_batch: &mut ResourceBatch) {

diff --git a/hydro_cli/src/core/hydroflow_crate/ports.rs → ..._deploy/core/src/hydroflow_crate/ports.rs b/hydro_cli/src/core/hydroflow_crate/ports.rs → ..._deploy/core/src/hydroflow_crate/ports.rs
@@ -11,7 +11,7 @@ use hydroflow_cli_integration::ServerPort;
 use tokio::sync::RwLock;
 
 use super::HydroflowCrate;
-use crate::core::{ClientStrategy, Host, HostStrategyGetter, LaunchedHost, ServerStrategy};
+use crate::{ClientStrategy, Host, HostStrategyGetter, LaunchedHost, ServerStrategy};
 
 pub trait HydroflowSource: Send + Sync {
     fn source_path(&self) -> SourcePath;

diff --git a/hydro_cli/src/core/mod.rs → hydro_deploy/core/src/lib.rs b/hydro_cli/src/core/mod.rs → hydro_deploy/core/src/lib.rs
diff --git a/hydro_cli/src/core/localhost.rs → hydro_deploy/core/src/localhost.rs b/hydro_cli/src/core/localhost.rs → hydro_deploy/core/src/localhost.rs
diff --git a/hydro_cli/src/core/progress.rs → hydro_deploy/core/src/progress.rs b/hydro_cli/src/core/progress.rs → hydro_deploy/core/src/progress.rs
diff --git a/hydro_cli/src/core/ssh.rs → hydro_deploy/core/src/ssh.rs b/hydro_cli/src/core/ssh.rs → hydro_deploy/core/src/ssh.rs
@@ -17,8 +17,8 @@ use tokio::sync::RwLock;
 
 use super::localhost::create_broadcast;
 use super::progress::ProgressTracker;
+use super::util::async_retry;
 use super::{LaunchedBinary, LaunchedHost, ResourceResult, ServerStrategy};
-use crate::core::util::async_retry;
 
 struct LaunchedSSHBinary {
     _resource_result: Arc<ResourceResult>,

diff --git a/hydro_cli/src/core/terraform.rs → hydro_deploy/core/src/terraform.rs b/hydro_cli/src/core/terraform.rs → hydro_deploy/core/src/terraform.rs
diff --git a/hydro_cli/src/core/util.rs → hydro_deploy/core/src/util.rs b/hydro_cli/src/core/util.rs → hydro_deploy/core/src/util.rs
diff --git a/hydro_deploy/hydro_cli/.gitignore b/hydro_deploy/hydro_cli/.gitignore
@@ -0,0 +1,3 @@
+.venv/
+.hydro/
+*.csv
diff --git a/hydro_cli/CHANGELOG.md → hydro_deploy/hydro_cli/CHANGELOG.md b/hydro_cli/CHANGELOG.md → hydro_deploy/hydro_cli/CHANGELOG.md
diff --git a/hydro_deploy/hydro_cli/Cargo.toml b/hydro_deploy/hydro_cli/Cargo.toml
@@ -0,0 +1,27 @@
+[package]
+name = "hydro_cli"
+publish = true
+version = "0.5.0"
+edition = "2021"
+license = "Apache-2.0"
+documentation = "https://docs.rs/hydro_cli/"
+description = "Hydro Deploy Command Line Interface"
+
+[lib]
+name = "hydro_cli"
+# "cdylib" is necessary to produce a shared library for Python to import from.
+crate-type = ["cdylib"]
+
+[dependencies]
+hydro_deploy = { path = "../core", version = "^0.5.0" }
+tokio = { version = "1.16", features = [ "full" ] }
+once_cell = "1.17"
+anyhow = { version = "1.0.69", features = [ "backtrace" ] }
+clap = { version = "4.1.8", features = ["derive"] }
+pyo3 = { version = "0.18", features = ["abi3-py37"] }
+pyo3-asyncio = { version = "0.18", features = ["attributes", "tokio-runtime"] }
+pythonize = "0.18"
+futures = "0.3.26"
+async-channel = "1.8.0"
+bytes = "1.1.0"
+hydroflow_cli_integration = { path = "../hydroflow_cli_integration", version = "^0.3.0" }
diff --git a/hydro_cli/README.md → hydro_deploy/hydro_cli/README.md b/hydro_cli/README.md → hydro_deploy/hydro_cli/README.md
diff --git a/hydro_cli/hydro/.gitignore → hydro_deploy/hydro_cli/hydro/.gitignore b/hydro_cli/hydro/.gitignore → hydro_deploy/hydro_cli/hydro/.gitignore
diff --git a/hydro_cli/hydro/__init__.py → hydro_deploy/hydro_cli/hydro/__init__.py b/hydro_cli/hydro/__init__.py → hydro_deploy/hydro_cli/hydro/__init__.py
diff --git a/hydro_cli/hydro/_core.pyi → hydro_deploy/hydro_cli/hydro/_core.pyi b/hydro_cli/hydro/_core.pyi → hydro_deploy/hydro_cli/hydro/_core.pyi
diff --git a/hydro_cli/hydro/async_wrapper.py → ...o_deploy/hydro_cli/hydro/async_wrapper.py b/hydro_cli/hydro/async_wrapper.py → ...o_deploy/hydro_cli/hydro/async_wrapper.py
diff --git a/hydro_cli/hydro/cli.py → hydro_deploy/hydro_cli/hydro/cli.py b/hydro_cli/hydro/cli.py → hydro_deploy/hydro_cli/hydro/cli.py
diff --git a/hydro_cli/pyproject.toml → hydro_deploy/hydro_cli/pyproject.toml b/hydro_cli/pyproject.toml → hydro_deploy/hydro_cli/pyproject.toml
diff --git a/hydro_cli/python_tests/basic.py → hydro_deploy/hydro_cli/python_tests/basic.py b/hydro_cli/python_tests/basic.py → hydro_deploy/hydro_cli/python_tests/basic.py
diff --git a/hydro_cli/python_tests/demux.py → hydro_deploy/hydro_cli/python_tests/demux.py b/hydro_cli/python_tests/demux.py → hydro_deploy/hydro_cli/python_tests/demux.py
diff --git a/hydro_cli/python_tests/requirements.txt → ...y/hydro_cli/python_tests/requirements.txt b/hydro_cli/python_tests/requirements.txt → ...y/hydro_cli/python_tests/requirements.txt
diff --git a/hydro_cli/python_tests/tagged.py → ...o_deploy/hydro_cli/python_tests/tagged.py b/hydro_cli/python_tests/tagged.py → ...o_deploy/hydro_cli/python_tests/tagged.py
diff --git a/hydro_cli/src/cli.rs → hydro_deploy/hydro_cli/src/cli.rs b/hydro_cli/src/cli.rs → hydro_deploy/hydro_cli/src/cli.rs
diff --git a/hydro_cli/src/lib.rs → hydro_deploy/hydro_cli/src/lib.rs b/hydro_cli/src/lib.rs → hydro_deploy/hydro_cli/src/lib.rs
@@ -20,7 +20,7 @@ use tokio::sync::RwLock;
 use crate::core::hydroflow_crate::ports::HydroflowSource;
 
 mod cli;
-pub mod core;
+use hydro_deploy as core;
 
 static TOKIO_RUNTIME: std::sync::RwLock<Option<tokio::runtime::Runtime>> =
     std::sync::RwLock::new(None);

diff --git a/hydro_cli_examples/2pc.hydro.py → hydro_deploy/hydro_cli_examples/2pc.hydro.py b/hydro_cli_examples/2pc.hydro.py → hydro_deploy/hydro_cli_examples/2pc.hydro.py
diff --git a/hydro_cli_examples/Cargo.toml → hydro_deploy/hydro_cli_examples/Cargo.toml b/hydro_cli_examples/Cargo.toml → hydro_deploy/hydro_cli_examples/Cargo.toml
@@ -35,8 +35,8 @@ name = "dedalus_2pc_participant"
 name = "ws_chat_server"
 
 [dev-dependencies]
-hydroflow = { path = "../hydroflow", features = [ "cli_integration" ] }
-hydroflow_datalog = { path = "../hydroflow_datalog" }
+hydroflow = { path = "../../hydroflow", features = [ "cli_integration" ] }
+hydroflow_datalog = { path = "../../hydroflow_datalog" }
 
 tokio = { version = "1.16", features = [ "full" ] }
 serde = { version = "1", features = ["rc"] }

diff --git a/...examples/dedalus_python_receiver.hydro.py → ...examples/dedalus_python_receiver.hydro.py b/...examples/dedalus_python_receiver.hydro.py → ...examples/dedalus_python_receiver.hydro.py
diff --git a/.../examples/dedalus_2pc_coordinator/main.rs → .../examples/dedalus_2pc_coordinator/main.rs b/.../examples/dedalus_2pc_coordinator/main.rs → .../examples/dedalus_2pc_coordinator/main.rs
diff --git a/.../examples/dedalus_2pc_participant/main.rs → .../examples/dedalus_2pc_participant/main.rs b/.../examples/dedalus_2pc_participant/main.rs → .../examples/dedalus_2pc_participant/main.rs
diff --git a/...xamples/examples/dedalus_receiver/main.rs → ...xamples/examples/dedalus_receiver/main.rs b/...xamples/examples/dedalus_receiver/main.rs → ...xamples/examples/dedalus_receiver/main.rs
diff --git a/..._examples/examples/dedalus_sender/main.rs → ..._examples/examples/dedalus_sender/main.rs b/..._examples/examples/dedalus_sender/main.rs → ..._examples/examples/dedalus_sender/main.rs
diff --git a/...ples/examples/dedalus_vote_leader/main.rs → ...ples/examples/dedalus_vote_leader/main.rs b/...ples/examples/dedalus_vote_leader/main.rs → ...ples/examples/dedalus_vote_leader/main.rs
diff --git a/...examples/dedalus_vote_participant/main.rs → ...examples/dedalus_vote_participant/main.rs b/...examples/dedalus_vote_participant/main.rs → ...examples/dedalus_vote_participant/main.rs
diff --git a/...i_examples/examples/empty_program/main.rs → ...i_examples/examples/empty_program/main.rs b/...i_examples/examples/empty_program/main.rs → ...i_examples/examples/empty_program/main.rs
diff --git a/...examples/examples/stdout_receiver/main.rs → ...examples/examples/stdout_receiver/main.rs b/...examples/examples/stdout_receiver/main.rs → ...examples/examples/stdout_receiver/main.rs
diff --git a/...s/examples/tagged_stdout_receiver/main.rs → ...s/examples/tagged_stdout_receiver/main.rs b/...s/examples/tagged_stdout_receiver/main.rs → ...s/examples/tagged_stdout_receiver/main.rs
diff --git a/..._examples/examples/ws_chat_server/main.rs → ..._examples/examples/ws_chat_server/main.rs b/..._examples/examples/ws_chat_server/main.rs → ..._examples/examples/ws_chat_server/main.rs
diff --git a/..._examples/examples/ws_chat_server/util.rs → ..._examples/examples/ws_chat_server/util.rs b/..._examples/examples/ws_chat_server/util.rs → ..._examples/examples/ws_chat_server/util.rs
diff --git a/...cli_examples/hydro_python_sender.hydro.py → ...cli_examples/hydro_python_sender.hydro.py b/...cli_examples/hydro_python_sender.hydro.py → ...cli_examples/hydro_python_sender.hydro.py
diff --git a/hydro_cli_examples/pn_counter.hydro.py → ...oy/hydro_cli_examples/pn_counter.hydro.py b/hydro_cli_examples/pn_counter.hydro.py → ...oy/hydro_cli_examples/pn_counter.hydro.py
diff --git a/hydro_cli_examples/simple_dedalus.hydro.py → ...ydro_cli_examples/simple_dedalus.hydro.py b/hydro_cli_examples/simple_dedalus.hydro.py → ...ydro_cli_examples/simple_dedalus.hydro.py
diff --git a/hydro_cli_examples/spin.hydro.py → ...o_deploy/hydro_cli_examples/spin.hydro.py b/hydro_cli_examples/spin.hydro.py → ...o_deploy/hydro_cli_examples/spin.hydro.py
diff --git a/...li_examples/tagged_python_sender.hydro.py → ...li_examples/tagged_python_sender.hydro.py b/...li_examples/tagged_python_sender.hydro.py → ...li_examples/tagged_python_sender.hydro.py
diff --git a/hydro_cli_examples/vote.hydro.py → ...o_deploy/hydro_cli_examples/vote.hydro.py b/hydro_cli_examples/vote.hydro.py → ...o_deploy/hydro_cli_examples/vote.hydro.py
diff --git a/hydro_cli_examples/ws_chat_server.py → ...ploy/hydro_cli_examples/ws_chat_server.py b/hydro_cli_examples/ws_chat_server.py → ...ploy/hydro_cli_examples/ws_chat_server.py
diff --git a/hydroflow_cli_integration/CHANGELOG.md → ...oy/hydroflow_cli_integration/CHANGELOG.md b/hydroflow_cli_integration/CHANGELOG.md → ...oy/hydroflow_cli_integration/CHANGELOG.md
-Original file line number
+Diff line change
@@ -0,0 +1,3 @@
+    .venv/
+    .hydro/
+    *.csv