add grpc for uploading and downloading segments

Cargo.lock

@@ -551,6 +551,34 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
 
+[[package]]
+name = "axum"
+version = "0.6.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
+dependencies = [
+ "async-trait",
+ "axum-core 0.3.4",
+ "bitflags 1.3.2",
+ "bytes",
+ "futures-util",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "hyper 0.14.29",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite 0.2.14",
+ "rustversion",
+ "serde",
+ "sync_wrapper 0.1.2",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "axum"
 version = "0.7.5"
@@ -558,7 +586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
 dependencies = [
  "async-trait",
- "axum-core",
+ "axum-core 0.4.5",
  "bytes",
  "futures-util",
  "http 1.2.0",
@@ -578,6 +606,23 @@ dependencies = [
  "tower-service",
 ]
 
+[[package]]
+name = "axum-core"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "mime",
+ "rustversion",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "axum-core"
 version = "0.4.5"
@@ -682,7 +727,7 @@ dependencies = [
  "lazy_static",
  "lazycell",
  "peeking_take_while",
- "prettyplease",
+ "prettyplease 0.2.20",
  "proc-macro2",
  "quote",
  "regex",
@@ -1189,7 +1234,7 @@ dependencies = [
  "futures-core",
  "prost 0.13.4",
  "prost-types 0.13.4",
- "tonic",
+ "tonic 0.12.3",
  "tracing-core",
 ]
 
@@ -1213,7 +1258,7 @@ dependencies = [
  "thread_local",
  "tokio",
  "tokio-stream",
- "tonic",
+ "tonic 0.12.3",
  "tracing",
  "tracing-core",
  "tracing-subscriber",
@@ -2293,7 +2338,7 @@ dependencies = [
  "ethers-core",
  "ethers-etherscan",
  "eyre",
- "prettyplease",
+ "prettyplease 0.2.20",
  "proc-macro2",
  "quote",
  "regex",
@@ -3435,6 +3480,18 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper 0.14.29",
+ "pin-project-lite 0.2.14",
+ "tokio",
+ "tokio-io-timeout",
+]
+
 [[package]]
 name = "hyper-timeout"
 version = "0.5.2"
@@ -6104,6 +6161,16 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 
+[[package]]
+name = "prettyplease"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86"
+dependencies = [
+ "proc-macro2",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "prettyplease"
 version = "0.2.20"
@@ -6277,6 +6344,16 @@ dependencies = [
  "prost-derive 0.10.1",
 ]
 
+[[package]]
+name = "prost"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
+dependencies = [
+ "bytes",
+ "prost-derive 0.11.9",
+]
+
 [[package]]
 name = "prost"
 version = "0.13.4"
@@ -6329,6 +6406,28 @@ dependencies = [
  "which",
 ]
 
+[[package]]
+name = "prost-build"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270"
+dependencies = [
+ "bytes",
+ "heck 0.4.1",
+ "itertools 0.10.5",
+ "lazy_static",
+ "log",
+ "multimap",
+ "petgraph",
+ "prettyplease 0.1.25",
+ "prost 0.11.9",
+ "prost-types 0.11.9",
+ "regex",
+ "syn 1.0.109",
+ "tempfile",
+ "which",
+]
+
 [[package]]
 name = "prost-codec"
 version = "0.1.0"
@@ -6368,6 +6467,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "prost-derive"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
+dependencies = [
+ "anyhow",
+ "itertools 0.10.5",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "prost-derive"
 version = "0.13.4"
@@ -6401,6 +6513,15 @@ dependencies = [
  "prost 0.10.4",
 ]
 
+[[package]]
+name = "prost-types"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
+dependencies = [
+ "prost 0.11.9",
+]
+
 [[package]]
 name = "prost-types"
 version = "0.13.4"
@@ -6866,7 +6987,7 @@ dependencies = [
 
 [[package]]
 name = "rpc"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "append_merkle",
  "base64 0.13.1",
@@ -6881,6 +7002,9 @@ dependencies = [
  "miner",
  "network",
  "parking_lot 0.12.3",
+ "prost 0.11.9",
+ "prost-build 0.11.9",
+ "prost-types 0.11.9",
  "serde",
  "serde_json",
  "shared_types",
@@ -6889,6 +7013,8 @@ dependencies = [
  "sync",
  "task_executor",
  "tokio",
+ "tonic 0.9.2",
+ "tonic-build",
  "tracing",
 ]
 
@@ -8188,6 +8314,34 @@ dependencies = [
  "winnow 0.6.13",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
+dependencies = [
+ "async-trait",
+ "axum 0.6.20",
+ "base64 0.21.7",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2 0.3.26",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "hyper 0.14.29",
+ "hyper-timeout 0.4.1",
+ "percent-encoding",
+ "pin-project 1.1.5",
+ "prost 0.11.9",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
 [[package]]
 name = "tonic"
 version = "0.12.3"
@@ -8196,7 +8350,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
 dependencies = [
  "async-stream",
  "async-trait",
- "axum",
+ "axum 0.7.5",
  "base64 0.22.1",
  "bytes",
  "h2 0.4.7",
@@ -8204,7 +8358,7 @@ dependencies = [
  "http-body 1.0.1",
  "http-body-util",
  "hyper 1.5.2",
- "hyper-timeout",
+ "hyper-timeout 0.5.2",
  "hyper-util",
  "percent-encoding",
  "pin-project 1.1.5",
@@ -8218,6 +8372,19 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "tonic-build"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07"
+dependencies = [
+ "prettyplease 0.1.25",
+ "proc-macro2",
+ "prost-build 0.11.9",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "tower"
 version = "0.4.13"
@@ -8246,9 +8413,9 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
 
 [[package]]
 name = "tower-service"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
 
 [[package]]
 name = "tracing"

node/rpc/Cargo.toml

@@ -1,7 +1,8 @@
 [package]
 name = "rpc"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
+build = "build.rs"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -28,3 +29,10 @@ merkle_tree = { path = "../../common/merkle_tree"}
 futures-channel = "^0.3"
 metrics = { workspace = true }
 parking_lot = "0.12.3"
+tonic = { version = "0.9.2", features = ["transport"] }
+prost = "0.11.9"
+prost-types = "0.11.9"
+
+[build-dependencies]
+tonic-build = "0.9.2"
+prost-build = "0.11.9"

node/rpc/build.rs

@@ -0,0 +1,5 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Compile proto/my_service.proto
+    tonic_build::compile_protos("proto/zgs_grpc.proto")?;
+    Ok(())
+}

node/rpc/proto/zgs_grpc.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+package zgs_grpc;
+
+// A trivial ping service
+service ZgsGrpcService {
+  rpc Ping (PingRequest) returns (PingReply);
+}
+
+message PingRequest {
+  string message = 1;
+}
+
+message PingReply {
+  string message = 1;
+}

node/rpc/src/config.rs

@@ -8,6 +8,7 @@ pub struct Config {
     pub enabled: bool,
     pub listen_address: SocketAddr,
     pub listen_address_admin: SocketAddr,
+    pub listen_address_grpc: SocketAddr,
     pub chunks_per_segment: usize,
     pub max_request_body_size: u32,
     pub max_cache_file_size: usize,
@@ -19,6 +20,7 @@ impl Default for Config {
             enabled: true,
             listen_address: SocketAddr::from_str("0.0.0.0:5678").unwrap(),
             listen_address_admin: SocketAddr::from_str("127.0.0.1:5679").unwrap(),
+            listen_address_grpc: SocketAddr::from_str("0.0.0.0:50051").unwrap(),
             chunks_per_segment: 1024,
             max_request_body_size: 100 * 1024 * 1024, // 100MB
             max_cache_file_size: 10 * 1024 * 1024,    // 10MB

node/rpc/src/lib.rs

@@ -12,6 +12,8 @@ pub mod types;
 mod zgs;
 
 use crate::miner::RpcServer as MinerRpcServer;
+use crate::zgs_grpc::zgs_grpc::ZgsGrpcServiceImpl;
+use crate::zgs_grpc_proto::zgs_grpc_service_server::ZgsGrpcServiceServer;
 use admin::RpcServer as AdminRpcServer;
 use chunk_pool::MemoryChunkPool;
 use file_location_cache::FileLocationCache;
@@ -33,6 +35,14 @@ pub use config::Config as RPCConfig;
 pub use miner::RpcClient as ZgsMinerRpcClient;
 pub use zgs::RpcClient as ZgsRPCClient;
 
+pub mod zgs_grpc_proto {
+    tonic::include_proto!("zgs_grpc");
+}
+
+mod zgs_grpc;
+
+use tonic::transport::Server;
+
 /// A wrapper around all the items required to spawn the HTTP server.
 ///
 /// The server will gracefully handle the case where any fields are `None`.
@@ -73,7 +83,7 @@ pub async fn run_server(
         (run_server_all(ctx).await?, None)
     };
 
-    info!("Server started");
+    info!("Rpc Server started");
 
     Ok(handles)
 }
@@ -133,3 +143,14 @@ async fn run_server_public_private(
 
     Ok((handle_public, Some(handle_private)))
 }
+
+pub async fn run_grpc_server(ctx: Context) -> Result<(), Box<dyn Error>> {
+    let grpc_addr = ctx.config.listen_address_grpc;
+    let server = ZgsGrpcServiceServer::new(ZgsGrpcServiceImpl { ctx });
+    Server::builder()
+        .add_service(server)
+        .serve(grpc_addr)
+        .await?;
+    Ok(())
+}
+

node/rpc/src/zgs_grpc/mod.rs

@@ -0,0 +1 @@
+pub mod zgs_grpc;

node/rpc/src/zgs_grpc/zgs_grpc.rs

@@ -0,0 +1,19 @@
+use crate::zgs_grpc_proto::{PingRequest, PingReply};
+use crate::zgs_grpc_proto::zgs_grpc_service_server::ZgsGrpcService;
+use crate::Context;
+
+pub struct ZgsGrpcServiceImpl {
+    pub ctx: Context,
+}
+
+#[tonic::async_trait]
+impl ZgsGrpcService for ZgsGrpcServiceImpl {
+    async fn ping(
+        &self,
+        request: tonic::Request<PingRequest>,
+    ) -> Result<tonic::Response<PingReply>, tonic::Status> {
+        let msg = request.into_inner().message;
+        let reply = PingReply { message: format!("Echo: {}", msg) };
+        Ok(tonic::Response::new(reply))
+    }
+}

node/src/client/builder.rs

@@ -298,7 +298,7 @@ impl ClientBuilder {
             mine_service_sender: mine_send,
         };
 
-        let (rpc_handle, maybe_admin_rpc_handle) = rpc::run_server(ctx)
+        let (rpc_handle, maybe_admin_rpc_handle) = rpc::run_server(ctx.clone())
             .await
             .map_err(|e| format!("Unable to start HTTP RPC server: {:?}", e))?;
 
@@ -307,6 +307,13 @@ impl ClientBuilder {
             executor.spawn(admin_rpc_handle, "rpc_admin");
         }
 
+        executor.spawn(
+            async move {
+                rpc::run_grpc_server(ctx.clone()).await.expect("Failed to start gRPC server");
+            },
+            "grpc",
+        );
+
         Ok(self)
     }
 

run/config-testnet-standard.toml

@@ -305,6 +305,9 @@ auto_sync_enabled = true
 # HTTP server address to bind for admin and debug RPC.
 # listen_address_admin = "127.0.0.1:5679"
 
+## Grpc server address to bind
+# listen_address_grpc = "0.0.0.0:50051"
+
 # Number of chunks for a single segment.
 # chunks_per_segment = 1024
 

run/config-testnet-turbo.toml

@@ -317,6 +317,9 @@ auto_sync_enabled = true
 # HTTP server address to bind for admin and debug RPC.
 # listen_address_admin = "127.0.0.1:5679"
 
+## Grpc server address to bind
+# listen_address_grpc = "0.0.0.0:50051"
+
 # Number of chunks for a single segment.
 # chunks_per_segment = 1024
 

run/config.toml

@@ -319,6 +319,9 @@
 # HTTP server address to bind for admin and debug RPC.
 # listen_address_admin = "127.0.0.1:5679"
 
+## Grpc server address to bind
+# listen_address_grpc = "0.0.0.0:50051"
+
 # Number of chunks for a single segment.
 # chunks_per_segment = 1024