mirror of
https://github.com/0glabs/0g-storage-node.git
synced 2025-11-03 08:07:27 +00:00
add grpc for uploading and downloading segments
This commit is contained in:
Peter Zhang
parent
28654efde1
commit
a4b0bc1dac
187
Cargo.lock
generated
187
Cargo.lock
generated
@ -551,6 +551,34 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.6.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core 0.3.4",
|
||||
"bitflags 1.3.2",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"hyper 0.14.29",
|
||||
"itoa",
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite 0.2.14",
|
||||
"rustversion",
|
||||
"serde",
|
||||
"sync_wrapper 0.1.2",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.7.5"
|
||||
@ -558,7 +586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core",
|
||||
"axum-core 0.4.5",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.2.0",
|
||||
@ -578,6 +606,23 @@ dependencies = [
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"mime",
|
||||
"rustversion",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.4.5"
|
||||
@ -682,7 +727,7 @@ dependencies = [
|
||||
"lazy_static",
|
||||
"lazycell",
|
||||
"peeking_take_while",
|
||||
"prettyplease",
|
||||
"prettyplease 0.2.20",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
@ -1189,7 +1234,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"prost 0.13.4",
|
||||
"prost-types 0.13.4",
|
||||
"tonic",
|
||||
"tonic 0.12.3",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
@ -1213,7 +1258,7 @@ dependencies = [
|
||||
"thread_local",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
"tonic 0.12.3",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-subscriber",
|
||||
@ -2293,7 +2338,7 @@ dependencies = [
|
||||
"ethers-core",
|
||||
"ethers-etherscan",
|
||||
"eyre",
|
||||
"prettyplease",
|
||||
"prettyplease 0.2.20",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
@ -3435,6 +3480,18 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-timeout"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
|
||||
dependencies = [
|
||||
"hyper 0.14.29",
|
||||
"pin-project-lite 0.2.14",
|
||||
"tokio",
|
||||
"tokio-io-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-timeout"
|
||||
version = "0.5.2"
|
||||
@ -6104,6 +6161,16 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.2.20"
|
||||
@ -6277,6 +6344,16 @@ dependencies = [
|
||||
"prost-derive 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost"
|
||||
version = "0.11.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost-derive 0.11.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost"
|
||||
version = "0.13.4"
|
||||
@ -6329,6 +6406,28 @@ dependencies = [
|
||||
"which",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-build"
|
||||
version = "0.11.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"heck 0.4.1",
|
||||
"itertools 0.10.5",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph",
|
||||
"prettyplease 0.1.25",
|
||||
"prost 0.11.9",
|
||||
"prost-types 0.11.9",
|
||||
"regex",
|
||||
"syn 1.0.109",
|
||||
"tempfile",
|
||||
"which",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-codec"
|
||||
version = "0.1.0"
|
||||
@ -6368,6 +6467,19 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-derive"
|
||||
version = "0.11.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.10.5",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-derive"
|
||||
version = "0.13.4"
|
||||
@ -6401,6 +6513,15 @@ dependencies = [
|
||||
"prost 0.10.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-types"
|
||||
version = "0.11.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
|
||||
dependencies = [
|
||||
"prost 0.11.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-types"
|
||||
version = "0.13.4"
|
||||
@ -6866,7 +6987,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rpc"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"append_merkle",
|
||||
"base64 0.13.1",
|
||||
@ -6881,6 +7002,9 @@ dependencies = [
|
||||
"miner",
|
||||
"network",
|
||||
"parking_lot 0.12.3",
|
||||
"prost 0.11.9",
|
||||
"prost-build 0.11.9",
|
||||
"prost-types 0.11.9",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"shared_types",
|
||||
@ -6889,6 +7013,8 @@ dependencies = [
|
||||
"sync",
|
||||
"task_executor",
|
||||
"tokio",
|
||||
"tonic 0.9.2",
|
||||
"tonic-build",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@ -8188,6 +8314,34 @@ dependencies = [
|
||||
"winnow 0.6.13",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum 0.6.20",
|
||||
"base64 0.21.7",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.3.26",
|
||||
"http 0.2.12",
|
||||
"http-body 0.4.6",
|
||||
"hyper 0.14.29",
|
||||
"hyper-timeout 0.4.1",
|
||||
"percent-encoding",
|
||||
"pin-project 1.1.5",
|
||||
"prost 0.11.9",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.12.3"
|
||||
@ -8196,7 +8350,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"axum 0.7.5",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"h2 0.4.7",
|
||||
@ -8204,7 +8358,7 @@ dependencies = [
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper 1.5.2",
|
||||
"hyper-timeout",
|
||||
"hyper-timeout 0.5.2",
|
||||
"hyper-util",
|
||||
"percent-encoding",
|
||||
"pin-project 1.1.5",
|
||||
@ -8218,6 +8372,19 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic-build"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07"
|
||||
dependencies = [
|
||||
"prettyplease 0.1.25",
|
||||
"proc-macro2",
|
||||
"prost-build 0.11.9",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower"
|
||||
version = "0.4.13"
|
||||
@ -8246,9 +8413,9 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
|
||||
|
||||
[[package]]
|
||||
name = "tower-service"
|
||||
version = "0.3.2"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
|
||||
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
[package]
|
||||
name = "rpc"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
edition = "2021"
|
||||
build = "build.rs"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
@ -28,3 +29,10 @@ merkle_tree = { path = "../../common/merkle_tree"}
|
||||
futures-channel = "^0.3"
|
||||
metrics = { workspace = true }
|
||||
parking_lot = "0.12.3"
|
||||
tonic = { version = "0.9.2", features = ["transport"] }
|
||||
prost = "0.11.9"
|
||||
prost-types = "0.11.9"
|
||||
|
||||
[build-dependencies]
|
||||
tonic-build = "0.9.2"
|
||||
prost-build = "0.11.9"
|
||||
|
||||
5
node/rpc/build.rs
Normal file
5
node/rpc/build.rs
Normal file
@ -0,0 +1,5 @@
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Compile proto/my_service.proto
|
||||
tonic_build::compile_protos("proto/zgs_grpc.proto")?;
|
||||
Ok(())
|
||||
}
|
||||
16
node/rpc/proto/zgs_grpc.proto
Normal file
16
node/rpc/proto/zgs_grpc.proto
Normal file
@ -0,0 +1,16 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package zgs_grpc;
|
||||
|
||||
// A trivial ping service
|
||||
service ZgsGrpcService {
|
||||
rpc Ping (PingRequest) returns (PingReply);
|
||||
}
|
||||
|
||||
message PingRequest {
|
||||
string message = 1;
|
||||
}
|
||||
|
||||
message PingReply {
|
||||
string message = 1;
|
||||
}
|
||||
@ -8,6 +8,7 @@ pub struct Config {
|
||||
pub enabled: bool,
|
||||
pub listen_address: SocketAddr,
|
||||
pub listen_address_admin: SocketAddr,
|
||||
pub listen_address_grpc: SocketAddr,
|
||||
pub chunks_per_segment: usize,
|
||||
pub max_request_body_size: u32,
|
||||
pub max_cache_file_size: usize,
|
||||
@ -19,6 +20,7 @@ impl Default for Config {
|
||||
enabled: true,
|
||||
listen_address: SocketAddr::from_str("0.0.0.0:5678").unwrap(),
|
||||
listen_address_admin: SocketAddr::from_str("127.0.0.1:5679").unwrap(),
|
||||
listen_address_grpc: SocketAddr::from_str("0.0.0.0:50051").unwrap(),
|
||||
chunks_per_segment: 1024,
|
||||
max_request_body_size: 100 * 1024 * 1024, // 100MB
|
||||
max_cache_file_size: 10 * 1024 * 1024, // 10MB
|
||||
|
||||
@ -12,6 +12,8 @@ pub mod types;
|
||||
mod zgs;
|
||||
|
||||
use crate::miner::RpcServer as MinerRpcServer;
|
||||
use crate::zgs_grpc::zgs_grpc::ZgsGrpcServiceImpl;
|
||||
use crate::zgs_grpc_proto::zgs_grpc_service_server::ZgsGrpcServiceServer;
|
||||
use admin::RpcServer as AdminRpcServer;
|
||||
use chunk_pool::MemoryChunkPool;
|
||||
use file_location_cache::FileLocationCache;
|
||||
@ -33,6 +35,14 @@ pub use config::Config as RPCConfig;
|
||||
pub use miner::RpcClient as ZgsMinerRpcClient;
|
||||
pub use zgs::RpcClient as ZgsRPCClient;
|
||||
|
||||
pub mod zgs_grpc_proto {
|
||||
tonic::include_proto!("zgs_grpc");
|
||||
}
|
||||
|
||||
mod zgs_grpc;
|
||||
|
||||
use tonic::transport::Server;
|
||||
|
||||
/// A wrapper around all the items required to spawn the HTTP server.
|
||||
///
|
||||
/// The server will gracefully handle the case where any fields are `None`.
|
||||
@ -73,7 +83,7 @@ pub async fn run_server(
|
||||
(run_server_all(ctx).await?, None)
|
||||
};
|
||||
|
||||
info!("Server started");
|
||||
info!("Rpc Server started");
|
||||
|
||||
Ok(handles)
|
||||
}
|
||||
@ -133,3 +143,14 @@ async fn run_server_public_private(
|
||||
|
||||
Ok((handle_public, Some(handle_private)))
|
||||
}
|
||||
|
||||
pub async fn run_grpc_server(ctx: Context) -> Result<(), Box<dyn Error>> {
|
||||
let grpc_addr = ctx.config.listen_address_grpc;
|
||||
let server = ZgsGrpcServiceServer::new(ZgsGrpcServiceImpl { ctx });
|
||||
Server::builder()
|
||||
.add_service(server)
|
||||
.serve(grpc_addr)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
1
node/rpc/src/zgs_grpc/mod.rs
Normal file
1
node/rpc/src/zgs_grpc/mod.rs
Normal file
@ -0,0 +1 @@
|
||||
pub mod zgs_grpc;
|
||||
19
node/rpc/src/zgs_grpc/zgs_grpc.rs
Normal file
19
node/rpc/src/zgs_grpc/zgs_grpc.rs
Normal file
@ -0,0 +1,19 @@
|
||||
use crate::zgs_grpc_proto::{PingRequest, PingReply};
|
||||
use crate::zgs_grpc_proto::zgs_grpc_service_server::ZgsGrpcService;
|
||||
use crate::Context;
|
||||
|
||||
pub struct ZgsGrpcServiceImpl {
|
||||
pub ctx: Context,
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl ZgsGrpcService for ZgsGrpcServiceImpl {
|
||||
async fn ping(
|
||||
&self,
|
||||
request: tonic::Request<PingRequest>,
|
||||
) -> Result<tonic::Response<PingReply>, tonic::Status> {
|
||||
let msg = request.into_inner().message;
|
||||
let reply = PingReply { message: format!("Echo: {}", msg) };
|
||||
Ok(tonic::Response::new(reply))
|
||||
}
|
||||
}
|
||||
@ -298,7 +298,7 @@ impl ClientBuilder {
|
||||
mine_service_sender: mine_send,
|
||||
};
|
||||
|
||||
let (rpc_handle, maybe_admin_rpc_handle) = rpc::run_server(ctx)
|
||||
let (rpc_handle, maybe_admin_rpc_handle) = rpc::run_server(ctx.clone())
|
||||
.await
|
||||
.map_err(|e| format!("Unable to start HTTP RPC server: {:?}", e))?;
|
||||
|
||||
@ -307,6 +307,13 @@ impl ClientBuilder {
|
||||
executor.spawn(admin_rpc_handle, "rpc_admin");
|
||||
}
|
||||
|
||||
executor.spawn(
|
||||
async move {
|
||||
rpc::run_grpc_server(ctx.clone()).await.expect("Failed to start gRPC server");
|
||||
},
|
||||
"grpc",
|
||||
);
|
||||
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
|
||||
@ -305,6 +305,9 @@ auto_sync_enabled = true
|
||||
# HTTP server address to bind for admin and debug RPC.
|
||||
# listen_address_admin = "127.0.0.1:5679"
|
||||
|
||||
## Grpc server address to bind
|
||||
# listen_address_grpc = "0.0.0.0:50051"
|
||||
|
||||
# Number of chunks for a single segment.
|
||||
# chunks_per_segment = 1024
|
||||
|
||||
|
||||
@ -317,6 +317,9 @@ auto_sync_enabled = true
|
||||
# HTTP server address to bind for admin and debug RPC.
|
||||
# listen_address_admin = "127.0.0.1:5679"
|
||||
|
||||
## Grpc server address to bind
|
||||
# listen_address_grpc = "0.0.0.0:50051"
|
||||
|
||||
# Number of chunks for a single segment.
|
||||
# chunks_per_segment = 1024
|
||||
|
||||
|
||||
@ -319,6 +319,9 @@
|
||||
# HTTP server address to bind for admin and debug RPC.
|
||||
# listen_address_admin = "127.0.0.1:5679"
|
||||
|
||||
## Grpc server address to bind
|
||||
# listen_address_grpc = "0.0.0.0:50051"
|
||||
|
||||
# Number of chunks for a single segment.
|
||||
# chunks_per_segment = 1024
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user