willsm/network-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial check-in of prover CLI

Browse Source
This commit is contained in:
Collin Jackson 2024-09-05 01:14:41 -07:00
parent 401d512ab5
commit 8f83323646
8 changed files with 3892 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cli/.gitignore vendored Normal file
View File

@ -0,0 +1,7 @@
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# These are backup files generated by rustfmt
**/*.rs.bk

3313
cli/Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

89
cli/Cargo.toml Normal file
View File

@ -0,0 +1,89 @@
[package]
name = "nexus-network"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "prover"
path = "src/prover.rs"
[build-dependencies]
prost-build = "0.13"
[dependencies]
async-stream = "0.3"
clap = { version = "4.5", features = ["derive"] }
futures = "0.3"
prost = "0.13"
rand = "0.8.5"
reqwest = { version = "0.12", features = ["blocking"] }
tokio = { version = "1.38", features = ["full"] }
tokio-tungstenite = { version = "0.23", features = ["native-tls"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["std", "env-filter"] }
uuid = { version = "1.9", features = ["v4", "fast-rng"] }
base64 = "0.22.1"
flutter_rust_bridge = "=2.1.0"
nexus-core = { git = "https://github.com/nexus-xyz/nexus-zkvm.git" }
getrandom = { version = "0.2", features = ["js"] }
# Workaround for "failed to resolve patches for `https://github.com/rust-lang/crates.io-index`"
zstd = { version = "=0.13.2", git = "https://github.com/gyscos/zstd-rs", features = ["wasm"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
elf = { version = "0.7", default-features = false, features = ["std"] }
jsonrpsee = { version = "0.23", default-features = false }
sha3 = { version = "0.10", default-features = false }
hex = { version = "0.4.3" }
ark-crypto-primitives = { version = "0.4.0", features = [
"r1cs",
"sponge",
"crh",
"merkle_tree",
] }
ark-std = "0.4.0"
ark-relations = { version = "0.4.0" }
ark-r1cs-std = { version = "0.4.0" }
ark-ff = "0.4.0"
ark-ec = { version = "0.4.0", default-features = false }
ark-serialize = { version = "0.4.0", features = ["derive"] }
ark-poly = "0.4.0"
ark-poly-commit = "0.4.0"
ark-bn254 = "0.4.0"
ark-grumpkin = "0.4.0"
ark-pallas = "0.4.0"
ark-vesta = "0.4.0"
ark-test-curves = { version = "0.4.2", features = ["bls12_381_curve"] }
[patch.crates-io]
ark-crypto-primitives = { git = "https://github.com/arkworks-rs/crypto-primitives", rev = "d27a5c8" }
ark-r1cs-std = { git = "https://github.com/arkworks-rs/r1cs-std/", rev = "2ca3bd7" }
ark-ff = { git = "https://github.com/arkworks-rs/algebra/", rev = "2a80c54" }
ark-ec = { git = "https://github.com/arkworks-rs/algebra/", rev = "2a80c54" }
ark-serialize = { git = "https://github.com/arkworks-rs/algebra/", rev = "2a80c54" }
ark-poly = { git = "https://github.com/arkworks-rs/algebra/", rev = "2a80c54" }
ark-test-curves = { git = "https://github.com/arkworks-rs/algebra/", rev = "2a80c54" }
ark-poly-commit = { git = "https://github.com/arkworks-rs/poly-commit/", rev = "12f5529" }
# note bls is using a different commit from the other curves
ark-bn254 = { git = "https://github.com/arkworks-rs/curves/", rev = "8c0256a" }
ark-grumpkin = { git = "https://github.com/arkworks-rs/curves/", rev = "8c0256a" }
ark-pallas = { git = "https://github.com/arkworks-rs/curves/", rev = "8c0256a" }
ark-vesta = { git = "https://github.com/arkworks-rs/curves/", rev = "8c0256a" }
ark-bls12-381 = { git = "https://github.com/arkworks-rs/curves/", rev = "3fded1f" }
zstd-sys = { git = "https://github.com/gyscos/zstd-rs" }
[profile.release]
strip = true
lto = true
codegen-units = 1

26
cli/README.md Normal file
View File

@ -0,0 +1,26 @@
# network-cli
Command line interface (CLI) for accessing the Nexus Network. Highest-performance option for proving.
## Quick start
Ensure that you have Rust.
```
curl https://sh.rustup.rs -sSf | sh
```
Get the CLI and run it:
```
git clone https://github.com/nexus-xyz/network-cli
cd network-cli
cargo run --bin prover -- dev.orchestrator.nexus.xyz
```
Note: Hostname will default to `orchestrator.nexus.xyz` in the public beta release.
## Known issues
Currently only proving is supported. Submitting programs to the network is in private beta.
To request an API key, [contact us](https://forms.gle/183D9bcDHUdbxCV5A).

14
cli/build.rs Normal file
View File

@ -0,0 +1,14 @@
use std::{error::Error, path::PathBuf};
fn main() -> Result<(), Box<dyn Error>> {
let out_dir: PathBuf = "./src/generated/".into();
let proto_file: PathBuf = "../../proto/orchestrator.proto".into();
let proto_dir = proto_file.parent().unwrap();
prost_build::Config::new()
.out_dir(out_dir)
.protoc_arg("--experimental_allow_proto3_optional")
.compile_protos(&[&proto_file], &[proto_dir])?;
Ok(())
}

4
cli/src/generated/mod.rs Normal file
View File

@ -0,0 +1,4 @@
#[allow(clippy::all, clippy::pedantic)]
pub mod pb {
include!("nexus.orchestrator.rs");
}

213
cli/src/generated/nexus.orchestrator.rs Normal file
View File

@ -0,0 +1,213 @@
// This file is @generated by prost-build.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProofRequest {
#[prost(message, optional, tag = "1")]
pub program: ::core::option::Option<CompiledProgram>,
#[prost(message, optional, tag = "2")]
pub input: ::core::option::Option<VmProgramInput>,
/// Step of the trace to start the proof, inclusive.
///
/// If missing, proving starts at the beginning of the trace.
#[prost(int32, optional, tag = "3")]
pub step_to_start: ::core::option::Option<i32>,
/// Number of steps for this proof request.
///
/// If zero, proving is skipped. If missing, all steps are proved.
#[prost(int32, optional, tag = "4")]
pub steps_to_prove: ::core::option::Option<i32>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProofResponse {
#[prost(message, optional, tag = "1")]
pub proof: ::core::option::Option<Proof>,
}
/// A message that always represents a program runnable on the Nexus VM.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CompiledProgram {
#[prost(oneof = "compiled_program::Program", tags = "1")]
pub program: ::core::option::Option<compiled_program::Program>,
}
/// Nested message and enum types in `CompiledProgram`.
pub mod compiled_program {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Program {
/// ELF binary containing a program to be proved, expressed in the RV32I ISA.
#[prost(bytes, tag = "1")]
Rv32iElfBytes(::prost::alloc::vec::Vec<u8>),
}
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct VmProgramInput {
#[prost(oneof = "vm_program_input::Input", tags = "1")]
pub input: ::core::option::Option<vm_program_input::Input>,
}
/// Nested message and enum types in `VMProgramInput`.
pub mod vm_program_input {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Input {
/// Input expressed as raw bytes to be read as-is off of the input tape.
#[prost(bytes, tag = "1")]
RawBytes(::prost::alloc::vec::Vec<u8>),
}
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Proof {
#[prost(oneof = "proof::Proof", tags = "1")]
pub proof: ::core::option::Option<proof::Proof>,
}
/// Nested message and enum types in `Proof`.
pub mod proof {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Proof {
#[prost(bytes, tag = "1")]
NovaBytes(::prost::alloc::vec::Vec<u8>),
}
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProgramSource {
/// The source code to be compiled. There will be a variety of languages and
/// ways to express everything a program needs for compilation (dependencies,
/// multiple files, etc.) as our scope expands.
#[prost(oneof = "program_source::Source", tags = "1")]
pub source: ::core::option::Option<program_source::Source>,
}
/// Nested message and enum types in `ProgramSource`.
pub mod program_source {
/// The source code to be compiled. There will be a variety of languages and
/// ways to express everything a program needs for compilation (dependencies,
/// multiple files, etc.) as our scope expands.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Source {
/// Option to use when the program in question can be expressed as a single
/// rust file (i.e., a program written in the playground).
#[prost(string, tag = "1")]
RustSingleFile(::prost::alloc::string::String),
}
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CompileRequest {
#[prost(message, optional, tag = "1")]
pub source: ::core::option::Option<ProgramSource>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CompileResponse {
#[prost(message, optional, tag = "1")]
pub program: ::core::option::Option<CompiledProgram>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct Progress {
/// Completion status expressed as a number between zero and one,
/// inclusive.
#[prost(float, tag = "1")]
pub completed_fraction: f32,
/// The total size of the execution trace in steps.
#[prost(int32, tag = "2")]
pub steps_in_trace: i32,
/// The number of steps of the execution trace to be proven.
#[prost(int32, tag = "3")]
pub steps_to_prove: i32,
/// The number of steps proven so far.
#[prost(int32, tag = "4")]
pub steps_proven: i32,
}
/// Streamed messages sent to the orchestrator to keep it updated with the
/// prover's status.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProverRequest {
#[prost(oneof = "prover_request::Contents", tags = "1, 2, 3, 4")]
pub contents: ::core::option::Option<prover_request::Contents>,
}
/// Nested message and enum types in `ProverRequest`.
pub mod prover_request {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Contents {
/// Details about this supply node for use by the orchestrator.
#[prost(message, tag = "1")]
Registration(super::ProverRequestRegistration),
/// A completed proof.
#[prost(message, tag = "2")]
Proof(super::Proof),
/// Periodic progress update for the current proof.
#[prost(message, tag = "3")]
Progress(super::Progress),
/// Periodic liveness indicator when no proof is being computed.
#[prost(message, tag = "4")]
Heartbeat(super::Heartbeat),
}
}
/// Metadata that helps the orchestrator schedule work to the requesting compute
/// supplier.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProverRequestRegistration {
/// What type of prover this is.
#[prost(enumeration = "ProverType", tag = "1")]
pub prover_type: i32,
/// A unique identifier for this prover, generated by the prover.
///
/// Distict provers must not share an identifier; do not use a constant value.
#[prost(string, tag = "2")]
pub prover_id: ::prost::alloc::string::String,
/// The number of proof cycles that this prover expects to compute
/// over the course of one second. Proof cycles are proof steps times k.
#[prost(double, optional, tag = "3")]
pub estimated_proof_cycles_hertz: ::core::option::Option<f64>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProverResponse {
/// Forward the literal request for now
#[prost(message, optional, tag = "1")]
pub to_prove: ::core::option::Option<ProofRequest>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct Heartbeat {}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
#[repr(i32)]
pub enum ProverType {
/// Experimental new prover types should leave the prover type unspecified.
Unspecified = 0,
/// The default prover type, used for volunteered compute resources.
Volunteer = 1,
/// Provers running on public continuous integration.
/// May restrict the types of programs that can be assigned.
Ci = 2,
}
impl ProverType {
/// String value of the enum field names used in the ProtoBuf definition.
///
/// The values are not transformed in any way and thus are considered stable
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
pub fn as_str_name(&self) -> &'static str {
match self {
ProverType::Unspecified => "PROVER_TYPE_UNSPECIFIED",
ProverType::Volunteer => "PROVER_TYPE_VOLUNTEER",
ProverType::Ci => "PROVER_TYPE_CI",
}
}
/// Creates an enum from field names used in the ProtoBuf definition.
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
match value {
"PROVER_TYPE_UNSPECIFIED" => Some(Self::Unspecified),
"PROVER_TYPE_VOLUNTEER" => Some(Self::Volunteer),
"PROVER_TYPE_CI" => Some(Self::Ci),
_ => None,
}
}
}

226
cli/src/prover.rs Normal file
View File

@ -0,0 +1,226 @@
// Copyright (c) 2024 Nexus. All rights reserved.
mod generated;
use std::borrow::Cow;
use clap::Parser;
use futures::{SinkExt, StreamExt};
use generated::pb::{
self, compiled_program::Program, proof, prover_request, vm_program_input::Input, Progress,
ProverRequest, ProverRequestRegistration, ProverResponse, ProverType,
};
use prost::Message as _;
use tokio_tungstenite::tungstenite::protocol::{frame::coding::CloseCode, CloseFrame, Message};
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::EnvFilter;
use nexus_core::{
nvm::{
interactive::{parse_elf, trace},
memory::MerkleTrie,
NexusVM,
},
prover::nova::{
init_circuit_trace,
key::{CanonicalSerialize},
prove_seq_step,
types::*,
pp::gen_vm_pp,
},
};
use zstd::stream::Encoder;
#[derive(Parser, Debug)]
struct Args {
/// Hostname at which Orchestrator can be reached
hostname: String,
/// Port over which to communicate with Orchestrator
#[arg(short, long, default_value_t = 443u16)]
port: u16,
/// Whether to connect using secure web sockets
#[arg(short, long, default_value_t = true)]
use_https: bool,
/// Whether to loop and keep the connection open
#[arg(short, long, default_value_t = true)]
keep_listening: bool,
}
#[tokio::main]
async fn main() {
// Configure the tracing subscriber
tracing_subscriber::fmt()
.with_env_filter(EnvFilter::from_default_env())
.with_span_events(FmtSpan::CLOSE)
.init();
let args = Args::parse();
let ws_addr_string = format!(
"{}://{}:{}/prove",
if args.use_https { "wss" } else { "ws" },
args.hostname,
args.port
);
let prover_id = format!("prover_{}", rand::random::<u32>());
let k = 4;
// TODO(collinjackson): Get parameters from a file or URL.
let pp = gen_vm_pp::<C1, seq::SetupParams<(G1, G2, C1, C2, RO, SC)>>(k as usize, &())
.expect("error generating public parameters");
println!(
"{} supplying proofs to Orchestrator at {}",
prover_id, &ws_addr_string
);
let registration = ProverRequest {
contents: Some(prover_request::Contents::Registration(
ProverRequestRegistration {
prover_type: ProverType::Volunteer.into(),
prover_id: prover_id,
estimated_proof_cycles_hertz: None,
},
)),
};
let (mut client, _) = tokio_tungstenite::connect_async(ws_addr_string)
.await
.unwrap();
client
.send(Message::Binary(registration.encode_to_vec()))
.await
.unwrap();
println!("Sent registration message...");
loop {
let program_message = match client.next().await.unwrap().unwrap() {
Message::Binary(b) => b,
_ => panic!("Unexpected message type"),
};
println!("Received program message...");
let program = ProverResponse::decode(program_message.as_slice()).unwrap();
let Program::Rv32iElfBytes(elf_bytes) = program
.to_prove
.clone()
.unwrap()
.program
.unwrap()
.program
.unwrap();
let to_prove = program.to_prove.unwrap();
let Input::RawBytes(input) = to_prove.input.unwrap().input.unwrap();
println!(
"Received a {} byte program to prove with {} bytes of input",
elf_bytes.len(),
input.len()
);
let mut vm: NexusVM<MerkleTrie> =
parse_elf(&elf_bytes.as_ref()).expect("error loading and parsing RISC-V instruction");
vm.syscalls.set_input(&input);
let k = 4;
// TODO(collinjackson): Get outputs
let completed_trace = trace(&mut vm, k as usize, false).expect("error generating trace");
let tr = init_circuit_trace(completed_trace).expect("error initializing circuit trace");
let total_steps = tr.steps();
println!("Program trace {} steps", total_steps);
let start: usize = 0;
match to_prove.step_to_start {
Some(step) => step as usize,
None => 0,
};
let steps_to_prove = to_prove.steps_to_prove;
let mut end: usize = match steps_to_prove {
Some(steps) => start + steps as usize,
None => total_steps,
};
if end > total_steps {
end = total_steps
}
let initial_progress = ProverRequest {
contents: Some(prover_request::Contents::Progress(Progress {
completed_fraction: 0.0,
steps_in_trace: total_steps as i32,
steps_to_prove: (end - start) as i32,
steps_proven: 0,
})),
};
client
.send(Message::Binary(initial_progress.encode_to_vec()))
.await
.unwrap();
let z_st = tr.input(start).expect("error starting circuit trace");
println!("Proving...");
let mut proof = IVCProof::new(&z_st);
println!("Proving from {} to {}", start, end);
for step in start..end {
proof = prove_seq_step(Some(proof), &pp, &tr).expect("error proving step");
println!("Proved step {}", step);
let steps_proven = step - start + 1;
let progress = ProverRequest {
contents: Some(prover_request::Contents::Progress(Progress {
completed_fraction: steps_proven as f32 / steps_to_prove.unwrap() as f32,
steps_in_trace: total_steps as i32,
steps_to_prove: (end - start) as i32,
steps_proven: steps_proven as i32,
})),
};
client
.send(Message::Binary(progress.encode_to_vec()))
.await
.unwrap();
if step == end - 1 {
let mut buf = Vec::new();
let mut writer = Box::new(&mut buf);
let mut encoder = Encoder::new(&mut writer, 0).expect("failed to create encoder");
proof
.serialize_compressed(&mut encoder)
.expect("failed to compress proof");
encoder.finish().expect("failed to finish encoder");
let response = ProverRequest {
contents: Some(prover_request::Contents::Proof(pb::Proof {
proof: Some(proof::Proof::NovaBytes(buf)),
})),
};
client
.send(Message::Binary(response.encode_to_vec()))
.await
.unwrap();
}
}
// TODO(collinjackson): Consider verifying the proof before sending it
// proof.verify(&public_params, proof.step_num() as _).expect("error verifying execution")
println!("Proof sent!");
if args.keep_listening {
println!("Waiting for another program to prove...");
} else {
break;
}
}
client
.close(Some(CloseFrame {
code: CloseCode::Normal,
reason: Cow::Borrowed("Finished proving."),
}))
.await
.unwrap();
println!("Sent proof and closed connection...");
}