diff --git a/.dockerignore b/.dockerignore index 473661b..e75d33e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,6 @@ github.env Taskfile.yaml +# Rust +target +vdf/generated diff --git a/.gitignore b/.gitignore index ae9c657..8b8f9ff 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ ceremony-client .env.signers .task node-tmp-* +build # Rust target diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..836326b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# Local development + +The following software is required for local development (assuming MacOS ARM): + +- Go 1.20 +- Rust toolchain +- GMP 6.3: `brew install gmp` +- Install the Go plugin for uniffi-rs: `cargo install uniffi-bindgen-go --git https://github.com/NordSecurity/uniffi-bindgen-go --tag v0.2.1+v0.25.0` + +# Building release binaries + +The following is software is required to build release binaries (assuming MacOS ARM) : + +- [Local development](#local-development) dependencies +- Docker +- [Taskfile](https://taskfile.dev/) + +Then from the repo root use the following commands to build the release binaries that statically link the [native VDF](./vdf) for the supported platforms: + +```shell +task build_node_arm64_macos +task build_node_arm64_linux +task build_node_arm64_macos +``` + +The output binaries will be in `node/build`. diff --git a/Dockerfile b/Dockerfile index 42fa039..4475ea3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,20 @@ -FROM golang:1.20.14-alpine3.19 as build +FROM golang:1.20.14-bullseye as build-base + +ENV PATH="${PATH}:/root/.cargo/bin/" + +# Install GMP 6.2 (6.3 which MacOS is using only available on Debian unstable) +RUN apt-get update && apt-get install -y \ + libgmp-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY docker/rustup-init.sh /opt/rustup-init.sh + +RUN /opt/rustup-init.sh -y --profile minimal + +# Install uniffi-bindgen-go +RUN cargo install uniffi-bindgen-go --git https://github.com/NordSecurity/uniffi-bindgen-go --tag v0.2.1+v0.25.0 + +FROM build-base as build ENV GOEXPERIMENT=arenas @@ -6,16 +22,27 @@ WORKDIR /opt/ceremonyclient COPY . . +## Generate Rust bindings for VDF +WORKDIR /opt/ceremonyclient/vdf +RUN ./generate.sh + +# Build and install the node WORKDIR /opt/ceremonyclient/node -RUN go install ./... +RUN ./build.sh && cp node /go/bin RUN go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest +# Build and install qclient WORKDIR /opt/ceremonyclient/client RUN go build -o qclient ./main.go -FROM alpine:3.19 +# Allows exporting single binary +FROM scratch AS node +COPY --from=build /go/bin/node /node +ENTRYPOINT [ "/node" ] + +FROM debian:bullseye ARG NODE_VERSION ARG GIT_REPO diff --git a/Dockerfile_build b/Dockerfile_build new file mode 100644 index 0000000..2ced8e3 --- /dev/null +++ b/Dockerfile_build @@ -0,0 +1,42 @@ +# syntax=docker/dockerfile:1 +ARG GO_VERSION=1.20.14 +FROM golang:${GO_VERSION}-bullseye AS build-base + +ENV PATH="${PATH}:/root/.cargo/bin/" + +# Install GMP 6.3 +RUN apt-get update && apt-get install -y \ + libgmp-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust toolchain +RUN rustup-init -y --profile minimal + +# Install uniffi-bindgen-go +RUN cargo install uniffi-bindgen-go --git https://github.com/NordSecurity/uniffi-bindgen-go --tag v0.2.1+v0.25.0 + +FROM base as build-node + +WORKDIR /opt/ceremonyclient +COPY . . + +# Generate Rust bindings for VDF +WORKDIR /opt/ceremonyclient/vdf +RUN go generate + +# Download node Go dependencies +WORKDIR /opt/ceremonyclient/node +RUN --mount=type=cache,target=/go/pkg/mod/ \ + --mount=type=bind,source=./node/go.sum,target=go.sum \ + --mount=type=bind,source=./node/go.mod,target=go.mod \ + go mod download -x + +# Build node Go dependencies +RUN --mount=type=cache,target=/go/pkg/mod/ \ + --mount=type=bind,target=. \ + go build -o /bin/node ./cmd/node + +FROM scratch AS node +COPY --from=build-node /bin/node /bin/ +ENTRYPOINT [ "/bin/node" ] + diff --git a/Taskfile.yaml b/Taskfile.yaml index a2de09e..a4dd7da 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -25,6 +25,22 @@ tasks: - echo -n "Branch :" && echo " {{.GIT_BRANCH}}" - echo -n "Commit :" && echo " {{.GIT_COMMIT}}" silent: true + + build_node_arm64_macos: + desc: Build the Quilibrium node binary for MacOS ARM. Assumes it's ran from the same platform. Outputs to node/build. + cmds: + - vdf/generate.sh + - node/build.sh -o build/arm64_macos/node + + build_node_arm64_linux: + desc: Build the Quilibrium node binary for ARM64 Linux. Outputs to node/build. + cmds: + - docker build --platform linux/arm64 -f Dockerfile --output node/build/arm64_linux --target=node . + + build_node_amd64_linux: + desc: Build the Quilibrium node binary for AMD64 Linux. Outputs to node/build. + cmds: + - docker build --platform linux/amd64 -f Dockerfile --output node/build/amd64_linux --target=node . build: desc: Build the Quilibrium docker image, unless it is already built. diff --git a/client/go.mod b/client/go.mod index caad573..feeaf2c 100644 --- a/client/go.mod +++ b/client/go.mod @@ -24,8 +24,8 @@ require ( github.com/minio/sha256-simd v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.18.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/sys v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect source.quilibrium.com/quilibrium/monorepo/nekryptology v0.0.0-00010101000000-000000000000 // indirect diff --git a/client/go.sum b/client/go.sum index 03655ac..69e7b3f 100644 --- a/client/go.sum +++ b/client/go.sum @@ -89,8 +89,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -103,8 +103,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/crates/classgroup/build.rs b/crates/classgroup/build.rs new file mode 100644 index 0000000..a4b0529 --- /dev/null +++ b/crates/classgroup/build.rs @@ -0,0 +1,16 @@ +use std::env; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + + let target = env::var("TARGET").expect("cargo should have set this"); + if target == "aarch64-apple-darwin" { + println!("cargo:rustc-link-search=/opt/homebrew/Cellar/gmp/6.3.0/lib"); + } else if target == "aarch64-unknown-linux-gnu" { + println!("cargo:rustc-link-search=/usr/lib/aarch64-linux-gnu/"); + } else if target == "x86_64-unknown-linux-gnu" { + println!("cargo:rustc-link-search=/usr/lib/x86_64-linux-gnu/"); + } else { + panic!("unsupported target {target}"); + } +} diff --git a/crates/classgroup/src/gmp/ffi.rs b/crates/classgroup/src/gmp/ffi.rs index 116cd4c..d8cc2b1 100644 --- a/crates/classgroup/src/gmp/ffi.rs +++ b/crates/classgroup/src/gmp/ffi.rs @@ -1,6 +1,6 @@ use super::mpz::*; -#[link(name = "gmp")] +#[link(name = "gmp", kind = "static")] extern "C" { pub fn __gmpz_fdiv_q(q: mpz_ptr, n: mpz_srcptr, d: mpz_srcptr); pub fn __gmpz_cdiv_q(q: mpz_ptr, n: mpz_srcptr, d: mpz_srcptr); diff --git a/crates/classgroup/src/gmp/mpz.rs b/crates/classgroup/src/gmp/mpz.rs index e11f94e..b8a1beb 100644 --- a/crates/classgroup/src/gmp/mpz.rs +++ b/crates/classgroup/src/gmp/mpz.rs @@ -26,7 +26,7 @@ pub type mp_bitcnt_t = c_ulong; pub type mpz_srcptr = *const mpz_struct; pub type mpz_ptr = *mut mpz_struct; -#[link(name = "gmp")] +#[link(name = "gmp", kind = "static")] extern "C" { static __gmp_bits_per_limb: c_int; fn __gmpz_init(x: mpz_ptr); diff --git a/crates/classgroup/src/gmp/test.rs b/crates/classgroup/src/gmp/test.rs index 69cd400..8036386 100644 --- a/crates/classgroup/src/gmp/test.rs +++ b/crates/classgroup/src/gmp/test.rs @@ -2,7 +2,7 @@ use super::mpz::mp_limb_t; use std; use libc::c_int; -#[link(name = "gmp")] +#[link(name = "gmp", kind = "static")] extern "C" { static __gmp_bits_per_limb: c_int; } diff --git a/crates/classgroup/src/gmp_classgroup/ffi.rs b/crates/classgroup/src/gmp_classgroup/ffi.rs index 874411f..7e7b31f 100644 --- a/crates/classgroup/src/gmp_classgroup/ffi.rs +++ b/crates/classgroup/src/gmp_classgroup/ffi.rs @@ -24,7 +24,7 @@ use libc::{c_int, c_long, c_ulong, c_void, size_t}; // pub use c_ulong; use std::{mem, usize}; // We use the unsafe versions to avoid unecessary allocations. -#[link(name = "gmp")] +#[link(name = "gmp", kind = "static")] extern "C" { fn __gmpz_gcdext(gcd: *mut Mpz, s: *mut Mpz, t: *mut Mpz, a: *const Mpz, b: *const Mpz); fn __gmpz_gcd(rop: *mut Mpz, op1: *const Mpz, op2: *const Mpz); diff --git a/crates/vdf/Cargo.toml b/crates/vdf/Cargo.toml index c2fdb7e..acc815d 100644 --- a/crates/vdf/Cargo.toml +++ b/crates/vdf/Cargo.toml @@ -23,7 +23,7 @@ edition = "2018" description = "An implementation of Verifiable Delay Functions (VDFs) in Rust" [lib] -crate-type = ["lib", "cdylib"] +crate-type = ["lib", "staticlib"] name = "vdf" [dependencies] diff --git a/crates/vdf/build.rs b/crates/vdf/build.rs index ad96740..a8de622 100644 --- a/crates/vdf/build.rs +++ b/crates/vdf/build.rs @@ -105,11 +105,6 @@ fn generate(f: &mut dyn Write) { fn main() { println!("cargo:rerun-if-changed=build.rs"); - let target = env::var("TARGET").expect("cargo should have set this"); - if target == "aarch64-apple-darwin" { - println!("cargo:rustc-link-search=/opt/homebrew/Cellar/gmp/6.3.0/lib"); - } - uniffi::generate_scaffolding("src/lib.udl").expect("uniffi generation failed"); let manifest_path = env::var("OUT_DIR").expect("cargo should have set this"); diff --git a/docker/rustup-init.sh b/docker/rustup-init.sh new file mode 100755 index 0000000..c49b6ab --- /dev/null +++ b/docker/rustup-init.sh @@ -0,0 +1,811 @@ +#!/bin/sh +# shellcheck shell=dash +# shellcheck disable=SC2039 # local is non-POSIX + +# This is just a little script that can be downloaded from the internet to +# install rustup. It just does platform detection, downloads the installer +# and runs it. + +# It runs on Unix shells like {a,ba,da,k,z}sh. It uses the common `local` +# extension. Note: Most shells limit `local` to 1 var per line, contra bash. + +# Some versions of ksh have no `local` keyword. Alias it to `typeset`, but +# beware this makes variables global with f()-style function syntax in ksh93. +# mksh has this alias by default. +has_local() { + # shellcheck disable=SC2034 # deliberately unused + local _has_local +} + +has_local 2>/dev/null || alias local=typeset + +is_zsh() { + [ -n "${ZSH_VERSION-}" ] +} + +set -u + +# If RUSTUP_UPDATE_ROOT is unset or empty, default it. +RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-https://static.rust-lang.org/rustup}" + +# NOTICE: If you change anything here, please make the same changes in setup_mode.rs +usage() { + cat < + Choose a default host triple + --default-toolchain + Choose a default toolchain to install. Use 'none' to not install any toolchains at all + --profile + [default: default] [possible values: minimal, default, complete] + -c, --component ... + Component name to also install + -t, --target ... + Target name to also install + --no-update-default-toolchain + Don't update any existing default toolchain after install + --no-modify-path + Don't configure the PATH environment variable + -h, --help + Print help + -V, --version + Print version +EOF +} + +main() { + downloader --check + need_cmd uname + need_cmd mktemp + need_cmd chmod + need_cmd mkdir + need_cmd rm + need_cmd rmdir + + get_architecture || return 1 + local _arch="$RETVAL" + assert_nz "$_arch" "arch" + + local _ext="" + case "$_arch" in + *windows*) + _ext=".exe" + ;; + esac + + local _url="${RUSTUP_UPDATE_ROOT}/dist/${_arch}/rustup-init${_ext}" + + local _dir + if ! _dir="$(ensure mktemp -d)"; then + # Because the previous command ran in a subshell, we must manually + # propagate exit status. + exit 1 + fi + local _file="${_dir}/rustup-init${_ext}" + + local _ansi_escapes_are_valid=false + if [ -t 2 ]; then + if [ "${TERM+set}" = 'set' ]; then + case "$TERM" in + xterm*|rxvt*|urxvt*|linux*|vt*) + _ansi_escapes_are_valid=true + ;; + esac + fi + fi + + # check if we have to use /dev/tty to prompt the user + local need_tty=yes + for arg in "$@"; do + case "$arg" in + --help) + usage + exit 0 + ;; + *) + OPTIND=1 + if [ "${arg%%--*}" = "" ]; then + # Long option (other than --help); + # don't attempt to interpret it. + continue + fi + while getopts :hy sub_arg "$arg"; do + case "$sub_arg" in + h) + usage + exit 0 + ;; + y) + # user wants to skip the prompt -- + # we don't need /dev/tty + need_tty=no + ;; + *) + ;; + esac + done + ;; + esac + done + + if $_ansi_escapes_are_valid; then + printf "\33[1minfo:\33[0m downloading installer\n" 1>&2 + else + printf '%s\n' 'info: downloading installer' 1>&2 + fi + + ensure mkdir -p "$_dir" + ensure downloader "$_url" "$_file" "$_arch" + ensure chmod u+x "$_file" + if [ ! -x "$_file" ]; then + printf '%s\n' "Cannot execute $_file (likely because of mounting /tmp as noexec)." 1>&2 + printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./rustup-init${_ext}." 1>&2 + exit 1 + fi + + if [ "$need_tty" = "yes" ] && [ ! -t 0 ]; then + # The installer is going to want to ask for confirmation by + # reading stdin. This script was piped into `sh` though and + # doesn't have stdin to pass to its children. Instead we're going + # to explicitly connect /dev/tty to the installer's stdin. + if [ ! -t 1 ]; then + err "Unable to run interactively. Run with -y to accept defaults, --help for additional options" + fi + + ignore "$_file" "$@" < /dev/tty + else + ignore "$_file" "$@" + fi + + local _retval=$? + + ignore rm "$_file" + ignore rmdir "$_dir" + + return "$_retval" +} + +check_proc() { + # Check for /proc by looking for the /proc/self/exe link + # This is only run on Linux + if ! test -L /proc/self/exe ; then + err "fatal: Unable to find /proc/self/exe. Is /proc mounted? Installation cannot proceed without /proc." + fi +} + +get_bitness() { + need_cmd head + # Architecture detection without dependencies beyond coreutils. + # ELF files start out "\x7fELF", and the following byte is + # 0x01 for 32-bit and + # 0x02 for 64-bit. + # The printf builtin on some shells like dash only supports octal + # escape sequences, so we use those. + local _current_exe_head + _current_exe_head=$(head -c 5 /proc/self/exe ) + if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then + echo 32 + elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then + echo 64 + else + err "unknown platform bitness" + fi +} + +is_host_amd64_elf() { + need_cmd head + need_cmd tail + # ELF e_machine detection without dependencies beyond coreutils. + # Two-byte field at offset 0x12 indicates the CPU, + # but we're interested in it being 0x3E to indicate amd64, or not that. + local _current_exe_machine + _current_exe_machine=$(head -c 19 /proc/self/exe | tail -c 1) + [ "$_current_exe_machine" = "$(printf '\076')" ] +} + +get_endianness() { + local cputype=$1 + local suffix_eb=$2 + local suffix_el=$3 + + # detect endianness without od/hexdump, like get_bitness() does. + need_cmd head + need_cmd tail + + local _current_exe_endianness + _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)" + if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then + echo "${cputype}${suffix_el}" + elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then + echo "${cputype}${suffix_eb}" + else + err "unknown platform endianness" + fi +} + +# Detect the Linux/LoongArch UAPI flavor, with all errors being non-fatal. +# Returns 0 or 234 in case of successful detection, 1 otherwise (/tmp being +# noexec, or other causes). +check_loongarch_uapi() { + need_cmd base64 + + local _tmp + if ! _tmp="$(ensure mktemp)"; then + return 1 + fi + + # Minimal Linux/LoongArch UAPI detection, exiting with 0 in case of + # upstream ("new world") UAPI, and 234 (-EINVAL truncated) in case of + # old-world (as deployed on several early commercial Linux distributions + # for LoongArch). + # + # See https://gist.github.com/xen0n/5ee04aaa6cecc5c7794b9a0c3b65fc7f for + # source to this helper binary. + ignore base64 -d > "$_tmp" <&2 + echo 'Your Linux kernel does not provide the ABI required by this Rust' >&2 + echo 'distribution. Please check with your OS provider for how to obtain a' >&2 + echo 'compatible Rust package for your system.' >&2 + echo >&2 + exit 1 + ;; + *) + echo "Warning: Cannot determine current system's ABI flavor, continuing anyway." >&2 + echo >&2 + echo 'Note that the official Rust distribution only works with the upstream' >&2 + echo 'kernel ABI. Installation will fail if your running kernel happens to be' >&2 + echo 'incompatible.' >&2 + ;; + esac +} + +get_architecture() { + local _ostype _cputype _bitness _arch _clibtype + _ostype="$(uname -s)" + _cputype="$(uname -m)" + _clibtype="gnu" + + if [ "$_ostype" = Linux ]; then + if [ "$(uname -o)" = Android ]; then + _ostype=Android + fi + if ldd --version 2>&1 | grep -q 'musl'; then + _clibtype="musl" + fi + fi + + if [ "$_ostype" = Darwin ]; then + # Darwin `uname -m` can lie due to Rosetta shenanigans. If you manage to + # invoke a native shell binary and then a native uname binary, you can + # get the real answer, but that's hard to ensure, so instead we use + # `sysctl` (which doesn't lie) to check for the actual architecture. + if [ "$_cputype" = i386 ]; then + # Handling i386 compatibility mode in older macOS versions (<10.15) + # running on x86_64-based Macs. + # Starting from 10.15, macOS explicitly bans all i386 binaries from running. + # See: + + # Avoid `sysctl: unknown oid` stderr output and/or non-zero exit code. + if sysctl hw.optional.x86_64 2> /dev/null || true | grep -q ': 1'; then + _cputype=x86_64 + fi + elif [ "$_cputype" = x86_64 ]; then + # Handling x86-64 compatibility mode (a.k.a. Rosetta 2) + # in newer macOS versions (>=11) running on arm64-based Macs. + # Rosetta 2 is built exclusively for x86-64 and cannot run i386 binaries. + + # Avoid `sysctl: unknown oid` stderr output and/or non-zero exit code. + if sysctl hw.optional.arm64 2> /dev/null || true | grep -q ': 1'; then + _cputype=arm64 + fi + fi + fi + + if [ "$_ostype" = SunOS ]; then + # Both Solaris and illumos presently announce as "SunOS" in "uname -s" + # so use "uname -o" to disambiguate. We use the full path to the + # system uname in case the user has coreutils uname first in PATH, + # which has historically sometimes printed the wrong value here. + if [ "$(/usr/bin/uname -o)" = illumos ]; then + _ostype=illumos + fi + + # illumos systems have multi-arch userlands, and "uname -m" reports the + # machine hardware name; e.g., "i86pc" on both 32- and 64-bit x86 + # systems. Check for the native (widest) instruction set on the + # running kernel: + if [ "$_cputype" = i86pc ]; then + _cputype="$(isainfo -n)" + fi + fi + + case "$_ostype" in + + Android) + _ostype=linux-android + ;; + + Linux) + check_proc + _ostype=unknown-linux-$_clibtype + _bitness=$(get_bitness) + ;; + + FreeBSD) + _ostype=unknown-freebsd + ;; + + NetBSD) + _ostype=unknown-netbsd + ;; + + DragonFly) + _ostype=unknown-dragonfly + ;; + + Darwin) + _ostype=apple-darwin + ;; + + illumos) + _ostype=unknown-illumos + ;; + + MINGW* | MSYS* | CYGWIN* | Windows_NT) + _ostype=pc-windows-gnu + ;; + + *) + err "unrecognized OS type: $_ostype" + ;; + + esac + + case "$_cputype" in + + i386 | i486 | i686 | i786 | x86) + _cputype=i686 + ;; + + xscale | arm) + _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + fi + ;; + + armv6l) + _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + + armv7l | armv8l) + _cputype=armv7 + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + + aarch64 | arm64) + _cputype=aarch64 + ;; + + x86_64 | x86-64 | x64 | amd64) + _cputype=x86_64 + ;; + + mips) + _cputype=$(get_endianness mips '' el) + ;; + + mips64) + if [ "$_bitness" -eq 64 ]; then + # only n64 ABI is supported for now + _ostype="${_ostype}abi64" + _cputype=$(get_endianness mips64 '' el) + fi + ;; + + ppc) + _cputype=powerpc + ;; + + ppc64) + _cputype=powerpc64 + ;; + + ppc64le) + _cputype=powerpc64le + ;; + + s390x) + _cputype=s390x + ;; + riscv64) + _cputype=riscv64gc + ;; + loongarch64) + _cputype=loongarch64 + ensure_loongarch_uapi + ;; + *) + err "unknown CPU type: $_cputype" + + esac + + # Detect 64-bit linux with 32-bit userland + if [ "${_ostype}" = unknown-linux-gnu ] && [ "${_bitness}" -eq 32 ]; then + case $_cputype in + x86_64) + if [ -n "${RUSTUP_CPUTYPE:-}" ]; then + _cputype="$RUSTUP_CPUTYPE" + else { + # 32-bit executable for amd64 = x32 + if is_host_amd64_elf; then { + echo "This host is running an x32 userland; as it stands, x32 support is poor," 1>&2 + echo "and there isn't a native toolchain -- you will have to install" 1>&2 + echo "multiarch compatibility with i686 and/or amd64, then select one" 1>&2 + echo "by re-running this script with the RUSTUP_CPUTYPE environment variable" 1>&2 + echo "set to i686 or x86_64, respectively." 1>&2 + echo 1>&2 + echo "You will be able to add an x32 target after installation by running" 1>&2 + echo " rustup target add x86_64-unknown-linux-gnux32" 1>&2 + exit 1 + }; else + _cputype=i686 + fi + }; fi + ;; + mips64) + _cputype=$(get_endianness mips '' el) + ;; + powerpc64) + _cputype=powerpc + ;; + aarch64) + _cputype=armv7 + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + riscv64gc) + err "riscv64 with 32-bit userland unsupported" + ;; + esac + fi + + # Detect armv7 but without the CPU features Rust needs in that build, + # and fall back to arm. + # See https://github.com/rust-lang/rustup.rs/issues/587. + if [ "$_ostype" = "unknown-linux-gnueabihf" ] && [ "$_cputype" = armv7 ]; then + if ensure grep '^Features' /proc/cpuinfo | grep -E -q -v 'neon|simd'; then + # At least one processor does not have NEON (which is asimd on armv8+). + _cputype=arm + fi + fi + + _arch="${_cputype}-${_ostype}" + + RETVAL="$_arch" +} + +say() { + printf 'rustup: %s\n' "$1" +} + +err() { + say "$1" >&2 + exit 1 +} + +need_cmd() { + if ! check_cmd "$1"; then + err "need '$1' (command not found)" + fi +} + +check_cmd() { + command -v "$1" > /dev/null 2>&1 +} + +assert_nz() { + if [ -z "$1" ]; then err "assert_nz $2"; fi +} + +# Run a command that should never fail. If the command fails execution +# will immediately terminate with an error showing the failing +# command. +ensure() { + if ! "$@"; then err "command failed: $*"; fi +} + +# This is just for indicating that commands' results are being +# intentionally ignored. Usually, because it's being executed +# as part of error handling. +ignore() { + "$@" +} + +# This wraps curl or wget. Try curl first, if not installed, +# use wget instead. +downloader() { + # zsh does not split words by default, Required for curl retry arguments below. + is_zsh && setopt local_options shwordsplit + + local _dld + local _ciphersuites + local _err + local _status + local _retry + if check_cmd curl; then + _dld=curl + elif check_cmd wget; then + _dld=wget + else + _dld='curl or wget' # to be used in error message of need_cmd + fi + + if [ "$1" = --check ]; then + need_cmd "$_dld" + elif [ "$_dld" = curl ]; then + check_curl_for_retry_support + _retry="$RETVAL" + get_ciphersuites_for_curl + _ciphersuites="$RETVAL" + if [ -n "$_ciphersuites" ]; then + _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + else + echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" + if ! check_help_for "$3" curl --proto --tlsv1.2; then + echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" + _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + else + _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + fi + fi + if [ -n "$_err" ]; then + echo "$_err" >&2 + if echo "$_err" | grep -q 404$; then + err "installer for platform '$3' not found, this may be unsupported" + fi + fi + return $_status + elif [ "$_dld" = wget ]; then + if [ "$(wget -V 2>&1|head -2|tail -1|cut -f1 -d" ")" = "BusyBox" ]; then + echo "Warning: using the BusyBox version of wget. Not enforcing strong cipher suites for TLS or TLS v1.2, this is potentially less secure" + _err=$(wget "$1" -O "$2" 2>&1) + _status=$? + else + get_ciphersuites_for_wget + _ciphersuites="$RETVAL" + if [ -n "$_ciphersuites" ]; then + _err=$(wget --https-only --secure-protocol=TLSv1_2 --ciphers "$_ciphersuites" "$1" -O "$2" 2>&1) + _status=$? + else + echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" + if ! check_help_for "$3" wget --https-only --secure-protocol; then + echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" + _err=$(wget "$1" -O "$2" 2>&1) + _status=$? + else + _err=$(wget --https-only --secure-protocol=TLSv1_2 "$1" -O "$2" 2>&1) + _status=$? + fi + fi + fi + if [ -n "$_err" ]; then + echo "$_err" >&2 + if echo "$_err" | grep -q ' 404 Not Found$'; then + err "installer for platform '$3' not found, this may be unsupported" + fi + fi + return $_status + else + err "Unknown downloader" # should not reach here + fi +} + +check_help_for() { + local _arch + local _cmd + local _arg + _arch="$1" + shift + _cmd="$1" + shift + + local _category + if "$_cmd" --help | grep -q 'For all options use the manual or "--help all".'; then + _category="all" + else + _category="" + fi + + case "$_arch" in + + *darwin*) + if check_cmd sw_vers; then + case $(sw_vers -productVersion) in + 10.*) + # If we're running on macOS, older than 10.13, then we always + # fail to find these options to force fallback + if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then + # Older than 10.13 + echo "Warning: Detected macOS platform older than 10.13" + return 1 + fi + ;; + 11.*) + # We assume Big Sur will be OK for now + ;; + *) + # Unknown product version, warn and continue + echo "Warning: Detected unknown macOS major version: $(sw_vers -productVersion)" + echo "Warning TLS capabilities detection may fail" + ;; + esac + fi + ;; + + esac + + for _arg in "$@"; do + if ! "$_cmd" --help "$_category" | grep -q -- "$_arg"; then + return 1 + fi + done + + true # not strictly needed +} + +# Check if curl supports the --retry flag, then pass it to the curl invocation. +check_curl_for_retry_support() { + local _retry_supported="" + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "curl" "--retry"; then + _retry_supported="--retry 3" + if check_help_for "notspecified" "curl" "--continue-at"; then + # "-C -" tells curl to automatically find where to resume the download when retrying. + _retry_supported="--retry 3 -C -" + fi + fi + + RETVAL="$_retry_supported" +} + +# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites +# if support by local tools is detected. Detection currently supports these curl backends: +# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. +get_ciphersuites_for_curl() { + if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then + # user specified custom cipher suites, assume they know what they're doing + RETVAL="$RUSTUP_TLS_CIPHERSUITES" + return + fi + + local _openssl_syntax="no" + local _gnutls_syntax="no" + local _backend_supported="yes" + if curl -V | grep -q ' OpenSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' LibreSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' BoringSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' GnuTLS/'; then + _gnutls_syntax="yes" + else + _backend_supported="no" + fi + + local _args_supported="no" + if [ "$_backend_supported" = "yes" ]; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "curl" "--tlsv1.2" "--ciphers" "--proto"; then + _args_supported="yes" + fi + fi + + local _cs="" + if [ "$_args_supported" = "yes" ]; then + if [ "$_openssl_syntax" = "yes" ]; then + _cs=$(get_strong_ciphersuites_for "openssl") + elif [ "$_gnutls_syntax" = "yes" ]; then + _cs=$(get_strong_ciphersuites_for "gnutls") + fi + fi + + RETVAL="$_cs" +} + +# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites +# if support by local tools is detected. Detection currently supports these wget backends: +# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. +get_ciphersuites_for_wget() { + if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then + # user specified custom cipher suites, assume they know what they're doing + RETVAL="$RUSTUP_TLS_CIPHERSUITES" + return + fi + + local _cs="" + if wget -V | grep -q '\-DHAVE_LIBSSL'; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then + _cs=$(get_strong_ciphersuites_for "openssl") + fi + elif wget -V | grep -q '\-DHAVE_LIBGNUTLS'; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then + _cs=$(get_strong_ciphersuites_for "gnutls") + fi + fi + + RETVAL="$_cs" +} + +# Return strong TLS 1.2-1.3 cipher suites in OpenSSL or GnuTLS syntax. TLS 1.2 +# excludes non-ECDHE and non-AEAD cipher suites. DHE is excluded due to bad +# DH params often found on servers (see RFC 7919). Sequence matches or is +# similar to Firefox 68 ESR with weak cipher suites disabled via about:config. +# $1 must be openssl or gnutls. +get_strong_ciphersuites_for() { + if [ "$1" = "openssl" ]; then + # OpenSSL is forgiving of unknown values, no problems with TLS 1.3 values on versions that don't support it yet. + echo "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" + elif [ "$1" = "gnutls" ]; then + # GnuTLS isn't forgiving of unknown values, so this may require a GnuTLS version that supports TLS 1.3 even if wget doesn't. + # Begin with SECURE128 (and higher) then remove/add to build cipher suites. Produces same 9 cipher suites as OpenSSL but in slightly different order. + echo "SECURE128:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-DTLS-ALL:-CIPHER-ALL:-MAC-ALL:-KX-ALL:+AEAD:+ECDHE-ECDSA:+ECDHE-RSA:+AES-128-GCM:+CHACHA20-POLY1305:+AES-256-GCM" + fi +} + +main "$@" || exit 1 diff --git a/node/build.sh b/node/build.sh new file mode 100755 index 0000000..f7606c5 --- /dev/null +++ b/node/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -euxo pipefail + +# This script builds the node binary for the current platform and statically links it with VDF static lib. + +ROOT_DIR="${ROOT_DIR:-$( cd "$(dirname "$(realpath "$( dirname "${BASH_SOURCE[0]}" )")")" >/dev/null 2>&1 && pwd )}" + +NODE_DIR="$ROOT_DIR/node" +BINARIES_DIR="$ROOT_DIR/target/release" + +pushd "$NODE_DIR" > /dev/null + +export CGO_ENABLED=1 +export GOEXPERIMENT=arenas + +os_type="$(uname)" +case "$os_type" in + "Darwin") + # Check if the architecture is ARM + if [[ "$(uname -m)" == "arm64" ]]; then + # MacOS ld doesn't support -Bstatic and -Bdynamic, so it's important that there is only a static version of the library + go build -ldflags "-linkmode 'external' -extldflags '-L$BINARIES_DIR -lvdf -ldl -lm'" "$@" + else + echo "Unsupported platform" + exit 1 + fi + ;; + "Linux") + go build -ldflags "-linkmode 'external' -extldflags '-L$BINARIES_DIR -Wl,-Bstatic -lvdf -Wl,-Bdynamic -ldl -lm'" "$@" + ;; + *) + echo "Unsupported platform" + exit 1 + ;; +esac diff --git a/node/test.sh b/node/test.sh new file mode 100755 index 0000000..0c3845e --- /dev/null +++ b/node/test.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -euxo pipefail + +# Run tests for the node package. Takes care of linking the native VDF. +# Assumes that the VDF library has been built by running the generate.sh script in the `../vdf` directory. + +ROOT_DIR="${ROOT_DIR:-$( cd "$(dirname "$(realpath "$( dirname "${BASH_SOURCE[0]}" )")")" >/dev/null 2>&1 && pwd )}" + +NODE_DIR="$ROOT_DIR/node" +BINARIES_DIR="$ROOT_DIR/target/release" + +# Link the native VDF and execute tests +pushd "$NODE_DIR" > /dev/null + CGO_LDFLAGS="-L$BINARIES_DIR -lvdf -ldl" \ + CGO_ENABLED=1 \ + GOEXPERIMENT=arenas \ + go test "$@" diff --git a/node/test_weso_frame_prover.sh b/node/test_weso_frame_prover.sh deleted file mode 100755 index 7ffffe0..0000000 --- a/node/test_weso_frame_prover.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -SCRIPT_DIR="${SCRIPT_DIR:-$( cd "../$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )}" -ROOT_DIR="$SCRIPT_DIR" - -BINDINGS_DIR="$ROOT_DIR/node" -BINARIES_DIR="$ROOT_DIR/target/release" - -go generate - -# Test the generated bindings -pushd "$BINDINGS_DIR" > /dev/null -LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:$BINARIES_DIR" \ - CGO_LDFLAGS="-lvdf -L$BINARIES_DIR -ldl" \ - CGO_ENABLED=1 \ - LC_RPATH="$BINARIES_DIR" \ - GOEXPERIMENT=arenas \ - go test -v crypto/wesolowski_frame_prover_test.go diff --git a/vdf/REAMDE.md b/vdf/REAMDE.md new file mode 100644 index 0000000..de9a57c --- /dev/null +++ b/vdf/REAMDE.md @@ -0,0 +1,9 @@ +# VDF + +Wrapper for the Rust implementation of the VDF (Verifiable Delay Function) in [crates/vdf](../crates/vdf). + +## Generate Go bindings + +```sh +go generate +``` diff --git a/vdf/generate.sh b/vdf/generate.sh index d57a1f8..45b706a 100755 --- a/vdf/generate.sh +++ b/vdf/generate.sh @@ -1,8 +1,7 @@ #!/bin/bash set -euxo pipefail -SCRIPT_DIR="${SCRIPT_DIR:-$( cd "../$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )}" -ROOT_DIR="$SCRIPT_DIR" +ROOT_DIR="${ROOT_DIR:-$( cd "$(dirname "$(realpath "$( dirname "${BASH_SOURCE[0]}" )")")" >/dev/null 2>&1 && pwd )}" RUST_VDF_PACKAGE="$ROOT_DIR/crates/vdf" BINDINGS_DIR="$ROOT_DIR/vdf" diff --git a/vdf/test.sh b/vdf/test.sh index 48479d6..cb42e34 100755 --- a/vdf/test.sh +++ b/vdf/test.sh @@ -1,18 +1,17 @@ #!/bin/bash set -euxo pipefail -SCRIPT_DIR="${SCRIPT_DIR:-$( cd "../$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )}" -ROOT_DIR="$SCRIPT_DIR" +# Run tests for the vdf package. Takes care of linking the native VDF. +# Assumes that the VDF library has been built by running the generate.sh script in the same directory. -BINDINGS_DIR="$ROOT_DIR/vdf" +ROOT_DIR="${ROOT_DIR:-$( cd "$(dirname "$(realpath "$( dirname "${BASH_SOURCE[0]}" )")")" >/dev/null 2>&1 && pwd )}" + +NODE_DIR="$ROOT_DIR/vdf" BINARIES_DIR="$ROOT_DIR/target/release" -go generate - -# Test the generated bindings -pushd "$BINDINGS_DIR" > /dev/null -LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:$BINARIES_DIR" \ - CGO_LDFLAGS="-lvdf -L$BINARIES_DIR -ldl" \ +# Link the native VDF and execute tests +pushd "$NODE_DIR" > /dev/null + CGO_LDFLAGS="-L$BINARIES_DIR -lvdf -ldl" \ CGO_ENABLED=1 \ - LC_RPATH="$BINARIES_DIR" \ - go test + GOEXPERIMENT=arenas \ + go test "$@"