From ae7c14fc2abe94667d4f46b4728a6967c2908448 Mon Sep 17 00:00:00 2001 From: Marius Scurtescu Date: Wed, 28 Feb 2024 22:02:49 -0800 Subject: [PATCH] better buffer instructions and other minor changes (#75) * add git commit to Dockerfile label * add deploy resource requirements * add default stats collection * fix file permissions * relax buffer warning * fix typos * add various deploy instructions --- DOCKER-README.md | 96 ++++++++++++++++++++++++++++++++++++++----- Dockerfile | 3 ++ docker-compose.yml | 10 ++++- node/config/config.go | 6 ++- node/main.go | 4 +- 5 files changed, 105 insertions(+), 14 deletions(-) diff --git a/DOCKER-README.md b/DOCKER-README.md index f2f64dc..3a198c9 100644 --- a/DOCKER-README.md +++ b/DOCKER-README.md @@ -3,39 +3,65 @@ ## WARNING > [!WARNING] -> Currently Docker cannot be used to run Quilibrium. +> The Quilibrium docker container requires host configuration changes. There are extreme buffering requirements, especially during sync, and these in turn require `sysctl` -configuration changes that unfortunately are not supported by Docker. +configuration changes that unfortunately are not supported by Docker. But if these changes are made on +the host machine, then luckily containers seem to automatically have the larger buffers. The buffer related `sysctl` settings are `net.core.rmem_max` and `net.core.wmem_max` and they both should be set to `600,000,000` bytes. This value allows pre-buffering of the entire maximum payload for sync. +You can tell that the buffer size is not large enough by noticing this log entry at beginning when +Quilibrium starts, a few lines below the large logo: +> failed to sufficiently increase receive buffer size (was: 208 kiB, wanted: 2048 kiB, got: 416 kiB). +> See https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes for details. + To read the currently set values: ```shell sysctl -n net.core.rmem_max sysctl -n net.core.wmem_max ``` -To set new values: +To set new values, this is not a persistent change: ```shell sudo sysctl -w net.core.rmem_max=600000000 sudo sysctl -w net.core.wmem_max=600000000 ``` -## Build +To persistently set the new values add a configuration file named `20-quilibrium.conf` to +`/etc/sysctl.d/`. The file content should be: +``` +# Quilibrium buffering requirements, especially during sync. +# The value could be as low as 26214400, but everything would be slower. -Build the docker image: -```shell -docker build -t quilibrium -t quilibrium:1.2.9 . +net.core.rmem_max = 600000000 +net.core.wmem_max = 600000000 ``` -Use latest version instead of `1.2.9`. + +## Build + +In the repository root folder, where the [Dockerfile](Dockerfile) file is, build the docker image: +```shell +docker build --build-arg GIT_COMMIT=$(git log -1 --format=%h) -t quilibrium -t quilibrium:1.2.15 . +``` + +Use latest version instead of `1.2.15`. ## Run +You can run Quilibrium on the same machine where you built the image, from the same repository root +folder where [docker-compose.yml](docker-compose.yml) is. + +You can also copy `docker-compose.yml` to a new folder on a server and run it there. In this case you +have to have a way to push your image to a Docker image repo and then pull that image on the server. +Github offers such an image repo and a way to push and pull images using special authentication +tokens. See +[Working with the Container registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). + Run Quilibrium in a container: ```shell docker compose up -d @@ -45,7 +71,57 @@ A `.config/` subfolder will be created under the current folder, this is mapped Make sure you backup `config.yml` and `keys.yml`. -## Intereact with a running cotainer +### Resource management +To ensure that your client performs optimally within a specific resource configuration, you can specify +resource limits and reservations in the node configuration as illustrated below. + +This configuration helps in deploying the client with controlled resource usage, such as CPU and memory, +to avoid overconsumption of resources in your environment. + +The [docker-compose.yml](docker-compose.yml) file already specifies resources following the currently +recommended hardware requirements. + +```yaml +services: + node: + # Some other configuration sections here + deploy: + resources: + limits: + cpus: '4' # Maximum CPU count that the container can use + memory: 16G # Maximum memory that the container can use + reservations: + cpus: '2' # CPU count that the container initially requests + memory: 8G # Memory that the container initially request +``` + + +### Customizing docker-compose.yml + +If you want to change certain parameters in [docker-compose.yml](docker-compose.yml) it is better not +to edit the file directly as new versions pushed through git would overwrite your changes. A more +flexible solution is to create another file called `docker-compose.override.yml` right next to it +and specifying the necessary overriding changes there. + +For example: +```yaml +services: + node: + image: ghcr.io/mscurtescu/ceremonyclient + restart: on-failure:7 +``` + +The above will override the image name and also the restart policy. + +To check if your overrides are being picked up run the following command: +```shell +docker compose config +``` + +This will output the merged and canonical compose file that will be used to run the container(s). + + +## Interact with a running container Drop into a shell inside a running container: ```shell @@ -62,7 +138,7 @@ Get the Peer ID: docker compose exec node go run ./... -peer-id ``` -Get the token ballance: +Get the token balance: ```shell docker compose exec node go run ./... -balance ``` diff --git a/Dockerfile b/Dockerfile index c401150..01768f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,13 @@ FROM golang:1.20.14-alpine3.19 +ARG GIT_COMMIT=unspecified + LABEL org.opencontainers.image.title="Quilibrium Network Node" LABEL org.opencontainers.image.description="Quilibrium is a decentralized alternative to platform as a service providers." LABEL org.opencontainers.image.vendor=Quilibrium LABEL org.opencontainers.image.url=https://quilibrium.com/ LABEL org.opencontainers.image.documentation=https://quilibrium.com/docs +LABEL org.opencontainers.image.revision=$GIT_COMMIT ENV GOEXPERIMENT=arenas diff --git a/docker-compose.yml b/docker-compose.yml index c2ab4cc..5b92b1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,16 +3,24 @@ version: "3.8" name: quilibrium # See sysctl related warning in DOCKER-README.md. -# Currently Docker cannot be used to run Quilibrium. +# Host configuration changes are required. services: node: build: ./ image: quilibrium restart: unless-stopped + deploy: + resources: + limits: + memory: 32G + reservations: + cpus: 12 + memory: 16G environment: - DEFAULT_LISTEN_GRPC_MULTIADDR=/ip4/0.0.0.0/tcp/8337 - DEFAULT_LISTEN_REST_MULTIADDR=/ip4/0.0.0.0/tcp/8338 + - DEFAULT_STATS_MULTIADDR=/dns/stats.quilibrium.com/tcp/443 ports: - '8336:8336/udp' # p2p - '127.0.0.1:8337:8337/tcp' # gRPC diff --git a/node/config/config.go b/node/config/config.go index 27a63b8..d5b4817 100644 --- a/node/config/config.go +++ b/node/config/config.go @@ -142,6 +142,10 @@ func LoadConfig(configPath string, proverKey string) (*Config, error) { config.ListenRestMultiaddr = os.Getenv("DEFAULT_LISTEN_REST_MULTIADDR") } + if multiAddr := os.Getenv("DEFAULT_STATS_MULTIADDR"); multiAddr != "" { + config.Engine.StatsMultiaddr = multiAddr + } + fmt.Println("Saving config...") if err = SaveConfig(configPath, config); err != nil { panic(err) @@ -150,7 +154,7 @@ func LoadConfig(configPath string, proverKey string) (*Config, error) { keyfile, err := os.OpenFile( filepath.Join(configPath, "keys.yml"), os.O_CREATE|os.O_RDWR, - fs.FileMode(0700), + fs.FileMode(0600), ) if err != nil { panic(err) diff --git a/node/main.go b/node/main.go index 90d7f2b..c372a86 100644 --- a/node/main.go +++ b/node/main.go @@ -202,7 +202,7 @@ func clearIfTestData(configDir string, nodeConfig *config.Config) { versionFile, err := os.OpenFile( filepath.Join(configDir, "RELEASE_VERSION"), os.O_CREATE|os.O_RDWR, - fs.FileMode(0700), + fs.FileMode(0600), ) if err != nil { panic(err) @@ -232,7 +232,7 @@ func migrate(configDir string, nodeConfig *config.Config) { migrationFile, err := os.OpenFile( filepath.Join(configDir, "MIGRATIONS"), os.O_CREATE|os.O_RDWR, - fs.FileMode(0700), + fs.FileMode(0600), ) if err != nil { panic(err)