smeb_y/0g-storage-node
1
0
Fork 0
mirror of https://github.com/0glabs/0g-storage-node.git synced 2024-11-20 15:05:19 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Automatically install dependent blockchain binary when run a single python test (#57)

Browse Source 
* Separate script to build blockchain binaries

* auto remove http_proxy env if configured

* build bsc binary in run all process

* auto install binary when run a single test

* Add necessary output when build or download binary

* create tmp dir if absent
This commit is contained in:
Bo QIU 2024-04-19 20:24:50 +08:00 committed by GitHub
parent b9c6e43617
commit 0bd9ef0ed8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 138 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -65,12 +65,6 @@ or, run any single test, e.g.
python sync_test.py
```
### Troubleshooting
1. Test failed due to blockchain full node rpc inaccessible.
* Traceback: `node.wait_for_rpc_connection()`
* Solution: unset the `http_proxy` and `https_proxy` environment variables if configured.
## Contributing
To make contributions to the project, please follow the guidelines [here](contributing.md).

58
tests/test_framework/bsc_node.py
View File

@ -1,8 +1,5 @@
import os
import platform
import requests
import shutil
import stat
from config.node_config import BSC_CONFIG
from eth_utils import encode_hex
@ -11,9 +8,9 @@ from test_framework.blockchain_node import BlockChainNodeType, BlockchainNode
from utility.utils import (
blockchain_p2p_port,
blockchain_rpc_port,
is_windows_platform,
wait_until,
)
from utility.build_binary import build_bsc
__file_path__ = os.path.dirname(os.path.realpath(__file__))
@ -29,6 +26,9 @@ class BSCNode(BlockchainNode):
log,
rpc_timeout=10,
):
if not os.path.exists(binary):
build_bsc(os.path.dirname(binary))
local_conf = BSC_CONFIG.copy()
indexed_config = {
"HTTPPort": blockchain_rpc_port(index),
@ -46,20 +46,6 @@ class BSCNode(BlockchainNode):
)
self.binary = binary
if not os.path.exists(self.binary):
log.info("binary does not exist")
dir_name = os.path.dirname(self.binary)
if not os.path.exists(dir_name):
os.makedirs(dir_name, exist_ok=True)
try:
with open(f"{self.binary}", "xb") as f:
self.__try_download_node(f, log)
except FileExistsError:
log.info("Binary is alrady under downloading")
wait_until(lambda: os.access(f"{self.binary}", os.X_OK), timeout=120)
self.node_id = encode_hex(priv_to_addr(ec_random_keys()[0]))
super().__init__(
@ -74,42 +60,6 @@ class BSCNode(BlockchainNode):
rpc_timeout,
)
def __try_download_node(self, f, log):
url = "https://api.github.com/repos/{}/{}/releases/79485895".format(
"bnb-chain", "bsc"
)
req = requests.get(url)
if req.ok:
asset_name = self.__get_asset_name()
url = ""
for asset in req.json()["assets"]:
if asset["name"].lower() == asset_name:
url = asset["browser_download_url"]
break
if url:
log.info("Try to download geth from %s", url)
f.write(requests.get(url).content)
f.close()
if not is_windows_platform():
st = os.stat(self.binary)
os.chmod(self.binary, st.st_mode | stat.S_IEXEC)
else:
log.info("Request failed with %s", req)
def __get_asset_name(self):
sys = platform.system().lower()
if sys == "linux":
return "geth_linux"
elif sys == "windows":
return "geth_windows.exe"
elif sys == "darwin":
return "geth_mac"
else:
raise RuntimeError("Unable to recognize platform")
def start(self):
self.args = [
self.binary,

4
tests/test_framework/conflux_node.py
View File

@ -18,6 +18,7 @@ from utility.utils import (
blockchain_rpc_port_core,
wait_until,
)
from utility.build_binary import build_conflux
from web3.exceptions import TransactionNotFound
@ -32,6 +33,9 @@ class ConfluxNode(BlockchainNode):
log,
rpc_timeout=10,
):
if not os.path.exists(binary):
build_conflux(os.path.dirname(binary))
local_conf = CONFLUX_CONFIG.copy()
indexed_config = {
"jsonrpc_http_eth_port": blockchain_rpc_port(index),

4
tests/test_framework/evmos_node.py
View File

@ -4,6 +4,7 @@ import tempfile
from test_framework.blockchain_node import BlockChainNodeType, BlockchainNode
from utility.utils import blockchain_rpc_port, arrange_port
from utility.build_binary import build_evmos
EVMOS_PORT_CATEGORY_WS = 0
EVMOS_PORT_CATEGORY_P2P = 1
@ -45,6 +46,9 @@ class EvmosNode(BlockchainNode):
log,
rpc_timeout=10,
):
if not os.path.exists(binary):
build_evmos(os.path.dirname(binary))
data_dir = os.path.join(root_dir, "evmosd", "node" + str(index))
rpc_url = "http://127.0.0.1:%s" % blockchain_rpc_port(index)

3
tests/test_framework/test_framework.py
View File

@ -35,8 +35,7 @@ TEST_EXIT_FAILED = 1
class TestFramework:
def __init__(self, blockchain_node_type=BlockChainNodeType.Evmos):
if "http_proxy" in os.environ:
# Print a warning message in yellow color
print("\n\033[93m ⚠️ Warning: You've set the environment variable 'http_proxy', which might lead to testing issues.\033[0m\n")
del os.environ["http_proxy"]
self.num_blockchain_nodes = None
self.num_nodes = None

120
tests/utility/build_binary.py Normal file
View File

@ -0,0 +1,120 @@
import os
import time
import shutil
import stat
import requests
import platform
from utility.utils import is_windows_platform, wait_until
def build_conflux(dir: str) -> bool:
return __build_from_github(
dir=dir,
binary_name="conflux.exe" if is_windows_platform() else "conflux",
github_url="https://github.com/Conflux-Chain/conflux-rust.git",
build_cmd="cargo build --release --bin conflux",
compiled_relative_path=["target", "release"],
)
def build_bsc(dir: str) -> bool:
sys = platform.system().lower()
if sys == "linux":
asset_name = "geth_linux"
elif sys == "windows":
asset_name = "geth_windows.exe"
elif sys == "darwin":
asset_name = "geth_mac"
else:
raise RuntimeError("Unable to recognize platform")
return __download_from_github(
dir=dir,
binary_name="geth.exe" if is_windows_platform() else "geth",
github_url="https://api.github.com/repos/bnb-chain/bsc/releases/79485895",
asset_name=asset_name,
)
def build_evmos(dir: str) -> bool:
return __build_from_github(
dir=dir,
binary_name="evmosd.exe" if is_windows_platform() else "evmosd",
github_url="-b testnet https://github.com/0glabs/0g-evmos.git",
build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .",
compiled_relative_path=[],
)
def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str]) -> bool:
if not os.path.exists(dir):
os.makedirs(dir, exist_ok=True)
binary_path = os.path.join(dir, binary_name)
if os.path.exists(binary_path):
return False
print("Begin to build binary from github: %s" % binary_name, flush=True)
start_time = time.time()
# clone code from github to a temp folder
code_tmp_dir_name = (binary_name[:-4] if is_windows_platform() else binary_name) + "_tmp"
code_tmp_dir = os.path.join(dir, code_tmp_dir_name)
if os.path.exists(code_tmp_dir):
shutil.rmtree(code_tmp_dir)
clone_command = "git clone " + github_url + " " + code_tmp_dir
os.system(clone_command)
# build binary
origin_path = os.getcwd()
os.chdir(code_tmp_dir)
os.system(build_cmd)
# copy compiled binary to right place
compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
shutil.copyfile(compiled_binary, binary_path)
if not is_windows_platform():
st = os.stat(binary_path)
os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
os.chdir(origin_path)
shutil.rmtree(code_tmp_dir, ignore_errors=True)
print("Completed to build binary, Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
return True
def __download_from_github(dir: str, binary_name: str, github_url: str, asset_name: str) -> bool:
if not os.path.exists(dir):
os.makedirs(dir, exist_ok=True)
binary_path = os.path.join(dir, binary_name)
if os.path.exists(binary_path):
return False
print("Begin to download binary from github: %s" % binary_name, flush=True)
start_time = time.time()
with open(binary_path, "xb") as f:
req = requests.get(github_url)
assert req.ok, "Failed to request: %s" % github_url
for asset in req.json()["assets"]:
if asset["name"].lower() == asset_name:
download_url = asset["browser_download_url"]
break
assert download_url is not None, "Cannot find binary to download by asset name [%s]" % asset_name
f.write(requests.get(download_url).content)
if not is_windows_platform():
st = os.stat(binary_path)
os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
wait_until(lambda: os.access(binary_path, os.X_OK), timeout=120)
print("Completed to download binary, Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
return True

64
tests/utility/run_all.py
View File

@ -3,20 +3,15 @@ import os
import time
import subprocess
import sys
import shutil
import stat
from concurrent.futures import ProcessPoolExecutor
from utility.utils import is_windows_platform
from utility.build_binary import build_conflux, build_bsc, build_evmos
DEFAULT_PORT_MIN = 11000
DEFAULT_PORT_MAX = 65535
DEFAULT_PORT_RANGE = 500
CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux"
EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd"
def print_testcase_result(color, glyph, script, start_time):
print(color[1] + glyph + " Testcase " + script + "\telapsed: " + str(int(time.time() - start_time)) + " seconds" + color[0], flush=True)
@ -64,23 +59,10 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir, exist_ok=True)
# Build conflux binary if absent
build_from_github(
dir=tmp_dir,
binary_name=CONFLUX_BINARY,
github_url="https://github.com/Conflux-Chain/conflux-rust.git",
build_cmd="cargo build --release --bin conflux",
compiled_relative_path=["target", "release"],
)
# Build evmos binary if absent
build_from_github(
dir=tmp_dir,
binary_name=EVMOS_BINARY,
github_url="-b testnet https://github.com/0glabs/0g-evmos.git",
build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .",
compiled_relative_path=[],
)
# Build blockchain binaries if absent
build_conflux(tmp_dir)
build_bsc(tmp_dir)
build_evmos(tmp_dir)
start_time = time.time()
@ -155,39 +137,3 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
for c in failed:
print(c)
sys.exit(1)
def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str]) -> bool:
binary_path = os.path.join(dir, binary_name)
if os.path.exists(binary_path):
return False
start_time = time.time()
# clone code from github to a temp folder
code_tmp_dir_name = (binary_name[:-4] if is_windows_platform() else binary_name) + "_tmp"
code_tmp_dir = os.path.join(dir, code_tmp_dir_name)
if os.path.exists(code_tmp_dir):
shutil.rmtree(code_tmp_dir)
clone_command = "git clone " + github_url + " " + code_tmp_dir
os.system(clone_command)
# build binary
origin_path = os.getcwd()
os.chdir(code_tmp_dir)
os.system(build_cmd)
# copy compiled binary to right place
compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
shutil.copyfile(compiled_binary, binary_path)
if not is_windows_platform():
st = os.stat(binary_path)
os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
os.chdir(origin_path)
shutil.rmtree(code_tmp_dir, ignore_errors=True)
print("Completed to build binary " + binary_name + ", Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
return True