From 0bd9ef0ed8b1a7507a42bcfe0932794aa156c77f Mon Sep 17 00:00:00 2001 From: Bo QIU <35757521+boqiu@users.noreply.github.com> Date: Fri, 19 Apr 2024 20:24:50 +0800 Subject: [PATCH] Automatically install dependent blockchain binary when run a single python test (#57) * Separate script to build blockchain binaries * auto remove http_proxy env if configured * build bsc binary in run all process * auto install binary when run a single test * Add necessary output when build or download binary * create tmp dir if absent --- README.md | 6 -- tests/test_framework/bsc_node.py | 58 +----------- tests/test_framework/conflux_node.py | 4 + tests/test_framework/evmos_node.py | 4 + tests/test_framework/test_framework.py | 3 +- tests/utility/build_binary.py | 120 +++++++++++++++++++++++++ tests/utility/run_all.py | 64 ++----------- 7 files changed, 138 insertions(+), 121 deletions(-) create mode 100644 tests/utility/build_binary.py diff --git a/README.md b/README.md index 3e73f34..d7e5921 100644 --- a/README.md +++ b/README.md @@ -65,12 +65,6 @@ or, run any single test, e.g. python sync_test.py ``` -### Troubleshooting - -1. Test failed due to blockchain full node rpc inaccessible. - * Traceback: `node.wait_for_rpc_connection()` - * Solution: unset the `http_proxy` and `https_proxy` environment variables if configured. - ## Contributing To make contributions to the project, please follow the guidelines [here](contributing.md). diff --git a/tests/test_framework/bsc_node.py b/tests/test_framework/bsc_node.py index 3eaae7a..11c8900 100644 --- a/tests/test_framework/bsc_node.py +++ b/tests/test_framework/bsc_node.py @@ -1,8 +1,5 @@ import os -import platform -import requests import shutil -import stat from config.node_config import BSC_CONFIG from eth_utils import encode_hex @@ -11,9 +8,9 @@ from test_framework.blockchain_node import BlockChainNodeType, BlockchainNode from utility.utils import ( blockchain_p2p_port, blockchain_rpc_port, - is_windows_platform, wait_until, ) +from utility.build_binary import build_bsc __file_path__ = os.path.dirname(os.path.realpath(__file__)) @@ -29,6 +26,9 @@ class BSCNode(BlockchainNode): log, rpc_timeout=10, ): + if not os.path.exists(binary): + build_bsc(os.path.dirname(binary)) + local_conf = BSC_CONFIG.copy() indexed_config = { "HTTPPort": blockchain_rpc_port(index), @@ -46,20 +46,6 @@ class BSCNode(BlockchainNode): ) self.binary = binary - if not os.path.exists(self.binary): - log.info("binary does not exist") - dir_name = os.path.dirname(self.binary) - if not os.path.exists(dir_name): - os.makedirs(dir_name, exist_ok=True) - - try: - with open(f"{self.binary}", "xb") as f: - self.__try_download_node(f, log) - except FileExistsError: - log.info("Binary is alrady under downloading") - - wait_until(lambda: os.access(f"{self.binary}", os.X_OK), timeout=120) - self.node_id = encode_hex(priv_to_addr(ec_random_keys()[0])) super().__init__( @@ -74,42 +60,6 @@ class BSCNode(BlockchainNode): rpc_timeout, ) - def __try_download_node(self, f, log): - url = "https://api.github.com/repos/{}/{}/releases/79485895".format( - "bnb-chain", "bsc" - ) - req = requests.get(url) - if req.ok: - asset_name = self.__get_asset_name() - - url = "" - for asset in req.json()["assets"]: - if asset["name"].lower() == asset_name: - url = asset["browser_download_url"] - break - - if url: - log.info("Try to download geth from %s", url) - f.write(requests.get(url).content) - f.close() - - if not is_windows_platform(): - st = os.stat(self.binary) - os.chmod(self.binary, st.st_mode | stat.S_IEXEC) - else: - log.info("Request failed with %s", req) - - def __get_asset_name(self): - sys = platform.system().lower() - if sys == "linux": - return "geth_linux" - elif sys == "windows": - return "geth_windows.exe" - elif sys == "darwin": - return "geth_mac" - else: - raise RuntimeError("Unable to recognize platform") - def start(self): self.args = [ self.binary, diff --git a/tests/test_framework/conflux_node.py b/tests/test_framework/conflux_node.py index fc55fe6..e4d8bea 100644 --- a/tests/test_framework/conflux_node.py +++ b/tests/test_framework/conflux_node.py @@ -18,6 +18,7 @@ from utility.utils import ( blockchain_rpc_port_core, wait_until, ) +from utility.build_binary import build_conflux from web3.exceptions import TransactionNotFound @@ -32,6 +33,9 @@ class ConfluxNode(BlockchainNode): log, rpc_timeout=10, ): + if not os.path.exists(binary): + build_conflux(os.path.dirname(binary)) + local_conf = CONFLUX_CONFIG.copy() indexed_config = { "jsonrpc_http_eth_port": blockchain_rpc_port(index), diff --git a/tests/test_framework/evmos_node.py b/tests/test_framework/evmos_node.py index 4943213..3701fb9 100644 --- a/tests/test_framework/evmos_node.py +++ b/tests/test_framework/evmos_node.py @@ -4,6 +4,7 @@ import tempfile from test_framework.blockchain_node import BlockChainNodeType, BlockchainNode from utility.utils import blockchain_rpc_port, arrange_port +from utility.build_binary import build_evmos EVMOS_PORT_CATEGORY_WS = 0 EVMOS_PORT_CATEGORY_P2P = 1 @@ -45,6 +46,9 @@ class EvmosNode(BlockchainNode): log, rpc_timeout=10, ): + if not os.path.exists(binary): + build_evmos(os.path.dirname(binary)) + data_dir = os.path.join(root_dir, "evmosd", "node" + str(index)) rpc_url = "http://127.0.0.1:%s" % blockchain_rpc_port(index) diff --git a/tests/test_framework/test_framework.py b/tests/test_framework/test_framework.py index 4a7160d..376d8c6 100644 --- a/tests/test_framework/test_framework.py +++ b/tests/test_framework/test_framework.py @@ -35,8 +35,7 @@ TEST_EXIT_FAILED = 1 class TestFramework: def __init__(self, blockchain_node_type=BlockChainNodeType.Evmos): if "http_proxy" in os.environ: - # Print a warning message in yellow color - print("\n\033[93m ⚠️ Warning: You've set the environment variable 'http_proxy', which might lead to testing issues.\033[0m\n") + del os.environ["http_proxy"] self.num_blockchain_nodes = None self.num_nodes = None diff --git a/tests/utility/build_binary.py b/tests/utility/build_binary.py new file mode 100644 index 0000000..7f8b240 --- /dev/null +++ b/tests/utility/build_binary.py @@ -0,0 +1,120 @@ +import os +import time +import shutil +import stat +import requests +import platform + +from utility.utils import is_windows_platform, wait_until + +def build_conflux(dir: str) -> bool: + return __build_from_github( + dir=dir, + binary_name="conflux.exe" if is_windows_platform() else "conflux", + github_url="https://github.com/Conflux-Chain/conflux-rust.git", + build_cmd="cargo build --release --bin conflux", + compiled_relative_path=["target", "release"], + ) + +def build_bsc(dir: str) -> bool: + sys = platform.system().lower() + if sys == "linux": + asset_name = "geth_linux" + elif sys == "windows": + asset_name = "geth_windows.exe" + elif sys == "darwin": + asset_name = "geth_mac" + else: + raise RuntimeError("Unable to recognize platform") + + return __download_from_github( + dir=dir, + binary_name="geth.exe" if is_windows_platform() else "geth", + github_url="https://api.github.com/repos/bnb-chain/bsc/releases/79485895", + asset_name=asset_name, + ) + +def build_evmos(dir: str) -> bool: + return __build_from_github( + dir=dir, + binary_name="evmosd.exe" if is_windows_platform() else "evmosd", + github_url="-b testnet https://github.com/0glabs/0g-evmos.git", + build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .", + compiled_relative_path=[], + ) + + +def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str]) -> bool: + if not os.path.exists(dir): + os.makedirs(dir, exist_ok=True) + + binary_path = os.path.join(dir, binary_name) + if os.path.exists(binary_path): + return False + + print("Begin to build binary from github: %s" % binary_name, flush=True) + + start_time = time.time() + + # clone code from github to a temp folder + code_tmp_dir_name = (binary_name[:-4] if is_windows_platform() else binary_name) + "_tmp" + code_tmp_dir = os.path.join(dir, code_tmp_dir_name) + if os.path.exists(code_tmp_dir): + shutil.rmtree(code_tmp_dir) + clone_command = "git clone " + github_url + " " + code_tmp_dir + os.system(clone_command) + + # build binary + origin_path = os.getcwd() + os.chdir(code_tmp_dir) + os.system(build_cmd) + + # copy compiled binary to right place + compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name) + shutil.copyfile(compiled_binary, binary_path) + + if not is_windows_platform(): + st = os.stat(binary_path) + os.chmod(binary_path, st.st_mode | stat.S_IEXEC) + + os.chdir(origin_path) + + shutil.rmtree(code_tmp_dir, ignore_errors=True) + + print("Completed to build binary, Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True) + + return True + +def __download_from_github(dir: str, binary_name: str, github_url: str, asset_name: str) -> bool: + if not os.path.exists(dir): + os.makedirs(dir, exist_ok=True) + + binary_path = os.path.join(dir, binary_name) + if os.path.exists(binary_path): + return False + + print("Begin to download binary from github: %s" % binary_name, flush=True) + + start_time = time.time() + + with open(binary_path, "xb") as f: + req = requests.get(github_url) + assert req.ok, "Failed to request: %s" % github_url + for asset in req.json()["assets"]: + if asset["name"].lower() == asset_name: + download_url = asset["browser_download_url"] + break + + assert download_url is not None, "Cannot find binary to download by asset name [%s]" % asset_name + + f.write(requests.get(download_url).content) + + if not is_windows_platform(): + st = os.stat(binary_path) + os.chmod(binary_path, st.st_mode | stat.S_IEXEC) + + wait_until(lambda: os.access(binary_path, os.X_OK), timeout=120) + + print("Completed to download binary, Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True) + + return True diff --git a/tests/utility/run_all.py b/tests/utility/run_all.py index c77f14c..6f51174 100644 --- a/tests/utility/run_all.py +++ b/tests/utility/run_all.py @@ -3,20 +3,15 @@ import os import time import subprocess import sys -import shutil -import stat from concurrent.futures import ProcessPoolExecutor -from utility.utils import is_windows_platform +from utility.build_binary import build_conflux, build_bsc, build_evmos DEFAULT_PORT_MIN = 11000 DEFAULT_PORT_MAX = 65535 DEFAULT_PORT_RANGE = 500 -CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux" -EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd" - def print_testcase_result(color, glyph, script, start_time): print(color[1] + glyph + " Testcase " + script + "\telapsed: " + str(int(time.time() - start_time)) + " seconds" + color[0], flush=True) @@ -64,23 +59,10 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={}, if not os.path.exists(tmp_dir): os.makedirs(tmp_dir, exist_ok=True) - # Build conflux binary if absent - build_from_github( - dir=tmp_dir, - binary_name=CONFLUX_BINARY, - github_url="https://github.com/Conflux-Chain/conflux-rust.git", - build_cmd="cargo build --release --bin conflux", - compiled_relative_path=["target", "release"], - ) - - # Build evmos binary if absent - build_from_github( - dir=tmp_dir, - binary_name=EVMOS_BINARY, - github_url="-b testnet https://github.com/0glabs/0g-evmos.git", - build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .", - compiled_relative_path=[], - ) + # Build blockchain binaries if absent + build_conflux(tmp_dir) + build_bsc(tmp_dir) + build_evmos(tmp_dir) start_time = time.time() @@ -155,39 +137,3 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={}, for c in failed: print(c) sys.exit(1) - -def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str]) -> bool: - binary_path = os.path.join(dir, binary_name) - if os.path.exists(binary_path): - return False - - start_time = time.time() - - # clone code from github to a temp folder - code_tmp_dir_name = (binary_name[:-4] if is_windows_platform() else binary_name) + "_tmp" - code_tmp_dir = os.path.join(dir, code_tmp_dir_name) - if os.path.exists(code_tmp_dir): - shutil.rmtree(code_tmp_dir) - clone_command = "git clone " + github_url + " " + code_tmp_dir - os.system(clone_command) - - # build binary - origin_path = os.getcwd() - os.chdir(code_tmp_dir) - os.system(build_cmd) - - # copy compiled binary to right place - compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name) - shutil.copyfile(compiled_binary, binary_path) - - if not is_windows_platform(): - st = os.stat(binary_path) - os.chmod(binary_path, st.st_mode | stat.S_IEXEC) - - os.chdir(origin_path) - - shutil.rmtree(code_tmp_dir, ignore_errors=True) - - print("Completed to build binary " + binary_name + ", Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True) - - return True