Automatically install dependent blockchain binary when run a single python test (#57)

* Separate script to build blockchain binaries

* auto remove http_proxy env if configured

* build bsc binary in run all process

* auto install binary when run a single test

* Add necessary output when build or download binary

* create tmp dir if absent

README.md

@@ -65,12 +65,6 @@ or, run any single test, e.g.
 python sync_test.py
 ```
 
-### Troubleshooting
-
-1. Test failed due to blockchain full node rpc inaccessible.
-   * Traceback: `node.wait_for_rpc_connection()`
-   * Solution: unset the `http_proxy` and `https_proxy` environment variables if configured.
-
 ## Contributing
 
 To make contributions to the project, please follow the guidelines [here](contributing.md).

tests/test_framework/bsc_node.py

@@ -1,8 +1,5 @@
 import os
-import platform
-import requests
 import shutil
-import stat
 
 from config.node_config import BSC_CONFIG
 from eth_utils import encode_hex
@@ -11,9 +8,9 @@ from test_framework.blockchain_node import BlockChainNodeType, BlockchainNode
 from utility.utils import (
     blockchain_p2p_port,
     blockchain_rpc_port,
-    is_windows_platform,
     wait_until,
 )
+from utility.build_binary import build_bsc
 
 __file_path__ = os.path.dirname(os.path.realpath(__file__))
 
@@ -29,6 +26,9 @@ class BSCNode(BlockchainNode):
         log,
         rpc_timeout=10,
     ):
+        if not os.path.exists(binary):
+            build_bsc(os.path.dirname(binary))
+
         local_conf = BSC_CONFIG.copy()
         indexed_config = {
             "HTTPPort": blockchain_rpc_port(index),
@@ -46,20 +46,6 @@ class BSCNode(BlockchainNode):
         )
         self.binary = binary
 
-        if not os.path.exists(self.binary):
-            log.info("binary does not exist")
-            dir_name = os.path.dirname(self.binary)
-            if not os.path.exists(dir_name):
-                os.makedirs(dir_name, exist_ok=True)
-
-            try:
-                with open(f"{self.binary}", "xb") as f:
-                    self.__try_download_node(f, log)
-            except FileExistsError:
-                log.info("Binary is alrady under downloading")
-
-        wait_until(lambda: os.access(f"{self.binary}", os.X_OK), timeout=120)
-
         self.node_id = encode_hex(priv_to_addr(ec_random_keys()[0]))
 
         super().__init__(
@@ -74,42 +60,6 @@ class BSCNode(BlockchainNode):
             rpc_timeout,
         )
 
-    def __try_download_node(self, f, log):
-        url = "https://api.github.com/repos/{}/{}/releases/79485895".format(
-            "bnb-chain", "bsc"
-        )
-        req = requests.get(url)
-        if req.ok:
-            asset_name = self.__get_asset_name()
-
-            url = ""
-            for asset in req.json()["assets"]:
-                if asset["name"].lower() == asset_name:
-                    url = asset["browser_download_url"]
-                    break
-
-            if url:
-                log.info("Try to download geth from %s", url)
-                f.write(requests.get(url).content)
-                f.close()
-
-                if not is_windows_platform():
-                    st = os.stat(self.binary)
-                    os.chmod(self.binary, st.st_mode | stat.S_IEXEC)
-        else:
-            log.info("Request failed with %s", req)
-
-    def __get_asset_name(self):
-        sys = platform.system().lower()
-        if sys == "linux":
-            return "geth_linux"
-        elif sys == "windows":
-            return "geth_windows.exe"
-        elif sys == "darwin":
-            return "geth_mac"
-        else:
-            raise RuntimeError("Unable to recognize platform")
-
     def start(self):
         self.args = [
             self.binary,

tests/test_framework/conflux_node.py

@@ -18,6 +18,7 @@ from utility.utils import (
     blockchain_rpc_port_core,
     wait_until,
 )
+from utility.build_binary import build_conflux
 from web3.exceptions import TransactionNotFound
 
 
@@ -32,6 +33,9 @@ class ConfluxNode(BlockchainNode):
         log,
         rpc_timeout=10,
     ):
+        if not os.path.exists(binary):
+            build_conflux(os.path.dirname(binary))
+
         local_conf = CONFLUX_CONFIG.copy()
         indexed_config = {
             "jsonrpc_http_eth_port": blockchain_rpc_port(index),

tests/test_framework/evmos_node.py

@@ -4,6 +4,7 @@ import tempfile
 
 from test_framework.blockchain_node import BlockChainNodeType, BlockchainNode
 from utility.utils import blockchain_rpc_port, arrange_port
+from utility.build_binary import build_evmos
 
 EVMOS_PORT_CATEGORY_WS = 0
 EVMOS_PORT_CATEGORY_P2P = 1
@@ -45,6 +46,9 @@ class EvmosNode(BlockchainNode):
         log,
         rpc_timeout=10,
     ):
+        if not os.path.exists(binary):
+            build_evmos(os.path.dirname(binary))
+
         data_dir = os.path.join(root_dir, "evmosd", "node" + str(index))
         rpc_url = "http://127.0.0.1:%s" % blockchain_rpc_port(index)
 

tests/test_framework/test_framework.py

@@ -35,8 +35,7 @@ TEST_EXIT_FAILED = 1
 class TestFramework:
     def __init__(self, blockchain_node_type=BlockChainNodeType.Evmos):
         if "http_proxy" in os.environ:
-            # Print a warning message in yellow color
+            del os.environ["http_proxy"]
-            print("\n\033[93m ⚠️   Warning: You've set the environment variable 'http_proxy', which might lead to testing issues.\033[0m\n")
 
         self.num_blockchain_nodes = None
         self.num_nodes = None

tests/utility/build_binary.py

@@ -0,0 +1,120 @@
+import os
+import time
+import shutil
+import stat
+import requests
+import platform
+
+from utility.utils import is_windows_platform, wait_until
+
+def build_conflux(dir: str) -> bool:
+    return __build_from_github(
+        dir=dir,
+        binary_name="conflux.exe" if is_windows_platform() else "conflux",
+        github_url="https://github.com/Conflux-Chain/conflux-rust.git",
+        build_cmd="cargo build --release --bin conflux",
+        compiled_relative_path=["target", "release"],
+    )
+
+def build_bsc(dir: str) -> bool:
+    sys = platform.system().lower()
+    if sys == "linux":
+        asset_name = "geth_linux"
+    elif sys == "windows":
+        asset_name = "geth_windows.exe"
+    elif sys == "darwin":
+        asset_name = "geth_mac"
+    else:
+        raise RuntimeError("Unable to recognize platform")
+
+    return __download_from_github(
+        dir=dir,
+        binary_name="geth.exe" if is_windows_platform() else "geth",
+        github_url="https://api.github.com/repos/bnb-chain/bsc/releases/79485895",
+        asset_name=asset_name,
+    )
+
+def build_evmos(dir: str) -> bool:
+    return __build_from_github(
+        dir=dir,
+        binary_name="evmosd.exe" if is_windows_platform() else "evmosd",
+        github_url="-b testnet https://github.com/0glabs/0g-evmos.git",
+        build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .",
+        compiled_relative_path=[],
+    )
+
+
+def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str]) -> bool:
+    if not os.path.exists(dir):
+        os.makedirs(dir, exist_ok=True)
+
+    binary_path = os.path.join(dir, binary_name)
+    if os.path.exists(binary_path):
+        return False
+    
+    print("Begin to build binary from github: %s" % binary_name, flush=True)
+    
+    start_time = time.time()
+    
+    # clone code from github to a temp folder
+    code_tmp_dir_name = (binary_name[:-4] if is_windows_platform() else binary_name) + "_tmp"
+    code_tmp_dir = os.path.join(dir, code_tmp_dir_name)
+    if os.path.exists(code_tmp_dir):
+        shutil.rmtree(code_tmp_dir)
+    clone_command = "git clone " + github_url + " " + code_tmp_dir
+    os.system(clone_command)
+
+    # build binary
+    origin_path = os.getcwd()
+    os.chdir(code_tmp_dir)
+    os.system(build_cmd)
+
+    # copy compiled binary to right place
+    compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
+    shutil.copyfile(compiled_binary, binary_path)
+
+    if not is_windows_platform():
+        st = os.stat(binary_path)
+        os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
+
+    os.chdir(origin_path)
+
+    shutil.rmtree(code_tmp_dir, ignore_errors=True)
+
+    print("Completed to build binary, Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
+
+    return True
+
+def __download_from_github(dir: str, binary_name: str, github_url: str, asset_name: str) -> bool:
+    if not os.path.exists(dir):
+        os.makedirs(dir, exist_ok=True)
+
+    binary_path = os.path.join(dir, binary_name)
+    if os.path.exists(binary_path):
+        return False
+    
+    print("Begin to download binary from github: %s" % binary_name, flush=True)
+    
+    start_time = time.time()
+
+    with open(binary_path, "xb") as f:
+        req = requests.get(github_url)
+        assert req.ok, "Failed to request: %s" % github_url
+        for asset in req.json()["assets"]:
+            if asset["name"].lower() == asset_name:
+                download_url = asset["browser_download_url"]
+                break
+
+        assert download_url is not None, "Cannot find binary to download by asset name [%s]" % asset_name
+
+        f.write(requests.get(download_url).content)
+
+        if not is_windows_platform():
+            st = os.stat(binary_path)
+            os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
+    
+    wait_until(lambda: os.access(binary_path, os.X_OK), timeout=120)
+
+    print("Completed to download binary, Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
+
+    return True

tests/utility/run_all.py

@@ -3,20 +3,15 @@ import os
 import time
 import subprocess
 import sys
-import shutil
-import stat
 
 from concurrent.futures import ProcessPoolExecutor
 
-from utility.utils import is_windows_platform
+from utility.build_binary import build_conflux, build_bsc, build_evmos
 
 DEFAULT_PORT_MIN = 11000
 DEFAULT_PORT_MAX = 65535
 DEFAULT_PORT_RANGE = 500
 
-CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux"
-EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd"
-
 def print_testcase_result(color, glyph, script, start_time):
     print(color[1] + glyph + " Testcase " + script + "\telapsed: " + str(int(time.time() - start_time)) + " seconds" + color[0], flush=True)
 
@@ -64,23 +59,10 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
     if not os.path.exists(tmp_dir):
         os.makedirs(tmp_dir, exist_ok=True)
 
-    # Build conflux binary if absent
+    # Build blockchain binaries if absent
-    build_from_github(
+    build_conflux(tmp_dir)
-        dir=tmp_dir,
+    build_bsc(tmp_dir)
-        binary_name=CONFLUX_BINARY,
+    build_evmos(tmp_dir)
-        github_url="https://github.com/Conflux-Chain/conflux-rust.git",
-        build_cmd="cargo build --release --bin conflux",
-        compiled_relative_path=["target", "release"],
-    )
-
-    # Build evmos binary if absent
-    build_from_github(
-        dir=tmp_dir,
-        binary_name=EVMOS_BINARY,
-        github_url="-b testnet https://github.com/0glabs/0g-evmos.git",
-        build_cmd="make install; cp $(go env GOPATH)/bin/evmosd .",
-        compiled_relative_path=[],
-    )
 
     start_time = time.time()
 
@@ -155,39 +137,3 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
         for c in failed:
             print(c)
         sys.exit(1)
-
-def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str]) -> bool:
-    binary_path = os.path.join(dir, binary_name)
-    if os.path.exists(binary_path):
-        return False
-    
-    start_time = time.time()
-    
-    # clone code from github to a temp folder
-    code_tmp_dir_name = (binary_name[:-4] if is_windows_platform() else binary_name) + "_tmp"
-    code_tmp_dir = os.path.join(dir, code_tmp_dir_name)
-    if os.path.exists(code_tmp_dir):
-        shutil.rmtree(code_tmp_dir)
-    clone_command = "git clone " + github_url + " " + code_tmp_dir
-    os.system(clone_command)
-
-    # build binary
-    origin_path = os.getcwd()
-    os.chdir(code_tmp_dir)
-    os.system(build_cmd)
-
-    # copy compiled binary to right place
-    compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
-    shutil.copyfile(compiled_binary, binary_path)
-
-    if not is_windows_platform():
-        st = os.stat(binary_path)
-        os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
-
-    os.chdir(origin_path)
-
-    shutil.rmtree(code_tmp_dir, ignore_errors=True)
-
-    print("Completed to build binary " + binary_name + ", Elapsed: " + str(int(time.time() - start_time)) + " seconds", flush=True)
-
-    return True