Supports to download blockchain binaries from github release (#61)

* build evmos before init genesis

* Supports to download conflux and evmos binary

* print unzip info

* download conflux windows binary in zip format

* unnecessary to install evmos when init node

* aaa

* bbb

* ccc

* ddd

* eee

* 111

* 2222

* aaa

* aaa

* aaa

* aaa

.github/workflows/tests.yml

@@ -53,4 +53,5 @@ jobs:
     - name: Run tests
       run: |
         cd tests
+        uname -a
         python test_all.py

tests/test_framework/evmos_node.py

@@ -11,9 +11,12 @@ EVMOS_PORT_CATEGORY_P2P = 1
 EVMOS_PORT_CATEGORY_RPC = 2
 EVMOS_PORT_CATEGORY_PPROF = 3
 
-def evmos_init_genesis(root_dir: str, num_nodes: int):
+def evmos_init_genesis(binary: str, root_dir: str, num_nodes: int):
     assert num_nodes > 0, "Invalid number of blockchain nodes: %s" % num_nodes
 
+    if not os.path.exists(binary):
+            build_evmos(os.path.dirname(binary))
+
     shell_script = os.path.join(
         os.path.dirname(os.path.realpath(__file__)), # test_framework folder
         "..", "config", "evmos-init-genesis.sh"
@@ -46,9 +49,6 @@ class EvmosNode(BlockchainNode):
         log,
         rpc_timeout=10,
     ):
-        if not os.path.exists(binary):
-            build_evmos(os.path.dirname(binary))
-
         data_dir = os.path.join(root_dir, "evmosd", "node" + str(index))
         rpc_url = "http://127.0.0.1:%s" % blockchain_rpc_port(index)
 

tests/test_framework/test_framework.py

@@ -72,7 +72,7 @@ class TestFramework:
 
     def __setup_blockchain_node(self):
         if self.blockchain_node_type == BlockChainNodeType.Evmos:
-            evmos_init_genesis(self.root_dir, self.num_blockchain_nodes)
+            evmos_init_genesis(self.blockchain_binary, self.root_dir, self.num_blockchain_nodes)
             self.log.info("Evmos genesis initialized for %s nodes" % self.num_blockchain_nodes)
 
         for i in range(self.num_blockchain_nodes):

tests/utility/build_binary.py

@@ -7,7 +7,11 @@ import platform
 
 from utility.utils import is_windows_platform, wait_until
 
+# v1.0.0-ci release
+GITHUB_DOWNLOAD_URL="https://api.github.com/repos/0glabs/0g-storage-node/releases/152560136"
+
 CONFLUX_BINARY = "conflux.exe" if is_windows_platform() else "conflux"
+BSC_BINARY = "geth.exe" if is_windows_platform() else "geth"
 EVMOS_BINARY = "evmosd.exe" if is_windows_platform() else "evmosd"
 CLIENT_BINARY = "0g-storage-client.exe" if is_windows_platform() else "0g-storage-client"
 
@@ -15,8 +19,16 @@ EVMOS_GIT_REV = "2ef76f6c9bdd73cd15dabd7397492dbebc311f98"
 CLI_GIT_REV = "1d09ec4f0b9c27428b2357de46b66e8c231b74df"
 
 def build_conflux(dir: str) -> bool:
-    # Build conflux binary if absent
-    build_from_github(
+    # Download or build conflux binary if absent
+    if __download_from_github(
+        dir=dir,
+        binary_name=CONFLUX_BINARY,
+        github_url=GITHUB_DOWNLOAD_URL, 
+        asset_name=__asset_name(CONFLUX_BINARY, zip=True),
+    ):
+        return True
+
+    return __build_from_github(
         dir=dir,
         binary_name=CONFLUX_BINARY,
         github_url="https://github.com/Conflux-Chain/conflux-rust.git",
@@ -25,26 +37,31 @@ def build_conflux(dir: str) -> bool:
     )
 
 def build_bsc(dir: str) -> bool:
-    sys = platform.system().lower()
-    if sys == "linux":
-        asset_name = "geth_linux"
-    elif sys == "windows":
-        asset_name = "geth_windows.exe"
-    elif sys == "darwin":
-        asset_name = "geth_mac"
-    else:
-        raise RuntimeError("Unable to recognize platform")
-
-    return __download_from_github(
+    # Download bsc binary if absent
+    downloaded = __download_from_github(
         dir=dir,
-        binary_name="geth.exe" if is_windows_platform() else "geth",
+        binary_name=BSC_BINARY,
         github_url="https://api.github.com/repos/bnb-chain/bsc/releases/79485895",
-        asset_name=asset_name,
+        asset_name=__asset_name(BSC_BINARY),
     )
 
+    # Requires to download binary successfully, since it is not ready to build
+    # binary from source code.
+    assert downloaded, "Cannot download download binary from github [%s]" % BSC_BINARY
+
+    return True
+
 def build_evmos(dir: str) -> bool:
-    # Build evmos binary if absent
-    build_from_github(
+    # Download or build evmos binary if absent
+    if __download_from_github(
+        dir=dir,
+        binary_name=EVMOS_BINARY,
+        github_url=GITHUB_DOWNLOAD_URL,
+        asset_name=__asset_name(EVMOS_BINARY, zip=True),
+    ):
+        return True
+
+    return __build_from_github(
         dir=dir,
         binary_name=EVMOS_BINARY,
         github_url="https://github.com/0glabs/0g-evmos.git",
@@ -55,7 +72,7 @@ def build_evmos(dir: str) -> bool:
 
 def build_cli(dir: str) -> bool:
     # Build 0g-storage-client binary if absent
-    build_from_github(
+    return __build_from_github(
         dir=dir,
         binary_name=CLIENT_BINARY,
         github_url="https://github.com/0glabs/0g-storage-client.git",
@@ -64,7 +81,19 @@ def build_cli(dir: str) -> bool:
         compiled_relative_path=[],
     )
 
-def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool:
+def __asset_name(binary_name: str, zip: bool = False) -> str:
+    sys = platform.system().lower()
+    if sys == "linux":
+        return f"{binary_name}_linux.zip" if zip else f"{binary_name}_linux"
+    elif sys == "windows":
+        binary_name = binary_name.removesuffix(".exe")
+        return f"{binary_name}_windows.zip" if zip else f"{binary_name}_windows.exe"
+    elif sys == "darwin":
+        return f"{binary_name}_mac.zip" if zip else f"{binary_name}_mac"
+    else:
+        raise RuntimeError("Unable to recognize platform")
+
+def __build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: str, compiled_relative_path: list[str], git_rev = None) -> bool:
     if git_rev is not None:
         versioned_binary_name = f"{binary_name}_{git_rev}"
     else:
@@ -73,7 +102,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
     binary_path = os.path.join(dir, binary_name)
     versioned_binary_path = os.path.join(dir, versioned_binary_name)
     if os.path.exists(versioned_binary_path):
-        create_sym_link(versioned_binary_name, binary_name, dir)
+        __create_sym_link(versioned_binary_name, binary_name, dir)
         return False
     
     start_time = time.time()
@@ -95,7 +124,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
     # copy compiled binary to right place
     compiled_binary = os.path.join(code_tmp_dir, *compiled_relative_path, binary_name)
     shutil.copyfile(compiled_binary, versioned_binary_path)
-    create_sym_link(versioned_binary_name, binary_name, dir)
+    __create_sym_link(versioned_binary_name, binary_name, dir)
 
     if not is_windows_platform():
         st = os.stat(binary_path)
@@ -109,7 +138,7 @@ def build_from_github(dir: str, binary_name: str, github_url: str, build_cmd: st
 
     return True
 
-def create_sym_link(src, dst, path = None):
+def __create_sym_link(src, dst, path = None):
     if src == dst:
         return
     
@@ -139,21 +168,34 @@ def __download_from_github(dir: str, binary_name: str, github_url: str, asset_na
     
     start_time = time.time()
 
-    with open(binary_path, "xb") as f:
-        req = requests.get(github_url)
-        assert req.ok, "Failed to request: %s" % github_url
-        for asset in req.json()["assets"]:
-            if asset["name"].lower() == asset_name:
-                download_url = asset["browser_download_url"]
-                break
+    req = requests.get(github_url)
+    assert req.ok, "Failed to request: %s" % github_url
+    download_url = None
+    for asset in req.json()["assets"]:
+        if asset["name"].lower() == asset_name:
+            download_url = asset["browser_download_url"]
+            break
 
-        assert download_url is not None, "Cannot find binary to download by asset name [%s]" % asset_name
+    if download_url is None:
+        print(f"Cannot find asset by name {asset_name}", flush=True)
+        return False
     
-        f.write(requests.get(download_url).content)
+    content = requests.get(download_url).content
 
-        if not is_windows_platform():
-            st = os.stat(binary_path)
-            os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
+    # Supports to read from zipped binary
+    if asset_name.endswith(".zip"):
+        asset_path = os.path.join(dir, asset_name)
+        with open(asset_path, "xb") as f:
+            f.write(content)
+        shutil.unpack_archive(asset_path, dir)
+        assert os.path.exists(binary_path), f"Cannot find binary after unzip, binary = {binary_name}, asset = {asset_name}"
+    else:
+        with open(binary_path, "xb") as f:
+            f.write(content)    
+
+    if not is_windows_platform():
+        st = os.stat(binary_path)
+        os.chmod(binary_path, st.st_mode | stat.S_IEXEC)
     
     wait_until(lambda: os.access(binary_path, os.X_OK), timeout=120)
 

tests/utility/run_all.py

@@ -59,7 +59,6 @@ def run_all(test_dir: str, test_subdirs: list[str]=[], slow_tests: set[str]={},
     if not os.path.exists(tmp_dir):
         os.makedirs(tmp_dir, exist_ok=True)
 
-
     # Build blockchain binaries if absent
     build_conflux(tmp_dir)
     build_bsc(tmp_dir)