Merge fd9c033176 into baf0521c99

README.md

@@ -2,69 +2,32 @@
 
 ## Overview
 
-0G Storage is the storage layer for the ZeroGravity data availability (DA) system. The 0G Storage layer holds three important features:
+0G Storage is a decentralized data storage system designed to address the challenges of high-throughput and low-latency data storage and retrieval, in areas such as AI and gaming. 
 
-* Buit-in - It is natively built into the ZeroGravity DA system for data storage and retrieval.
+In addition, it forms the storage layer for the 0G data availability (DA) system, with the cross-layer integration abstracted away from Rollup and AppChain builders.
-* General purpose - It is designed to support atomic transactions, mutable kv stores as well as archive log systems to enable wide range of applications with various data types.
-* Incentive - Instead of being just a decentralized database, 0G Storage introduces PoRA mining algorithm to incentivize storage network participants.
 
-To dive deep into the technical details, continue reading [0G Storage Spec.](docs/)
+## System Architecture
 
-## Integration
+0G Storage consists of two main components:
 
-We provide a [SDK](https://github.com/0glabs/0g-js-storage-sdk) for users to easily integrate 0G Storage in their applications with the following features:
+1. **Data Publishing Lane**: Ensures quick data availability and verification through the 0G Consensus network.
+2. **Data Storage Lane**: Manages large data transfers and storage using an erasure-coding mechanism for redundancy and reliability.
 
-* File Merkle Tree Class
+Across the two lanes, 0G Storage supports the following features:
-* Flow Contract Types
-* RPC methods support
-* File upload
-* Support browser environment
-* Tests for different environments (In Progress)
-* File download (In Progress)
 
-## Deployment
+* **General Purpose Design**: Supports atomic transactions, mutable key-value stores, and archive log systems, enabling a wide range of applications with various data types.
+* **Incentivized Participation**: Utilizes the PoRA (Proof of Random Access) mining algorithm to incentivize storage network participants.
 
-Please refer to [Deployment](docs/run.md) page for detailed steps to compile and start a 0G Storage node.
+For in-depth technical details about 0G Storage, please read our [Intro to 0G Storage](https://docs.0g.ai/og-storage).
 
-## Test
+## Documentation
 
-### Prerequisites
+- If you want to run a node, please refer to the [Running a Node](https://docs.0g.ai/run-a-node/storage-node) guide.
+- If you want build a project using 0G storage, please refer to the [0G Storage SDK](https://docs.0g.ai/build-with-0g/storage-sdk) guide.
 
-* Required python version: 3.8, 3.9, 3.10, higher version is not guaranteed (e.g. failed to install `pysha3`).
+## Support and Additional Resources
-* Install dependencies under root folder: `pip3 install -r requirements.txt`
+We want to do everything we can to help you be successful while working on your contribution and projects. Here you'll find various resources and communities that may help you complete a project or contribute to 0G. 
 
-### Dependencies
+### Communities
-
+- [0G Telegram](https://t.me/web3_0glabs)
-Python test framework will launch blockchain fullnodes at local for storage node to interact with. There are 2 kinds of fullnodes supported:
+- [0G Discord](https://discord.com/invite/0glabs)
-
-* Conflux eSpace node (by default).
-* BSC node (geth).
-
-For Conflux eSpace node, the test framework will automatically compile the binary at runtime, and copy the binary to `tests/tmp` folder. For BSC node, the test framework will automatically download the latest version binary from [github](https://github.com/bnb-chain/bsc/releases) to `tests/tmp` folder.
-
-Alternatively, you could also manually copy specific version binaries (conflux or geth) to the `tests/tmp` folder. Note, do **NOT** copy released conflux binary on github, since block height of some CIPs are hardcoded.
-
-For testing, it's also dependent on the following repos:
-
-* [0G Storage Contract](https://github.com/0glabs/0g-storage-contracts): It essentially provides two abi interfaces for 0G Storage Node to interact with the on-chain contracts.
-  * ZgsFlow: It contains apis to submit chunk data.
-  * PoraMine: It contains apis to submit PoRA answers.
-* [0G Storage Client](https://github.com/0glabs/0g-storage-client): It is used to interact with certain 0G Storage Nodes to upload/download files.
-
-### Run Tests
-
-Go to the `tests` folder and run the following command to run all tests:
-
-```
-python test_all.py
-```
-
-or, run any single test, e.g.
-
-```
-python sync_test.py
-```
-
-## Contributing
-
-To make contributions to the project, please follow the guidelines [here](contributing.md).

tests/config/node_config.py

@@ -1,3 +1,5 @@
+import os
+
 from web3 import Web3
 
 ZGS_CONFIG = {
@@ -22,6 +24,8 @@ ZGS_CONFIG = {
     }
 }
 
+CONFIG_DIR = os.path.dirname(__file__)
+ZGS_KEY_FILE = os.path.join(CONFIG_DIR, "zgs", "network", "key")
 ZGS_NODEID = "16Uiu2HAmLkGFUbNFYdhuSbTQ5hmnPjFXx2zUDtwQ2uihHpN9YNNe"
 
 BSC_CONFIG = dict(

tests/config/zgs/network/enr.dat

@@ -1 +0,0 @@
-enr:-Ly4QJZwz9htAorBIx_otqoaRFPohX7NQJ31iBB6mcEhBiuPWsOnigc1ABQsg6tLU1OirQdLR6aEvv8SlkkfIbV72T8CgmlkgnY0gmlwhH8AAAGQbmV0d29ya19pZGVudGl0eZ8oIwAAAAAAADPyz8cpvYcPpUtQMmYOBrTPKn-UAAIAiXNlY3AyNTZrMaEDeDdgnDgLPkxNxB39jKb9f1Na30t6R9vVolpTk5zu-hODdGNwgir4g3VkcIIq-A

tests/network_discovery_test.py

@@ -3,7 +3,7 @@
 import os
 import time
 
-from config.node_config import ZGS_NODEID
+from config.node_config import ZGS_KEY_FILE, ZGS_NODEID
 from test_framework.test_framework import TestFramework
 from utility.utils import p2p_port
 
@@ -17,13 +17,9 @@ class NetworkDiscoveryTest(TestFramework):
         self.num_nodes = 3
 
         # setup for node 0 as bootnode
-        tests_dir = os.path.dirname(__file__)
+        self.zgs_node_key_files = [ZGS_KEY_FILE]
-        network_dir = os.path.join(tests_dir, "config", "zgs", "network")
         bootnode_port = p2p_port(0)
         self.zgs_node_configs[0] = {
-            # load pre-defined keypair
-            "network_dir": network_dir,
-
             # enable UDP discovery relevant configs
             "network_enr_address": "127.0.0.1",
             "network_enr_tcp_port": bootnode_port,

tests/network_discovery_upgrade_test.py

@@ -3,7 +3,7 @@
 import os
 import time
 
-from config.node_config import ZGS_NODEID
+from config.node_config import ZGS_KEY_FILE, ZGS_NODEID
 from test_framework.test_framework import TestFramework
 from utility.utils import p2p_port
 
@@ -17,13 +17,9 @@ class NetworkDiscoveryUpgradeTest(TestFramework):
         self.num_nodes = 2
 
         # setup for node 0 as bootnode
-        tests_dir = os.path.dirname(__file__)
+        self.zgs_node_key_files = [ZGS_KEY_FILE]
-        network_dir = os.path.join(tests_dir, "config", "zgs", "network")
         bootnode_port = p2p_port(0)
         self.zgs_node_configs[0] = {
-            # load pre-defined keypair
-            "network_dir": network_dir,
-
             # enable UDP discovery relevant configs
             "network_enr_address": "127.0.0.1",
             "network_enr_tcp_port": bootnode_port,

tests/test_framework/test_framework.py

@@ -55,6 +55,7 @@ class TestFramework:
         self.lifetime_seconds = 3600
         self.launch_wait_seconds = 1
         self.num_deployed_contracts = 0
+        self.zgs_node_key_files = []
 
         # Set default binary path
         binary_ext = ".exe" if is_windows_platform() else ""
@@ -190,6 +191,10 @@ class TestFramework:
             else:
                 updated_config = {}
 
+            zgs_node_key_file = None
+            if i < len(self.zgs_node_key_files):
+                zgs_node_key_file = self.zgs_node_key_files[i]
+
             assert os.path.exists(self.zgs_binary), (
                 "%s should be exist" % self.zgs_binary
             )
@@ -202,6 +207,7 @@ class TestFramework:
                 self.mine_contract.address(),
                 self.reward_contract.address(),
                 self.log,
+                key_file=zgs_node_key_file,
             )
             self.nodes.append(node)
             node.setup_config()

tests/test_framework/zgs_node.py

@@ -24,6 +24,7 @@ class ZgsNode(TestNode):
         log,
         rpc_timeout=10,
         libp2p_nodes=None,
+        key_file=None,
     ):
         local_conf = ZGS_CONFIG.copy()
         if libp2p_nodes is None:
@@ -54,6 +55,7 @@ class ZgsNode(TestNode):
         # Overwrite with personalized configs.
         update_config(local_conf, updated_config)
         data_dir = os.path.join(root_dir, "zgs_node" + str(index))
+        self.key_file = key_file
         rpc_url = "http://" + rpc_listen_address
         super().__init__(
             NodeType.Zgs,
@@ -68,10 +70,16 @@ class ZgsNode(TestNode):
 
     def setup_config(self):
         os.mkdir(self.data_dir)
+
         log_config_path = os.path.join(self.data_dir, self.config["log_config_file"])
         with open(log_config_path, "w") as f:
             f.write("trace,hyper=info,h2=info")
 
+        if self.key_file is not None:
+            network_dir = os.path.join(self.data_dir, "network")
+            os.mkdir(network_dir)
+            shutil.copy(self.key_file, network_dir)
+
         initialize_toml_config(self.config_file, self.config)
 
     def wait_for_rpc_connection(self):