Test mining with the simple market (#31)

* Test mining with the simple market

* Support reward donation

0g-storage-contracts

@@ -1 +1 @@
-Subproject commit 1f3f759236b48dd8682deba91011292cb5966f4c
+Subproject commit d466311abb6f629a6489450f2e684b2c6e7b1089

node/miner/src/pora.rs

@@ -43,6 +43,10 @@ impl<'a> Miner<'a> {
     pub async fn iteration(&self, nonce: H256) -> Option<AnswerWithoutProof> {
         let (scratch_pad, recall_seed) = self.make_scratch_pad(&nonce);
 
+        if self.mining_length == 0 {
+            return None;
+        }
+
         let (_, recall_offset) = U256::from_big_endian(&recall_seed)
             .div_mod(U256::from((self.mining_length as usize) / SECTORS_PER_LOAD));
         let recall_offset = recall_offset.as_u64();

tests/mine_with_market_test.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+from test_framework.blockchain_node import BlockChainNodeType
+from test_framework.test_framework import TestFramework
+from config.node_config import MINER_ID, GENESIS_PRIV_KEY
+from test_framework.blockchain_node import BlockChainNodeType
+from utility.submission import create_submission, submit_data
+from utility.utils import wait_until, assert_equal, assert_greater_than
+
+import math
+
+PRICE_PER_SECTOR = math.ceil(10 * (10 ** 18) / (2 ** 30) * 256 / 12)
+
+class MineTest(TestFramework):
+    def setup_params(self):
+        self.num_blockchain_nodes = 1
+        self.num_nodes = 1
+        self.zgs_node_configs[0] = {
+            "miner_id": MINER_ID,
+            "miner_key": GENESIS_PRIV_KEY,
+        }
+        self.enable_market = True
+        self.mine_period = 150
+
+    def submit_data(self, item, size, no_submit = False):
+        submissions_before = self.contract.num_submissions()
+        client = self.nodes[0]
+        chunk_data = item * 256 * size
+        submissions, data_root = create_submission(chunk_data)
+        value = int(size * PRICE_PER_SECTOR * 1.1)
+        self.contract.submit(submissions, tx_prarams = {"value": value})
+        wait_until(lambda: self.contract.num_submissions() == submissions_before + 1)
+        
+        if not no_submit:
+            wait_until(lambda: client.zgs_get_file_info(data_root) is not None)
+            segment = submit_data(client, chunk_data)
+            wait_until(lambda: client.zgs_get_file_info(data_root)["finalized"])
+
+    def run_test(self):
+        blockchain = self.blockchain_nodes[0]
+
+        self.log.info("flow address: %s", self.contract.address())
+        self.log.info("mine address: %s", self.mine_contract.address())
+
+        quality = int(2**256 / 4096)
+        self.mine_contract.set_quality(quality)
+
+        SECTORS_PER_PRICING = int(8 * ( 2 ** 30 ) / 256)
+
+        self.log.info("Submit the actual data chunk (256 MB)")
+        self.submit_data(b"\x11", int(SECTORS_PER_PRICING / 32))
+
+        self.log.info("Submit the data hash only (8 GB)")
+        self.submit_data(b"\x11", int(SECTORS_PER_PRICING), no_submit=True)
+
+        self.log.info("Sumission Done, Current block number %d", int(blockchain.eth_blockNumber(), 16))
+        self.log.info("Wait for mine context release")
+        wait_until(lambda: self.contract.get_mine_context()[0] > 0, timeout=180)
+        self.log.info("Current flow length: %d", self.contract.get_mine_context()[3])
+
+        self.log.info("Wait for mine answer")
+        wait_until(lambda: self.mine_contract.last_mined_epoch() == 1)
+
+        rewards = self.reward_contract.reward_distributes()
+        assert_equal(len(self.reward_contract.reward_distributes()), 1)
+        firstReward = rewards[0].args.amount
+        self.log.info("Received reward %d Gwei", firstReward / (10**9))
+
+        self.reward_contract.transfer(10000 * 10 ** 18)
+        self.log.info("Donation Done")
+        self.log.info("Submit the data hash only (8 GB)")
+        self.submit_data(b"\x11", int(SECTORS_PER_PRICING), no_submit=True)
+        self.log.info("Sumission Done, Current block number %d", int(blockchain.eth_blockNumber(), 16))
+
+        
+        self.log.info("Wait for mine context release")
+        wait_until(lambda: self.contract.get_mine_context()[0] > 1, timeout=180)
+
+        self.log.info("Wait for mine answer")
+        wait_until(lambda: self.mine_contract.last_mined_epoch() == 2)
+        rewards = self.reward_contract.reward_distributes()
+        assert_equal(len(self.reward_contract.reward_distributes()), 2)
+        secondReward = rewards[1].args.amount
+        self.log.info("Received reward %d Gwei", secondReward / (10**9))
+
+        assert_greater_than(secondReward, 100 * firstReward)
+
+
+
+if __name__ == "__main__":
+    MineTest(blockchain_node_type=BlockChainNodeType.BSC).main()

tests/test_framework/blockchain_node.py

@@ -248,7 +248,7 @@ class BlockchainNode(TestNode):
     def wait_for_transaction_receipt(self, w3, tx_hash, timeout=120, parent_hash=None):
         return w3.eth.wait_for_transaction_receipt(tx_hash, timeout)
 
-    def setup_contract(self):
+    def setup_contract(self, enable_market, mine_period):
         w3 = Web3(HTTPProvider(self.rpc_url))
 
         account1 = w3.eth.account.from_key(GENESIS_PRIV_KEY)
@@ -285,32 +285,61 @@ class BlockchainNode(TestNode):
             contract_address = '0x' + contract_address_bytes.hex()           
             return Web3.to_checksum_address(contract_address)
         
+        def deploy_no_market():
+            flowAddress = predict_contract_address(1)
+            mineAddress = predict_contract_address(2)
 
-        flowAddress = predict_contract_address(1)
+            ZERO = "0x0000000000000000000000000000000000000000"
-        mineAddress = predict_contract_address(2)
 
-        ZERO = "0x0000000000000000000000000000000000000000"
+            self.log.debug("Start deploy contracts")
+            book, _ = deploy_contract("AddressBook", [flowAddress, ZERO, ZERO, mineAddress]);
+            self.log.debug("AddressBook deployed")
 
-        self.log.debug("Start deploy contracts")
+            flow_contract, flow_contract_hash = deploy_contract("Flow", [book.address, mine_period, 0])
-        book, _ = deploy_contract("AddressBook", [flowAddress, ZERO, ZERO, mineAddress]);
+            self.log.debug("Flow deployed")
-        self.log.debug("AddressBook deployed")
 
-        flow_contract, flow_contract_hash = deploy_contract(
+            mine_contract, _ = deploy_contract("PoraMineTest", [book.address, 3])
-            "Flow", [book.address, 100, 0]
+            self.log.debug("Mine deployed")
-        )
+            self.log.info("All contracts deployed")
-        self.log.debug("Flow deployed")
 
-        mine_contract, _ = deploy_contract(
+            tx_hash = mine_contract.functions.setMiner(decode_hex(MINER_ID)).transact(TX_PARAMS)
-            "PoraMineTest",
+            self.wait_for_transaction_receipt(w3, tx_hash)
-            [book.address, 3],
-        )
-        self.log.debug("Mine deployed")
-        self.log.info("All contracts deployed")
             
-        tx_hash = mine_contract.functions.setMiner(decode_hex(MINER_ID)).transact(TX_PARAMS)
+            dummy_reward_contract = w3.eth.contract(
-        self.wait_for_transaction_receipt(w3, tx_hash)
+                address = book.functions.reward().call(),
+                abi=load_contract_metadata(base_path=self.contract_path, name="IReward")["abi"],
+            )
 
-        return flow_contract, flow_contract_hash, mine_contract
+            return flow_contract, flow_contract_hash, mine_contract, dummy_reward_contract
+        
+        def deploy_with_market():
+            mineAddress = predict_contract_address(1)
+            marketAddress = predict_contract_address(2)
+            rewardAddress = predict_contract_address(3)
+            flowAddress = predict_contract_address(4)
+
+            LIFETIME_MONTH = 1
+
+            self.log.debug("Start deploy contracts")
+            book, _ = deploy_contract("AddressBook", [flowAddress, marketAddress, rewardAddress, mineAddress]);
+            self.log.debug("AddressBook deployed")
+
+            mine_contract, _ = deploy_contract("PoraMineTest", [book.address, 3])
+            deploy_contract("FixedPrice", [book.address, LIFETIME_MONTH])
+            reward_contract, _ =deploy_contract("OnePoolReward", [book.address, LIFETIME_MONTH])
+            flow_contract, flow_contract_hash = deploy_contract("FixedPriceFlow", [book.address, mine_period, 0])
+
+            self.log.info("All contracts deployed")
+
+            tx_hash = mine_contract.functions.setMiner(decode_hex(MINER_ID)).transact(TX_PARAMS)
+            self.wait_for_transaction_receipt(w3, tx_hash)
+
+            return flow_contract, flow_contract_hash, mine_contract, reward_contract
+        
+        if enable_market:
+            return deploy_with_market()
+        else:
+            return deploy_no_market()
 
     def get_contract(self, contract_address):
         w3 = Web3(HTTPProvider(self.rpc_url))

tests/test_framework/contract_proxy.py

@@ -1,6 +1,7 @@
 from gettext import npgettext
 from config.node_config import TX_PARAMS
 from utility.utils import assert_equal
+from copy import copy
 
 
 class ContractProxy:
@@ -28,18 +29,39 @@ class ContractProxy:
         contract = self._get_contract(node_idx)
         return getattr(contract.functions, fn_name)(**args).transact(TX_PARAMS)
     
+    def _logs(self, event_name, node_idx, **args):
+        assert node_idx < len(self.blockchain_nodes)
+
+        contract = self._get_contract(node_idx)
+    
+        return getattr(contract.events, event_name).create_filter(fromBlock =0, toBlock="latest").get_all_entries()
+
+    def transfer(self, value, node_idx = 0):
+        tx_params = TX_PARAMS
+        tx_params["value"] = value
+
+        contract = self._get_contract(node_idx)
+        contract.receive.transact(tx_params)
+    
     def address(self):
         return self.contract_address
 
 
 class FlowContractProxy(ContractProxy):
     def submit(
-        self, submission_nodes, node_idx=0, tx_prarams=TX_PARAMS, parent_hash=None
+        self, submission_nodes, node_idx=0, tx_prarams=None, parent_hash=None,
     ):
         assert node_idx < len(self.blockchain_nodes)
 
+        combined_tx_prarams = TX_PARAMS
+
+        if tx_prarams is not None:
+            combined_tx_prarams.update(tx_prarams)
+            
+
         contract = self._get_contract(node_idx)
-        tx_hash = contract.functions.submit(submission_nodes).transact(tx_prarams)
+        # print(contract.functions.submit(submission_nodes).estimate_gas(combined_tx_prarams))
+        tx_hash = contract.functions.submit(submission_nodes).transact(combined_tx_prarams)
         receipt = self.blockchain_nodes[node_idx].wait_for_transaction_receipt(
             contract.w3, tx_hash, parent_hash=parent_hash
         )
@@ -57,6 +79,9 @@ class FlowContractProxy(ContractProxy):
     def epoch(self, node_idx=0):
         return self._call("epoch", node_idx)
 
+    def get_mine_context(self, node_idx=0):
+        return self._call("makeContextWithResult", node_idx)
+
 
 class MineContractProxy(ContractProxy):
     def last_mined_epoch(self, node_idx=0):
@@ -64,3 +89,9 @@ class MineContractProxy(ContractProxy):
 
     def set_quality(self, quality, node_idx=0):
         return self._send("setQuality", node_idx, _targetQuality=quality)
+    
+
+
+class IRewardContractProxy(ContractProxy):
+    def reward_distributes(self, node_idx=0):
+        return self._logs("DistributeReward", node_idx)

tests/test_framework/test_framework.py

@@ -13,7 +13,7 @@ import traceback
 
 from eth_utils import encode_hex
 from test_framework.bsc_node import BSCNode
-from test_framework.contract_proxy import FlowContractProxy, MineContractProxy
+from test_framework.contract_proxy import FlowContractProxy, MineContractProxy, IRewardContractProxy
 from test_framework.zgs_node import ZgsNode
 from test_framework.blockchain_node import BlockChainNodeType
 from test_framework.conflux_node import ConfluxNode, connect_sample_nodes, sync_blocks
@@ -34,6 +34,10 @@ TEST_EXIT_FAILED = 1
 
 class TestFramework:
     def __init__(self, blockchain_node_type=BlockChainNodeType.Conflux):
+        if "http_proxy" in os.environ:
+            # Print a warning message in yellow color
+            print("\n\033[93m ⚠️   Warning: You've set the environment variable 'http_proxy', which might lead to testing issues.\033[0m\n")
+
         self.num_blockchain_nodes = None
         self.num_nodes = None
         self.blockchain_nodes = []
@@ -42,6 +46,8 @@ class TestFramework:
         self.blockchain_node_configs = {}
         self.zgs_node_configs = {}
         self.blockchain_node_type = blockchain_node_type
+        self.enable_market = False
+        self.mine_period = 100
 
         binary_ext = ".exe" if is_windows_platform() else ""
         tests_dir = os.path.dirname(__file_path__)
@@ -134,9 +140,11 @@ class TestFramework:
                 connect_sample_nodes(self.blockchain_nodes, self.log)
                 sync_blocks(self.blockchain_nodes)
 
-        contract, tx_hash, mine_contract = self.blockchain_nodes[0].setup_contract()
+        contract, tx_hash, mine_contract, reward_contract = self.blockchain_nodes[0].setup_contract(self.enable_market, self.mine_period)
         self.contract = FlowContractProxy(contract, self.blockchain_nodes)
         self.mine_contract = MineContractProxy(mine_contract, self.blockchain_nodes)
+        self.reward_contract = IRewardContractProxy(reward_contract, self.blockchain_nodes)
+
 
         for node in self.blockchain_nodes[1:]:
             node.wait_for_transaction(tx_hash)

tests/utility/merkle_tree.py

@@ -1,6 +1,7 @@
 import sha3
 
 from math import log2
+from utility.spec import ENTRY_SIZE
 
 
 def decompose(num):
@@ -154,6 +155,28 @@ class MerkleTree:
         leaf = Leaf.from_data(data, self.hasher)
         self.add_leaf(leaf)
 
+    @classmethod
+    def from_data_list(cls, data, encoding="utf-8"):
+        tree = cls(encoding)
+
+        n = len(data)
+        if n < ENTRY_SIZE or (n & (n - 1)) != 0:
+            raise Exception("Input length is not power of 2")
+        
+        leaves = [Leaf.from_data(data[i:i + ENTRY_SIZE], tree.hasher) for i in range(0, n, ENTRY_SIZE)]
+        tree.__leaves = leaves
+
+        nodes = leaves
+        while len(nodes) > 1:
+            next_nodes = []
+            for i in range(0, len(nodes), 2):
+                next_nodes.append(Node.from_children(nodes[i], nodes[i+1], tree.hasher))
+
+            nodes = next_nodes
+
+        tree.__root = nodes[0]
+        return tree
+
     def add_leaf(self, leaf):
         if self:
             subroot = self.get_last_subroot()

tests/utility/spec.py

@@ -0,0 +1,2 @@
+ENTRY_SIZE = 256
+PORA_CHUNK_SIZE = 1024

tests/utility/submission.py

@@ -3,11 +3,7 @@ import base64
 from eth_utils import encode_hex, decode_hex
 from math import log2
 from utility.merkle_tree import add_0x_prefix, Leaf, MerkleTree
-
+from utility.spec import ENTRY_SIZE, PORA_CHUNK_SIZE
-
-ENTRY_SIZE = 256
-PORA_CHUNK_SIZE = 1024
-
 
 def log2_pow2(n):
     return int(log2(((n ^ (n - 1)) >> 1) + 1))
@@ -98,9 +94,8 @@ def create_node(data, offset, chunks):
 
 def create_segment_node(data, offset, batch, size):
     tree = MerkleTree()
-    i = offset
     n = len(data)
-    while i < offset + size:
+    for i in range(offset, offset + size, batch):
         start = i
         end = min(offset + size, i + batch)
 
@@ -111,21 +106,34 @@ def create_segment_node(data, offset, batch, size):
         else:
             tree.add_leaf(Leaf(segment_root(data[start:end])))
 
-        i += batch
 
     return tree.get_root_hash()
 
 
+
+segment_root_cached_chunks = None
+segment_root_cached_output = None
 def segment_root(chunks):
+    global segment_root_cached_chunks, segment_root_cached_output
+
+    if segment_root_cached_chunks == chunks:
+        return segment_root_cached_output
+
+
     data_len = len(chunks)
     if data_len == 0:
         return b"\x00" * 32
 
+
     tree = MerkleTree()
     for i in range(0, data_len, ENTRY_SIZE):
         tree.encrypt(chunks[i : i + ENTRY_SIZE])
     
-    return tree.get_root_hash()
+    digest = tree.get_root_hash()
+
+    segment_root_cached_chunks = chunks
+    segment_root_cached_output = digest
+    return digest
 
 
 def generate_merkle_tree(data):