diff --git a/Cargo.lock b/Cargo.lock index 05c4002..aaca1b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5834,6 +5834,7 @@ dependencies = [ "task_executor", "tokio", "tracing", + "zgs_spec", ] [[package]] diff --git a/common/contract-interface/src/lib.rs b/common/contract-interface/src/lib.rs index 1417dc6..5e3badb 100644 --- a/common/contract-interface/src/lib.rs +++ b/common/contract-interface/src/lib.rs @@ -11,7 +11,7 @@ abigen!(PoraMine, "../../storage-contracts-abis/PoraMine.json"); #[cfg(not(feature = "dev"))] abigen!( ChunkLinearReward, - "../../0g-storage-contracts/artifacts/contracts/reward/ChunkLinearReward.sol/ChunkLinearReward.json" + "../../storage-contracts-abis/ChunkLinearReward.json" ); #[cfg(feature = "dev")] @@ -29,5 +29,5 @@ abigen!( #[cfg(feature = "dev")] abigen!( ChunkLinearReward, - "../../0g-storage-contracts/artifacts/contracts/reward/ChunkLinearReward.sol/ChunkLinearReward.json" + "../../0g-storage-contracts-dev/artifacts/contracts/reward/ChunkLinearReward.sol/ChunkLinearReward.json" ); diff --git a/node/pruner/Cargo.toml b/node/pruner/Cargo.toml index c9848a1..5a50b72 100644 --- a/node/pruner/Cargo.toml +++ b/node/pruner/Cargo.toml @@ -14,4 +14,5 @@ task_executor = { path = "../../common/task_executor" } tracing = "0.1.40" ethereum-types = "0.14.1" contract-interface = { path = "../../common/contract-interface" } -ethers = "^2" \ No newline at end of file +ethers = "^2" +zgs_spec = { path = "../../common/spec" } \ No newline at end of file diff --git a/node/pruner/src/lib.rs b/node/pruner/src/lib.rs index bd45f9c..fb126d1 100644 --- a/node/pruner/src/lib.rs +++ b/node/pruner/src/lib.rs @@ -8,16 +8,20 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use storage::config::{ShardConfig, SHARD_CONFIG_KEY}; +use storage::log_store::log_manager::PORA_CHUNK_SIZE; use storage_async::Store; use task_executor::TaskExecutor; use tokio::sync::{broadcast, mpsc}; use tracing::{debug, info}; +use zgs_spec::SECTORS_PER_PRICING; // Start pruning when the db directory size exceeds 0.9 * limit. const PRUNE_THRESHOLD: f32 = 0.9; const FIRST_REWARDABLE_CHUNK_KEY: &str = "first_rewardable_chunk"; +const CHUNKS_PER_PRICING: u64 = (SECTORS_PER_PRICING / PORA_CHUNK_SIZE) as u64; + #[derive(Debug)] pub struct PrunerConfig { pub shard_config: ShardConfig, @@ -110,6 +114,7 @@ impl Pruner { self.prune_in_batch(no_reward_list).await?; self.put_first_rewardable_chunk_index(new_first_rewardable) .await?; + self.first_rewardable_chunk = new_first_rewardable; } tokio::time::sleep(self.config.check_time).await; } @@ -140,7 +145,12 @@ impl Pruner { config.num_shard *= 2; // Generate delete list - let flow_len = self.store.get_context().await?.1; + let flow_len = self + .store + .get_context() + .await? + .1 + .div_ceil(PORA_CHUNK_SIZE as u64); let start_index = old_shard_id + (!rand_bit) as usize * old_num_shard; Ok(Some(Box::new( (start_index as u64..flow_len).step_by(config.num_shard), @@ -162,7 +172,8 @@ impl Pruner { ); } else { Ok(Some(Box::new( - self.first_rewardable_chunk..new_first_rewardable, + self.first_rewardable_chunk * CHUNKS_PER_PRICING + ..new_first_rewardable * CHUNKS_PER_PRICING, ))) } } diff --git a/node/src/config/mod.rs b/node/src/config/mod.rs index d7701b9..713fa4a 100644 --- a/node/src/config/mod.rs +++ b/node/src/config/mod.rs @@ -65,7 +65,7 @@ build_config! { (db_dir, (String), "db".to_string()) (db_max_num_chunks, (Option), None) (prune_check_time_s, (u64), 60) - (prune_batch_size, (usize), 1024) + (prune_batch_size, (usize), 16 * 1024) (prune_batch_wait_time_ms, (u64), 1000) // misc diff --git a/tests/pruner_test.py b/tests/pruner_test.py index dd65738..21a58cc 100755 --- a/tests/pruner_test.py +++ b/tests/pruner_test.py @@ -14,24 +14,24 @@ class PrunerTest(TestFramework): self.num_blockchain_nodes = 1 self.num_nodes = 1 self.zgs_node_configs[0] = { - "db_max_num_chunks": 16 * 1024, - "miner_key": GENESIS_PRIV_KEY, + "db_max_num_chunks": 32 * 1024 * 1024, + # "miner_key": GENESIS_PRIV_KEY, "prune_check_time_s": 1, - "prune_batch_wait_time_ms": 10, + "prune_batch_wait_time_ms": 1, } self.enable_market = True self.mine_period = int(45 / self.block_time) - self.lifetime_seconds = 60 + self.lifetime_seconds = 240 self.launch_wait_seconds = 15 self.log.info("Contract Info: Est. block time %.2f, Mine period %d", self.block_time, self.mine_period) def run_test(self): - difficulty = int(2**256 / 5 / estimate_st_performance()) - self.mine_contract.set_quality(difficulty) + # difficulty = int(2**256 / 5 / estimate_st_performance()) + # self.mine_contract.set_quality(difficulty) client = self.nodes[0] - chunk_data = b"\x02" * 16 * 256 * 1024 + chunk_data = b"\x02" * 5 * 1024 * 1024 * 1024 submissions, data_root = create_submission(chunk_data) self.contract.submit(submissions, tx_prarams = {"value": int(len(chunk_data) / 256 * PRICE_PER_SECTOR * 1.1)}) wait_until(lambda: self.contract.num_submissions() == 1) @@ -45,19 +45,14 @@ class PrunerTest(TestFramework): shard_id = int(shard_config["shardId"]) num_shard = int(shard_config["numShard"]) + wait_until(lambda: self.reward_contract.first_rewardable_chunk() != 0, timeout=180) + first_rewardable = self.reward_contract.first_rewardable_chunk() * 32 * 1024 + # Wait for 1 sec for the no reward segments to be pruned. + time.sleep(1) + # Wait for chunks to be removed. for i in range(len(segment)): seg = client.zgs_download_segment(data_root, i * 1024, (i + 1) * 1024) - if i % num_shard == shard_id: - # base64 encoding size - assert_equal(len(seg), 349528) - else: - assert_equal(seg, None) - - wait_until(lambda: self.reward_contract.first_rewardable_chunk() != 0) - first_rewardable = self.reward_contract.first_rewardable_chunk() - for i in range(shard_id, len(segment), num_shard): - seg = client.zgs_download_segment(data_root, i * 1024, (i + 1) * 1024) - if i < first_rewardable: + if i < first_rewardable or i % num_shard != shard_id: assert_equal(seg, None) else: assert_equal(len(seg), 349528) diff --git a/tests/test_framework/contract_proxy.py b/tests/test_framework/contract_proxy.py index 8269b24..a9e05f1 100644 --- a/tests/test_framework/contract_proxy.py +++ b/tests/test_framework/contract_proxy.py @@ -17,11 +17,11 @@ class ContractProxy: else self.blockchain_nodes[node_idx].get_contract(self.contract_address) ) - def _call(self, fn_name, node_idx, **args): + def _call(self, fn_name, node_idx, *args): assert node_idx < len(self.blockchain_nodes) contract = self._get_contract(node_idx) - return getattr(contract.functions, fn_name)(**args).call() + return getattr(contract.functions, fn_name)(*args).call() def _send(self, fn_name, node_idx, **args): assert node_idx < len(self.blockchain_nodes) @@ -113,4 +113,10 @@ class RewardContractProxy(ContractProxy): return self._send_payable("donate", node_idx, value) def base_reward(self, node_idx = 0): - return self._call("baseReward", node_idx) \ No newline at end of file + return self._call("baseReward", node_idx) + + def first_rewardable_chunk(self, node_idx = 0): + return self._call("firstRewardableChunk", node_idx) + + def reward_deadline(self, node_idx = 0): + return self._call("rewardDeadline", node_idx, 0)