smeb_y/0g-storage-node
1
0
Fork 0
mirror of https://github.com/0glabs/0g-storage-node.git synced 2024-12-25 07:45:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix tests.

Browse Source
This commit is contained in:
Peilun Li 2024-07-29 23:35:41 +08:00
parent 9e0beb6e05
commit e0bb902194
7 changed files with 41 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -5834,6 +5834,7 @@ dependencies = [
"task_executor",
"tokio",
"tracing",
"zgs_spec",
]
[[package]]

4
common/contract-interface/src/lib.rs
View File

@ -11,7 +11,7 @@ abigen!(PoraMine, "../../storage-contracts-abis/PoraMine.json");
#[cfg(not(feature = "dev"))]
abigen!(
ChunkLinearReward,
"../../0g-storage-contracts/artifacts/contracts/reward/ChunkLinearReward.sol/ChunkLinearReward.json"
"../../storage-contracts-abis/ChunkLinearReward.json"
);
#[cfg(feature = "dev")]
@ -29,5 +29,5 @@ abigen!(
#[cfg(feature = "dev")]
abigen!(
ChunkLinearReward,
"../../0g-storage-contracts/artifacts/contracts/reward/ChunkLinearReward.sol/ChunkLinearReward.json"
"../../0g-storage-contracts-dev/artifacts/contracts/reward/ChunkLinearReward.sol/ChunkLinearReward.json"
);

3
node/pruner/Cargo.toml
View File

@ -14,4 +14,5 @@ task_executor = { path = "../../common/task_executor" }
tracing = "0.1.40"
ethereum-types = "0.14.1"
contract-interface = { path = "../../common/contract-interface" }
ethers = "^2"
ethers = "^2"
zgs_spec = { path = "../../common/spec" }

15
node/pruner/src/lib.rs
View File

@ -8,16 +8,20 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use storage::config::{ShardConfig, SHARD_CONFIG_KEY};
use storage::log_store::log_manager::PORA_CHUNK_SIZE;
use storage_async::Store;
use task_executor::TaskExecutor;
use tokio::sync::{broadcast, mpsc};
use tracing::{debug, info};
use zgs_spec::SECTORS_PER_PRICING;
// Start pruning when the db directory size exceeds 0.9 * limit.
const PRUNE_THRESHOLD: f32 = 0.9;
const FIRST_REWARDABLE_CHUNK_KEY: &str = "first_rewardable_chunk";
const CHUNKS_PER_PRICING: u64 = (SECTORS_PER_PRICING / PORA_CHUNK_SIZE) as u64;
#[derive(Debug)]
pub struct PrunerConfig {
pub shard_config: ShardConfig,
@ -110,6 +114,7 @@ impl Pruner {
self.prune_in_batch(no_reward_list).await?;
self.put_first_rewardable_chunk_index(new_first_rewardable)
.await?;
self.first_rewardable_chunk = new_first_rewardable;
}
tokio::time::sleep(self.config.check_time).await;
}
@ -140,7 +145,12 @@ impl Pruner {
config.num_shard *= 2;
// Generate delete list
let flow_len = self.store.get_context().await?.1;
let flow_len = self
.store
.get_context()
.await?
.1
.div_ceil(PORA_CHUNK_SIZE as u64);
let start_index = old_shard_id + (!rand_bit) as usize * old_num_shard;
Ok(Some(Box::new(
(start_index as u64..flow_len).step_by(config.num_shard),
@ -162,7 +172,8 @@ impl Pruner {
);
} else {
Ok(Some(Box::new(
self.first_rewardable_chunk..new_first_rewardable,
self.first_rewardable_chunk * CHUNKS_PER_PRICING
..new_first_rewardable * CHUNKS_PER_PRICING,
)))
}
}

2
node/src/config/mod.rs
View File

@ -65,7 +65,7 @@ build_config! {
(db_dir, (String), "db".to_string())
(db_max_num_chunks, (Option<usize>), None)
(prune_check_time_s, (u64), 60)
(prune_batch_size, (usize), 1024)
(prune_batch_size, (usize), 16 * 1024)
(prune_batch_wait_time_ms, (u64), 1000)
// misc

31
tests/pruner_test.py
View File

@ -14,24 +14,24 @@ class PrunerTest(TestFramework):
self.num_blockchain_nodes = 1
self.num_nodes = 1
self.zgs_node_configs[0] = {
"db_max_num_chunks": 16 * 1024,
"miner_key": GENESIS_PRIV_KEY,
"db_max_num_chunks": 32 * 1024 * 1024,
# "miner_key": GENESIS_PRIV_KEY,
"prune_check_time_s": 1,
"prune_batch_wait_time_ms": 10,
"prune_batch_wait_time_ms": 1,
}
self.enable_market = True
self.mine_period = int(45 / self.block_time)
self.lifetime_seconds = 60
self.lifetime_seconds = 240
self.launch_wait_seconds = 15
self.log.info("Contract Info: Est. block time %.2f, Mine period %d", self.block_time, self.mine_period)
def run_test(self):
difficulty = int(2**256 / 5 / estimate_st_performance())
self.mine_contract.set_quality(difficulty)
# difficulty = int(2**256 / 5 / estimate_st_performance())
# self.mine_contract.set_quality(difficulty)
client = self.nodes[0]
chunk_data = b"\x02" * 16 * 256 * 1024
chunk_data = b"\x02" * 5 * 1024 * 1024 * 1024
submissions, data_root = create_submission(chunk_data)
self.contract.submit(submissions, tx_prarams = {"value": int(len(chunk_data) / 256 * PRICE_PER_SECTOR * 1.1)})
wait_until(lambda: self.contract.num_submissions() == 1)
@ -45,19 +45,14 @@ class PrunerTest(TestFramework):
shard_id = int(shard_config["shardId"])
num_shard = int(shard_config["numShard"])
wait_until(lambda: self.reward_contract.first_rewardable_chunk() != 0, timeout=180)
first_rewardable = self.reward_contract.first_rewardable_chunk() * 32 * 1024
# Wait for 1 sec for the no reward segments to be pruned.
time.sleep(1)
# Wait for chunks to be removed.
for i in range(len(segment)):
seg = client.zgs_download_segment(data_root, i * 1024, (i + 1) * 1024)
if i % num_shard == shard_id:
# base64 encoding size
assert_equal(len(seg), 349528)
else:
assert_equal(seg, None)
wait_until(lambda: self.reward_contract.first_rewardable_chunk() != 0)
first_rewardable = self.reward_contract.first_rewardable_chunk()
for i in range(shard_id, len(segment), num_shard):
seg = client.zgs_download_segment(data_root, i * 1024, (i + 1) * 1024)
if i < first_rewardable:
if i < first_rewardable or i % num_shard != shard_id:
assert_equal(seg, None)
else:
assert_equal(len(seg), 349528)

12
tests/test_framework/contract_proxy.py
View File

@ -17,11 +17,11 @@ class ContractProxy:
else self.blockchain_nodes[node_idx].get_contract(self.contract_address)
)
def _call(self, fn_name, node_idx, **args):
def _call(self, fn_name, node_idx, *args):
assert node_idx < len(self.blockchain_nodes)
contract = self._get_contract(node_idx)
return getattr(contract.functions, fn_name)(**args).call()
return getattr(contract.functions, fn_name)(*args).call()
def _send(self, fn_name, node_idx, **args):
assert node_idx < len(self.blockchain_nodes)
@ -113,4 +113,10 @@ class RewardContractProxy(ContractProxy):
return self._send_payable("donate", node_idx, value)
def base_reward(self, node_idx = 0):
return self._call("baseReward", node_idx)
return self._call("baseReward", node_idx)
def first_rewardable_chunk(self, node_idx = 0):
return self._call("firstRewardableChunk", node_idx)
def reward_deadline(self, node_idx = 0):
return self._call("rewardDeadline", node_idx, 0)