Add new docc page for local testing (#283)

* fix tx store

* fix params

Cargo.lock

@@ -223,7 +223,10 @@ dependencies = [
  "eth2_ssz",
  "eth2_ssz_derive",
  "ethereum-types 0.14.1",
+ "itertools 0.13.0",
  "lazy_static",
+ "lru 0.12.5",
+ "metrics",
  "once_cell",
  "serde",
  "tiny-keccak",
@@ -910,7 +913,9 @@ dependencies = [
  "anyhow",
  "async-lock 2.8.0",
  "hashlink 0.8.4",
+ "lazy_static",
  "log_entry_sync",
+ "metrics",
  "network",
  "shared_types",
  "storage-async",
@@ -1673,7 +1678,7 @@ dependencies = [
  "hkdf",
  "lazy_static",
  "libp2p-core 0.30.2",
- "lru",
+ "lru 0.7.8",
  "parking_lot 0.11.2",
  "rand 0.8.5",
  "rlp",
@@ -2514,6 +2519,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2"
+
 [[package]]
 name = "foreign-types"
 version = "0.3.2"
@@ -2946,6 +2957,17 @@ dependencies = [
  "allocator-api2",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash",
+]
+
 [[package]]
 name = "hashers"
 version = "1.0.1"
@@ -4117,7 +4139,7 @@ dependencies = [
  "libp2p-core 0.33.0",
  "libp2p-swarm",
  "log",
- "lru",
+ "lru 0.7.8",
  "prost 0.10.4",
  "prost-build 0.10.4",
  "prost-codec",
@@ -4633,12 +4655,14 @@ dependencies = [
  "jsonrpsee",
  "lazy_static",
  "metrics",
+ "reqwest",
  "serde_json",
  "shared_types",
  "storage",
  "task_executor",
  "thiserror",
  "tokio",
+ "url",
 ]
 
 [[package]]
@@ -4650,6 +4674,15 @@ dependencies = [
  "hashbrown 0.12.3",
 ]
 
+[[package]]
+name = "lru"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
+dependencies = [
+ "hashbrown 0.15.0",
+]
+
 [[package]]
 name = "lru-cache"
 version = "0.1.2"
@@ -4715,9 +4748,10 @@ dependencies = [
 [[package]]
 name = "metrics"
 version = "0.1.0"
-source = "git+https://github.com/Conflux-Chain/conflux-rust.git?rev=992ebc5483d937c8f6b883e266f8ed2a67a7fa9a#992ebc5483d937c8f6b883e266f8ed2a67a7fa9a"
+source = "git+https://github.com/Conflux-Chain/conflux-rust.git?rev=c4734e337c66d38e6396742cd5117b596e8d2603#c4734e337c66d38e6396742cd5117b596e8d2603"
 dependencies = [
  "chrono",
+ "duration-str",
  "futures",
  "influx_db_client",
  "lazy_static",
@@ -4999,6 +5033,7 @@ dependencies = [
 name = "network"
 version = "0.2.0"
 dependencies = [
+ "channel",
  "directory",
  "dirs 4.0.0",
  "discv5",
@@ -5018,7 +5053,7 @@ dependencies = [
  "lazy_static",
  "libp2p",
  "lighthouse_metrics",
- "lru",
+ "lru 0.7.8",
  "parking_lot 0.12.3",
  "rand 0.8.5",
  "regex",
@@ -7271,8 +7306,11 @@ dependencies = [
  "kvdb",
  "kvdb-memorydb",
  "kvdb-rocksdb",
+ "lazy_static",
  "merkle_light",
  "merkle_tree",
+ "metrics",
+ "once_cell",
  "parking_lot 0.12.3",
  "rand 0.8.5",
  "rayon",
@@ -7294,6 +7332,7 @@ name = "storage-async"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "backtrace",
  "eth2_ssz",
  "shared_types",
  "storage",

Cargo.toml

@@ -28,7 +28,7 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
-metrics = { git = "https://github.com/Conflux-Chain/conflux-rust.git", rev = "992ebc5483d937c8f6b883e266f8ed2a67a7fa9a" }
+metrics = { git = "https://github.com/Conflux-Chain/conflux-rust.git", rev = "c4734e337c66d38e6396742cd5117b596e8d2603" }
 
 [patch.crates-io]
 discv5 = { path = "version-meld/discv5" }
@@ -37,3 +37,7 @@ enr = { path = "version-meld/enr" }
 
 [profile.bench.package.'storage']
 debug = true
+
+[profile.dev]
+# enabling debug_assertions will make node fail to start because of checks in `clap`.
+debug-assertions = false

README.md

@@ -2,32 +2,34 @@
 
 ## Overview
 
-0G Storage is a decentralized data storage system designed to address the challenges of high-throughput and low-latency data storage and retrieval, in areas such as AI and gaming. 
+0G Storage is the storage layer for the ZeroGravity data availability (DA) system. The 0G Storage layer holds three important features:
 
-In addition, it forms the storage layer for the 0G data availability (DA) system, with the cross-layer integration abstracted away from Rollup and AppChain builders.
+* Built-in - It is natively built into the ZeroGravity DA system for data storage and retrieval.
+* General purpose - It is designed to support atomic transactions, mutable kv stores as well as archive log systems to enable wide range of applications with various data types.
+* Incentive - Instead of being just a decentralized database, 0G Storage introduces PoRA mining algorithm to incentivize storage network participants.
 
-## System Architecture
+To dive deep into the technical details, continue reading [0G Storage Spec.](docs/)
 
-0G Storage consists of two main components:
+## Integration
 
-1. **Data Publishing Lane**: Ensures quick data availability and verification through the 0G Consensus network.
-2. **Data Storage Lane**: Manages large data transfers and storage using an erasure-coding mechanism for redundancy and reliability.
+We provide a [SDK](https://github.com/0glabs/0g-js-storage-sdk) for users to easily integrate 0G Storage in their applications with the following features:
 
-Across the two lanes, 0G Storage supports the following features:
+* File Merkle Tree Class
+* Flow Contract Types
+* RPC methods support
+* File upload
+* Support browser environment
+* Tests for different environments (In Progress)
+* File download (In Progress)
 
-* **General Purpose Design**: Supports atomic transactions, mutable key-value stores, and archive log systems, enabling a wide range of applications with various data types.
-* **Incentivized Participation**: Utilizes the PoRA (Proof of Random Access) mining algorithm to incentivize storage network participants.
+## Deployment
 
-For in-depth technical details about 0G Storage, please read our [Intro to 0G Storage](https://docs.0g.ai/og-storage).
+Please refer to [Deployment](docs/run.md) page for detailed steps to compile and start a 0G Storage node.
 
-## Documentation
+## Test
 
-- If you want to run a node, please refer to the [Running a Node](https://docs.0g.ai/run-a-node/storage-node) guide.
-- If you want build a project using 0G storage, please refer to the [0G Storage SDK](https://docs.0g.ai/build-with-0g/storage-sdk) guide.
+Please refer to the [One Box Test](docs/onebox-test.md) page for local testing purpose.
 
-## Support and Additional Resources
-We want to do everything we can to help you be successful while working on your contribution and projects. Here you'll find various resources and communities that may help you complete a project or contribute to 0G. 
+## Contributing
 
-### Communities
-- [0G Telegram](https://t.me/web3_0glabs)
-- [0G Discord](https://discord.com/invite/0glabs)
+To make contributions to the project, please follow the guidelines [here](contributing.md).

common/append_merkle/Cargo.toml

@@ -13,3 +13,8 @@ serde = { version = "1.0.137", features = ["derive"] }
 lazy_static = "1.4.0"
 tracing = "0.1.36"
 once_cell = "1.19.0"
+
+metrics = { workspace = true }
+
+itertools = "0.13.0"
+lru = "0.12.5"

common/append_merkle/src/lib.rs

@@ -1,23 +1,30 @@
 mod merkle_tree;
+mod metrics;
+mod node_manager;
 mod proof;
 mod sha3;
 
 use anyhow::{anyhow, bail, Result};
+use itertools::Itertools;
 use std::cmp::Ordering;
 use std::collections::{BTreeMap, HashMap};
 use std::fmt::Debug;
 use std::marker::PhantomData;
+use std::sync::Arc;
+use std::time::Instant;
 use tracing::{trace, warn};
 
+use crate::merkle_tree::MerkleTreeWrite;
 pub use crate::merkle_tree::{
     Algorithm, HashElement, MerkleTreeInitialData, MerkleTreeRead, ZERO_HASHES,
 };
+pub use crate::node_manager::{EmptyNodeDatabase, NodeDatabase, NodeManager, NodeTransaction};
 pub use proof::{Proof, RangeProof};
 pub use sha3::Sha3Algorithm;
 
 pub struct AppendMerkleTree<E: HashElement, A: Algorithm<E>> {
     /// Keep all the nodes in the latest version. `layers[0]` is the layer of leaves.
-    layers: Vec<Vec<E>>,
+    node_manager: NodeManager<E>,
     /// Keep the delta nodes that can be used to construct a history tree.
     /// The key is the root node of that version.
     delta_nodes_map: BTreeMap<u64, DeltaNodes<E>>,
@@ -35,13 +42,16 @@ pub struct AppendMerkleTree<E: HashElement, A: Algorithm<E>> {
 impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     pub fn new(leaves: Vec<E>, leaf_height: usize, start_tx_seq: Option<u64>) -> Self {
         let mut merkle = Self {
-            layers: vec![leaves],
+            node_manager: NodeManager::new_dummy(),
             delta_nodes_map: BTreeMap::new(),
             root_to_tx_seq_map: HashMap::new(),
             min_depth: None,
             leaf_height,
             _a: Default::default(),
         };
+        merkle.node_manager.start_transaction();
+        merkle.node_manager.add_layer();
+        merkle.node_manager.append_nodes(0, &leaves);
         if merkle.leaves() == 0 {
             if let Some(seq) = start_tx_seq {
                 merkle.delta_nodes_map.insert(
@@ -51,10 +61,12 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
                     },
                 );
             }
+            merkle.node_manager.commit();
             return merkle;
         }
         // Reconstruct the whole tree.
         merkle.recompute(0, 0, None);
+        merkle.node_manager.commit();
         // Commit the first version in memory.
         // TODO(zz): Check when the roots become available.
         merkle.commit(start_tx_seq);
@@ -62,53 +74,44 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     }
 
     pub fn new_with_subtrees(
-        initial_data: MerkleTreeInitialData<E>,
+        node_db: Arc<dyn NodeDatabase<E>>,
+        node_cache_capacity: usize,
         leaf_height: usize,
-        start_tx_seq: Option<u64>,
     ) -> Result<Self> {
         let mut merkle = Self {
-            layers: vec![vec![]],
+            node_manager: NodeManager::new(node_db, node_cache_capacity)?,
             delta_nodes_map: BTreeMap::new(),
             root_to_tx_seq_map: HashMap::new(),
             min_depth: None,
             leaf_height,
             _a: Default::default(),
         };
-        if initial_data.subtree_list.is_empty() {
-            if let Some(seq) = start_tx_seq {
-                merkle.delta_nodes_map.insert(
-                    seq,
-                    DeltaNodes {
-                        right_most_nodes: vec![],
-                    },
-                );
-            }
-            return Ok(merkle);
-        }
-        merkle.append_subtree_list(initial_data.subtree_list)?;
-        merkle.commit(start_tx_seq);
-        for (index, h) in initial_data.known_leaves {
-            merkle.fill_leaf(index, h);
-        }
-        for (layer_index, position, h) in initial_data.extra_mpt_nodes {
-            // TODO: Delete duplicate nodes from DB.
-            merkle.layers[layer_index][position] = h;
+        if merkle.height() == 0 {
+            merkle.node_manager.start_transaction();
+            merkle.node_manager.add_layer();
+            merkle.node_manager.commit();
         }
         Ok(merkle)
     }
 
     /// This is only used for the last chunk, so `leaf_height` is always 0 so far.
     pub fn new_with_depth(leaves: Vec<E>, depth: usize, start_tx_seq: Option<u64>) -> Self {
+        let mut node_manager = NodeManager::new_dummy();
+        node_manager.start_transaction();
         if leaves.is_empty() {
             // Create an empty merkle tree with `depth`.
             let mut merkle = Self {
-                layers: vec![vec![]; depth],
+                // dummy node manager for the last chunk.
+                node_manager,
                 delta_nodes_map: BTreeMap::new(),
                 root_to_tx_seq_map: HashMap::new(),
                 min_depth: Some(depth),
                 leaf_height: 0,
                 _a: Default::default(),
             };
+            for _ in 0..depth {
+                merkle.node_manager.add_layer();
+            }
             if let Some(seq) = start_tx_seq {
                 merkle.delta_nodes_map.insert(
                     seq,
@@ -117,20 +120,26 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
                     },
                 );
             }
+            merkle.node_manager.commit();
             merkle
         } else {
-            let mut layers = vec![vec![]; depth];
-            layers[0] = leaves;
             let mut merkle = Self {
-                layers,
+                // dummy node manager for the last chunk.
+                node_manager,
                 delta_nodes_map: BTreeMap::new(),
                 root_to_tx_seq_map: HashMap::new(),
                 min_depth: Some(depth),
                 leaf_height: 0,
                 _a: Default::default(),
             };
+            merkle.node_manager.add_layer();
+            merkle.append_nodes(0, &leaves);
+            for _ in 1..depth {
+                merkle.node_manager.add_layer();
+            }
             // Reconstruct the whole tree.
             merkle.recompute(0, 0, None);
+            merkle.node_manager.commit();
             // Commit the first version in memory.
             merkle.commit(start_tx_seq);
             merkle
@@ -138,22 +147,31 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     }
 
     pub fn append(&mut self, new_leaf: E) {
+        let start_time = Instant::now();
         if new_leaf == E::null() {
             // appending null is not allowed.
             return;
         }
-        self.layers[0].push(new_leaf);
+        self.node_manager.start_transaction();
+        self.node_manager.push_node(0, new_leaf);
         self.recompute_after_append_leaves(self.leaves() - 1);
+
+        self.node_manager.commit();
+        metrics::APPEND.update_since(start_time);
     }
 
-    pub fn append_list(&mut self, mut leaf_list: Vec<E>) {
+    pub fn append_list(&mut self, leaf_list: Vec<E>) {
+        let start_time = Instant::now();
         if leaf_list.contains(&E::null()) {
             // appending null is not allowed.
             return;
         }
+        self.node_manager.start_transaction();
         let start_index = self.leaves();
-        self.layers[0].append(&mut leaf_list);
+        self.node_manager.append_nodes(0, &leaf_list);
         self.recompute_after_append_leaves(start_index);
+        self.node_manager.commit();
+        metrics::APPEND_LIST.update_since(start_time);
     }
 
     /// Append a leaf list by providing their intermediate node hash.
@@ -162,43 +180,57 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     /// Other nodes in the subtree will be set to `null` nodes.
     /// TODO: Optimize to avoid storing the `null` nodes?
     pub fn append_subtree(&mut self, subtree_depth: usize, subtree_root: E) -> Result<()> {
+        let start_time = Instant::now();
         if subtree_root == E::null() {
             // appending null is not allowed.
             bail!("subtree_root is null");
         }
+        self.node_manager.start_transaction();
         let start_index = self.leaves();
         self.append_subtree_inner(subtree_depth, subtree_root)?;
         self.recompute_after_append_subtree(start_index, subtree_depth - 1);
+        self.node_manager.commit();
+        metrics::APPEND_SUBTREE.update_since(start_time);
+
         Ok(())
     }
 
     pub fn append_subtree_list(&mut self, subtree_list: Vec<(usize, E)>) -> Result<()> {
+        let start_time = Instant::now();
         if subtree_list.iter().any(|(_, root)| root == &E::null()) {
             // appending null is not allowed.
             bail!("subtree_list contains null");
         }
+        self.node_manager.start_transaction();
         for (subtree_depth, subtree_root) in subtree_list {
             let start_index = self.leaves();
             self.append_subtree_inner(subtree_depth, subtree_root)?;
             self.recompute_after_append_subtree(start_index, subtree_depth - 1);
         }
+        self.node_manager.commit();
+        metrics::APPEND_SUBTREE_LIST.update_since(start_time);
+
         Ok(())
     }
 
     /// Change the value of the last leaf and return the new merkle root.
     /// This is needed if our merkle-tree in memory only keeps intermediate nodes instead of real leaves.
     pub fn update_last(&mut self, updated_leaf: E) {
+        let start_time = Instant::now();
         if updated_leaf == E::null() {
             // updating to null is not allowed.
             return;
         }
-        if self.layers[0].is_empty() {
+        self.node_manager.start_transaction();
+        if self.layer_len(0) == 0 {
             // Special case for the first data.
-            self.layers[0].push(updated_leaf);
+            self.push_node(0, updated_leaf);
         } else {
-            *self.layers[0].last_mut().unwrap() = updated_leaf;
+            self.update_node(0, self.layer_len(0) - 1, updated_leaf);
         }
         self.recompute_after_append_leaves(self.leaves() - 1);
+        self.node_manager.commit();
+        metrics::UPDATE_LAST.update_since(start_time);
     }
 
     /// Fill an unknown `null` leaf with its real value.
@@ -207,13 +239,17 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     pub fn fill_leaf(&mut self, index: usize, leaf: E) {
         if leaf == E::null() {
             // fill leaf with null is not allowed.
-        } else if self.layers[0][index] == E::null() {
-            self.layers[0][index] = leaf;
+        } else if self.node(0, index) == E::null() {
+            self.node_manager.start_transaction();
+            self.update_node(0, index, leaf);
             self.recompute_after_fill_leaves(index, index + 1);
-        } else if self.layers[0][index] != leaf {
+            self.node_manager.commit();
+        } else if self.node(0, index) != leaf {
             panic!(
                 "Fill with invalid leaf, index={} was={:?} get={:?}",
-                index, self.layers[0][index], leaf
+                index,
+                self.node(0, index),
+                leaf
             );
         }
     }
@@ -226,18 +262,20 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
         &mut self,
         proof: RangeProof<E>,
     ) -> Result<Vec<(usize, usize, E)>> {
-        self.fill_with_proof(
-            proof
-                .left_proof
-                .proof_nodes_in_tree()
-                .split_off(self.leaf_height),
-        )?;
-        self.fill_with_proof(
-            proof
-                .right_proof
-                .proof_nodes_in_tree()
-                .split_off(self.leaf_height),
-        )
+        self.node_manager.start_transaction();
+        let mut updated_nodes = Vec::new();
+        let mut left_nodes = proof.left_proof.proof_nodes_in_tree();
+        if left_nodes.len() >= self.leaf_height {
+            updated_nodes
+                .append(&mut self.fill_with_proof(left_nodes.split_off(self.leaf_height))?);
+        }
+        let mut right_nodes = proof.right_proof.proof_nodes_in_tree();
+        if right_nodes.len() >= self.leaf_height {
+            updated_nodes
+                .append(&mut self.fill_with_proof(right_nodes.split_off(self.leaf_height))?);
+        }
+        self.node_manager.commit();
+        Ok(updated_nodes)
     }
 
     pub fn fill_with_file_proof(
@@ -262,13 +300,16 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
         if tx_merkle_nodes.is_empty() {
             return Ok(Vec::new());
         }
+        self.node_manager.start_transaction();
         let mut position_and_data =
             proof.file_proof_nodes_in_tree(tx_merkle_nodes, tx_merkle_nodes_size);
         let start_index = (start_index >> self.leaf_height) as usize;
         for (i, (position, _)) in position_and_data.iter_mut().enumerate() {
             *position += start_index >> i;
         }
-        self.fill_with_proof(position_and_data)
+        let updated_nodes = self.fill_with_proof(position_and_data)?;
+        self.node_manager.commit();
+        Ok(updated_nodes)
     }
 
     /// This assumes that the proof leaf is no lower than the tree leaf. It holds for both SegmentProof and ChunkProof.
@@ -280,28 +321,27 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
         let mut updated_nodes = Vec::new();
         // A valid proof should not fail the following checks.
         for (i, (position, data)) in position_and_data.into_iter().enumerate() {
-            let layer = &mut self.layers[i];
-            if position > layer.len() {
+            if position > self.layer_len(i) {
                 bail!(
                     "proof position out of range, position={} layer.len()={}",
                     position,
-                    layer.len()
+                    self.layer_len(i)
                 );
             }
-            if position == layer.len() {
+            if position == self.layer_len(i) {
                 // skip padding node.
                 continue;
             }
-            if layer[position] == E::null() {
-                layer[position] = data.clone();
+            if self.node(i, position) == E::null() {
+                self.update_node(i, position, data.clone());
                 updated_nodes.push((i, position, data))
-            } else if layer[position] != data {
+            } else if self.node(i, position) != data {
                 // The last node in each layer may have changed in the tree.
                 trace!(
                     "conflict data layer={} position={} tree_data={:?} proof_data={:?}",
                     i,
                     position,
-                    layer[position],
+                    self.node(i, position),
                     data
                 );
             }
@@ -317,8 +357,8 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
         if position >= self.leaves() {
             bail!("Out of bound: position={} end={}", position, self.leaves());
         }
-        if self.layers[0][position] != E::null() {
-            Ok(Some(self.layers[0][position].clone()))
+        if self.node(0, position) != E::null() {
+            Ok(Some(self.node(0, position)))
         } else {
             // The leaf hash is unknown.
             Ok(None)
@@ -366,10 +406,11 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
                 return;
             }
             let mut right_most_nodes = Vec::new();
-            for layer in &self.layers {
-                right_most_nodes.push((layer.len() - 1, layer.last().unwrap().clone()));
+            for height in 0..self.height() {
+                let pos = self.layer_len(height) - 1;
+                right_most_nodes.push((pos, self.node(height, pos)));
             }
-            let root = self.root().clone();
+            let root = self.root();
             self.delta_nodes_map
                 .insert(tx_seq, DeltaNodes::new(right_most_nodes));
             self.root_to_tx_seq_map.insert(root, tx_seq);
@@ -377,8 +418,8 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     }
 
     fn before_extend_layer(&mut self, height: usize) {
-        if height == self.layers.len() {
-            self.layers.push(Vec::new());
+        if height == self.height() {
+            self.node_manager.add_layer()
         }
     }
 
@@ -395,7 +436,6 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     }
 
     /// Given a range of changed leaf nodes and recompute the tree.
-    /// Since this tree is append-only, we always compute to the end.
     fn recompute(
         &mut self,
         mut start_index: usize,
@@ -405,22 +445,29 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
         start_index >>= height;
         maybe_end_index = maybe_end_index.map(|end| end >> height);
         // Loop until we compute the new root and reach `tree_depth`.
-        while self.layers[height].len() > 1 || height < self.layers.len() - 1 {
+        while self.layer_len(height) > 1 || height < self.height() - 1 {
             let next_layer_start_index = start_index >> 1;
             if start_index % 2 == 1 {
                 start_index -= 1;
             }
 
-            let mut end_index = maybe_end_index.unwrap_or(self.layers[height].len());
-            if end_index % 2 == 1 && end_index != self.layers[height].len() {
+            let mut end_index = maybe_end_index.unwrap_or(self.layer_len(height));
+            if end_index % 2 == 1 && end_index != self.layer_len(height) {
                 end_index += 1;
             }
             let mut i = 0;
-            let mut iter = self.layers[height][start_index..end_index].chunks_exact(2);
+            let iter = self
+                .node_manager
+                .get_nodes(height, start_index, end_index)
+                .chunks(2);
             // We cannot modify the parent layer while iterating the child layer,
             // so just keep the changes and update them later.
             let mut parent_update = Vec::new();
-            while let Some([left, right]) = iter.next() {
+            for chunk_iter in &iter {
+                let chunk: Vec<_> = chunk_iter.collect();
+                if chunk.len() == 2 {
+                    let left = &chunk[0];
+                    let right = &chunk[1];
                     // If either left or right is null (unknown), we cannot compute the parent hash.
                     // Note that if we are recompute a range of an existing tree,
                     // we do not need to keep these possibly null parent. This is only saved
@@ -432,8 +479,9 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
                     };
                     parent_update.push((next_layer_start_index + i, parent));
                     i += 1;
-            }
-            if let [r] = iter.remainder() {
+                } else {
+                    assert_eq!(chunk.len(), 1);
+                    let r = &chunk[0];
                     // Same as above.
                     let parent = if *r == E::null() {
                         E::null()
@@ -442,6 +490,7 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
                     };
                     parent_update.push((next_layer_start_index + i, parent));
                 }
+            }
             if !parent_update.is_empty() {
                 self.before_extend_layer(height + 1);
             }
@@ -449,27 +498,27 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
             // we can just overwrite `last_changed_parent_index` with new values.
             let mut last_changed_parent_index = None;
             for (parent_index, parent) in parent_update {
-                match parent_index.cmp(&self.layers[height + 1].len()) {
+                match parent_index.cmp(&self.layer_len(height + 1)) {
                     Ordering::Less => {
                         // We do not overwrite with null.
                         if parent != E::null() {
-                            if self.layers[height + 1][parent_index] == E::null()
+                            if self.node(height + 1, parent_index) == E::null()
                                 // The last node in a layer can be updated.
-                                || (self.layers[height + 1][parent_index] != parent
-                                    && parent_index == self.layers[height + 1].len() - 1)
+                                || (self.node(height + 1, parent_index) != parent
+                                    && parent_index == self.layer_len(height + 1) - 1)
                             {
-                                self.layers[height + 1][parent_index] = parent;
+                                self.update_node(height + 1, parent_index, parent);
                                 last_changed_parent_index = Some(parent_index);
-                            } else if self.layers[height + 1][parent_index] != parent {
+                            } else if self.node(height + 1, parent_index) != parent {
                                 // Recompute changes a node in the middle. This should be impossible
                                 // if the inputs are valid.
                                 panic!("Invalid append merkle tree! height={} index={} expected={:?} get={:?}",
-                                       height + 1, parent_index, self.layers[height + 1][parent_index], parent);
+                                       height + 1, parent_index, self.node(height + 1, parent_index), parent);
                             }
                         }
                     }
                     Ordering::Equal => {
-                        self.layers[height + 1].push(parent);
+                        self.push_node(height + 1, parent);
                         last_changed_parent_index = Some(parent_index);
                     }
                     Ordering::Greater => {
@@ -500,10 +549,10 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
         for height in 0..(subtree_depth - 1) {
             self.before_extend_layer(height);
             let subtree_layer_size = 1 << (subtree_depth - 1 - height);
-            self.layers[height].append(&mut vec![E::null(); subtree_layer_size]);
+            self.append_nodes(height, &vec![E::null(); subtree_layer_size]);
         }
         self.before_extend_layer(subtree_depth - 1);
-        self.layers[subtree_depth - 1].push(subtree_root);
+        self.push_node(subtree_depth - 1, subtree_root);
         Ok(())
     }
 
@@ -514,23 +563,45 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     }
 
     pub fn revert_to(&mut self, tx_seq: u64) -> Result<()> {
-        if self.layers[0].is_empty() {
+        if self.layer_len(0) == 0 {
             // Any previous state of an empty tree is always empty.
             return Ok(());
         }
+        self.node_manager.start_transaction();
         let delta_nodes = self
             .delta_nodes_map
             .get(&tx_seq)
-            .ok_or_else(|| anyhow!("tx_seq unavailable, root={:?}", tx_seq))?;
+            .ok_or_else(|| anyhow!("tx_seq unavailable, root={:?}", tx_seq))?
+            .clone();
         // Dropping the upper layers that are not in the old merkle tree.
-        self.layers.truncate(delta_nodes.right_most_nodes.len());
+        for height in (delta_nodes.right_most_nodes.len()..self.height()).rev() {
+            self.node_manager.truncate_layer(height);
+        }
         for (height, (last_index, right_most_node)) in
             delta_nodes.right_most_nodes.iter().enumerate()
         {
-            self.layers[height].truncate(*last_index + 1);
-            self.layers[height][*last_index] = right_most_node.clone();
+            self.node_manager.truncate_nodes(height, *last_index + 1);
+            self.update_node(height, *last_index, right_most_node.clone())
         }
         self.clear_after(tx_seq);
+        self.node_manager.commit();
+        Ok(())
+    }
+
+    // Revert to a tx_seq not in `delta_nodes_map`.
+    // This is needed to revert the last unfinished tx after restart.
+    pub fn revert_to_leaves(&mut self, leaves: usize) -> Result<()> {
+        self.node_manager.start_transaction();
+        for height in (0..self.height()).rev() {
+            let kept_nodes = leaves >> height;
+            if kept_nodes == 0 {
+                self.node_manager.truncate_layer(height);
+            } else {
+                self.node_manager.truncate_nodes(height, kept_nodes + 1);
+            }
+        }
+        self.recompute_after_append_leaves(leaves);
+        self.node_manager.commit();
         Ok(())
     }
 
@@ -550,17 +621,25 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
             bail!("empty tree");
         }
         Ok(HistoryTree {
-            layers: &self.layers,
+            node_manager: &self.node_manager,
             delta_nodes,
             leaf_height: self.leaf_height,
         })
     }
 
     pub fn reset(&mut self) {
-        self.layers = match self.min_depth {
-            None => vec![vec![]],
-            Some(depth) => vec![vec![]; depth],
-        };
+        self.node_manager.start_transaction();
+        for height in (0..self.height()).rev() {
+            self.node_manager.truncate_layer(height);
+        }
+        if let Some(depth) = self.min_depth {
+            for _ in 0..depth {
+                self.node_manager.add_layer();
+            }
+        } else {
+            self.node_manager.add_layer();
+        }
+        self.node_manager.commit();
     }
 
     fn clear_after(&mut self, tx_seq: u64) {
@@ -580,10 +659,10 @@ impl<E: HashElement, A: Algorithm<E>> AppendMerkleTree<E, A> {
     fn first_known_root_at(&self, index: usize) -> (usize, E) {
         let mut height = 0;
         let mut index_in_layer = index;
-        while height < self.layers.len() {
+        while height < self.height() {
             let node = self.node(height, index_in_layer);
             if !node.is_null() {
-                return (height + 1, node.clone());
+                return (height + 1, node);
             }
             height += 1;
             index_in_layer /= 2;
@@ -628,7 +707,7 @@ impl<E: HashElement> DeltaNodes<E> {
 
 pub struct HistoryTree<'m, E: HashElement> {
     /// A reference to the global tree nodes.
-    layers: &'m Vec<Vec<E>>,
+    node_manager: &'m NodeManager<E>,
     /// The delta nodes that are difference from `layers`.
     /// This could be a reference, we just take ownership for convenience.
     delta_nodes: &'m DeltaNodes<E>,
@@ -639,16 +718,18 @@ pub struct HistoryTree<'m, E: HashElement> {
 impl<E: HashElement, A: Algorithm<E>> MerkleTreeRead for AppendMerkleTree<E, A> {
     type E = E;
 
-    fn node(&self, layer: usize, index: usize) -> &Self::E {
-        &self.layers[layer][index]
+    fn node(&self, layer: usize, index: usize) -> Self::E {
+        self.node_manager
+            .get_node(layer, index)
+            .expect("index checked")
     }
 
     fn height(&self) -> usize {
-        self.layers.len()
+        self.node_manager.num_layers()
     }
 
     fn layer_len(&self, layer_height: usize) -> usize {
-        self.layers[layer_height].len()
+        self.node_manager.layer_size(layer_height)
     }
 
     fn padding_node(&self, height: usize) -> Self::E {
@@ -658,10 +739,13 @@ impl<E: HashElement, A: Algorithm<E>> MerkleTreeRead for AppendMerkleTree<E, A>
 
 impl<'a, E: HashElement> MerkleTreeRead for HistoryTree<'a, E> {
     type E = E;
-    fn node(&self, layer: usize, index: usize) -> &Self::E {
+    fn node(&self, layer: usize, index: usize) -> Self::E {
         match self.delta_nodes.get(layer, index).expect("range checked") {
-            Some(node) if *node != E::null() => node,
-            _ => &self.layers[layer][index],
+            Some(node) if *node != E::null() => node.clone(),
+            _ => self
+                .node_manager
+                .get_node(layer, index)
+                .expect("index checked"),
         }
     }
 
@@ -678,6 +762,22 @@ impl<'a, E: HashElement> MerkleTreeRead for HistoryTree<'a, E> {
     }
 }
 
+impl<E: HashElement, A: Algorithm<E>> MerkleTreeWrite for AppendMerkleTree<E, A> {
+    type E = E;
+
+    fn push_node(&mut self, layer: usize, node: Self::E) {
+        self.node_manager.push_node(layer, node);
+    }
+
+    fn append_nodes(&mut self, layer: usize, nodes: &[Self::E]) {
+        self.node_manager.append_nodes(layer, nodes);
+    }
+
+    fn update_node(&mut self, layer: usize, pos: usize, node: Self::E) {
+        self.node_manager.add_node(layer, pos, node);
+    }
+}
+
 #[macro_export]
 macro_rules! ensure_eq {
     ($given:expr, $expected:expr) => {
@@ -699,6 +799,7 @@ macro_rules! ensure_eq {
 #[cfg(test)]
 mod tests {
     use crate::merkle_tree::MerkleTreeRead;
+
     use crate::sha3::Sha3Algorithm;
     use crate::AppendMerkleTree;
     use ethereum_types::H256;

common/append_merkle/src/merkle_tree.rs

@@ -49,7 +49,7 @@ pub trait Algorithm<E: HashElement> {
 
 pub trait MerkleTreeRead {
     type E: HashElement;
-    fn node(&self, layer: usize, index: usize) -> &Self::E;
+    fn node(&self, layer: usize, index: usize) -> Self::E;
     fn height(&self) -> usize;
     fn layer_len(&self, layer_height: usize) -> usize;
     fn padding_node(&self, height: usize) -> Self::E;
@@ -58,7 +58,7 @@ pub trait MerkleTreeRead {
         self.layer_len(0)
     }
 
-    fn root(&self) -> &Self::E {
+    fn root(&self) -> Self::E {
         self.node(self.height() - 1, 0)
     }
 
@@ -70,16 +70,16 @@ pub trait MerkleTreeRead {
                 self.leaves()
             );
         }
-        if self.node(0, leaf_index) == &Self::E::null() {
+        if self.node(0, leaf_index) == Self::E::null() {
             bail!("Not ready to generate proof for leaf_index={}", leaf_index);
         }
         if self.height() == 1 {
-            return Proof::new(vec![self.root().clone(), self.root().clone()], vec![]);
+            return Proof::new(vec![self.root(), self.root().clone()], vec![]);
         }
         let mut lemma: Vec<Self::E> = Vec::with_capacity(self.height()); // path + root
         let mut path: Vec<bool> = Vec::with_capacity(self.height() - 2); // path - 1
         let mut index_in_layer = leaf_index;
-        lemma.push(self.node(0, leaf_index).clone());
+        lemma.push(self.node(0, leaf_index));
         for height in 0..(self.height() - 1) {
             trace!(
                 "gen_proof: height={} index={} hash={:?}",
@@ -93,15 +93,15 @@ pub trait MerkleTreeRead {
                     // TODO: This can be skipped if the tree size is available in validation.
                     lemma.push(self.padding_node(height));
                 } else {
-                    lemma.push(self.node(height, index_in_layer + 1).clone());
+                    lemma.push(self.node(height, index_in_layer + 1));
                 }
             } else {
                 path.push(false);
-                lemma.push(self.node(height, index_in_layer - 1).clone());
+                lemma.push(self.node(height, index_in_layer - 1));
             }
             index_in_layer >>= 1;
         }
-        lemma.push(self.root().clone());
+        lemma.push(self.root());
         if lemma.contains(&Self::E::null()) {
             bail!(
                 "Not enough data to generate proof, lemma={:?} path={:?}",
@@ -130,6 +130,13 @@ pub trait MerkleTreeRead {
     }
 }
 
+pub trait MerkleTreeWrite {
+    type E: HashElement;
+    fn push_node(&mut self, layer: usize, node: Self::E);
+    fn append_nodes(&mut self, layer: usize, nodes: &[Self::E]);
+    fn update_node(&mut self, layer: usize, pos: usize, node: Self::E);
+}
+
 /// This includes the data to reconstruct an `AppendMerkleTree` root where some nodes
 /// are `null`. Other intermediate nodes will be computed based on these known nodes.
 pub struct MerkleTreeInitialData<E: HashElement> {

common/append_merkle/src/metrics.rs

@@ -0,0 +1,11 @@
+use std::sync::Arc;
+
+use metrics::{register_timer, Timer};
+
+lazy_static::lazy_static! {
+    pub static ref APPEND: Arc<dyn Timer> = register_timer("append_merkle_append");
+    pub static ref APPEND_LIST: Arc<dyn Timer> = register_timer("append_merkle_append_list");
+    pub static ref APPEND_SUBTREE: Arc<dyn Timer> = register_timer("append_merkle_append_subtree");
+    pub static ref APPEND_SUBTREE_LIST: Arc<dyn Timer> = register_timer("append_merkle_append_subtree_list");
+    pub static ref UPDATE_LAST: Arc<dyn Timer> = register_timer("append_merkle_update_last");
+}

common/append_merkle/src/node_manager.rs

@@ -0,0 +1,219 @@
+use crate::HashElement;
+use anyhow::Result;
+use lru::LruCache;
+use std::any::Any;
+use std::num::NonZeroUsize;
+use std::sync::Arc;
+use tracing::error;
+
+pub struct NodeManager<E: HashElement> {
+    cache: LruCache<(usize, usize), E>,
+    layer_size: Vec<usize>,
+    db: Arc<dyn NodeDatabase<E>>,
+    db_tx: Option<Box<dyn NodeTransaction<E>>>,
+}
+
+impl<E: HashElement> NodeManager<E> {
+    pub fn new(db: Arc<dyn NodeDatabase<E>>, capacity: usize) -> Result<Self> {
+        let mut layer = 0;
+        let mut layer_size = Vec::new();
+        while let Some(size) = db.get_layer_size(layer)? {
+            layer_size.push(size);
+            layer += 1;
+        }
+        Ok(Self {
+            cache: LruCache::new(NonZeroUsize::new(capacity).expect("capacity should be non-zero")),
+            layer_size,
+            db,
+            db_tx: None,
+        })
+    }
+
+    pub fn new_dummy() -> Self {
+        Self {
+            cache: LruCache::unbounded(),
+            layer_size: vec![],
+            db: Arc::new(EmptyNodeDatabase {}),
+            db_tx: None,
+        }
+    }
+
+    pub fn push_node(&mut self, layer: usize, node: E) {
+        self.add_node(layer, self.layer_size[layer], node);
+        self.set_layer_size(layer, self.layer_size[layer] + 1);
+    }
+
+    pub fn append_nodes(&mut self, layer: usize, nodes: &[E]) {
+        let mut pos = self.layer_size[layer];
+        let mut saved_nodes = Vec::with_capacity(nodes.len());
+        for node in nodes {
+            self.cache.put((layer, pos), node.clone());
+            saved_nodes.push((layer, pos, node));
+            pos += 1;
+        }
+        self.set_layer_size(layer, pos);
+        self.db_tx().save_node_list(&saved_nodes);
+    }
+
+    pub fn get_node(&self, layer: usize, pos: usize) -> Option<E> {
+        match self.cache.peek(&(layer, pos)) {
+            Some(node) => Some(node.clone()),
+            None => self.db.get_node(layer, pos).unwrap_or_else(|e| {
+                error!("Failed to get node: {}", e);
+                None
+            }),
+        }
+    }
+
+    pub fn get_nodes(&self, layer: usize, start_pos: usize, end_pos: usize) -> NodeIterator<E> {
+        NodeIterator {
+            node_manager: self,
+            layer,
+            start_pos,
+            end_pos,
+        }
+    }
+
+    pub fn add_node(&mut self, layer: usize, pos: usize, node: E) {
+        // No need to insert if the value is unchanged.
+        if self.cache.get(&(layer, pos)) != Some(&node) {
+            self.db_tx().save_node(layer, pos, &node);
+            self.cache.put((layer, pos), node);
+        }
+    }
+
+    pub fn add_layer(&mut self) {
+        self.layer_size.push(0);
+        let layer = self.layer_size.len() - 1;
+        self.db_tx().save_layer_size(layer, 0);
+    }
+
+    pub fn layer_size(&self, layer: usize) -> usize {
+        self.layer_size[layer]
+    }
+
+    pub fn num_layers(&self) -> usize {
+        self.layer_size.len()
+    }
+
+    pub fn truncate_nodes(&mut self, layer: usize, pos_end: usize) {
+        let mut removed_nodes = Vec::new();
+        for pos in pos_end..self.layer_size[layer] {
+            self.cache.pop(&(layer, pos));
+            removed_nodes.push((layer, pos));
+        }
+        self.db_tx().remove_node_list(&removed_nodes);
+        self.set_layer_size(layer, pos_end);
+    }
+
+    pub fn truncate_layer(&mut self, layer: usize) {
+        self.truncate_nodes(layer, 0);
+        if layer == self.num_layers() - 1 {
+            self.layer_size.pop();
+            self.db_tx().remove_layer_size(layer);
+        }
+    }
+
+    pub fn start_transaction(&mut self) {
+        if self.db_tx.is_some() {
+            error!("start new tx before commit");
+            panic!("start new tx before commit");
+        }
+        self.db_tx = Some(self.db.start_transaction());
+    }
+
+    pub fn commit(&mut self) {
+        let tx = match self.db_tx.take() {
+            Some(tx) => tx,
+            None => {
+                error!("db_tx is None");
+                return;
+            }
+        };
+        if let Err(e) = self.db.commit(tx) {
+            error!("Failed to commit db transaction: {}", e);
+        }
+    }
+
+    fn db_tx(&mut self) -> &mut dyn NodeTransaction<E> {
+        (*self.db_tx.as_mut().expect("tx checked")).as_mut()
+    }
+
+    fn set_layer_size(&mut self, layer: usize, size: usize) {
+        self.layer_size[layer] = size;
+        self.db_tx().save_layer_size(layer, size);
+    }
+}
+
+pub struct NodeIterator<'a, E: HashElement> {
+    node_manager: &'a NodeManager<E>,
+    layer: usize,
+    start_pos: usize,
+    end_pos: usize,
+}
+
+impl<'a, E: HashElement> Iterator for NodeIterator<'a, E> {
+    type Item = E;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.start_pos < self.end_pos {
+            let r = self.node_manager.get_node(self.layer, self.start_pos);
+            self.start_pos += 1;
+            r
+        } else {
+            None
+        }
+    }
+}
+
+pub trait NodeDatabase<E: HashElement>: Send + Sync {
+    fn get_node(&self, layer: usize, pos: usize) -> Result<Option<E>>;
+    fn get_layer_size(&self, layer: usize) -> Result<Option<usize>>;
+    fn start_transaction(&self) -> Box<dyn NodeTransaction<E>>;
+    fn commit(&self, tx: Box<dyn NodeTransaction<E>>) -> Result<()>;
+}
+
+pub trait NodeTransaction<E: HashElement>: Send + Sync {
+    fn save_node(&mut self, layer: usize, pos: usize, node: &E);
+    /// `nodes` are a list of tuples `(layer, pos, node)`.
+    fn save_node_list(&mut self, nodes: &[(usize, usize, &E)]);
+    fn remove_node_list(&mut self, nodes: &[(usize, usize)]);
+    fn save_layer_size(&mut self, layer: usize, size: usize);
+    fn remove_layer_size(&mut self, layer: usize);
+
+    fn into_any(self: Box<Self>) -> Box<dyn Any>;
+}
+
+/// A dummy database structure for in-memory merkle tree that will not read/write db.
+pub struct EmptyNodeDatabase {}
+pub struct EmptyNodeTransaction {}
+impl<E: HashElement> NodeDatabase<E> for EmptyNodeDatabase {
+    fn get_node(&self, _layer: usize, _pos: usize) -> Result<Option<E>> {
+        Ok(None)
+    }
+    fn get_layer_size(&self, _layer: usize) -> Result<Option<usize>> {
+        Ok(None)
+    }
+    fn start_transaction(&self) -> Box<dyn NodeTransaction<E>> {
+        Box::new(EmptyNodeTransaction {})
+    }
+    fn commit(&self, _tx: Box<dyn NodeTransaction<E>>) -> Result<()> {
+        Ok(())
+    }
+}
+
+impl<E: HashElement> NodeTransaction<E> for EmptyNodeTransaction {
+    fn save_node(&mut self, _layer: usize, _pos: usize, _node: &E) {}
+
+    fn save_node_list(&mut self, _nodes: &[(usize, usize, &E)]) {}
+
+    fn remove_node_list(&mut self, _nodes: &[(usize, usize)]) {}
+
+    fn save_layer_size(&mut self, _layer: usize, _size: usize) {}
+
+    fn remove_layer_size(&mut self, _layer: usize) {}
+
+    fn into_any(self: Box<Self>) -> Box<dyn Any> {
+        self
+    }
+}

common/contract-interface/src/lib.rs

@@ -15,19 +15,13 @@ abigen!(
 );
 
 #[cfg(feature = "dev")]
-abigen!(
-    ZgsFlow,
-    "../../0g-storage-contracts-dev/artifacts/contracts/dataFlow/Flow.sol/Flow.json"
-);
+abigen!(ZgsFlow, "../../storage-contracts-abis/Flow.json");
 
 #[cfg(feature = "dev")]
-abigen!(
-    PoraMine,
-    "../../0g-storage-contracts-dev/artifacts/contracts/miner/Mine.sol/PoraMine.json"
-);
+abigen!(PoraMine, "../../storage-contracts-abis/PoraMine.json");
 
 #[cfg(feature = "dev")]
 abigen!(
     ChunkLinearReward,
-    "../../0g-storage-contracts-dev/artifacts/contracts/reward/ChunkLinearReward.sol/ChunkLinearReward.json"
+    "../../storage-contracts-abis/ChunkLinearReward.json"
 );

common/task_executor/Cargo.toml

@@ -9,5 +9,5 @@ exit-future = "0.2.0"
 futures = "0.3.21"
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../lighthouse_metrics" }
-tokio = { version = "1.19.2", features = ["rt"] }
+tokio = { version = "1.38.0", features = ["full"] }
 tracing = "0.1.35"

docs/incentive-mechanism/proof-of-random-access.md

@@ -1,6 +1,6 @@
 # Proof of Random Access
 
-The ZeroGravity network adopts a Proof of Random Access (PoRA) mechanism to incentivize miners to store data. By requiring miners to answer randomly produced queries to archived data chunks, the PoRA mechanism establishes the relation between mining proof generation power and data storage. Miners answer the queries repeatedly and computes an output digest for each loaded chunk util find a digest that satisfies the mining difficulty (i.e., has enough leading zeros). PoRA will stress the miners' disk I/O and reduce their capability to respond user queries. So 0G Storage adopts intermittent mining, in which a mining epoch starts with a block generation at a specific block height on the host chain and stops when a valid PoRA is submitted to the 0G Storage contract.
+The ZeroGravity network adopts a Proof of Random Access (PoRA) mechanism to incentivize miners to store data. By requiring miners to answer randomly produced queries to archived data chunks, the PoRA mechanism establishes the relation between mining proof generation power and data storage. Miners answer the queries repeatedly and computes an output digest for each loaded chunk until find a digest that satisfies the mining difficulty (i.e., has enough leading zeros). PoRA will stress the miners' disk I/O and reduce their capability to respond user queries. So 0G Storage adopts intermittent mining, in which a mining epoch starts with a block generation at a specific block height on the host chain and stops when a valid PoRA is submitted to the 0G Storage contract.
 
 In a strawman design, a PoRA iteration consists of a computing stage and a loading stage. In the computing stage, a miner computes a random recall position (the universal offset in the flow) based on an arbitrary picked random nonce and a mining status read from the host chain. In the loading stage, a miner loads the archived data chunks at the given recall position, and computes output digest by hashing the tuple of mining status and the data chunks. If the output digest satisfies the target difficulty, the miner can construct a legitimate PoRA consists of the chosen random nonce, the loaded data chunk and the proof for the correctness of data chunk to the mining contract.
 

docs/log-system.md

@@ -27,4 +27,4 @@ The mining process of 0G Storage requires to prove data accessibility to random
 
 ## Data Flow
 
-In 0G Storage, committed data are organized sequentially. Such a sequence of data is called a data flow, which can be interpreted as a list of data entries or equivalently a sequence of fixed-size data sectors. Thus, every piece of data in ZeroGravity can be indexed conveniently with a universal offset. This offset will be used to sample challenges in the mining process of PoRA. The default data flow is called the "main flow" of ZeroGravity. It incorporates all new log entries (unless otherwise specified) in an append-only manner. There are also specialized flows that only accept some category of log entries, e.g. data related to a specifc application. The most significant advantage of specialized flows is a consecutive addressing space, which may be crucial in some use cases. Furthermore, a specialized flow can apply customized storage price, which is typically significantly higher than the floor price of the default flow, and hence achieves better data availability and reliability.
+In 0G Storage, committed data are organized sequentially. Such a sequence of data is called a data flow, which can be interpreted as a list of data entries or equivalently a sequence of fixed-size data sectors. Thus, every piece of data in ZeroGravity can be indexed conveniently with a universal offset. This offset will be used to sample challenges in the mining process of PoRA. The default data flow is called the "main flow" of ZeroGravity. It incorporates all new log entries (unless otherwise specified) in an append-only manner. There are also specialized flows that only accept some category of log entries, e.g. data related to a specific application. The most significant advantage of specialized flows is a consecutive addressing space, which may be crucial in some use cases. Furthermore, a specialized flow can apply customized storage price, which is typically significantly higher than the floor price of the default flow, and hence achieves better data availability and reliability.

docs/onebox-test.md

@@ -0,0 +1,38 @@
+# One Box Test
+
+0G storage node provides one box test framework for developers to verify system functionalities via RPC.
+
+## Prerequisites
+
+- Requires python version: 3.8, 3.9 or 3.10, higher version is not guaranteed (e.g. failed to install `pysha3`).
+- Installs dependencies under root folder: `pip3 install -r requirements.txt`
+
+## Install Blockchain Nodes
+
+Python test framework will launch blockchain nodes at local machine for storage nodes to interact with. There are 3 kinds of blockchains available:
+
+- 0G blockchain (by default).
+- Conflux eSpace (for chain reorg test purpose).
+- BSC node (geth).
+
+The blockchain node binaries will be compiled or downloaded from github to `tests/tmp` folder automatically. Alternatively, developers could also manually copy binaries of specific version to the `tests/tmp` folder.
+
+## Run Tests
+
+Changes to the `tests` folder and run following command to run all tests:
+
+```
+python test_all.py
+```
+
+or, run any single test, e.g.
+
+```
+python example_test.py
+```
+
+*Note, please ensure blockchain nodes installed before running any single test, e.g. run all tests at first.*
+
+## Add New Test
+
+Please follow the `example_test.py` to add a new `xxx_test.py` file under `tests` folder.

node/chunk_pool/Cargo.toml

@@ -13,3 +13,5 @@ tokio = { version = "1.19.2", features = ["sync"] }
 async-lock = "2.5.0"
 hashlink = "0.8.0"
 tracing = "0.1.35"
+lazy_static = "1.4.0"
+metrics = { workspace = true }

node/chunk_pool/src/handler.rs

@@ -1,11 +1,16 @@
 use super::mem_pool::MemoryChunkPool;
 use crate::mem_pool::FileID;
 use anyhow::Result;
-use network::NetworkMessage;
+use metrics::{Histogram, Sample};
+use network::{NetworkMessage, NetworkSender};
 use shared_types::{ChunkArray, FileProof};
-use std::{sync::Arc, time::SystemTime};
+use std::{sync::Arc, time::Instant};
 use storage_async::{ShardConfig, Store};
-use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
+use tokio::sync::mpsc::UnboundedReceiver;
+
+lazy_static::lazy_static! {
+    pub static ref FINALIZE_FILE_LATENCY: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register("chunk_pool_finalize_file_latency", 1024);
+}
 
 /// Handle the cached file when uploaded completely and verified from blockchain.
 /// Generally, the file will be persisted into log store.
@@ -13,7 +18,7 @@ pub struct ChunkPoolHandler {
     receiver: UnboundedReceiver<ChunkPoolMessage>,
     mem_pool: Arc<MemoryChunkPool>,
     log_store: Arc<Store>,
-    sender: UnboundedSender<NetworkMessage>,
+    sender: NetworkSender,
 }
 
 impl ChunkPoolHandler {
@@ -21,7 +26,7 @@ impl ChunkPoolHandler {
         receiver: UnboundedReceiver<ChunkPoolMessage>,
         mem_pool: Arc<MemoryChunkPool>,
         log_store: Arc<Store>,
-        sender: UnboundedSender<NetworkMessage>,
+        sender: NetworkSender,
     ) -> Self {
         ChunkPoolHandler {
             receiver,
@@ -68,7 +73,7 @@ impl ChunkPoolHandler {
             }
         }
 
-        let start = SystemTime::now();
+        let start = Instant::now();
         if !self
             .log_store
             .finalize_tx_with_hash(id.tx_id.seq, id.tx_id.hash)
@@ -77,8 +82,9 @@ impl ChunkPoolHandler {
             return Ok(false);
         }
 
-        let elapsed = start.elapsed()?;
+        let elapsed = start.elapsed();
         debug!(?id, ?elapsed, "Transaction finalized");
+        FINALIZE_FILE_LATENCY.update_since(start);
 
         // always remove file from pool after transaction finalized
         self.mem_pool.remove_file(&id.root).await;

node/chunk_pool/src/lib.rs

@@ -29,7 +29,7 @@ impl Config {
 pub fn unbounded(
     config: Config,
     log_store: Arc<storage_async::Store>,
-    network_send: tokio::sync::mpsc::UnboundedSender<network::NetworkMessage>,
+    network_send: network::NetworkSender,
 ) -> (Arc<MemoryChunkPool>, ChunkPoolHandler) {
     let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
 

node/log_entry_sync/Cargo.toml

@@ -24,3 +24,5 @@ futures-util = "0.3.28"
 thiserror = "1.0.44"
 lazy_static = "1.4.0"
 metrics = { workspace = true }
+reqwest = {version = "0.11", features = ["json"]}
+url = { version = "2.4", default-features = false }

node/log_entry_sync/src/sync_manager/config.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use crate::ContractAddress;
 
 pub struct LogSyncConfig {
@@ -34,6 +36,9 @@ pub struct LogSyncConfig {
     pub watch_loop_wait_time_ms: u64,
     // force to sync log from start block number
     pub force_log_sync_from_start_block_number: bool,
+
+    // the timeout for blockchain rpc connection
+    pub blockchain_rpc_timeout: Duration,
 }
 
 #[derive(Clone)]
@@ -61,6 +66,7 @@ impl LogSyncConfig {
         remove_finalized_block_interval_minutes: u64,
         watch_loop_wait_time_ms: u64,
         force_log_sync_from_start_block_number: bool,
+        blockchain_rpc_timeout: Duration,
     ) -> Self {
         Self {
             rpc_endpoint_url,
@@ -77,6 +83,7 @@ impl LogSyncConfig {
             remove_finalized_block_interval_minutes,
             watch_loop_wait_time_ms,
             force_log_sync_from_start_block_number,
+            blockchain_rpc_timeout,
         }
     }
 }

node/log_entry_sync/src/sync_manager/log_entry_fetcher.rs

@@ -1,6 +1,6 @@
 use crate::sync_manager::log_query::LogQuery;
-use crate::sync_manager::RETRY_WAIT_MS;
-use crate::ContractAddress;
+use crate::sync_manager::{metrics, RETRY_WAIT_MS};
+use crate::{ContractAddress, LogSyncConfig};
 use anyhow::{anyhow, bail, Result};
 use append_merkle::{Algorithm, Sha3Algorithm};
 use contract_interface::{SubmissionNode, SubmitFilter, ZgsFlow};
@@ -12,17 +12,13 @@ use futures::StreamExt;
 use jsonrpsee::tracing::{debug, error, info, warn};
 use shared_types::{DataRoot, Transaction};
 use std::collections::{BTreeMap, HashMap};
-use std::str::FromStr;
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 use storage::log_store::{tx_store::BlockHashAndSubmissionIndex, Store};
 use task_executor::TaskExecutor;
-use tokio::{
-    sync::{
+use tokio::sync::{
     mpsc::{UnboundedReceiver, UnboundedSender},
     RwLock,
-    },
-    time::Instant,
 };
 
 pub struct LogEntryFetcher {
@@ -34,28 +30,29 @@ pub struct LogEntryFetcher {
 }
 
 impl LogEntryFetcher {
-    pub async fn new(
-        url: &str,
-        contract_address: ContractAddress,
-        log_page_size: u64,
-        confirmation_delay: u64,
-        rate_limit_retries: u32,
-        timeout_retries: u32,
-        initial_backoff: u64,
-    ) -> Result<Self> {
+    pub async fn new(config: &LogSyncConfig) -> Result<Self> {
         let provider = Arc::new(Provider::new(
             RetryClientBuilder::default()
-                .rate_limit_retries(rate_limit_retries)
-                .timeout_retries(timeout_retries)
-                .initial_backoff(Duration::from_millis(initial_backoff))
-                .build(Http::from_str(url)?, Box::new(HttpRateLimitRetryPolicy)),
+                .rate_limit_retries(config.rate_limit_retries)
+                .timeout_retries(config.timeout_retries)
+                .initial_backoff(Duration::from_millis(config.initial_backoff))
+                .build(
+                    Http::new_with_client(
+                        url::Url::parse(&config.rpc_endpoint_url)?,
+                        reqwest::Client::builder()
+                            .timeout(config.blockchain_rpc_timeout)
+                            .connect_timeout(config.blockchain_rpc_timeout)
+                            .build()?,
+                    ),
+                    Box::new(HttpRateLimitRetryPolicy),
+                ),
         ));
         // TODO: `error` types are removed from the ABI json file.
         Ok(Self {
-            contract_address,
+            contract_address: config.contract_address,
             provider,
-            log_page_size,
-            confirmation_delay,
+            log_page_size: config.log_page_size,
+            confirmation_delay: config.confirmation_block_count,
         })
     }
 
@@ -222,7 +219,7 @@ impl LogEntryFetcher {
     ) -> UnboundedReceiver<LogFetchProgress> {
         let provider = self.provider.clone();
         let (recover_tx, recover_rx) = tokio::sync::mpsc::unbounded_channel();
-        let contract = ZgsFlow::new(self.contract_address, provider.clone());
+        let contract = self.flow_contract();
         let log_page_size = self.log_page_size;
 
         executor.spawn(
@@ -236,15 +233,20 @@ impl LogEntryFetcher {
                     .filter;
                 let mut stream = LogQuery::new(&provider, &filter, log_query_delay)
                     .with_page_size(log_page_size);
-                debug!(
+                info!(
                     "start_recover starts, start={} end={}",
                     start_block_number, end_block_number
                 );
+                let (mut block_hash_sent, mut block_number_sent) = (None, None);
                 while let Some(maybe_log) = stream.next().await {
+                    let start_time = Instant::now();
                     match maybe_log {
                         Ok(log) => {
                             let sync_progress =
                                 if log.block_hash.is_some() && log.block_number.is_some() {
+                                    if block_hash_sent != log.block_hash
+                                        || block_number_sent != log.block_number
+                                    {
                                         let synced_block = LogFetchProgress::SyncedBlock((
                                             log.block_number.unwrap().as_u64(),
                                             log.block_hash.unwrap(),
@@ -254,6 +256,9 @@ impl LogEntryFetcher {
                                         Some(synced_block)
                                     } else {
                                         None
+                                    }
+                                } else {
+                                    None
                                 };
                             debug!("recover: progress={:?}", sync_progress);
 
@@ -268,11 +273,17 @@ impl LogEntryFetcher {
                                             log.block_number.expect("block number exist").as_u64(),
                                         ))
                                         .and_then(|_| match sync_progress {
-                                            Some(b) => recover_tx.send(b),
+                                            Some(b) => {
+                                                recover_tx.send(b)?;
+                                                block_hash_sent = log.block_hash;
+                                                block_number_sent = log.block_number;
+                                                Ok(())
+                                            }
                                             None => Ok(()),
                                         })
                                     {
                                         error!("send error: e={:?}", e);
+                                        break;
                                     }
                                 }
                                 Err(e) => {
@@ -288,7 +299,10 @@ impl LogEntryFetcher {
                             tokio::time::sleep(Duration::from_millis(RETRY_WAIT_MS)).await;
                         }
                     }
+                    metrics::RECOVER_LOG.update_since(start_time);
                 }
+
+                info!("log recover end");
             },
             "log recover",
         );
@@ -305,7 +319,7 @@ impl LogEntryFetcher {
         mut watch_progress_rx: UnboundedReceiver<u64>,
     ) -> UnboundedReceiver<LogFetchProgress> {
         let (watch_tx, watch_rx) = tokio::sync::mpsc::unbounded_channel();
-        let contract = ZgsFlow::new(self.contract_address, self.provider.clone());
+        let contract = self.flow_contract();
         let provider = self.provider.clone();
         let confirmation_delay = self.confirmation_delay;
         let log_page_size = self.log_page_size;
@@ -583,6 +597,10 @@ impl LogEntryFetcher {
     pub fn provider(&self) -> &Provider<RetryClient<Http>> {
         self.provider.as_ref()
     }
+
+    pub fn flow_contract(&self) -> ZgsFlow<Provider<RetryClient<Http>>> {
+        ZgsFlow::new(self.contract_address, self.provider.clone())
+    }
 }
 
 async fn check_watch_process(
@@ -658,6 +676,7 @@ async fn check_watch_process(
                     "get block hash for block {} from RPC, assume there is no org",
                     *progress - 1
                 );
+                let hash = loop {
                     match provider.get_block(*progress - 1).await {
                         Ok(Some(v)) => {
                             break v.hash.expect("parent block hash expect exist");
@@ -666,11 +685,17 @@ async fn check_watch_process(
                             panic!("parent block {} expect exist", *progress - 1);
                         }
                         Err(e) => {
+                            if e.to_string().contains("server is too busy") {
+                                warn!("server busy, wait for parent block {}", *progress - 1);
+                            } else {
                                 panic!("parent block {} expect exist, error {}", *progress - 1, e);
                             }
                         }
                     }
                 };
+                break hash;
+            }
+        };
     }
 
     progress_reset_history.retain(|k, _| k + 1000 >= *progress);

node/log_entry_sync/src/sync_manager/log_query.rs

@@ -14,7 +14,7 @@ use thiserror::Error;
 pub(crate) type PinBoxFut<'a, T> =
     Pin<Box<dyn Future<Output = Result<T, ProviderError>> + Send + 'a>>;
 
-const TOO_MANY_LOGS_ERROR_MSG: [&str; 2] = ["query returned more than", "too large with more than"];
+const TOO_MANY_LOGS_ERROR_MSG: [&str; 2] = ["exceeds the max limit of", "too large with more than"];
 
 /// A log query provides streaming access to historical logs via a paginated
 /// request. For streaming access to future logs, use [`Middleware::watch`] or

node/log_entry_sync/src/sync_manager/metrics.rs

@@ -1,7 +1,13 @@
 use std::sync::Arc;
 
-use metrics::{register_timer, Timer};
+use metrics::{register_timer, Gauge, GaugeUsize, Timer};
 
 lazy_static::lazy_static! {
-    pub static ref STORE_PUT_TX: Arc<dyn Timer> = register_timer("log_entry_sync_store_put_tx");
+    pub static ref LOG_MANAGER_HANDLE_DATA_TRANSACTION: Arc<dyn Timer> = register_timer("log_manager_handle_data_transaction");
+
+    pub static ref STORE_PUT_TX: Arc<dyn Timer> = register_timer("log_entry_sync_manager_put_tx_inner");
+
+    pub static ref STORE_PUT_TX_SPEED_IN_BYTES: Arc<dyn Gauge<usize>> = GaugeUsize::register("log_entry_sync_manager_put_tx_speed_in_bytes");
+
+    pub static ref RECOVER_LOG: Arc<dyn Timer> = register_timer("log_entry_sync_manager_recover_log");
 }

node/log_entry_sync/src/sync_manager/mod.rs

@@ -26,6 +26,7 @@ const RETRY_WAIT_MS: u64 = 500;
 // Each tx has less than 10KB, so the cache size should be acceptable.
 const BROADCAST_CHANNEL_CAPACITY: usize = 25000;
 const CATCH_UP_END_GAP: u64 = 10;
+const CHECK_ROOT_INTERVAL: u64 = 500;
 
 /// Errors while handle data
 #[derive(Error, Debug)]
@@ -86,16 +87,7 @@ impl LogSyncManager {
                         .expect("shutdown send error")
                 },
                 async move {
-                    let log_fetcher = LogEntryFetcher::new(
-                        &config.rpc_endpoint_url,
-                        config.contract_address,
-                        config.log_page_size,
-                        config.confirmation_block_count,
-                        config.rate_limit_retries,
-                        config.timeout_retries,
-                        config.initial_backoff,
-                    )
-                    .await?;
+                    let log_fetcher = LogEntryFetcher::new(&config).await?;
                     let data_cache = DataCache::new(config.cache_config.clone());
 
                     let block_hash_cache = Arc::new(RwLock::new(
@@ -277,6 +269,9 @@ impl LogSyncManager {
                                 .remove_finalized_block_interval_minutes,
                         );
 
+                    // start the pad data store
+                    log_sync_manager.store.start_padding(&executor_clone);
+
                     let (watch_progress_tx, watch_progress_rx) =
                         tokio::sync::mpsc::unbounded_channel();
                     let watch_rx = log_sync_manager.log_fetcher.start_watch(
@@ -408,6 +403,7 @@ impl LogSyncManager {
                 }
                 LogFetchProgress::Transaction((tx, block_number)) => {
                     let mut stop = false;
+                    let start_time = Instant::now();
                     match self.put_tx(tx.clone()).await {
                         Some(false) => stop = true,
                         Some(true) => {
@@ -441,6 +437,8 @@ impl LogSyncManager {
                         // no receivers will be created.
                         warn!("log sync broadcast error, error={:?}", e);
                     }
+
+                    metrics::LOG_MANAGER_HANDLE_DATA_TRANSACTION.update_since(start_time);
                 }
                 LogFetchProgress::Reverted(reverted) => {
                     self.process_reverted(reverted).await;
@@ -453,7 +451,6 @@ impl LogSyncManager {
     async fn put_tx_inner(&mut self, tx: Transaction) -> bool {
         let start_time = Instant::now();
         let result = self.store.put_tx(tx.clone());
-        metrics::STORE_PUT_TX.update_since(start_time);
 
         if let Err(e) = result {
             error!("put_tx error: e={:?}", e);
@@ -509,7 +506,50 @@ impl LogSyncManager {
                 }
             }
             self.data_cache.garbage_collect(self.next_tx_seq);
+
             self.next_tx_seq += 1;
+
+            // Check if the computed data root matches on-chain state.
+            // If the call fails, we won't check the root here and return `true` directly.
+            if self.next_tx_seq % CHECK_ROOT_INTERVAL == 0 {
+                let flow_contract = self.log_fetcher.flow_contract();
+
+                match flow_contract
+                    .get_flow_root_by_tx_seq(tx.seq.into())
+                    .call()
+                    .await
+                {
+                    Ok(contract_root_bytes) => {
+                        let contract_root = H256::from_slice(&contract_root_bytes);
+                        // contract_root is zero for tx submitted before upgrading.
+                        if !contract_root.is_zero() {
+                            match self.store.get_context() {
+                                Ok((local_root, _)) => {
+                                    if contract_root != local_root {
+                                        error!(
+                                            ?contract_root,
+                                            ?local_root,
+                                            "local flow root and on-chain flow root mismatch"
+                                        );
+                                        return false;
+                                    }
+                                }
+                                Err(e) => {
+                                    warn!(?e, "fail to read the local flow root");
+                                }
+                            }
+                        }
+                    }
+                    Err(e) => {
+                        warn!(?e, "fail to read the on-chain flow root");
+                    }
+                }
+            }
+
+            metrics::STORE_PUT_TX_SPEED_IN_BYTES
+                .update((tx.size * 1000 / start_time.elapsed().as_micros() as u64) as usize);
+            metrics::STORE_PUT_TX.update_since(start_time);
+
             true
         }
     }

node/miner/src/config.rs

@@ -67,8 +67,8 @@ impl MinerConfig {
         })
     }
 
-    pub(crate) async fn make_provider(&self) -> Result<MineServiceMiddleware, String> {
-        let provider = Arc::new(Provider::new(
+    pub(crate) fn make_provider(&self) -> Result<Arc<Provider<RetryClient<Http>>>, String> {
+        Ok(Arc::new(Provider::new(
             RetryClientBuilder::default()
                 .rate_limit_retries(self.rate_limit_retries)
                 .timeout_retries(self.timeout_retries)
@@ -78,7 +78,11 @@ impl MinerConfig {
                         .map_err(|e| format!("Cannot parse blockchain endpoint: {:?}", e))?,
                     Box::new(HttpRateLimitRetryPolicy),
                 ),
-        ));
+        )))
+    }
+
+    pub(crate) async fn make_signing_provider(&self) -> Result<MineServiceMiddleware, String> {
+        let provider = self.make_provider()?;
         let chain_id = provider
             .get_chainid()
             .await

node/miner/src/miner_id.rs

@@ -5,17 +5,20 @@ use ethereum_types::Address;
 use ethers::contract::ContractCall;
 use ethers::contract::EthEvent;
 use std::sync::Arc;
+use storage::log_store::log_manager::DATA_DB_KEY;
 use storage::H256;
 use storage_async::Store;
 
 const MINER_ID: &str = "mine.miner_id";
 
 pub async fn load_miner_id(store: &Store) -> storage::error::Result<Option<H256>> {
-    store.get_config_decoded(&MINER_ID).await
+    store.get_config_decoded(&MINER_ID, DATA_DB_KEY).await
 }
 
 async fn set_miner_id(store: &Store, miner_id: &H256) -> storage::error::Result<()> {
-    store.set_config_encoded(&MINER_ID, miner_id).await
+    store
+        .set_config_encoded(&MINER_ID, miner_id, DATA_DB_KEY)
+        .await
 }
 
 pub(crate) async fn check_and_request_miner_id(

node/miner/src/sealer.rs

@@ -1,6 +1,7 @@
 use std::{collections::BTreeMap, sync::Arc};
 
 use ethereum_types::H256;
+use ethers::prelude::{Http, Provider, RetryClient};
 use tokio::time::{sleep, Duration, Instant};
 
 use contract_interface::{EpochRangeWithContextDigest, ZgsFlow};
@@ -12,14 +13,14 @@ use storage_async::Store;
 use task_executor::TaskExecutor;
 use zgs_spec::SECTORS_PER_SEAL;
 
-use crate::config::{MineServiceMiddleware, MinerConfig};
+use crate::config::MinerConfig;
 
 const DB_QUERY_PERIOD_ON_NO_TASK: u64 = 1;
 const DB_QUERY_PERIOD_ON_ERROR: u64 = 5;
 const CHAIN_STATUS_QUERY_PERIOD: u64 = 5;
 
 pub struct Sealer {
-    flow_contract: ZgsFlow<MineServiceMiddleware>,
+    flow_contract: ZgsFlow<Provider<RetryClient<Http>>>,
     store: Arc<Store>,
     context_cache: BTreeMap<u128, EpochRangeWithContextDigest>,
     last_context_flow_length: u64,
@@ -29,7 +30,7 @@ pub struct Sealer {
 impl Sealer {
     pub fn spawn(
         executor: TaskExecutor,
-        provider: Arc<MineServiceMiddleware>,
+        provider: Arc<Provider<RetryClient<Http>>>,
         store: Arc<Store>,
         config: &MinerConfig,
         miner_id: H256,

node/miner/src/service.rs

@@ -3,13 +3,12 @@ use crate::monitor::Monitor;
 use crate::sealer::Sealer;
 use crate::submitter::Submitter;
 use crate::{config::MinerConfig, mine::PoraService, watcher::MineContextWatcher};
-use network::NetworkMessage;
+use network::NetworkSender;
 use std::sync::Arc;
 use std::time::Duration;
 use storage::config::ShardConfig;
 use storage_async::Store;
 use tokio::sync::broadcast;
-use tokio::sync::mpsc;
 
 #[derive(Clone, Debug)]
 pub enum MinerMessage {
@@ -29,15 +28,17 @@ pub struct MineService;
 impl MineService {
     pub async fn spawn(
         executor: task_executor::TaskExecutor,
-        _network_send: mpsc::UnboundedSender<NetworkMessage>,
+        _network_send: NetworkSender,
         config: MinerConfig,
         store: Arc<Store>,
     ) -> Result<broadcast::Sender<MinerMessage>, String> {
-        let provider = Arc::new(config.make_provider().await?);
+        let provider = config.make_provider()?;
+        let signing_provider = Arc::new(config.make_signing_provider().await?);
 
         let (msg_send, msg_recv) = broadcast::channel(1024);
 
-        let miner_id = check_and_request_miner_id(&config, store.as_ref(), &provider).await?;
+        let miner_id =
+            check_and_request_miner_id(&config, store.as_ref(), &signing_provider).await?;
         debug!("miner id setting complete.");
 
         let mine_context_receiver = MineContextWatcher::spawn(
@@ -61,6 +62,7 @@ impl MineService {
             mine_answer_receiver,
             mine_context_receiver,
             provider.clone(),
+            signing_provider,
             store.clone(),
             &config,
         );

node/miner/src/submitter.rs

@@ -2,6 +2,7 @@ use contract_interface::PoraAnswer;
 use contract_interface::{PoraMine, ZgsFlow};
 use ethereum_types::U256;
 use ethers::contract::ContractCall;
+use ethers::prelude::{Http, Provider, RetryClient};
 use ethers::providers::PendingTransaction;
 use hex::ToHex;
 use shared_types::FlowRangeProof;
@@ -24,7 +25,7 @@ pub struct Submitter {
     mine_answer_receiver: mpsc::UnboundedReceiver<AnswerWithoutProof>,
     mine_context_receiver: broadcast::Receiver<MineContextMessage>,
     mine_contract: PoraMine<MineServiceMiddleware>,
-    flow_contract: ZgsFlow<MineServiceMiddleware>,
+    flow_contract: ZgsFlow<Provider<RetryClient<Http>>>,
     default_gas_limit: Option<U256>,
     store: Arc<Store>,
 }
@@ -34,11 +35,12 @@ impl Submitter {
         executor: TaskExecutor,
         mine_answer_receiver: mpsc::UnboundedReceiver<AnswerWithoutProof>,
         mine_context_receiver: broadcast::Receiver<MineContextMessage>,
-        provider: Arc<MineServiceMiddleware>,
+        provider: Arc<Provider<RetryClient<Http>>>,
+        signing_provider: Arc<MineServiceMiddleware>,
         store: Arc<Store>,
         config: &MinerConfig,
     ) {
-        let mine_contract = PoraMine::new(config.mine_address, provider.clone());
+        let mine_contract = PoraMine::new(config.mine_address, signing_provider);
         let flow_contract = ZgsFlow::new(config.flow_address, provider);
         let default_gas_limit = config.submission_gas;
 

node/miner/src/watcher.rs

@@ -14,13 +14,13 @@ use tokio::{
     try_join,
 };
 
+use crate::{config::MineServiceMiddleware, mine::PoraPuzzle, MinerConfig, MinerMessage};
+use ethers::prelude::{Http, RetryClient};
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
 use std::{ops::DerefMut, str::FromStr};
 
-use crate::{config::MineServiceMiddleware, mine::PoraPuzzle, MinerConfig, MinerMessage};
-
 pub type MineContextMessage = Option<PoraPuzzle>;
 
 lazy_static! {
@@ -29,9 +29,9 @@ lazy_static! {
 }
 
 pub struct MineContextWatcher {
-    provider: Arc<MineServiceMiddleware>,
-    flow_contract: ZgsFlow<MineServiceMiddleware>,
-    mine_contract: PoraMine<MineServiceMiddleware>,
+    provider: Arc<Provider<RetryClient<Http>>>,
+    flow_contract: ZgsFlow<Provider<RetryClient<Http>>>,
+    mine_contract: PoraMine<Provider<RetryClient<Http>>>,
 
     mine_context_sender: broadcast::Sender<MineContextMessage>,
     last_report: MineContextMessage,
@@ -44,7 +44,7 @@ impl MineContextWatcher {
     pub fn spawn(
         executor: TaskExecutor,
         msg_recv: broadcast::Receiver<MinerMessage>,
-        provider: Arc<MineServiceMiddleware>,
+        provider: Arc<Provider<RetryClient<Http>>>,
         config: &MinerConfig,
     ) -> broadcast::Receiver<MineContextMessage> {
         let mine_contract = PoraMine::new(config.mine_address, provider.clone());

node/network/Cargo.toml

@@ -41,6 +41,7 @@ if-addrs = "0.10.1"
 slog = "2.7.0"
 igd = "0.12.1"
 duration-str = "0.5.1"
+channel = { path = "../../common/channel" }
 
 [dependencies.libp2p]
 version = "0.45.1"

node/network/src/behaviour/gossip_cache.rs

@@ -20,6 +20,8 @@ pub struct GossipCache {
     topic_msgs: HashMap<GossipTopic, HashMap<Vec<u8>, Key>>,
     /// Timeout for Example messages.
     example: Option<Duration>,
+    /// Timeout for NewFile messages.
+    new_file: Option<Duration>,
     /// Timeout for FindFile messages.
     find_file: Option<Duration>,
     /// Timeout for FindChunks messages.
@@ -37,6 +39,8 @@ pub struct GossipCacheBuilder {
     default_timeout: Option<Duration>,
     /// Timeout for Example messages.
     example: Option<Duration>,
+    /// Timeout for NewFile messages.
+    new_file: Option<Duration>,
     /// Timeout for blocks FindFile messages.
     find_file: Option<Duration>,
     /// Timeout for blocks FindChunks messages.
@@ -64,6 +68,12 @@ impl GossipCacheBuilder {
         self
     }
 
+    /// Timeout for NewFile messages.
+    pub fn new_file_timeout(mut self, timeout: Duration) -> Self {
+        self.new_file = Some(timeout);
+        self
+    }
+
     /// Timeout for FindFile messages.
     pub fn find_file_timeout(mut self, timeout: Duration) -> Self {
         self.find_file = Some(timeout);
@@ -98,6 +108,7 @@ impl GossipCacheBuilder {
         let GossipCacheBuilder {
             default_timeout,
             example,
+            new_file,
             find_file,
             find_chunks,
             announce_file,
@@ -109,6 +120,7 @@ impl GossipCacheBuilder {
             expirations: DelayQueue::default(),
             topic_msgs: HashMap::default(),
             example: example.or(default_timeout),
+            new_file: new_file.or(default_timeout),
             find_file: find_file.or(default_timeout),
             find_chunks: find_chunks.or(default_timeout),
             announce_file: announce_file.or(default_timeout),
@@ -129,6 +141,7 @@ impl GossipCache {
     pub fn insert(&mut self, topic: GossipTopic, data: Vec<u8>) {
         let expire_timeout = match topic.kind() {
             GossipKind::Example => self.example,
+            GossipKind::NewFile => self.new_file,
             GossipKind::FindFile => self.find_file,
             GossipKind::FindChunks => self.find_chunks,
             GossipKind::AnnounceFile => self.announce_file,

node/network/src/behaviour/mod.rs

@@ -6,6 +6,7 @@ use crate::peer_manager::{
     ConnectionDirection, PeerManager, PeerManagerEvent,
 };
 use crate::rpc::methods::DataByHashRequest;
+use crate::rpc::methods::FileAnnouncement;
 use crate::rpc::methods::GetChunksRequest;
 use crate::rpc::*;
 use crate::service::Context as ServiceContext;
@@ -232,6 +233,9 @@ impl<AppReqId: ReqId> Behaviour<AppReqId> {
             let topic: Topic = GossipTopic::new(kind, GossipEncoding::default()).into();
             topic.hash()
         };
+        params
+            .topics
+            .insert(get_hash(GossipKind::NewFile), TopicScoreParams::default());
         params
             .topics
             .insert(get_hash(GossipKind::FindFile), TopicScoreParams::default());
@@ -263,7 +267,7 @@ impl<AppReqId: ReqId> Behaviour<AppReqId> {
             discovery_enabled: !config.disable_discovery,
             metrics_enabled: config.metrics_enabled,
             target_peer_count: config.target_peers,
-            ..Default::default()
+            ..config.peer_manager
         };
 
         let slot_duration = std::time::Duration::from_secs(12);
@@ -543,6 +547,9 @@ impl<AppReqId: ReqId> Behaviour<AppReqId> {
             Request::DataByHash { .. } => {
                 metrics::inc_counter_vec(&metrics::TOTAL_RPC_REQUESTS, &["data_by_hash"])
             }
+            Request::AnnounceFile { .. } => {
+                metrics::inc_counter_vec(&metrics::TOTAL_RPC_REQUESTS, &["announce_file"])
+            }
             Request::GetChunks { .. } => {
                 metrics::inc_counter_vec(&metrics::TOTAL_RPC_REQUESTS, &["get_chunks"])
             }
@@ -585,7 +592,7 @@ where
                 // peer that originally published the message.
                 match PubsubMessage::decode(&gs_msg.topic, &gs_msg.data) {
                     Err(e) => {
-                        debug!(topic = ?gs_msg.topic, error = ?e, "Could not decode gossipsub message");
+                        debug!(topic = ?gs_msg.topic, %propagation_source, error = ?e, "Could not decode gossipsub message");
                         //reject the message
                         if let Err(e) = self.gossipsub.report_message_validation_result(
                             &id,
@@ -594,6 +601,24 @@ where
                         ) {
                             warn!(message_id = %id, peer_id = %propagation_source, error = ?e, "Failed to report message validation");
                         }
+
+                        self.peer_manager.report_peer(
+                            &propagation_source,
+                            PeerAction::Fatal,
+                            ReportSource::Gossipsub,
+                            None,
+                            "gossipsub message decode error",
+                        );
+
+                        if let Some(source) = &gs_msg.source {
+                            self.peer_manager.report_peer(
+                                source,
+                                PeerAction::Fatal,
+                                ReportSource::Gossipsub,
+                                None,
+                                "gossipsub message decode error",
+                            );
+                        }
                     }
                     Ok(msg) => {
                         // Notify the network
@@ -755,6 +780,9 @@ where
                     InboundRequest::DataByHash(req) => {
                         self.propagate_request(peer_request_id, peer_id, Request::DataByHash(req))
                     }
+                    InboundRequest::AnnounceFile(req) => {
+                        self.propagate_request(peer_request_id, peer_id, Request::AnnounceFile(req))
+                    }
                     InboundRequest::GetChunks(req) => {
                         self.propagate_request(peer_request_id, peer_id, Request::GetChunks(req))
                     }
@@ -969,6 +997,8 @@ pub enum Request {
     Status(StatusMessage),
     /// A data by hash request.
     DataByHash(DataByHashRequest),
+    /// An AnnounceFile message.
+    AnnounceFile(FileAnnouncement),
     /// A GetChunks request.
     GetChunks(GetChunksRequest),
 }
@@ -978,6 +1008,7 @@ impl std::convert::From<Request> for OutboundRequest {
         match req {
             Request::Status(s) => OutboundRequest::Status(s),
             Request::DataByHash(r) => OutboundRequest::DataByHash(r),
+            Request::AnnounceFile(r) => OutboundRequest::AnnounceFile(r),
             Request::GetChunks(r) => OutboundRequest::GetChunks(r),
         }
     }

node/network/src/config.rs

@@ -1,6 +1,5 @@
-use crate::peer_manager::peerdb::PeerDBConfig;
 use crate::types::GossipKind;
-use crate::{Enr, PeerIdSerialized};
+use crate::{peer_manager, Enr, PeerIdSerialized};
 use directory::{
     DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_NETWORK, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
 };
@@ -128,7 +127,12 @@ pub struct Config {
     /// The id of the storage network.
     pub network_id: NetworkIdentity,
 
-    pub peer_db: PeerDBConfig,
+    pub peer_db: peer_manager::peerdb::PeerDBConfig,
+    pub peer_manager: peer_manager::config::Config,
+
+    /// Whether to disable network identity in ENR.
+    /// This is for test purpose only.
+    pub disable_enr_network_id: bool,
 }
 
 impl Default for Config {
@@ -153,8 +157,8 @@ impl Default for Config {
         let filter_rate_limiter = Some(
             discv5::RateLimiterBuilder::new()
                 .total_n_every(300, Duration::from_secs(1)) // Allow bursts, average 300 per second
-                .ip_n_every(300, Duration::from_secs(1)) // Allow bursts, average 300 per second
-                .node_n_every(300, Duration::from_secs(1)) // Allow bursts, average 300 per second
+                .ip_n_every(9, Duration::from_secs(1)) // Allow bursts, average 9 per second
+                .node_n_every(8, Duration::from_secs(1)) // Allow bursts, average 8 per second
                 .build()
                 .expect("The total rate limit has been specified"),
         );
@@ -208,6 +212,8 @@ impl Default for Config {
             metrics_enabled: false,
             network_id: Default::default(),
             peer_db: Default::default(),
+            peer_manager: Default::default(),
+            disable_enr_network_id: false,
         }
     }
 }

node/network/src/discovery/enr.rs

@@ -1,9 +1,10 @@
 //! Helper functions and an extension trait for Ethereum 2 ENRs.
 
 pub use discv5::enr::{CombinedKey, EnrBuilder};
+use ssz::Encode;
 
-use super::enr_ext::CombinedKeyExt;
-use super::ENR_FILENAME;
+use super::enr_ext::{CombinedKeyExt, ENR_CONTENT_KEY_NETWORK_ID};
+use super::{EnrExt, ENR_FILENAME};
 use crate::types::Enr;
 use crate::NetworkConfig;
 use discv5::enr::EnrKey;
@@ -32,7 +33,9 @@ pub fn use_or_load_enr(
                     Ok(disk_enr) => {
                         // if the same node id, then we may need to update our sequence number
                         if local_enr.node_id() == disk_enr.node_id() {
-                            if compare_enr(local_enr, &disk_enr) {
+                            if compare_enr(local_enr, &disk_enr)
+                                && is_disk_enr_network_id_unchanged(&disk_enr, config)
+                            {
                                 debug!(file = ?enr_f, "ENR loaded from disk");
                                 // the stored ENR has the same configuration, use it
                                 *local_enr = disk_enr;
@@ -94,6 +97,13 @@ pub fn create_enr_builder_from_config<T: EnrKey>(
         let tcp_port = config.enr_tcp_port.unwrap_or(config.libp2p_port);
         builder.tcp(tcp_port);
     }
+    // add network identity info in ENR if not disabled
+    if !config.disable_enr_network_id {
+        builder.add_value(
+            ENR_CONTENT_KEY_NETWORK_ID,
+            &config.network_id.as_ssz_bytes(),
+        );
+    }
     builder
 }
 
@@ -117,6 +127,14 @@ fn compare_enr(local_enr: &Enr, disk_enr: &Enr) -> bool {
         && (local_enr.udp().is_none() || local_enr.udp() == disk_enr.udp())
 }
 
+fn is_disk_enr_network_id_unchanged(disk_enr: &Enr, config: &NetworkConfig) -> bool {
+    match disk_enr.network_identity() {
+        Some(Ok(id)) => !config.disable_enr_network_id && id == config.network_id,
+        Some(Err(_)) => false,
+        None => config.disable_enr_network_id,
+    }
+}
+
 /// Loads enr from the given directory
 pub fn load_enr_from_disk(dir: &Path) -> Result<Enr, String> {
     let enr_f = dir.join(ENR_FILENAME);

node/network/src/discovery/enr_ext.rs

@@ -2,8 +2,12 @@
 use crate::{Enr, Multiaddr, PeerId};
 use discv5::enr::{CombinedKey, CombinedPublicKey};
 use libp2p::core::{identity::Keypair, identity::PublicKey, multiaddr::Protocol};
+use shared_types::NetworkIdentity;
+use ssz::Decode;
 use tiny_keccak::{Hasher, Keccak};
 
+pub(crate) const ENR_CONTENT_KEY_NETWORK_ID: &'static str = "network_identity";
+
 /// Extend ENR for libp2p types.
 pub trait EnrExt {
     /// The libp2p `PeerId` for the record.
@@ -24,6 +28,9 @@ pub trait EnrExt {
 
     /// Returns any multiaddrs that contain the TCP protocol.
     fn multiaddr_tcp(&self) -> Vec<Multiaddr>;
+
+    /// Returns network identity in content.
+    fn network_identity(&self) -> Option<Result<NetworkIdentity, ssz::DecodeError>>;
 }
 
 /// Extend ENR CombinedPublicKey for libp2p types.
@@ -189,6 +196,12 @@ impl EnrExt for Enr {
         }
         multiaddrs
     }
+
+    /// Returns network identity in content.
+    fn network_identity(&self) -> Option<Result<NetworkIdentity, ssz::DecodeError>> {
+        let value = self.get(ENR_CONTENT_KEY_NETWORK_ID)?;
+        Some(NetworkIdentity::from_ssz_bytes(value))
+    }
 }
 
 impl CombinedKeyPublicExt for CombinedPublicKey {

node/network/src/discovery/mod.rs

@@ -139,6 +139,7 @@ impl Discovery {
             udp = ?local_enr.udp(),
             tcp = ?local_enr.tcp(),
             udp4_socket = ?local_enr.udp_socket(),
+            network_id = ?local_enr.network_identity(),
             "ENR Initialised",
         );
 
@@ -158,6 +159,7 @@ impl Discovery {
                 ip = ?bootnode_enr.ip(),
                 udp = ?bootnode_enr.udp(),
                 tcp = ?bootnode_enr.tcp(),
+                network_id = ?bootnode_enr.network_identity(),
                 "Adding node to routing table",
             );
             let repr = bootnode_enr.to_string();
@@ -205,13 +207,37 @@ impl Discovery {
             match result {
                 Ok(enr) => {
                     debug!(
+                        multiaddr = %original_addr.to_string(),
                         node_id = %enr.node_id(),
                         peer_id = %enr.peer_id(),
                         ip = ?enr.ip(),
                         udp = ?enr.udp(),
                         tcp = ?enr.tcp(),
-                        "Adding node to routing table",
+                        network_id = ?enr.network_identity(),
+                        "Adding bootnode to routing table",
                     );
+
+                    // check network identity in bootnode ENR if required
+                    if !config.disable_enr_network_id {
+                        match enr.network_identity() {
+                            Some(Ok(id)) => {
+                                if id != config.network_id {
+                                    error!(bootnode=?id, local=?config.network_id, "Bootnode network identity mismatch");
+                                    continue;
+                                }
+                            }
+                            Some(Err(err)) => {
+                                error!(?err, "Failed to decode bootnode network identity");
+                                continue;
+                            }
+                            None => {
+                                error!("Bootnode has no network identity");
+                                continue;
+                            }
+                        }
+                    }
+
+                    // add bootnode into routing table
                     let _ = discv5.add_enr(enr).map_err(|e| {
                         error!(
                             addr = %original_addr.to_string(),
@@ -401,10 +427,16 @@ impl Discovery {
         // Generate a random target node id.
         let random_node = NodeId::random();
 
+        // only discover nodes with same network identity
+        let local_network_id = self.network_globals.network_id();
+        let predicate = move |enr: &Enr| -> bool {
+            matches!(enr.network_identity(), Some(Ok(id)) if id == local_network_id)
+        };
+
         // Build the future
         let query_future = self
             .discv5
-            .find_node_predicate(random_node, Box::new(|_| true), target_peers)
+            .find_node_predicate(random_node, Box::new(predicate), target_peers)
             .map(|v| QueryResult {
                 query_type: query,
                 result: v,

node/network/src/lib.rs

@@ -93,7 +93,11 @@ pub use peer_manager::{
 };
 pub use service::{load_private_key, Context, Libp2pEvent, Service, NETWORK_KEY_FILENAME};
 
-pub const PROTOCOL_VERSION: [u8; 3] = [0, 1, 0];
+/// Defines the current P2P protocol version.
+/// - v1: Broadcast FindFile & AnnounceFile messages in the whole network, which caused network too heavey.
+/// - v2: Publish NewFile to neighbors only and announce file via RPC message.
+pub const PROTOCOL_VERSION_V1: [u8; 3] = [0, 1, 1];
+pub const PROTOCOL_VERSION_V2: [u8; 3] = [0, 2, 1];
 
 /// Application level requests sent to the network.
 #[derive(Debug, Clone, Copy)]
@@ -156,3 +160,10 @@ pub enum NetworkMessage {
         udp_socket: Option<SocketAddr>,
     },
 }
+
+pub type NetworkSender = channel::metrics::Sender<NetworkMessage>;
+pub type NetworkReceiver = channel::metrics::Receiver<NetworkMessage>;
+
+pub fn new_network_channel() -> (NetworkSender, NetworkReceiver) {
+    channel::metrics::unbounded_channel("network")
+}

node/network/src/nat.rs

@@ -3,10 +3,9 @@
 //! Currently supported strategies:
 //! - UPnP
 
-use crate::{NetworkConfig, NetworkMessage};
+use crate::{NetworkConfig, NetworkMessage, NetworkSender};
 use if_addrs::get_if_addrs;
 use std::net::{IpAddr, SocketAddr, SocketAddrV4};
-use tokio::sync::mpsc;
 
 /// Configuration required to construct the UPnP port mappings.
 pub struct UPnPConfig {
@@ -36,10 +35,7 @@ impl UPnPConfig {
 }
 
 /// Attempts to construct external port mappings with UPnP.
-pub fn construct_upnp_mappings(
-    config: UPnPConfig,
-    network_send: mpsc::UnboundedSender<NetworkMessage>,
-) {
+pub fn construct_upnp_mappings(config: UPnPConfig, network_send: NetworkSender) {
     info!("UPnP Attempting to initialise routes");
     match igd::search_gateway(Default::default()) {
         Err(e) => info!(error = %e, "UPnP not available"),

node/network/src/peer_manager/config.rs

@@ -1,3 +1,8 @@
+use std::time::Duration;
+
+use duration_str::deserialize_duration;
+use serde::{Deserialize, Serialize};
+
 /// The time in seconds between re-status's peers.
 pub const DEFAULT_STATUS_INTERVAL: u64 = 300;
 
@@ -11,9 +16,14 @@ pub const DEFAULT_PING_INTERVAL_INBOUND: u64 = 20;
 pub const DEFAULT_TARGET_PEERS: usize = 50;
 
 /// Configurations for the PeerManager.
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+#[serde(default)]
 pub struct Config {
     /* Peer count related configurations */
+    /// The heartbeat performs regular updates such as updating reputations and performing discovery
+    /// requests. This defines the interval in seconds.
+    #[serde(deserialize_with = "deserialize_duration")]
+    pub heartbeat_interval: Duration,
     /// Whether discovery is enabled.
     pub discovery_enabled: bool,
     /// Whether metrics are enabled.
@@ -35,6 +45,7 @@ pub struct Config {
 impl Default for Config {
     fn default() -> Self {
         Config {
+            heartbeat_interval: Duration::from_secs(30),
             discovery_enabled: true,
             metrics_enabled: false,
             target_peer_count: DEFAULT_TARGET_PEERS,

node/network/src/peer_manager/mod.rs

@@ -30,10 +30,6 @@ use std::net::IpAddr;
 pub mod config;
 mod network_behaviour;
 
-/// The heartbeat performs regular updates such as updating reputations and performing discovery
-/// requests. This defines the interval in seconds.
-const HEARTBEAT_INTERVAL: u64 = 30;
-
 /// This is used in the pruning logic. We avoid pruning peers on sync-committees if doing so would
 /// lower our peer count below this number. Instead we favour a non-uniform distribution of subnet
 /// peers.
@@ -105,6 +101,7 @@ impl PeerManager {
         network_globals: Arc<NetworkGlobals>,
     ) -> error::Result<Self> {
         let config::Config {
+            heartbeat_interval,
             discovery_enabled,
             metrics_enabled,
             target_peer_count,
@@ -114,7 +111,7 @@ impl PeerManager {
         } = cfg;
 
         // Set up the peer manager heartbeat interval
-        let heartbeat = tokio::time::interval(tokio::time::Duration::from_secs(HEARTBEAT_INTERVAL));
+        let heartbeat = tokio::time::interval(heartbeat_interval);
 
         Ok(PeerManager {
             network_globals,
@@ -460,6 +457,7 @@ impl PeerManager {
                     Protocol::Goodbye => PeerAction::LowToleranceError,
                     Protocol::Status => PeerAction::LowToleranceError,
                     Protocol::DataByHash => PeerAction::MidToleranceError,
+                    Protocol::AnnounceFile => PeerAction::MidToleranceError,
                     Protocol::GetChunks => PeerAction::MidToleranceError,
                 },
             },
@@ -474,6 +472,7 @@ impl PeerManager {
                     Protocol::Goodbye => return,
                     Protocol::Status => PeerAction::LowToleranceError,
                     Protocol::DataByHash => return,
+                    Protocol::AnnounceFile => return,
                     Protocol::GetChunks => return,
                 }
             }
@@ -488,6 +487,7 @@ impl PeerManager {
                     Protocol::Goodbye => return,
                     Protocol::Status => return,
                     Protocol::DataByHash => PeerAction::MidToleranceError,
+                    Protocol::AnnounceFile => PeerAction::MidToleranceError,
                     Protocol::GetChunks => PeerAction::MidToleranceError,
                 },
             },

node/network/src/rpc/codec/ssz_snappy.rs

@@ -159,6 +159,7 @@ impl Encoder<OutboundRequest> for SSZSnappyOutboundCodec {
             OutboundRequest::Goodbye(req) => req.as_ssz_bytes(),
             OutboundRequest::Ping(req) => req.as_ssz_bytes(),
             OutboundRequest::DataByHash(req) => req.hashes.as_ssz_bytes(),
+            OutboundRequest::AnnounceFile(req) => req.as_ssz_bytes(),
             OutboundRequest::GetChunks(req) => req.as_ssz_bytes(),
         };
         // SSZ encoded bytes should be within `max_packet_size`
@@ -346,6 +347,9 @@ fn handle_v1_request(
         Protocol::DataByHash => Ok(Some(InboundRequest::DataByHash(DataByHashRequest {
             hashes: VariableList::from_ssz_bytes(decoded_buffer)?,
         }))),
+        Protocol::AnnounceFile => Ok(Some(InboundRequest::AnnounceFile(
+            FileAnnouncement::from_ssz_bytes(decoded_buffer)?,
+        ))),
         Protocol::GetChunks => Ok(Some(InboundRequest::GetChunks(
             GetChunksRequest::from_ssz_bytes(decoded_buffer)?,
         ))),
@@ -373,6 +377,10 @@ fn handle_v1_response(
         Protocol::DataByHash => Ok(Some(RPCResponse::DataByHash(Box::new(
             ZgsData::from_ssz_bytes(decoded_buffer)?,
         )))),
+        // This case should be unreachable as `AnnounceFile` has no response.
+        Protocol::AnnounceFile => Err(RPCError::InvalidData(
+            "AnnounceFile RPC message has no valid response".to_string(),
+        )),
         Protocol::GetChunks => Ok(Some(RPCResponse::Chunks(
             ChunkArrayWithProof::from_ssz_bytes(decoded_buffer)?,
         ))),

node/network/src/rpc/methods.rs

@@ -178,6 +178,14 @@ pub struct DataByHashRequest {
     pub hashes: VariableList<Hash256, MaxRequestBlocks>,
 }
 
+// The message of `AnnounceFile` RPC message.
+#[derive(Encode, Decode, Clone, Debug, PartialEq, Eq)]
+pub struct FileAnnouncement {
+    pub tx_id: TxID,
+    pub num_shard: usize,
+    pub shard_id: usize,
+}
+
 /// Request a chunk array from a peer.
 #[derive(Encode, Decode, Clone, Debug, PartialEq, Eq)]
 pub struct GetChunksRequest {

node/network/src/rpc/mod.rs

@@ -118,6 +118,7 @@ impl<Id: ReqId> RPC<Id> {
             .n_every(Protocol::Status, 5, Duration::from_secs(15))
             .one_every(Protocol::Goodbye, Duration::from_secs(10))
             .n_every(Protocol::DataByHash, 128, Duration::from_secs(10))
+            .n_every(Protocol::AnnounceFile, 256, Duration::from_secs(10))
             .n_every(Protocol::GetChunks, 4096, Duration::from_secs(10))
             .build()
             .expect("Configuration parameters are valid");

node/network/src/rpc/outbound.rs

@@ -34,6 +34,7 @@ pub enum OutboundRequest {
     Goodbye(GoodbyeReason),
     Ping(Ping),
     DataByHash(DataByHashRequest),
+    AnnounceFile(FileAnnouncement),
     GetChunks(GetChunksRequest),
 }
 
@@ -72,6 +73,11 @@ impl OutboundRequest {
                 Version::V1,
                 Encoding::SSZSnappy,
             )],
+            OutboundRequest::AnnounceFile(_) => vec![ProtocolId::new(
+                Protocol::AnnounceFile,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
             OutboundRequest::GetChunks(_) => vec![ProtocolId::new(
                 Protocol::GetChunks,
                 Version::V1,
@@ -89,6 +95,7 @@ impl OutboundRequest {
             OutboundRequest::Goodbye(_) => 0,
             OutboundRequest::Ping(_) => 1,
             OutboundRequest::DataByHash(req) => req.hashes.len() as u64,
+            OutboundRequest::AnnounceFile(_) => 0,
             OutboundRequest::GetChunks(_) => 1,
         }
     }
@@ -100,6 +107,7 @@ impl OutboundRequest {
             OutboundRequest::Goodbye(_) => Protocol::Goodbye,
             OutboundRequest::Ping(_) => Protocol::Ping,
             OutboundRequest::DataByHash(_) => Protocol::DataByHash,
+            OutboundRequest::AnnounceFile(_) => Protocol::AnnounceFile,
             OutboundRequest::GetChunks(_) => Protocol::GetChunks,
         }
     }
@@ -114,6 +122,7 @@ impl OutboundRequest {
             OutboundRequest::Status(_) => unreachable!(),
             OutboundRequest::Goodbye(_) => unreachable!(),
             OutboundRequest::Ping(_) => unreachable!(),
+            OutboundRequest::AnnounceFile(_) => unreachable!(),
             OutboundRequest::GetChunks(_) => unreachable!(),
         }
     }
@@ -170,6 +179,9 @@ impl std::fmt::Display for OutboundRequest {
             OutboundRequest::DataByHash(req) => {
                 write!(f, "Data by hash: {:?}", req)
             }
+            OutboundRequest::AnnounceFile(req) => {
+                write!(f, "AnnounceFile: {:?}", req)
+            }
             OutboundRequest::GetChunks(req) => {
                 write!(f, "GetChunks: {:?}", req)
             }

node/network/src/rpc/protocol.rs

@@ -91,6 +91,8 @@ pub enum Protocol {
     /// TODO
     DataByHash,
 
+    /// The file announce protocol.
+    AnnounceFile,
     /// The Chunk sync protocol.
     GetChunks,
 }
@@ -115,6 +117,7 @@ impl std::fmt::Display for Protocol {
             Protocol::Goodbye => "goodbye",
             Protocol::Ping => "ping",
             Protocol::DataByHash => "data_by_hash",
+            Protocol::AnnounceFile => "announce_file",
             Protocol::GetChunks => "get_chunks",
         };
         f.write_str(repr)
@@ -155,6 +158,7 @@ impl UpgradeInfo for RPCProtocol {
             ProtocolId::new(Protocol::Goodbye, Version::V1, Encoding::SSZSnappy),
             ProtocolId::new(Protocol::Ping, Version::V1, Encoding::SSZSnappy),
             ProtocolId::new(Protocol::DataByHash, Version::V1, Encoding::SSZSnappy),
+            ProtocolId::new(Protocol::AnnounceFile, Version::V1, Encoding::SSZSnappy),
             ProtocolId::new(Protocol::GetChunks, Version::V1, Encoding::SSZSnappy),
         ]
     }
@@ -216,6 +220,10 @@ impl ProtocolId {
                 // TODO
                 RpcLimits::new(1, *DATA_BY_HASH_REQUEST_MAX)
             }
+            Protocol::AnnounceFile => RpcLimits::new(
+                <FileAnnouncement as Encode>::ssz_fixed_len(),
+                <FileAnnouncement as Encode>::ssz_fixed_len(),
+            ),
             Protocol::GetChunks => RpcLimits::new(
                 <GetChunksRequest as Encode>::ssz_fixed_len(),
                 <GetChunksRequest as Encode>::ssz_fixed_len(),
@@ -243,6 +251,7 @@ impl ProtocolId {
                 <ZgsData as Encode>::ssz_fixed_len(),
             ),
 
+            Protocol::AnnounceFile => RpcLimits::new(0, 0), // AnnounceFile request has no response
             Protocol::GetChunks => RpcLimits::new(*CHUNKS_RESPONSE_MIN, *CHUNKS_RESPONSE_MAX),
         }
     }
@@ -325,6 +334,7 @@ pub enum InboundRequest {
     Goodbye(GoodbyeReason),
     Ping(Ping),
     DataByHash(DataByHashRequest),
+    AnnounceFile(FileAnnouncement),
     GetChunks(GetChunksRequest),
 }
 
@@ -363,6 +373,11 @@ impl InboundRequest {
                 Version::V1,
                 Encoding::SSZSnappy,
             )],
+            InboundRequest::AnnounceFile(_) => vec![ProtocolId::new(
+                Protocol::AnnounceFile,
+                Version::V1,
+                Encoding::SSZSnappy,
+            )],
             InboundRequest::GetChunks(_) => vec![ProtocolId::new(
                 Protocol::GetChunks,
                 Version::V1,
@@ -380,6 +395,7 @@ impl InboundRequest {
             InboundRequest::Goodbye(_) => 0,
             InboundRequest::DataByHash(req) => req.hashes.len() as u64,
             InboundRequest::Ping(_) => 1,
+            InboundRequest::AnnounceFile(_) => 0,
             InboundRequest::GetChunks(_) => 1,
         }
     }
@@ -391,6 +407,7 @@ impl InboundRequest {
             InboundRequest::Goodbye(_) => Protocol::Goodbye,
             InboundRequest::Ping(_) => Protocol::Ping,
             InboundRequest::DataByHash(_) => Protocol::DataByHash,
+            InboundRequest::AnnounceFile(_) => Protocol::AnnounceFile,
             InboundRequest::GetChunks(_) => Protocol::GetChunks,
         }
     }
@@ -405,6 +422,7 @@ impl InboundRequest {
             InboundRequest::Status(_) => unreachable!(),
             InboundRequest::Goodbye(_) => unreachable!(),
             InboundRequest::Ping(_) => unreachable!(),
+            InboundRequest::AnnounceFile(_) => unreachable!(),
             InboundRequest::GetChunks(_) => unreachable!(),
         }
     }
@@ -523,6 +541,9 @@ impl std::fmt::Display for InboundRequest {
             InboundRequest::DataByHash(req) => {
                 write!(f, "Data by hash: {:?}", req)
             }
+            InboundRequest::AnnounceFile(req) => {
+                write!(f, "Announce File: {:?}", req)
+            }
             InboundRequest::GetChunks(req) => {
                 write!(f, "Get Chunks: {:?}", req)
             }

node/network/src/rpc/rate_limiter.rs

@@ -68,6 +68,8 @@ pub struct RPCRateLimiter {
     status_rl: Limiter<PeerId>,
     /// DataByHash rate limiter.
     data_by_hash_rl: Limiter<PeerId>,
+    /// AnnounceFile rate limiter.
+    announce_file_rl: Limiter<PeerId>,
     /// GetChunks rate limiter.
     get_chunks_rl: Limiter<PeerId>,
 }
@@ -91,6 +93,8 @@ pub struct RPCRateLimiterBuilder {
     status_quota: Option<Quota>,
     /// Quota for the DataByHash protocol.
     data_by_hash_quota: Option<Quota>,
+    /// Quota for the AnnounceFile protocol.
+    announce_file_quota: Option<Quota>,
     /// Quota for the GetChunks protocol.
     get_chunks_quota: Option<Quota>,
 }
@@ -109,6 +113,7 @@ impl RPCRateLimiterBuilder {
             Protocol::Status => self.status_quota = q,
             Protocol::Goodbye => self.goodbye_quota = q,
             Protocol::DataByHash => self.data_by_hash_quota = q,
+            Protocol::AnnounceFile => self.announce_file_quota = q,
             Protocol::GetChunks => self.get_chunks_quota = q,
         }
         self
@@ -145,6 +150,9 @@ impl RPCRateLimiterBuilder {
         let data_by_hash_quota = self
             .data_by_hash_quota
             .ok_or("DataByHash quota not specified")?;
+        let announce_file_quota = self
+            .announce_file_quota
+            .ok_or("AnnounceFile quota not specified")?;
         let get_chunks_quota = self
             .get_chunks_quota
             .ok_or("GetChunks quota not specified")?;
@@ -154,6 +162,7 @@ impl RPCRateLimiterBuilder {
         let status_rl = Limiter::from_quota(status_quota)?;
         let goodbye_rl = Limiter::from_quota(goodbye_quota)?;
         let data_by_hash_rl = Limiter::from_quota(data_by_hash_quota)?;
+        let announce_file_rl = Limiter::from_quota(announce_file_quota)?;
         let get_chunks_rl = Limiter::from_quota(get_chunks_quota)?;
 
         // check for peers to prune every 30 seconds, starting in 30 seconds
@@ -166,6 +175,7 @@ impl RPCRateLimiterBuilder {
             status_rl,
             goodbye_rl,
             data_by_hash_rl,
+            announce_file_rl,
             get_chunks_rl,
             init_time: Instant::now(),
         })
@@ -210,6 +220,7 @@ impl RPCRateLimiter {
             Protocol::Status => &mut self.status_rl,
             Protocol::Goodbye => &mut self.goodbye_rl,
             Protocol::DataByHash => &mut self.data_by_hash_rl,
+            Protocol::AnnounceFile => &mut self.announce_file_rl,
             Protocol::GetChunks => &mut self.get_chunks_rl,
         };
         check(limiter)

node/network/src/service.rs

@@ -4,7 +4,7 @@ use crate::discovery::enr;
 use crate::multiaddr::Protocol;
 use crate::rpc::{GoodbyeReason, RPCResponseErrorCode, ReqId};
 use crate::types::{error, GossipKind};
-use crate::{EnrExt, NetworkMessage};
+use crate::{EnrExt, NetworkSender};
 use crate::{NetworkConfig, NetworkGlobals, PeerAction, ReportSource};
 use futures::prelude::*;
 use libp2p::core::{
@@ -21,7 +21,6 @@ use std::io::prelude::*;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::mpsc::UnboundedSender;
 
 use crate::peer_manager::{MIN_OUTBOUND_ONLY_FACTOR, PEER_EXCESS_FACTOR, PRIORITY_PEER_EXCESS};
 
@@ -60,7 +59,7 @@ pub struct Context<'a> {
 impl<AppReqId: ReqId> Service<AppReqId> {
     pub async fn new(
         executor: task_executor::TaskExecutor,
-        network_sender: UnboundedSender<NetworkMessage>,
+        network_sender: NetworkSender,
         ctx: Context<'_>,
     ) -> error::Result<(Arc<NetworkGlobals>, Keypair, Self)> {
         trace!("Libp2p Service starting");

node/network/src/types/mod.rs

@@ -7,7 +7,7 @@ pub type Enr = discv5::enr::Enr<discv5::enr::CombinedKey>;
 
 pub use globals::NetworkGlobals;
 pub use pubsub::{
-    AnnounceChunks, AnnounceFile, AnnounceShardConfig, FindChunks, FindFile, HasSignature,
+    AnnounceChunks, AnnounceFile, AnnounceShardConfig, FindChunks, FindFile, HasSignature, NewFile,
     PubsubMessage, SignedAnnounceChunks, SignedAnnounceFile, SignedAnnounceShardConfig,
     SignedMessage, SnappyTransform,
 };

node/network/src/types/pubsub.rs

@@ -114,9 +114,22 @@ impl ssz::Decode for WrappedPeerId {
     }
 }
 
+/// Published when file uploaded or completed to sync from other peers.
+#[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)]
+pub struct NewFile {
+    pub tx_id: TxID,
+    pub num_shard: usize,
+    pub shard_id: usize,
+    pub timestamp: u32,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)]
 pub struct FindFile {
     pub tx_id: TxID,
+    pub num_shard: usize,
+    pub shard_id: usize,
+    /// Indicates whether publish to neighboar nodes only.
+    pub neighbors_only: bool,
     pub timestamp: u32,
 }
 
@@ -205,6 +218,7 @@ type SignedAnnounceFiles = Vec<SignedAnnounceFile>;
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum PubsubMessage {
     ExampleMessage(u64),
+    NewFile(NewFile),
     FindFile(FindFile),
     FindChunks(FindChunks),
     AnnounceFile(Vec<SignedAnnounceFile>),
@@ -283,6 +297,7 @@ impl PubsubMessage {
     pub fn kind(&self) -> GossipKind {
         match self {
             PubsubMessage::ExampleMessage(_) => GossipKind::Example,
+            PubsubMessage::NewFile(_) => GossipKind::NewFile,
             PubsubMessage::FindFile(_) => GossipKind::FindFile,
             PubsubMessage::FindChunks(_) => GossipKind::FindChunks,
             PubsubMessage::AnnounceFile(_) => GossipKind::AnnounceFile,
@@ -309,6 +324,9 @@ impl PubsubMessage {
                     GossipKind::Example => Ok(PubsubMessage::ExampleMessage(
                         u64::from_ssz_bytes(data).map_err(|e| format!("{:?}", e))?,
                     )),
+                    GossipKind::NewFile => Ok(PubsubMessage::NewFile(
+                        NewFile::from_ssz_bytes(data).map_err(|e| format!("{:?}", e))?,
+                    )),
                     GossipKind::FindFile => Ok(PubsubMessage::FindFile(
                         FindFile::from_ssz_bytes(data).map_err(|e| format!("{:?}", e))?,
                     )),
@@ -341,6 +359,7 @@ impl PubsubMessage {
         // messages for us.
         match &self {
             PubsubMessage::ExampleMessage(data) => data.as_ssz_bytes(),
+            PubsubMessage::NewFile(data) => data.as_ssz_bytes(),
             PubsubMessage::FindFile(data) => data.as_ssz_bytes(),
             PubsubMessage::FindChunks(data) => data.as_ssz_bytes(),
             PubsubMessage::AnnounceFile(data) => data.as_ssz_bytes(),
@@ -356,6 +375,9 @@ impl std::fmt::Display for PubsubMessage {
             PubsubMessage::ExampleMessage(msg) => {
                 write!(f, "Example message: {}", msg)
             }
+            PubsubMessage::NewFile(msg) => {
+                write!(f, "NewFile message: {:?}", msg)
+            }
             PubsubMessage::FindFile(msg) => {
                 write!(f, "FindFile message: {:?}", msg)
             }

node/network/src/types/topics.rs

@@ -8,13 +8,15 @@ use strum::AsRefStr;
 pub const TOPIC_PREFIX: &str = "eth2";
 pub const SSZ_SNAPPY_ENCODING_POSTFIX: &str = "ssz_snappy";
 pub const EXAMPLE_TOPIC: &str = "example";
+pub const NEW_FILE_TOPIC: &str = "new_file";
 pub const FIND_FILE_TOPIC: &str = "find_file";
 pub const FIND_CHUNKS_TOPIC: &str = "find_chunks";
 pub const ANNOUNCE_FILE_TOPIC: &str = "announce_file";
 pub const ANNOUNCE_CHUNKS_TOPIC: &str = "announce_chunks";
 pub const ANNOUNCE_SHARD_CONFIG_TOPIC: &str = "announce_shard_config";
 
-pub const CORE_TOPICS: [GossipKind; 4] = [
+pub const CORE_TOPICS: [GossipKind; 5] = [
+    GossipKind::NewFile,
     GossipKind::FindFile,
     GossipKind::FindChunks,
     GossipKind::AnnounceFile,
@@ -37,6 +39,7 @@ pub struct GossipTopic {
 #[strum(serialize_all = "snake_case")]
 pub enum GossipKind {
     Example,
+    NewFile,
     FindFile,
     FindChunks,
     AnnounceFile,
@@ -77,6 +80,7 @@ impl GossipTopic {
 
             let kind = match topic_parts[2] {
                 EXAMPLE_TOPIC => GossipKind::Example,
+                NEW_FILE_TOPIC => GossipKind::NewFile,
                 FIND_FILE_TOPIC => GossipKind::FindFile,
                 FIND_CHUNKS_TOPIC => GossipKind::FindChunks,
                 ANNOUNCE_FILE_TOPIC => GossipKind::AnnounceFile,
@@ -106,6 +110,7 @@ impl From<GossipTopic> for String {
 
         let kind = match topic.kind {
             GossipKind::Example => EXAMPLE_TOPIC,
+            GossipKind::NewFile => NEW_FILE_TOPIC,
             GossipKind::FindFile => FIND_FILE_TOPIC,
             GossipKind::FindChunks => FIND_CHUNKS_TOPIC,
             GossipKind::AnnounceFile => ANNOUNCE_FILE_TOPIC,
@@ -125,6 +130,7 @@ impl std::fmt::Display for GossipTopic {
 
         let kind = match self.kind {
             GossipKind::Example => EXAMPLE_TOPIC,
+            GossipKind::NewFile => NEW_FILE_TOPIC,
             GossipKind::FindFile => FIND_FILE_TOPIC,
             GossipKind::FindChunks => FIND_CHUNKS_TOPIC,
             GossipKind::AnnounceFile => ANNOUNCE_FILE_TOPIC,

node/network/tests/common/mod.rs

@@ -1,6 +1,7 @@
 #![cfg(test)]
 
 use libp2p::gossipsub::GossipsubConfigBuilder;
+use network::new_network_channel;
 use network::Enr;
 use network::EnrExt;
 use network::Multiaddr;
@@ -22,7 +23,6 @@ pub mod swarm;
 type ReqId = usize;
 
 use tempfile::Builder as TempBuilder;
-use tokio::sync::mpsc::unbounded_channel;
 
 #[allow(unused)]
 pub struct Libp2pInstance(LibP2PService<ReqId>, exit_future::Signal);
@@ -72,7 +72,7 @@ pub async fn build_libp2p_instance(rt: Weak<Runtime>, boot_nodes: Vec<Enr>) -> L
     let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
     let executor = task_executor::TaskExecutor::new(rt, exit, shutdown_tx);
     let libp2p_context = network::Context { config: &config };
-    let (sender, _) = unbounded_channel();
+    let (sender, _) = new_network_channel();
     Libp2pInstance(
         LibP2PService::new(executor, sender, libp2p_context)
             .await

node/pruner/src/lib.rs

@@ -11,7 +11,7 @@ use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
 use storage::config::{ShardConfig, SHARD_CONFIG_KEY};
-use storage::log_store::log_manager::PORA_CHUNK_SIZE;
+use storage::log_store::log_manager::{DATA_DB_KEY, PORA_CHUNK_SIZE};
 use storage_async::Store;
 use task_executor::TaskExecutor;
 use tokio::sync::{broadcast, mpsc};
@@ -223,7 +223,8 @@ impl Pruner {
     }
 
     async fn prune_tx(&mut self, start_sector: u64, end_sector: u64) -> Result<()> {
-        while let Some(tx) = self.store.get_tx_by_seq_number(self.first_tx_seq).await? {
+        loop {
+            if let Some(tx) = self.store.get_tx_by_seq_number(self.first_tx_seq).await? {
                 // If a part of the tx data is pruned, we mark the tx as pruned.
                 if tx.start_entry_index() >= start_sector && tx.start_entry_index() < end_sector {
                     self.store.prune_tx(tx.seq).await?;
@@ -238,6 +239,10 @@ impl Pruner {
                     );
                 }
                 self.first_tx_seq += 1;
+            } else {
+                // Wait for `first_tx_seq` to be processed.
+                tokio::time::sleep(Duration::from_secs(60)).await;
+            }
         }
         Ok(())
     }
@@ -252,7 +257,7 @@ impl Pruner {
             .update_shard_config(self.config.shard_config)
             .await;
         self.store
-            .set_config_encoded(&SHARD_CONFIG_KEY, &self.config.shard_config)
+            .set_config_encoded(&SHARD_CONFIG_KEY, &self.config.shard_config, DATA_DB_KEY)
             .await
     }
 
@@ -265,17 +270,22 @@ impl Pruner {
             .set_config_encoded(
                 &FIRST_REWARDABLE_CHUNK_KEY,
                 &(new_first_rewardable_chunk, new_first_tx_seq),
+                DATA_DB_KEY,
             )
             .await
     }
 }
 
 async fn get_shard_config(store: &Store) -> Result<Option<ShardConfig>> {
-    store.get_config_decoded(&SHARD_CONFIG_KEY).await
+    store
+        .get_config_decoded(&SHARD_CONFIG_KEY, DATA_DB_KEY)
+        .await
 }
 
 async fn get_first_rewardable_chunk(store: &Store) -> Result<Option<(u64, u64)>> {
-    store.get_config_decoded(&FIRST_REWARDABLE_CHUNK_KEY).await
+    store
+        .get_config_decoded(&FIRST_REWARDABLE_CHUNK_KEY, DATA_DB_KEY)
+        .await
 }
 
 #[derive(Debug)]

node/router/src/lib.rs

@@ -10,7 +10,7 @@ mod service;
 use duration_str::deserialize_duration;
 use network::Multiaddr;
 use serde::Deserialize;
-use std::time::Duration;
+use std::{net::IpAddr, time::Duration};
 
 pub use crate::service::RouterService;
 
@@ -26,6 +26,7 @@ pub struct Config {
     pub libp2p_nodes: Vec<Multiaddr>,
     pub private_ip_enabled: bool,
     pub check_announced_ip: bool,
+    pub public_address: Option<IpAddr>,
 
     // batcher
     /// Timeout to publish messages in batch
@@ -47,6 +48,7 @@ impl Default for Config {
             libp2p_nodes: vec![],
             private_ip_enabled: false,
             check_announced_ip: false,
+            public_address: None,
 
             batcher_timeout: Duration::from_secs(1),
             batcher_file_capacity: 1,

node/router/src/libp2p_event_handler.rs

@@ -5,7 +5,8 @@ use std::{ops::Neg, sync::Arc};
 use chunk_pool::ChunkPoolMessage;
 use file_location_cache::FileLocationCache;
 use network::multiaddr::Protocol;
-use network::types::{AnnounceShardConfig, SignedAnnounceShardConfig};
+use network::rpc::methods::FileAnnouncement;
+use network::types::{AnnounceShardConfig, NewFile, SignedAnnounceShardConfig};
 use network::{
     rpc::StatusMessage,
     types::{
@@ -15,7 +16,7 @@ use network::{
     Keypair, MessageAcceptance, MessageId, NetworkGlobals, NetworkMessage, PeerId, PeerRequestId,
     PublicKey, PubsubMessage, Request, RequestId, Response,
 };
-use network::{Multiaddr, PeerAction, ReportSource};
+use network::{Multiaddr, NetworkSender, PeerAction, ReportSource};
 use shared_types::{bytes_to_chunks, timestamp_now, NetworkIdentity, TxID};
 use storage::config::ShardConfig;
 use storage_async::Store;
@@ -29,6 +30,11 @@ use crate::peer_manager::PeerManager;
 use crate::Config;
 
 lazy_static::lazy_static! {
+    /// Timeout to publish NewFile message to neighbor nodes.
+    pub static ref NEW_FILE_TIMEOUT: chrono::Duration = chrono::Duration::seconds(30);
+    /// Timeout to publish FindFile message to neighbor nodes.
+    pub static ref FIND_FILE_NEIGHBORS_TIMEOUT: chrono::Duration = chrono::Duration::seconds(30);
+    /// Timeout to publish FindFile message in the whole network.
     pub static ref FIND_FILE_TIMEOUT: chrono::Duration = chrono::Duration::minutes(5);
     pub static ref ANNOUNCE_FILE_TIMEOUT: chrono::Duration = chrono::Duration::minutes(5);
     pub static ref ANNOUNCE_SHARD_CONFIG_TIMEOUT: chrono::Duration = chrono::Duration::minutes(5);
@@ -82,7 +88,7 @@ pub struct Libp2pEventHandler {
     /// A collection of global variables, accessible outside of the network service.
     network_globals: Arc<NetworkGlobals>,
     /// A channel to the router service.
-    network_send: mpsc::UnboundedSender<NetworkMessage>,
+    network_send: NetworkSender,
     /// A channel to the syncing service.
     sync_send: SyncSender,
     /// A channel to the RPC chunk pool service.
@@ -106,7 +112,7 @@ impl Libp2pEventHandler {
     pub fn new(
         config: Config,
         network_globals: Arc<NetworkGlobals>,
-        network_send: mpsc::UnboundedSender<NetworkMessage>,
+        network_send: NetworkSender,
         sync_send: SyncSender,
         chunk_pool_send: UnboundedSender<ChunkPoolMessage>,
         local_keypair: Keypair,
@@ -219,6 +225,25 @@ impl Libp2pEventHandler {
                 });
                 metrics::LIBP2P_HANDLE_GET_CHUNKS_REQUEST.mark(1);
             }
+            Request::AnnounceFile(announcement) => {
+                match ShardConfig::new(announcement.shard_id, announcement.num_shard) {
+                    Ok(v) => {
+                        self.file_location_cache.insert_peer_config(peer_id, v);
+
+                        self.send_to_sync(SyncMessage::AnnounceFile {
+                            peer_id,
+                            request_id,
+                            announcement,
+                        });
+                    }
+                    Err(_) => self.send_to_network(NetworkMessage::ReportPeer {
+                        peer_id,
+                        action: PeerAction::Fatal,
+                        source: ReportSource::RPC,
+                        msg: "Invalid shard config in AnnounceFile RPC message",
+                    }),
+                }
+            }
             Request::DataByHash(_) => {
                 // ignore
             }
@@ -316,9 +341,13 @@ impl Libp2pEventHandler {
 
         match message {
             PubsubMessage::ExampleMessage(_) => MessageAcceptance::Ignore,
+            PubsubMessage::NewFile(msg) => {
+                metrics::LIBP2P_HANDLE_PUBSUB_NEW_FILE.mark(1);
+                self.on_new_file(propagation_source, msg).await
+            }
             PubsubMessage::FindFile(msg) => {
                 metrics::LIBP2P_HANDLE_PUBSUB_FIND_FILE.mark(1);
-                self.on_find_file(msg).await
+                self.on_find_file(propagation_source, msg).await
             }
             PubsubMessage::FindChunks(msg) => {
                 metrics::LIBP2P_HANDLE_PUBSUB_FIND_CHUNKS.mark(1);
@@ -348,17 +377,83 @@ impl Libp2pEventHandler {
         }
     }
 
-    async fn get_listen_addr_or_add(&self) -> Option<Multiaddr> {
+    /// Handle NewFile pubsub message `msg` that published by `from` peer.
+    async fn on_new_file(&self, from: PeerId, msg: NewFile) -> MessageAcceptance {
+        // verify timestamp
+        let d = duration_since(
+            msg.timestamp,
+            metrics::LIBP2P_HANDLE_PUBSUB_NEW_FILE_LATENCY.clone(),
+        );
+        if d < TOLERABLE_DRIFT.neg() || d > *NEW_FILE_TIMEOUT {
+            debug!(?d, ?msg, "Invalid timestamp, ignoring NewFile message");
+            metrics::LIBP2P_HANDLE_PUBSUB_NEW_FILE_TIMEOUT.mark(1);
+            self.send_to_network(NetworkMessage::ReportPeer {
+                peer_id: from,
+                action: PeerAction::LowToleranceError,
+                source: ReportSource::Gossipsub,
+                msg: "Received out of date NewFile message",
+            });
+            return MessageAcceptance::Ignore;
+        }
+
+        // verify announced shard config
+        let announced_shard_config = match ShardConfig::new(msg.shard_id, msg.num_shard) {
+            Ok(v) => v,
+            Err(_) => return MessageAcceptance::Reject,
+        };
+
+        // ignore if shard config mismatch
+        let my_shard_config = self.store.get_store().get_shard_config();
+        if !my_shard_config.intersect(&announced_shard_config) {
+            return MessageAcceptance::Ignore;
+        }
+
+        // ignore if already exists
+        match self.store.check_tx_completed(msg.tx_id.seq).await {
+            Ok(true) => return MessageAcceptance::Ignore,
+            Ok(false) => {}
+            Err(err) => {
+                warn!(?err, tx_seq = %msg.tx_id.seq, "Failed to check tx completed");
+                return MessageAcceptance::Ignore;
+            }
+        }
+
+        // ignore if already pruned
+        match self.store.check_tx_pruned(msg.tx_id.seq).await {
+            Ok(true) => return MessageAcceptance::Ignore,
+            Ok(false) => {}
+            Err(err) => {
+                warn!(?err, tx_seq = %msg.tx_id.seq, "Failed to check tx pruned");
+                return MessageAcceptance::Ignore;
+            }
+        }
+
+        // notify sync layer to handle in advance
+        self.send_to_sync(SyncMessage::NewFile { from, msg });
+
+        MessageAcceptance::Ignore
+    }
+
+    async fn construct_announced_ip(&self) -> Option<Multiaddr> {
+        // public address configured
+        if let Some(ip) = self.config.public_address {
+            let mut addr = Multiaddr::empty();
+            addr.push(ip.into());
+            addr.push(Protocol::Tcp(self.network_globals.listen_port_tcp()));
+            return Some(addr);
+        }
+
+        // public listen address
         if let Some(addr) = self.get_listen_addr() {
             return Some(addr);
         }
 
+        // auto detect public IP address
         let ipv4_addr = public_ip::addr_v4().await?;
 
         let mut addr = Multiaddr::empty();
         addr.push(Protocol::Ip4(ipv4_addr));
         addr.push(Protocol::Tcp(self.network_globals.listen_port_tcp()));
-        addr.push(Protocol::P2p(self.network_globals.local_peer_id().into()));
 
         self.network_globals
             .listen_multiaddrs
@@ -420,7 +515,7 @@ impl Libp2pEventHandler {
 
         let peer_id = *self.network_globals.peer_id.read();
 
-        let addr = self.get_listen_addr_or_add().await?;
+        let addr = self.construct_announced_ip().await?;
 
         let timestamp = timestamp_now();
         let shard_config = self.store.get_store().get_shard_config();
@@ -452,7 +547,7 @@ impl Libp2pEventHandler {
         shard_config: ShardConfig,
     ) -> Option<PubsubMessage> {
         let peer_id = *self.network_globals.peer_id.read();
-        let addr = self.get_listen_addr_or_add().await?;
+        let addr = self.construct_announced_ip().await?;
         let timestamp = timestamp_now();
 
         let msg = AnnounceShardConfig {
@@ -476,27 +571,69 @@ impl Libp2pEventHandler {
         Some(PubsubMessage::AnnounceShardConfig(signed))
     }
 
-    async fn on_find_file(&self, msg: FindFile) -> MessageAcceptance {
-        let FindFile { tx_id, timestamp } = msg;
+    async fn on_find_file(&self, from: PeerId, msg: FindFile) -> MessageAcceptance {
+        let FindFile {
+            tx_id, timestamp, ..
+        } = msg;
 
         // verify timestamp
         let d = duration_since(
             timestamp,
             metrics::LIBP2P_HANDLE_PUBSUB_FIND_FILE_LATENCY.clone(),
         );
-        if d < TOLERABLE_DRIFT.neg() || d > *FIND_FILE_TIMEOUT {
+        let timeout = if msg.neighbors_only {
+            *FIND_FILE_NEIGHBORS_TIMEOUT
+        } else {
+            *FIND_FILE_TIMEOUT
+        };
+        if d < TOLERABLE_DRIFT.neg() || d > timeout {
             debug!(%timestamp, ?d, "Invalid timestamp, ignoring FindFile message");
             metrics::LIBP2P_HANDLE_PUBSUB_FIND_FILE_TIMEOUT.mark(1);
+            if msg.neighbors_only {
+                self.send_to_network(NetworkMessage::ReportPeer {
+                    peer_id: from,
+                    action: PeerAction::LowToleranceError,
+                    source: ReportSource::Gossipsub,
+                    msg: "Received out of date FindFile message",
+                });
+            }
             return MessageAcceptance::Ignore;
         }
 
+        // verify announced shard config
+        let announced_shard_config = match ShardConfig::new(msg.shard_id, msg.num_shard) {
+            Ok(v) => v,
+            Err(_) => return MessageAcceptance::Reject,
+        };
+
+        // handle on shard config mismatch
+        let my_shard_config = self.store.get_store().get_shard_config();
+        if !my_shard_config.intersect(&announced_shard_config) {
+            return if msg.neighbors_only {
+                MessageAcceptance::Ignore
+            } else {
+                MessageAcceptance::Accept
+            };
+        }
+
         // check if we have it
         if matches!(self.store.check_tx_completed(tx_id.seq).await, Ok(true)) {
             if let Ok(Some(tx)) = self.store.get_tx_by_seq_number(tx_id.seq).await {
                 if tx.id() == tx_id {
                     trace!(?tx_id, "Found file locally, responding to FindFile query");
 
-                    if self.publish_file(tx_id).await.is_some() {
+                    if msg.neighbors_only {
+                        // announce file via RPC to avoid flooding pubsub message
+                        self.send_to_network(NetworkMessage::SendRequest {
+                            peer_id: from,
+                            request: Request::AnnounceFile(FileAnnouncement {
+                                tx_id,
+                                num_shard: my_shard_config.num_shard,
+                                shard_id: my_shard_config.shard_id,
+                            }),
+                            request_id: RequestId::Router(Instant::now()),
+                        });
+                    } else if self.publish_file(tx_id).await.is_some() {
                         metrics::LIBP2P_HANDLE_PUBSUB_FIND_FILE_STORE.mark(1);
                         return MessageAcceptance::Ignore;
                     }
@@ -504,6 +641,11 @@ impl Libp2pEventHandler {
             }
         }
 
+        // do not forward to whole network if only find file from neighbor nodes
+        if msg.neighbors_only {
+            return MessageAcceptance::Ignore;
+        }
+
         // try from cache
         if let Some(mut msg) = self.file_location_cache.get_one(tx_id) {
             trace!(?tx_id, "Found file in cache, responding to FindFile query");
@@ -528,7 +670,7 @@ impl Libp2pEventHandler {
         index_end: u64,
     ) -> Option<PubsubMessage> {
         let peer_id = *self.network_globals.peer_id.read();
-        let addr = self.get_listen_addr_or_add().await?;
+        let addr = self.construct_announced_ip().await?;
         let timestamp = timestamp_now();
 
         let msg = AnnounceChunks {
@@ -825,7 +967,7 @@ impl Libp2pEventHandler {
         }
     }
 
-    pub async fn publish_file(&self, tx_id: TxID) -> Option<bool> {
+    async fn publish_file(&self, tx_id: TxID) -> Option<bool> {
         match self.file_batcher.write().await.add(tx_id) {
             Some(batch) => {
                 let announcement = self.construct_announce_file_message(batch).await?;
@@ -868,10 +1010,12 @@ mod tests {
     use network::{
         discovery::{CombinedKey, ConnectionId},
         discv5::enr::EnrBuilder,
+        new_network_channel,
         rpc::{GetChunksRequest, StatusMessage, SubstreamId},
         types::FindFile,
         CombinedKeyExt, Keypair, MessageAcceptance, MessageId, Multiaddr, NetworkGlobals,
-        NetworkMessage, PeerId, PubsubMessage, Request, RequestId, Response, SyncId,
+        NetworkMessage, NetworkReceiver, PeerId, PubsubMessage, Request, RequestId, Response,
+        SyncId,
     };
     use shared_types::{timestamp_now, ChunkArray, ChunkArrayWithProof, FlowRangeProof, TxID};
     use storage::{
@@ -893,8 +1037,8 @@ mod tests {
         runtime: TestRuntime,
         network_globals: Arc<NetworkGlobals>,
         keypair: Keypair,
-        network_send: mpsc::UnboundedSender<NetworkMessage>,
-        network_recv: mpsc::UnboundedReceiver<NetworkMessage>,
+        network_send: NetworkSender,
+        network_recv: NetworkReceiver,
         sync_send: SyncSender,
         sync_recv: SyncReceiver,
         chunk_pool_send: mpsc::UnboundedSender<ChunkPoolMessage>,
@@ -908,12 +1052,11 @@ mod tests {
         fn default() -> Self {
             let runtime = TestRuntime::default();
             let (network_globals, keypair) = Context::new_network_globals();
-            let (network_send, network_recv) = mpsc::unbounded_channel();
+            let (network_send, network_recv) = new_network_channel();
             let (sync_send, sync_recv) = channel::Channel::unbounded("test");
             let (chunk_pool_send, _chunk_pool_recv) = mpsc::unbounded_channel();
 
-            let executor = runtime.task_executor.clone();
-            let store = LogManager::memorydb(LogConfig::default(), executor).unwrap();
+            let store = LogManager::memorydb(LogConfig::default()).unwrap();
             Self {
                 runtime,
                 network_globals: Arc::new(network_globals),
@@ -1194,7 +1337,13 @@ mod tests {
     ) -> MessageAcceptance {
         let (alice, bob) = (PeerId::random(), PeerId::random());
         let id = MessageId::new(b"dummy message");
-        let message = PubsubMessage::FindFile(FindFile { tx_id, timestamp });
+        let message = PubsubMessage::FindFile(FindFile {
+            tx_id,
+            num_shard: 1,
+            shard_id: 0,
+            neighbors_only: false,
+            timestamp,
+        });
         handler.on_pubsub_message(alice, bob, &id, message).await
     }
 

node/router/src/metrics.rs

@@ -44,6 +44,11 @@ lazy_static::lazy_static! {
     pub static ref LIBP2P_HANDLE_RESPONSE_ERROR: Arc<dyn Meter> = register_meter_with_group("router_libp2p_handle_response_error", "qps");
     pub static ref LIBP2P_HANDLE_RESPONSE_ERROR_LATENCY: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register_with_group("router_libp2p_handle_response_error", "latency", 1024);
 
+    // libp2p_event_handler: new file
+    pub static ref LIBP2P_HANDLE_PUBSUB_NEW_FILE: Arc<dyn Meter> = register_meter_with_group("router_libp2p_handle_pubsub_new_file", "qps");
+    pub static ref LIBP2P_HANDLE_PUBSUB_NEW_FILE_LATENCY: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register_with_group("router_libp2p_handle_pubsub_new_file", "latency", 1024);
+    pub static ref LIBP2P_HANDLE_PUBSUB_NEW_FILE_TIMEOUT: Arc<dyn Meter> = register_meter_with_group("router_libp2p_handle_pubsub_new_file", "timeout");
+
     // libp2p_event_handler: find & announce file
     pub static ref LIBP2P_HANDLE_PUBSUB_FIND_FILE: Arc<dyn Meter> = register_meter_with_group("router_libp2p_handle_pubsub_find_file", "qps");
     pub static ref LIBP2P_HANDLE_PUBSUB_FIND_FILE_LATENCY: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register_with_group("router_libp2p_handle_pubsub_find_file", "latency", 1024);

node/router/src/service.rs

@@ -6,10 +6,11 @@ use file_location_cache::FileLocationCache;
 use futures::{channel::mpsc::Sender, prelude::*};
 use miner::MinerMessage;
 use network::{
-    BehaviourEvent, Keypair, Libp2pEvent, NetworkGlobals, NetworkMessage, RequestId,
-    Service as LibP2PService, Swarm,
+    types::NewFile, BehaviourEvent, Keypair, Libp2pEvent, NetworkGlobals, NetworkMessage,
+    NetworkReceiver, NetworkSender, PubsubMessage, RequestId, Service as LibP2PService, Swarm,
 };
 use pruner::PrunerMessage;
+use shared_types::timestamp_now;
 use std::sync::Arc;
 use storage::log_store::Store as LogStore;
 use storage_async::Store;
@@ -30,7 +31,7 @@ pub struct RouterService {
     network_globals: Arc<NetworkGlobals>,
 
     /// The receiver channel for Zgs to communicate with the network service.
-    network_recv: mpsc::UnboundedReceiver<NetworkMessage>,
+    network_recv: NetworkReceiver,
 
     /// The receiver channel for Zgs to communicate with the pruner service.
     pruner_recv: Option<mpsc::UnboundedReceiver<PrunerMessage>>,
@@ -44,6 +45,8 @@ pub struct RouterService {
     /// Stores potentially created UPnP mappings to be removed on shutdown. (TCP port and UDP
     /// port).
     upnp_mappings: (Option<u16>, Option<u16>),
+
+    store: Arc<dyn LogStore>,
 }
 
 impl RouterService {
@@ -52,8 +55,8 @@ impl RouterService {
         executor: task_executor::TaskExecutor,
         libp2p: LibP2PService<RequestId>,
         network_globals: Arc<NetworkGlobals>,
-        network_recv: mpsc::UnboundedReceiver<NetworkMessage>,
-        network_send: mpsc::UnboundedSender<NetworkMessage>,
+        network_recv: NetworkReceiver,
+        network_send: NetworkSender,
         sync_send: SyncSender,
         _miner_send: Option<broadcast::Sender<MinerMessage>>,
         chunk_pool_send: UnboundedSender<ChunkPoolMessage>,
@@ -63,7 +66,6 @@ impl RouterService {
         local_keypair: Keypair,
         config: Config,
     ) {
-        let store = Store::new(store, executor.clone());
         let peers = Arc::new(RwLock::new(PeerManager::new(config.clone())));
 
         // create the network service and spawn the task
@@ -81,11 +83,12 @@ impl RouterService {
                 sync_send,
                 chunk_pool_send,
                 local_keypair,
-                store,
+                Store::new(store.clone(), executor.clone()),
                 file_location_cache,
                 peers,
             ),
             upnp_mappings: (None, None),
+            store,
         };
 
         // spawn service
@@ -328,15 +331,16 @@ impl RouterService {
                 }
             }
             NetworkMessage::AnnounceLocalFile { tx_id } => {
-                if self
-                    .libp2p_event_handler
-                    .publish_file(tx_id)
-                    .await
-                    .is_some()
-                {
+                let shard_config = self.store.get_shard_config();
+                let msg = PubsubMessage::NewFile(NewFile {
+                    tx_id,
+                    num_shard: shard_config.num_shard,
+                    shard_id: shard_config.shard_id,
+                    timestamp: timestamp_now(),
+                });
+                self.libp2p.swarm.behaviour_mut().publish(vec![msg]);
                 metrics::SERVICE_ROUTE_NETWORK_MESSAGE_ANNOUNCE_LOCAL_FILE.mark(1);
             }
-            }
             NetworkMessage::UPnPMappingEstablished {
                 tcp_socket,
                 udp_socket,

node/rpc/src/lib.rs

@@ -17,15 +17,13 @@ use file_location_cache::FileLocationCache;
 use futures::channel::mpsc::Sender;
 use jsonrpsee::core::RpcResult;
 use jsonrpsee::http_server::{HttpServerBuilder, HttpServerHandle};
-use network::NetworkGlobals;
-use network::NetworkMessage;
+use network::{NetworkGlobals, NetworkMessage, NetworkSender};
 use std::error::Error;
 use std::sync::Arc;
 use storage_async::Store;
 use sync::{SyncRequest, SyncResponse, SyncSender};
 use task_executor::ShutdownReason;
 use tokio::sync::broadcast;
-use tokio::sync::mpsc::UnboundedSender;
 use zgs::RpcServer as ZgsRpcServer;
 use zgs_miner::MinerMessage;
 
@@ -42,7 +40,7 @@ pub struct Context {
     pub config: RPCConfig,
     pub file_location_cache: Arc<FileLocationCache>,
     pub network_globals: Arc<NetworkGlobals>,
-    pub network_send: UnboundedSender<NetworkMessage>,
+    pub network_send: NetworkSender,
     pub sync_send: SyncSender,
     pub chunk_pool: Arc<MemoryChunkPool>,
     pub log_store: Arc<Store>,

node/rpc/src/types.rs

@@ -53,6 +53,8 @@ pub struct FileInfo {
     pub finalized: bool,
     pub is_cached: bool,
     pub uploaded_seg_num: usize,
+    /// Whether file is pruned, in which case `finalized` will be `false`.
+    pub pruned: bool,
 }
 
 #[derive(Debug, Serialize, Deserialize)]

node/rpc/src/zgs/api.rs

@@ -2,7 +2,7 @@ use crate::types::{FileInfo, Segment, SegmentWithProof, Status};
 use jsonrpsee::core::RpcResult;
 use jsonrpsee::proc_macros::rpc;
 use shared_types::{DataRoot, FlowProof, TxSeqOrRoot};
-use storage::config::ShardConfig;
+use storage::{config::ShardConfig, H256};
 
 #[rpc(server, client, namespace = "zgs")]
 pub trait Rpc {
@@ -77,4 +77,7 @@ pub trait Rpc {
         sector_index: u64,
         flow_root: Option<DataRoot>,
     ) -> RpcResult<FlowProof>;
+
+    #[method(name = "getFlowContext")]
+    async fn get_flow_context(&self) -> RpcResult<(H256, u64)>;
 }

node/rpc/src/zgs/impl.rs

@@ -8,7 +8,8 @@ use jsonrpsee::core::RpcResult;
 use shared_types::{DataRoot, FlowProof, Transaction, TxSeqOrRoot, CHUNK_SIZE};
 use std::fmt::{Debug, Formatter, Result};
 use storage::config::ShardConfig;
-use storage::try_option;
+use storage::log_store::tx_store::TxStatus;
+use storage::{try_option, H256};
 
 pub struct RpcServerImpl {
     pub ctx: Context,
@@ -198,6 +199,10 @@ impl RpcServer for RpcServerImpl {
         assert_eq!(proof.left_proof, proof.right_proof);
         Ok(proof.right_proof)
     }
+
+    async fn get_flow_context(&self) -> RpcResult<(H256, u64)> {
+        Ok(self.ctx.log_store.get_context().await?)
+    }
 }
 
 impl RpcServerImpl {
@@ -241,7 +246,12 @@ impl RpcServerImpl {
     }
 
     async fn get_file_info_by_tx(&self, tx: Transaction) -> RpcResult<FileInfo> {
-        let finalized = self.ctx.log_store.check_tx_completed(tx.seq).await?;
+        let (finalized, pruned) = match self.ctx.log_store.get_store().get_tx_status(tx.seq)? {
+            Some(TxStatus::Finalized) => (true, false),
+            Some(TxStatus::Pruned) => (false, true),
+            None => (false, false),
+        };
+
         let (uploaded_seg_num, is_cached) = match self
             .ctx
             .chunk_pool
@@ -250,7 +260,7 @@ impl RpcServerImpl {
         {
             Some(v) => v,
             _ => (
-                if finalized {
+                if finalized || pruned {
                     let chunks_per_segment = self.ctx.config.chunks_per_segment;
                     let (num_segments, _) = SegmentWithProof::split_file_into_segments(
                         tx.size as usize,
@@ -269,6 +279,7 @@ impl RpcServerImpl {
             finalized,
             is_cached,
             uploaded_seg_num,
+            pruned,
         })
     }
 

node/shared_types/src/lib.rs

@@ -9,13 +9,11 @@ use merkle_light::merkle::MerkleTree;
 use merkle_light::proof::Proof as RawFileProof;
 use merkle_light::{hash::Algorithm, merkle::next_pow2};
 use merkle_tree::RawLeafSha3Algorithm;
-use serde::de::Visitor;
 use serde::{Deserialize, Serialize};
 use ssz::Encode;
 use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode};
 use std::fmt;
 use std::hash::Hasher;
-use std::str::FromStr;
 use tiny_keccak::{Hasher as KeccakHasher, Keccak};
 use tracing::debug;
 
@@ -398,82 +396,39 @@ pub struct ProtocolVersion {
     pub build: u8,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(untagged)]
 pub enum TxSeqOrRoot {
     TxSeq(u64),
     Root(DataRoot),
 }
 
-impl Serialize for TxSeqOrRoot {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        match self {
-            TxSeqOrRoot::TxSeq(seq) => seq.serialize(serializer),
-            TxSeqOrRoot::Root(root) => root.serialize(serializer),
-        }
-    }
-}
-
-impl<'a> Deserialize<'a> for TxSeqOrRoot {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'a>,
-    {
-        deserializer.deserialize_any(TxSeqOrRootVisitor)
-    }
-}
-
-struct TxSeqOrRootVisitor;
-
-impl<'a> Visitor<'a> for TxSeqOrRootVisitor {
-    type Value = TxSeqOrRoot;
-
-    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(formatter, "an u64 integer or a hex64 value")
-    }
-
-    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(TxSeqOrRoot::TxSeq(v))
-    }
-
-    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        let root: H256 = H256::from_str(v).map_err(E::custom)?;
-        Ok(TxSeqOrRoot::Root(root))
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    use std::str::FromStr;
+
     use super::*;
 
     #[test]
     fn test_tx_seq_or_root_serde() {
-        // serialize tx seq
+        // serialize tx seq as number
         let tx_seq = TxSeqOrRoot::TxSeq(666);
         assert_eq!(serde_json::to_string(&tx_seq).unwrap(), "666".to_string());
 
-        // serialize root
+        // serialize root as quoted string
         let hash_str = "0xa906f46f8b9f15908dbee7adc5492ff30779c3abe114ccdb7079ecdcb72eb855";
         let hash_quoted = format!("\"{}\"", hash_str);
         let hash = H256::from_str(hash_str).unwrap();
         let root = TxSeqOrRoot::Root(hash);
         assert_eq!(serde_json::to_string(&root).unwrap(), hash_quoted);
 
-        // deserialize tx seq
+        // deserialize tx seq from number
         assert!(matches!(
             serde_json::from_str::<TxSeqOrRoot>("777").unwrap(),
             TxSeqOrRoot::TxSeq(777)
         ));
 
-        // deserialize root
+        // deserialize root from quoted string
         assert!(matches!(
             serde_json::from_str::<TxSeqOrRoot>(hash_quoted.as_str()).unwrap(),
             TxSeqOrRoot::Root(v) if v == hash,

node/src/client/builder.rs

@@ -1,11 +1,11 @@
 use super::{Client, RuntimeContext};
-use chunk_pool::{ChunkPoolMessage, Config as ChunkPoolConfig, MemoryChunkPool};
+use chunk_pool::{Config as ChunkPoolConfig, MemoryChunkPool};
 use file_location_cache::FileLocationCache;
 use log_entry_sync::{LogSyncConfig, LogSyncEvent, LogSyncManager};
-use miner::{MineService, MinerConfig, MinerMessage};
+use miner::{MineService, MinerConfig, MinerMessage, ShardConfig};
 use network::{
-    self, Keypair, NetworkConfig, NetworkGlobals, NetworkMessage, RequestId,
-    Service as LibP2PService,
+    self, new_network_channel, Keypair, NetworkConfig, NetworkGlobals, NetworkReceiver,
+    NetworkSender, RequestId, Service as LibP2PService,
 };
 use pruner::{Pruner, PrunerConfig, PrunerMessage};
 use router::RouterService;
@@ -27,15 +27,12 @@ macro_rules! require {
 }
 
 struct NetworkComponents {
-    send: mpsc::UnboundedSender<NetworkMessage>,
+    send: NetworkSender,
     globals: Arc<NetworkGlobals>,
     keypair: Keypair,
 
     // note: these will be owned by the router service
-    owned: Option<(
-        LibP2PService<RequestId>,
-        mpsc::UnboundedReceiver<NetworkMessage>,
-    )>,
+    owned: Option<(LibP2PService<RequestId>, NetworkReceiver)>,
 }
 
 struct SyncComponents {
@@ -57,7 +54,7 @@ struct PrunerComponents {
 }
 
 struct ChunkPoolComponents {
-    send: mpsc::UnboundedSender<ChunkPoolMessage>,
+    chunk_pool: Arc<MemoryChunkPool>,
 }
 
 /// Builds a `Client` instance.
@@ -89,10 +86,9 @@ impl ClientBuilder {
 
     /// Initializes in-memory storage.
     pub fn with_memory_store(mut self) -> Result<Self, String> {
-        let executor = require!("sync", self, runtime_context).clone().executor;
         // TODO(zz): Set config.
         let store = Arc::new(
-            LogManager::memorydb(LogConfig::default(), executor)
+            LogManager::memorydb(LogConfig::default())
                 .map_err(|e| format!("Unable to start in-memory store: {:?}", e))?,
         );
 
@@ -110,9 +106,12 @@ impl ClientBuilder {
 
     /// Initializes RocksDB storage.
     pub fn with_rocksdb_store(mut self, config: &StorageConfig) -> Result<Self, String> {
-        let executor = require!("sync", self, runtime_context).clone().executor;
         let store = Arc::new(
-            LogManager::rocksdb(LogConfig::default(), &config.db_dir, executor)
+            LogManager::rocksdb(
+                config.log_config.clone(),
+                config.db_dir.join("flow_db"),
+                config.db_dir.join("data_db"),
+            )
             .map_err(|e| format!("Unable to start RocksDB store: {:?}", e))?,
         );
 
@@ -142,7 +141,7 @@ impl ClientBuilder {
         let service_context = network::Context { config };
 
         // construct communication channel
-        let (send, recv) = mpsc::unbounded_channel::<NetworkMessage>();
+        let (send, recv) = new_network_channel();
 
         // launch libp2p service
         let (globals, keypair, libp2p) =
@@ -216,12 +215,22 @@ impl ClientBuilder {
         Ok(self)
     }
 
+    pub async fn with_shard(self, config: ShardConfig) -> Result<Self, String> {
+        self.async_store
+            .as_ref()
+            .unwrap()
+            .update_shard_config(config)
+            .await;
+
+        Ok(self)
+    }
+
     /// Starts the networking stack.
     pub fn with_router(mut self, router_config: router::Config) -> Result<Self, String> {
         let executor = require!("router", self, runtime_context).clone().executor;
         let sync_send = require!("router", self, sync).send.clone(); // note: we can make this optional in the future
         let miner_send = self.miner.as_ref().map(|x| x.send.clone());
-        let chunk_pool_send = require!("router", self, chunk_pool).send.clone();
+        let chunk_pool_send = require!("router", self, chunk_pool).chunk_pool.sender();
         let store = require!("router", self, store).clone();
         let file_location_cache = require!("router", self, file_location_cache).clone();
 
@@ -251,11 +260,7 @@ impl ClientBuilder {
         Ok(self)
     }
 
-    pub async fn with_rpc(
-        mut self,
-        rpc_config: RPCConfig,
-        chunk_pool_config: ChunkPoolConfig,
-    ) -> Result<Self, String> {
+    pub async fn with_rpc(self, rpc_config: RPCConfig) -> Result<Self, String> {
         if !rpc_config.enabled {
             return Ok(self);
         }
@@ -264,16 +269,9 @@ impl ClientBuilder {
         let async_store = require!("rpc", self, async_store).clone();
         let network_send = require!("rpc", self, network).send.clone();
         let mine_send = self.miner.as_ref().map(|x| x.send.clone());
-        let synced_tx_recv = require!("rpc", self, log_sync).send.subscribe();
         let file_location_cache = require!("rpc", self, file_location_cache).clone();
+        let chunk_pool = require!("rpc", self, chunk_pool).chunk_pool.clone();
 
-        let (chunk_pool, chunk_pool_handler) =
-            chunk_pool::unbounded(chunk_pool_config, async_store.clone(), network_send.clone());
-        let chunk_pool_components = ChunkPoolComponents {
-            send: chunk_pool.sender(),
-        };
-
-        let chunk_pool_clone = chunk_pool.clone();
         let ctx = rpc::Context {
             config: rpc_config,
             file_location_cache,
@@ -286,7 +284,7 @@ impl ClientBuilder {
             mine_service_sender: mine_send,
         };
 
-        let (rpc_handle, maybe_admin_rpc_handle) = rpc::run_server(ctx.clone())
+        let (rpc_handle, maybe_admin_rpc_handle) = rpc::run_server(ctx)
             .await
             .map_err(|e| format!("Unable to start HTTP RPC server: {:?}", e))?;
 
@@ -294,13 +292,29 @@ impl ClientBuilder {
         if let Some(admin_rpc_handle) = maybe_admin_rpc_handle {
             executor.spawn(admin_rpc_handle, "rpc_admin");
         }
+
+        Ok(self)
+    }
+
+    pub async fn with_chunk_pool(
+        mut self,
+        chunk_pool_config: ChunkPoolConfig,
+    ) -> Result<Self, String> {
+        let executor = require!("rpc", self, runtime_context).clone().executor;
+        let async_store = require!("rpc", self, async_store).clone();
+        let network_send = require!("rpc", self, network).send.clone();
+        let synced_tx_recv = require!("rpc", self, log_sync).send.subscribe();
+
+        let (chunk_pool, chunk_pool_handler) =
+            chunk_pool::unbounded(chunk_pool_config, async_store.clone(), network_send.clone());
+
         executor.spawn(chunk_pool_handler.run(), "chunk_pool_handler");
         executor.spawn(
-            MemoryChunkPool::monitor_log_entry(chunk_pool_clone, synced_tx_recv),
+            MemoryChunkPool::monitor_log_entry(chunk_pool.clone(), synced_tx_recv),
             "chunk_pool_log_monitor",
         );
 
-        self.chunk_pool = Some(chunk_pool_components);
+        self.chunk_pool = Some(ChunkPoolComponents { chunk_pool });
 
         Ok(self)
     }

node/src/config/convert.rs

@@ -5,12 +5,14 @@ use ethereum_types::{H256, U256};
 use ethers::prelude::{Http, Middleware, Provider};
 use log_entry_sync::{CacheConfig, ContractAddress, LogSyncConfig};
 use miner::MinerConfig;
-use network::NetworkConfig;
+use network::{EnrExt, NetworkConfig};
 use pruner::PrunerConfig;
 use shared_types::{NetworkIdentity, ProtocolVersion};
 use std::net::IpAddr;
+use std::sync::Arc;
 use std::time::Duration;
 use storage::config::ShardConfig;
+use storage::log_store::log_manager::LogConfig;
 use storage::StorageConfig;
 
 impl ZgsConfig {
@@ -37,15 +39,21 @@ impl ZgsConfig {
             .await
             .map_err(|e| format!("Unable to get chain id: {:?}", e))?
             .as_u64();
-        network_config.network_id = NetworkIdentity {
+        let network_protocol_version = if self.sync.neighbors_only {
+            network::PROTOCOL_VERSION_V2
+        } else {
+            network::PROTOCOL_VERSION_V1
+        };
+        let local_network_id = NetworkIdentity {
             chain_id,
             flow_address,
             p2p_protocol_version: ProtocolVersion {
-                major: network::PROTOCOL_VERSION[0],
-                minor: network::PROTOCOL_VERSION[1],
-                build: network::PROTOCOL_VERSION[2],
+                major: network_protocol_version[0],
+                minor: network_protocol_version[1],
+                build: network_protocol_version[2],
             },
         };
+        network_config.network_id = local_network_id.clone();
 
         if !self.network_disable_discovery {
             network_config.enr_tcp_port = Some(self.network_enr_tcp_port);
@@ -81,7 +89,13 @@ impl ZgsConfig {
             .collect::<Result<_, _>>()
             .map_err(|e| format!("Unable to parse network_libp2p_nodes: {:?}", e))?;
 
-        network_config.discv5_config.table_filter = |_| true;
+        network_config.discv5_config.table_filter = if self.discv5_disable_enr_network_id {
+            Arc::new(|_| true)
+        } else {
+            Arc::new(
+                move |enr| matches!(enr.network_identity(), Some(Ok(id)) if id == local_network_id),
+            )
+        };
         network_config.discv5_config.request_timeout =
             Duration::from_secs(self.discv5_request_timeout_secs);
         network_config.discv5_config.query_peer_timeout =
@@ -96,13 +110,18 @@ impl ZgsConfig {
         network_config.private = self.network_private;
 
         network_config.peer_db = self.network_peer_db;
+        network_config.peer_manager = self.network_peer_manager;
+        network_config.disable_enr_network_id = self.discv5_disable_enr_network_id;
 
         Ok(network_config)
     }
 
     pub fn storage_config(&self) -> Result<StorageConfig, String> {
+        let mut log_config = LogConfig::default();
+        log_config.flow.merkle_node_cache_capacity = self.merkle_node_cache_capacity;
         Ok(StorageConfig {
             db_dir: self.db_dir.clone().into(),
+            log_config,
         })
     }
 
@@ -132,6 +151,7 @@ impl ZgsConfig {
             self.remove_finalized_block_interval_minutes,
             self.watch_loop_wait_time_ms,
             self.force_log_sync_from_start_block_number,
+            Duration::from_secs(self.blockchain_rpc_timeout_secs),
         ))
     }
 
@@ -200,6 +220,13 @@ impl ZgsConfig {
     pub fn router_config(&self, network_config: &NetworkConfig) -> Result<router::Config, String> {
         let mut router_config = self.router.clone();
         router_config.libp2p_nodes = network_config.libp2p_nodes.to_vec();
+
+        if router_config.public_address.is_none() {
+            if let Some(addr) = &self.network_enr_address {
+                router_config.public_address = Some(addr.parse().unwrap());
+            }
+        }
+
         Ok(router_config)
     }
 
@@ -228,7 +255,7 @@ impl ZgsConfig {
         }
     }
 
-    fn shard_config(&self) -> Result<ShardConfig, String> {
+    pub fn shard_config(&self) -> Result<ShardConfig, String> {
         self.shard_position.clone().try_into()
     }
 }

node/src/config/mod.rs

@@ -28,6 +28,7 @@ build_config! {
     (discv5_report_discovered_peers, (bool), false)
     (discv5_disable_packet_filter, (bool), false)
     (discv5_disable_ip_limit, (bool), false)
+    (discv5_disable_enr_network_id, (bool), false)
 
     // log sync
     (blockchain_rpc_endpoint, (String), "http://127.0.0.1:8545".to_string())
@@ -48,6 +49,8 @@ build_config! {
     (remove_finalized_block_interval_minutes, (u64), 30)
     (watch_loop_wait_time_ms, (u64), 500)
 
+    (blockchain_rpc_timeout_secs, (u64), 120)
+
     // chunk pool
     (chunk_pool_write_window_size, (usize), 4)
     (chunk_pool_max_cached_chunks_all, (usize), 4*1024*1024)    // 1G
@@ -60,6 +63,7 @@ build_config! {
     (prune_check_time_s, (u64), 60)
     (prune_batch_size, (usize), 16 * 1024)
     (prune_batch_wait_time_ms, (u64), 1000)
+    (merkle_node_cache_capacity, (usize), 32 * 1024 * 1024)
 
     // misc
     (log_config_file, (String), "log_config".to_string())
@@ -86,6 +90,9 @@ pub struct ZgsConfig {
     /// Network peer db config, configured by [network_peer_db] section by `config` crate.
     pub network_peer_db: network::peer_manager::peerdb::PeerDBConfig,
 
+    /// Network peer manager config, configured by [network_peer_manager] section by `config` crate.
+    pub network_peer_manager: network::peer_manager::config::Config,
+
     // router config, configured by [router] section by `config` crate.
     pub router: router::Config,
 

node/src/main.rs

@@ -14,9 +14,11 @@ async fn start_node(context: RuntimeContext, config: ZgsConfig) -> Result<Client
     let network_config = config.network_config().await?;
     let storage_config = config.storage_config()?;
     let log_sync_config = config.log_sync_config()?;
+    let chunk_pool_config = config.chunk_pool_config()?;
     let miner_config = config.mine_config()?;
     let router_config = config.router_config(&network_config)?;
     let pruner_config = config.pruner_config()?;
+    let shard_config = config.shard_config()?;
 
     ClientBuilder::default()
         .with_runtime_context(context)
@@ -26,13 +28,17 @@ async fn start_node(context: RuntimeContext, config: ZgsConfig) -> Result<Client
         .with_file_location_cache(config.file_location_cache)
         .with_network(&network_config)
         .await?
+        .with_chunk_pool(chunk_pool_config)
+        .await?
         .with_sync(config.sync)
         .await?
         .with_miner(miner_config)
         .await?
+        .with_shard(shard_config)
+        .await?
         .with_pruner(pruner_config)
         .await?
-        .with_rpc(config.rpc, config.chunk_pool_config()?)
+        .with_rpc(config.rpc)
         .await?
         .with_router(router_config)?
         .build()

node/storage-async/Cargo.toml

@@ -11,3 +11,4 @@ task_executor = { path = "../../common/task_executor" }
 tokio = { version = "1.19.2", features = ["sync"] }
 tracing = "0.1.35"
 eth2_ssz = "0.4.0"
+backtrace = "0.3"

node/storage-async/src/lib.rs

@@ -2,6 +2,7 @@
 extern crate tracing;
 
 use anyhow::bail;
+use backtrace::Backtrace;
 use shared_types::{
     Chunk, ChunkArray, ChunkArrayWithProof, DataRoot, FlowProof, FlowRangeProof, Transaction,
 };
@@ -74,9 +75,11 @@ impl Store {
     pub async fn get_config_decoded<K: AsRef<[u8]> + Send + Sync, T: Decode + Send + 'static>(
         &self,
         key: &K,
+        dest: &str,
     ) -> Result<Option<T>> {
         let key = key.as_ref().to_vec();
-        self.spawn(move |store| store.get_config_decoded(&key))
+        let dest = dest.to_string();
+        self.spawn(move |store| store.get_config_decoded(&key, &dest))
             .await
     }
 
@@ -84,10 +87,12 @@ impl Store {
         &self,
         key: &K,
         value: &T,
+        dest: &str,
     ) -> anyhow::Result<()> {
         let key = key.as_ref().to_vec();
         let value = value.as_ssz_bytes();
-        self.spawn(move |store| store.set_config(&key, &value))
+        let dest = dest.to_string();
+        self.spawn(move |store| store.set_config(&key, &value, &dest))
             .await
     }
 
@@ -135,6 +140,9 @@ impl Store {
     {
         let store = self.store.clone();
         let (tx, rx) = oneshot::channel();
+        let mut backtrace = Backtrace::new();
+        let frames = backtrace.frames().to_vec();
+        backtrace = frames.into();
 
         self.executor.spawn_blocking(
             move || {
@@ -142,6 +150,7 @@ impl Store {
                 let res = f(&*store);
 
                 if tx.send(res).is_err() {
+                    warn!("Backtrace: {:?}", backtrace);
                     error!("Unable to complete async storage operation: the receiver dropped");
                 }
             },

node/storage/Cargo.toml

@@ -29,8 +29,11 @@ itertools = "0.13.0"
 serde = { version = "1.0.197", features = ["derive"] }
 parking_lot = "0.12.3"
 serde_json = "1.0.127"
-tokio = { version = "1.10.0", features = ["sync"] }
+tokio = { version = "1.38.0", features = ["full"] }
 task_executor = { path = "../../common/task_executor" }
+lazy_static = "1.4.0"
+metrics = { workspace = true }
+once_cell = { version = "1.19.0", features = [] }
 
 [dev-dependencies]
 rand = "0.8.5"

node/storage/benches/benchmark.rs

@@ -14,18 +14,14 @@ use storage::{
     },
     LogManager,
 };
-use task_executor::test_utils::TestRuntime;
 
 fn write_performance(c: &mut Criterion) {
     if Path::new("db_write").exists() {
         fs::remove_dir_all("db_write").unwrap();
     }
-    let runtime = TestRuntime::default();
-
-    let executor = runtime.task_executor.clone();
 
     let store: Arc<RwLock<dyn Store>> = Arc::new(RwLock::new(
-        LogManager::rocksdb(LogConfig::default(), "db_write", executor)
+        LogManager::rocksdb(LogConfig::default(), "db_flow_write", "db_data_write")
             .map_err(|e| format!("Unable to start RocksDB store: {:?}", e))
             .unwrap(),
     ));
@@ -109,12 +105,8 @@ fn read_performance(c: &mut Criterion) {
         fs::remove_dir_all("db_read").unwrap();
     }
 
-    let runtime = TestRuntime::default();
-
-    let executor = runtime.task_executor.clone();
-
     let store: Arc<RwLock<dyn Store>> = Arc::new(RwLock::new(
-        LogManager::rocksdb(LogConfig::default(), "db_read", executor)
+        LogManager::rocksdb(LogConfig::default(), "db_flow_read", "db_data_read")
             .map_err(|e| format!("Unable to start RocksDB store: {:?}", e))
             .unwrap(),
     ));

node/storage/src/config.rs

@@ -1,3 +1,4 @@
+use crate::log_store::log_manager::LogConfig;
 use serde::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use std::{cell::RefCell, path::PathBuf, rc::Rc, str::FromStr};
@@ -7,6 +8,7 @@ pub const SHARD_CONFIG_KEY: &str = "shard_config";
 #[derive(Clone)]
 pub struct Config {
     pub db_dir: PathBuf,
+    pub log_config: LogConfig,
 }
 
 #[derive(Clone, Copy, Debug, Decode, Encode, Serialize, Deserialize, Eq, PartialEq)]

node/storage/src/log_store/config.rs

@@ -2,16 +2,55 @@ use anyhow::{anyhow, Result};
 use kvdb::{DBKey, DBOp};
 use ssz::{Decode, Encode};
 
+use crate::log_store::log_manager::{COL_MISC, DATA_DB_KEY, FLOW_DB_KEY};
 use crate::LogManager;
 
-use super::log_manager::COL_MISC;
+macro_rules! db_operation {
+    ($self:expr, $dest:expr, get, $key:expr) => {{
+        let db = match $dest {
+            DATA_DB_KEY => &$self.data_db,
+            FLOW_DB_KEY => &$self.flow_db,
+            _ => return Err(anyhow!("Invalid destination")),
+        };
+        Ok(db.get(COL_MISC, $key)?)
+    }};
+
+    ($self:expr, $dest:expr, put, $key:expr, $value:expr) => {{
+        let db = match $dest {
+            DATA_DB_KEY => &$self.data_db,
+            FLOW_DB_KEY => &$self.flow_db,
+            _ => return Err(anyhow!("Invalid destination")),
+        };
+        Ok(db.put(COL_MISC, $key, $value)?)
+    }};
+
+    ($self:expr, $dest:expr, delete, $key:expr) => {{
+        let db = match $dest {
+            DATA_DB_KEY => &$self.data_db,
+            FLOW_DB_KEY => &$self.flow_db,
+            _ => return Err(anyhow!("Invalid destination")),
+        };
+        Ok(db.delete(COL_MISC, $key)?)
+    }};
+
+    ($self:expr, $dest:expr, transaction, $tx:expr) => {{
+        let db = match $dest {
+            DATA_DB_KEY => &$self.data_db,
+            FLOW_DB_KEY => &$self.flow_db,
+            _ => return Err(anyhow!("Invalid destination")),
+        };
+        let mut db_tx = db.transaction();
+        db_tx.ops = $tx.ops;
+        Ok(db.write(db_tx)?)
+    }};
+}
 
 pub trait Configurable {
-    fn get_config(&self, key: &[u8]) -> Result<Option<Vec<u8>>>;
-    fn set_config(&self, key: &[u8], value: &[u8]) -> Result<()>;
-    fn remove_config(&self, key: &[u8]) -> Result<()>;
+    fn get_config(&self, key: &[u8], dest: &str) -> Result<Option<Vec<u8>>>;
+    fn set_config(&self, key: &[u8], value: &[u8], dest: &str) -> Result<()>;
+    fn remove_config(&self, key: &[u8], dest: &str) -> Result<()>;
 
-    fn exec_configs(&self, tx: ConfigTx) -> Result<()>;
+    fn exec_configs(&self, tx: ConfigTx, dest: &str) -> Result<()>;
 }
 
 #[derive(Default)]
@@ -41,8 +80,12 @@ impl ConfigTx {
 }
 
 pub trait ConfigurableExt: Configurable {
-    fn get_config_decoded<K: AsRef<[u8]>, T: Decode>(&self, key: &K) -> Result<Option<T>> {
-        match self.get_config(key.as_ref())? {
+    fn get_config_decoded<K: AsRef<[u8]>, T: Decode>(
+        &self,
+        key: &K,
+        dest: &str,
+    ) -> Result<Option<T>> {
+        match self.get_config(key.as_ref(), dest)? {
             Some(val) => Ok(Some(
                 T::from_ssz_bytes(&val).map_err(|e| anyhow!("SSZ decode error: {:?}", e))?,
             )),
@@ -50,36 +93,36 @@ pub trait ConfigurableExt: Configurable {
         }
     }
 
-    fn set_config_encoded<K: AsRef<[u8]>, T: Encode>(&self, key: &K, value: &T) -> Result<()> {
-        self.set_config(key.as_ref(), &value.as_ssz_bytes())
+    fn set_config_encoded<K: AsRef<[u8]>, T: Encode>(
+        &self,
+        key: &K,
+        value: &T,
+        dest: &str,
+    ) -> Result<()> {
+        self.set_config(key.as_ref(), &value.as_ssz_bytes(), dest)
     }
 
-    fn remove_config_by_key<K: AsRef<[u8]>>(&self, key: &K) -> Result<()> {
-        self.remove_config(key.as_ref())
+    fn remove_config_by_key<K: AsRef<[u8]>>(&self, key: &K, dest: &str) -> Result<()> {
+        self.remove_config(key.as_ref(), dest)
     }
 }
 
 impl<T: ?Sized + Configurable> ConfigurableExt for T {}
 
 impl Configurable for LogManager {
-    fn get_config(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
-        Ok(self.db.get(COL_MISC, key)?)
+    fn get_config(&self, key: &[u8], dest: &str) -> Result<Option<Vec<u8>>> {
+        db_operation!(self, dest, get, key)
     }
 
-    fn set_config(&self, key: &[u8], value: &[u8]) -> Result<()> {
-        self.db.put(COL_MISC, key, value)?;
-        Ok(())
+    fn set_config(&self, key: &[u8], value: &[u8], dest: &str) -> Result<()> {
+        db_operation!(self, dest, put, key, value)
     }
 
-    fn remove_config(&self, key: &[u8]) -> Result<()> {
-        Ok(self.db.delete(COL_MISC, key)?)
+    fn remove_config(&self, key: &[u8], dest: &str) -> Result<()> {
+        db_operation!(self, dest, delete, key)
     }
 
-    fn exec_configs(&self, tx: ConfigTx) -> Result<()> {
-        let mut db_tx = self.db.transaction();
-        db_tx.ops = tx.ops;
-        self.db.write(db_tx)?;
-
-        Ok(())
+    fn exec_configs(&self, tx: ConfigTx, dest: &str) -> Result<()> {
+        db_operation!(self, dest, transaction, tx)
     }
 }

node/storage/src/log_store/flow_store.rs

@@ -1,66 +1,69 @@
-use super::load_chunk::EntryBatch;
-use super::seal_task_manager::SealTaskManager;
-use super::{MineLoadChunk, SealAnswer, SealTask};
 use crate::config::ShardConfig;
 use crate::error::Error;
+use crate::log_store::load_chunk::EntryBatch;
 use crate::log_store::log_manager::{
-    bytes_to_entries, data_to_merkle_leaves, COL_ENTRY_BATCH, COL_ENTRY_BATCH_ROOT,
-    COL_FLOW_MPT_NODES, ENTRY_SIZE, PORA_CHUNK_SIZE,
+    bytes_to_entries, COL_ENTRY_BATCH, COL_FLOW_MPT_NODES, COL_PAD_DATA_LIST,
+    COL_PAD_DATA_SYNC_HEIGH, PORA_CHUNK_SIZE,
+};
+use crate::log_store::seal_task_manager::SealTaskManager;
+use crate::log_store::{
+    metrics, FlowRead, FlowSeal, FlowWrite, MineLoadChunk, SealAnswer, SealTask,
 };
-use crate::log_store::{FlowRead, FlowSeal, FlowWrite};
 use crate::{try_option, ZgsKeyValueDB};
+use any::Any;
 use anyhow::{anyhow, bail, Result};
-use append_merkle::{MerkleTreeInitialData, MerkleTreeRead};
+use append_merkle::{MerkleTreeRead, NodeDatabase, NodeTransaction};
 use itertools::Itertools;
+use kvdb::DBTransaction;
 use parking_lot::RwLock;
-use shared_types::{ChunkArray, DataRoot, FlowProof, Merkle};
+use shared_types::{ChunkArray, DataRoot, FlowProof};
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode};
-use std::cmp::Ordering;
-use std::collections::BTreeMap;
+
 use std::fmt::Debug;
 use std::sync::Arc;
-use std::{cmp, mem};
+use std::time::Instant;
+use std::{any, cmp};
 use tracing::{debug, error, trace};
 use zgs_spec::{BYTES_PER_SECTOR, SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_SEAL};
 
 pub struct FlowStore {
-    db: FlowDBStore,
+    flow_db: Arc<FlowDBStore>,
+    data_db: Arc<FlowDBStore>,
     seal_manager: SealTaskManager,
     config: FlowConfig,
 }
 
 impl FlowStore {
-    pub fn new(db: Arc<dyn ZgsKeyValueDB>, config: FlowConfig) -> Self {
+    pub fn new(flow_db: Arc<FlowDBStore>, data_db: Arc<FlowDBStore>, config: FlowConfig) -> Self {
         Self {
-            db: FlowDBStore::new(db),
+            flow_db,
+            data_db,
             seal_manager: Default::default(),
             config,
         }
     }
 
-    pub fn put_batch_root_list(&self, root_map: BTreeMap<usize, (DataRoot, usize)>) -> Result<()> {
-        self.db.put_batch_root_list(root_map)
-    }
-
     pub fn insert_subtree_list_for_batch(
         &self,
         batch_index: usize,
         subtree_list: Vec<(usize, usize, DataRoot)>,
     ) -> Result<()> {
+        let start_time = Instant::now();
         let mut batch = self
-            .db
+            .data_db
             .get_entry_batch(batch_index as u64)?
             .unwrap_or_else(|| EntryBatch::new(batch_index as u64));
         batch.set_subtree_list(subtree_list);
-        self.db.put_entry_raw(vec![(batch_index as u64, batch)])?;
-
+        self.data_db
+            .put_entry_raw(vec![(batch_index as u64, batch)])?;
+        metrics::INSERT_SUBTREE_LIST.update_since(start_time);
         Ok(())
     }
 
     pub fn gen_proof_in_batch(&self, batch_index: usize, sector_index: usize) -> Result<FlowProof> {
         let batch = self
-            .db
+            .data_db
             .get_entry_batch(batch_index as u64)?
             .ok_or_else(|| anyhow!("batch missing, index={}", batch_index))?;
         let merkle = batch.to_merkle_tree(batch_index == 0)?.ok_or_else(|| {
@@ -72,27 +75,16 @@ impl FlowStore {
         merkle.gen_proof(sector_index)
     }
 
-    pub fn put_mpt_node_list(&self, node_list: Vec<(usize, usize, DataRoot)>) -> Result<()> {
-        self.db.put_mpt_node_list(node_list)
-    }
-
     pub fn delete_batch_list(&self, batch_list: &[u64]) -> Result<()> {
         self.seal_manager.delete_batch_list(batch_list);
-        self.db.delete_batch_list(batch_list)
-    }
-
-    pub fn get_raw_batch(&self, batch_index: u64) -> Result<Option<EntryBatch>> {
-        self.db.get_entry_batch(batch_index)
-    }
-
-    pub fn get_batch_root(&self, batch_index: u64) -> Result<Option<DataRoot>> {
-        self.db.get_batch_root(batch_index)
+        self.data_db.delete_batch_list(batch_list)
     }
 }
 
 #[derive(Clone, Debug)]
 pub struct FlowConfig {
     pub batch_size: usize,
+    pub merkle_node_cache_capacity: usize,
     pub shard_config: Arc<RwLock<ShardConfig>>,
 }
 
@@ -100,6 +92,8 @@ impl Default for FlowConfig {
     fn default() -> Self {
         Self {
             batch_size: SECTORS_PER_LOAD,
+            // Each node takes (8+8+32=)48 Bytes, so the default value is 1.5 GB memory size.
+            merkle_node_cache_capacity: 32 * 1024 * 1024,
             shard_config: Default::default(),
         }
     }
@@ -129,7 +123,7 @@ impl FlowRead for FlowStore {
                 length -= 1;
             }
 
-            let entry_batch = try_option!(self.db.get_entry_batch(chunk_index)?);
+            let entry_batch = try_option!(self.data_db.get_entry_batch(chunk_index)?);
             let mut entry_batch_data =
                 try_option!(entry_batch.get_unsealed_data(offset as usize, length as usize));
             data.append(&mut entry_batch_data);
@@ -158,7 +152,7 @@ impl FlowRead for FlowStore {
             let chunk_index = start_entry_index / self.config.batch_size as u64;
 
             if let Some(mut data_list) = self
-                .db
+                .data_db
                 .get_entry_batch(chunk_index)?
                 .map(|b| b.into_data_list(start_entry_index))
             {
@@ -182,13 +176,8 @@ impl FlowRead for FlowStore {
         Ok(entry_list)
     }
 
-    /// Return the list of all stored chunk roots.
-    fn get_chunk_root_list(&self) -> Result<MerkleTreeInitialData<DataRoot>> {
-        self.db.get_batch_root_list()
-    }
-
     fn load_sealed_data(&self, chunk_index: u64) -> Result<Option<MineLoadChunk>> {
-        let batch = try_option!(self.db.get_entry_batch(chunk_index)?);
+        let batch = try_option!(self.data_db.get_entry_batch(chunk_index)?);
         let mut mine_chunk = MineLoadChunk::default();
         for (seal_index, (sealed, validity)) in mine_chunk
             .loaded_chunk
@@ -206,7 +195,7 @@ impl FlowRead for FlowStore {
 
     fn get_num_entries(&self) -> Result<u64> {
         // This is an over-estimation as it assumes each batch is full.
-        self.db
+        self.data_db
             .kvdb
             .num_keys(COL_ENTRY_BATCH)
             .map(|num_batches| num_batches * PORA_CHUNK_SIZE as u64)
@@ -216,12 +205,21 @@ impl FlowRead for FlowStore {
     fn get_shard_config(&self) -> ShardConfig {
         *self.config.shard_config.read()
     }
+
+    fn get_pad_data(&self, start_index: u64) -> crate::error::Result<Option<Vec<PadPair>>> {
+        self.flow_db.get_pad_data(start_index)
+    }
+
+    fn get_pad_data_sync_height(&self) -> Result<Option<u64>> {
+        self.data_db.get_pad_data_sync_height()
+    }
 }
 
 impl FlowWrite for FlowStore {
     /// Return the roots of completed chunks. The order is guaranteed to be increasing
     /// by chunk index.
     fn append_entries(&self, data: ChunkArray) -> Result<Vec<(u64, DataRoot)>> {
+        let start_time = Instant::now();
         let mut to_seal_set = self.seal_manager.to_seal_set.write();
         trace!("append_entries: {} {}", data.start_index, data.data.len());
         if data.data.len() % BYTES_PER_SECTOR != 0 {
@@ -246,7 +244,7 @@ impl FlowWrite for FlowStore {
 
             // TODO: Try to avoid loading from db if possible.
             let mut batch = self
-                .db
+                .data_db
                 .get_entry_batch(chunk_index)?
                 .unwrap_or_else(|| EntryBatch::new(chunk_index));
             let completed_seals = batch.insert_data(
@@ -264,12 +262,14 @@ impl FlowWrite for FlowStore {
 
             batch_list.push((chunk_index, batch));
         }
-        self.db.put_entry_batch_list(batch_list)
+
+        metrics::APPEND_ENTRIES.update_since(start_time);
+        self.data_db.put_entry_batch_list(batch_list)
     }
 
     fn truncate(&self, start_index: u64) -> crate::error::Result<()> {
         let mut to_seal_set = self.seal_manager.to_seal_set.write();
-        let to_reseal = self.db.truncate(start_index, self.config.batch_size)?;
+        let to_reseal = self.data_db.truncate(start_index, self.config.batch_size)?;
 
         to_seal_set.split_off(&(start_index as usize / SECTORS_PER_SEAL));
         let new_seal_version = self.seal_manager.inc_seal_version();
@@ -283,6 +283,14 @@ impl FlowWrite for FlowStore {
     fn update_shard_config(&self, shard_config: ShardConfig) {
         *self.config.shard_config.write() = shard_config;
     }
+
+    fn put_pad_data(&self, data_sizes: &[PadPair], tx_seq: u64) -> crate::error::Result<()> {
+        self.flow_db.put_pad_data(data_sizes, tx_seq)
+    }
+
+    fn put_pad_data_sync_height(&self, sync_index: u64) -> crate::error::Result<()> {
+        self.data_db.put_pad_data_sync_height(sync_index)
+    }
 }
 
 impl FlowSeal for FlowStore {
@@ -299,7 +307,7 @@ impl FlowSeal for FlowStore {
         let mut tasks = Vec::with_capacity(SEALS_PER_LOAD);
 
         let batch_data = self
-            .db
+            .data_db
             .get_entry_batch((first_index / SEALS_PER_LOAD) as u64)?
             .expect("Lost data chunk in to_seal_set");
 
@@ -338,7 +346,7 @@ impl FlowSeal for FlowStore {
             .chunk_by(|answer| answer.seal_index / SEALS_PER_LOAD as u64)
         {
             let mut batch_chunk = self
-                .db
+                .data_db
                 .get_entry_batch(load_index)?
                 .expect("Can not find chunk data");
             for answer in answers_in_chunk {
@@ -354,12 +362,18 @@ impl FlowSeal for FlowStore {
             to_seal_set.remove(&idx);
         }
 
-        self.db.put_entry_raw(updated_chunk)?;
+        self.data_db.put_entry_raw(updated_chunk)?;
 
         Ok(())
     }
 }
 
+#[derive(Debug, PartialEq, DeriveEncode, DeriveDecode)]
+pub struct PadPair {
+    pub start_index: u64,
+    pub data_size: u64,
+}
+
 pub struct FlowDBStore {
     kvdb: Arc<dyn ZgsKeyValueDB>,
 }
@@ -373,6 +387,7 @@ impl FlowDBStore {
         &self,
         batch_list: Vec<(u64, EntryBatch)>,
     ) -> Result<Vec<(u64, DataRoot)>> {
+        let start_time = Instant::now();
         let mut completed_batches = Vec::new();
         let mut tx = self.kvdb.transaction();
         for (batch_index, batch) in batch_list {
@@ -383,16 +398,11 @@ impl FlowDBStore {
             );
             if let Some(root) = batch.build_root(batch_index == 0)? {
                 trace!("complete batch: index={}", batch_index);
-                tx.put(
-                    COL_ENTRY_BATCH_ROOT,
-                    // (batch_index, subtree_depth)
-                    &encode_batch_root_key(batch_index as usize, 1),
-                    root.as_bytes(),
-                );
                 completed_batches.push((batch_index, root));
             }
         }
         self.kvdb.write(tx)?;
+        metrics::PUT_ENTRY_BATCH_LIST.update_since(start_time);
         Ok(completed_batches)
     }
 
@@ -414,94 +424,6 @@ impl FlowDBStore {
         Ok(Some(EntryBatch::from_ssz_bytes(&raw).map_err(Error::from)?))
     }
 
-    fn put_batch_root_list(&self, root_map: BTreeMap<usize, (DataRoot, usize)>) -> Result<()> {
-        let mut tx = self.kvdb.transaction();
-        for (batch_index, (root, subtree_depth)) in root_map {
-            tx.put(
-                COL_ENTRY_BATCH_ROOT,
-                &encode_batch_root_key(batch_index, subtree_depth),
-                root.as_bytes(),
-            );
-        }
-        Ok(self.kvdb.write(tx)?)
-    }
-
-    fn get_batch_root_list(&self) -> Result<MerkleTreeInitialData<DataRoot>> {
-        let mut range_root = None;
-        // A list of `BatchRoot` that can reconstruct the whole merkle tree structure.
-        let mut root_list = Vec::new();
-        // A list of leaf `(index, root_hash)` in the subtrees of some nodes in `root_list`,
-        // and they will be updated in the merkle tree with `fill_leaf` by the caller.
-        let mut leaf_list = Vec::new();
-        let mut expected_index = 0;
-
-        let empty_data = vec![0; PORA_CHUNK_SIZE * ENTRY_SIZE];
-        let empty_root = *Merkle::new(data_to_merkle_leaves(&empty_data)?, 0, None).root();
-
-        for r in self.kvdb.iter(COL_ENTRY_BATCH_ROOT) {
-            let (index_bytes, root_bytes) = r?;
-            let (batch_index, subtree_depth) = decode_batch_root_key(index_bytes.as_ref())?;
-            let root = DataRoot::from_slice(root_bytes.as_ref());
-            debug!(
-                "load root depth={}, index expected={} get={} root={:?}",
-                subtree_depth, expected_index, batch_index, root,
-            );
-            if subtree_depth == 1 {
-                if range_root.is_none() {
-                    // This is expected to be the next leaf.
-                    if batch_index == expected_index {
-                        root_list.push((1, root));
-                        expected_index += 1;
-                    } else {
-                        bail!(
-                            "unexpected chunk leaf, expected={}, get={}",
-                            expected_index,
-                            batch_index
-                        );
-                    }
-                } else {
-                    match batch_index.cmp(&expected_index) {
-                        Ordering::Less => {
-                            // This leaf is within a subtree whose root is known.
-                            leaf_list.push((batch_index, root));
-                        }
-                        Ordering::Equal => {
-                            // A subtree range ends.
-                            range_root = None;
-                            root_list.push((1, root));
-                            expected_index += 1;
-                        }
-                        Ordering::Greater => {
-                            while batch_index > expected_index {
-                                // Fill the gap with empty leaves.
-                                root_list.push((1, empty_root));
-                                expected_index += 1;
-                            }
-                            range_root = None;
-                            root_list.push((1, root));
-                            expected_index += 1;
-                        }
-                    }
-                }
-            } else {
-                while batch_index > expected_index {
-                    // Fill the gap with empty leaves.
-                    root_list.push((1, empty_root));
-                    expected_index += 1;
-                }
-                range_root = Some(BatchRoot::Multiple((subtree_depth, root)));
-                root_list.push((subtree_depth, root));
-                expected_index += 1 << (subtree_depth - 1);
-            }
-        }
-        let extra_node_list = self.get_mpt_node_list()?;
-        Ok(MerkleTreeInitialData {
-            subtree_list: root_list,
-            known_leaves: leaf_list,
-            extra_mpt_nodes: extra_node_list,
-        })
-    }
-
     fn truncate(&self, start_index: u64, batch_size: usize) -> crate::error::Result<Vec<usize>> {
         let mut tx = self.kvdb.transaction();
         let mut start_batch_index = start_index / batch_size as u64;
@@ -542,38 +464,11 @@ impl FlowDBStore {
         };
         for batch_index in start_batch_index as usize..=end {
             tx.delete(COL_ENTRY_BATCH, &batch_index.to_be_bytes());
-            tx.delete_prefix(COL_ENTRY_BATCH_ROOT, &batch_index.to_be_bytes());
         }
         self.kvdb.write(tx)?;
         Ok(index_to_reseal)
     }
 
-    fn put_mpt_node_list(&self, mpt_node_list: Vec<(usize, usize, DataRoot)>) -> Result<()> {
-        let mut tx = self.kvdb.transaction();
-        for (layer_index, position, data) in mpt_node_list {
-            tx.put(
-                COL_FLOW_MPT_NODES,
-                &encode_mpt_node_key(layer_index, position),
-                data.as_bytes(),
-            );
-        }
-        Ok(self.kvdb.write(tx)?)
-    }
-
-    fn get_mpt_node_list(&self) -> Result<Vec<(usize, usize, DataRoot)>> {
-        let mut node_list = Vec::new();
-        for r in self.kvdb.iter(COL_FLOW_MPT_NODES) {
-            let (index_bytes, node_bytes) = r?;
-            let (layer_index, position) = decode_mpt_node_key(index_bytes.as_ref())?;
-            node_list.push((
-                layer_index,
-                position,
-                DataRoot::from_slice(node_bytes.as_ref()),
-            ));
-        }
-        Ok(node_list)
-    }
-
     fn delete_batch_list(&self, batch_list: &[u64]) -> Result<()> {
         let mut tx = self.kvdb.transaction();
         for i in batch_list {
@@ -582,14 +477,46 @@ impl FlowDBStore {
         Ok(self.kvdb.write(tx)?)
     }
 
-    fn get_batch_root(&self, batch_index: u64) -> Result<Option<DataRoot>> {
-        Ok(self
-            .kvdb
-            .get(
-                COL_ENTRY_BATCH_ROOT,
-                &encode_batch_root_key(batch_index as usize, 1),
-            )?
-            .map(|v| DataRoot::from_slice(&v)))
+    fn put_pad_data(&self, data_sizes: &[PadPair], tx_seq: u64) -> Result<()> {
+        let mut tx = self.kvdb.transaction();
+
+        let mut buffer = Vec::new();
+        for item in data_sizes {
+            buffer.extend(item.as_ssz_bytes());
+        }
+
+        tx.put(COL_PAD_DATA_LIST, &tx_seq.to_be_bytes(), &buffer);
+        self.kvdb.write(tx)?;
+        Ok(())
+    }
+
+    fn put_pad_data_sync_height(&self, tx_seq: u64) -> Result<()> {
+        let mut tx = self.kvdb.transaction();
+        tx.put(
+            COL_PAD_DATA_SYNC_HEIGH,
+            b"sync_height",
+            &tx_seq.to_be_bytes(),
+        );
+        self.kvdb.write(tx)?;
+        Ok(())
+    }
+
+    fn get_pad_data_sync_height(&self) -> Result<Option<u64>> {
+        match self.kvdb.get(COL_PAD_DATA_SYNC_HEIGH, b"sync_height")? {
+            Some(v) => Ok(Some(u64::from_be_bytes(
+                v.try_into().map_err(|e| anyhow!("{:?}", e))?,
+            ))),
+            None => Ok(None),
+        }
+    }
+
+    fn get_pad_data(&self, tx_seq: u64) -> Result<Option<Vec<PadPair>>> {
+        match self.kvdb.get(COL_PAD_DATA_LIST, &tx_seq.to_be_bytes())? {
+            Some(v) => Ok(Some(
+                Vec::<PadPair>::from_ssz_bytes(&v).map_err(Error::from)?,
+            )),
+            None => Ok(None),
+        }
     }
 }
 
@@ -636,33 +563,89 @@ fn decode_batch_index(data: &[u8]) -> Result<usize> {
     try_decode_usize(data)
 }
 
-/// For the same batch_index, we want to process the larger subtree_depth first in iteration.
-fn encode_batch_root_key(batch_index: usize, subtree_depth: usize) -> Vec<u8> {
-    let mut key = batch_index.to_be_bytes().to_vec();
-    key.extend_from_slice(&(usize::MAX - subtree_depth).to_be_bytes());
-    key
-}
-
-fn decode_batch_root_key(data: &[u8]) -> Result<(usize, usize)> {
-    if data.len() != mem::size_of::<usize>() * 2 {
-        bail!("invalid data length");
-    }
-    let batch_index = try_decode_usize(&data[..mem::size_of::<u64>()])?;
-    let subtree_depth = usize::MAX - try_decode_usize(&data[mem::size_of::<u64>()..])?;
-    Ok((batch_index, subtree_depth))
-}
-
 fn encode_mpt_node_key(layer_index: usize, position: usize) -> Vec<u8> {
     let mut key = layer_index.to_be_bytes().to_vec();
     key.extend_from_slice(&position.to_be_bytes());
     key
 }
 
-fn decode_mpt_node_key(data: &[u8]) -> Result<(usize, usize)> {
-    if data.len() != mem::size_of::<usize>() * 2 {
-        bail!("invalid data length");
+fn layer_size_key(layer: usize) -> Vec<u8> {
+    let mut key = "layer_size".as_bytes().to_vec();
+    key.extend_from_slice(&layer.to_be_bytes());
+    key
+}
+
+pub struct NodeDBTransaction(DBTransaction);
+
+impl NodeDatabase<DataRoot> for FlowDBStore {
+    fn get_node(&self, layer: usize, pos: usize) -> Result<Option<DataRoot>> {
+        Ok(self
+            .kvdb
+            .get(COL_FLOW_MPT_NODES, &encode_mpt_node_key(layer, pos))?
+            .map(|v| DataRoot::from_slice(&v)))
+    }
+
+    fn get_layer_size(&self, layer: usize) -> Result<Option<usize>> {
+        match self.kvdb.get(COL_FLOW_MPT_NODES, &layer_size_key(layer))? {
+            Some(v) => Ok(Some(try_decode_usize(&v)?)),
+            None => Ok(None),
+        }
+    }
+
+    fn start_transaction(&self) -> Box<dyn NodeTransaction<DataRoot>> {
+        Box::new(NodeDBTransaction(self.kvdb.transaction()))
+    }
+
+    fn commit(&self, tx: Box<dyn NodeTransaction<DataRoot>>) -> Result<()> {
+        let db_tx: Box<NodeDBTransaction> = tx
+            .into_any()
+            .downcast()
+            .map_err(|e| anyhow!("downcast failed, e={:?}", e))?;
+        self.kvdb.write(db_tx.0).map_err(Into::into)
+    }
+}
+
+impl NodeTransaction<DataRoot> for NodeDBTransaction {
+    fn save_node(&mut self, layer: usize, pos: usize, node: &DataRoot) {
+        self.0.put(
+            COL_FLOW_MPT_NODES,
+            &encode_mpt_node_key(layer, pos),
+            node.as_bytes(),
+        );
+    }
+
+    fn save_node_list(&mut self, nodes: &[(usize, usize, &DataRoot)]) {
+        for (layer_index, position, data) in nodes {
+            self.0.put(
+                COL_FLOW_MPT_NODES,
+                &encode_mpt_node_key(*layer_index, *position),
+                data.as_bytes(),
+            );
+        }
+    }
+
+    fn remove_node_list(&mut self, nodes: &[(usize, usize)]) {
+        for (layer_index, position) in nodes {
+            self.0.delete(
+                COL_FLOW_MPT_NODES,
+                &encode_mpt_node_key(*layer_index, *position),
+            );
+        }
+    }
+
+    fn save_layer_size(&mut self, layer: usize, size: usize) {
+        self.0.put(
+            COL_FLOW_MPT_NODES,
+            &layer_size_key(layer),
+            &size.to_be_bytes(),
+        );
+    }
+
+    fn remove_layer_size(&mut self, layer: usize) {
+        self.0.delete(COL_FLOW_MPT_NODES, &layer_size_key(layer));
+    }
+
+    fn into_any(self: Box<Self>) -> Box<dyn Any> {
+        self
     }
-    let layer_index = try_decode_usize(&data[..mem::size_of::<u64>()])?;
-    let position = try_decode_usize(&data[mem::size_of::<u64>()..])?;
-    Ok((layer_index, position))
 }

node/storage/src/log_store/load_chunk/chunk_data.rs

@@ -128,6 +128,12 @@ impl DataRange for Subtree {
     }
 }
 
+impl Default for EntryBatchData {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl EntryBatchData {
     pub fn new() -> Self {
         EntryBatchData::Incomplete(IncompleteData {

node/storage/src/log_store/load_chunk/mod.rs

@@ -4,11 +4,10 @@ mod seal;
 mod serde;
 
 use ::serde::{Deserialize, Serialize};
-use std::cmp::min;
-
 use anyhow::Result;
 use ethereum_types::H256;
 use ssz_derive::{Decode, Encode};
+use std::cmp::min;
 
 use crate::log_store::log_manager::data_to_merkle_leaves;
 use crate::try_option;
@@ -21,7 +20,7 @@ use zgs_spec::{
 };
 
 use super::SealAnswer;
-use chunk_data::EntryBatchData;
+pub use chunk_data::EntryBatchData;
 use seal::SealInfo;
 
 #[derive(Debug, Encode, Decode, Deserialize, Serialize)]
@@ -206,7 +205,7 @@ impl EntryBatch {
             }
         }
         Ok(Some(
-            *try_option!(self.to_merkle_tree(is_first_chunk)?).root(),
+            try_option!(self.to_merkle_tree(is_first_chunk)?).root(),
         ))
     }
 

node/storage/src/log_store/log_manager.rs

@@ -1,8 +1,11 @@
 use crate::config::ShardConfig;
-use crate::log_store::flow_store::{batch_iter_sharded, FlowConfig, FlowStore};
-use crate::log_store::tx_store::TransactionStore;
+use crate::log_store::flow_store::{
+    batch_iter_sharded, FlowConfig, FlowDBStore, FlowStore, PadPair,
+};
+use crate::log_store::tx_store::{BlockHashAndSubmissionIndex, TransactionStore, TxStatus};
 use crate::log_store::{
-    FlowRead, FlowWrite, LogStoreChunkRead, LogStoreChunkWrite, LogStoreRead, LogStoreWrite,
+    FlowRead, FlowSeal, FlowWrite, LogStoreChunkRead, LogStoreChunkWrite, LogStoreRead,
+    LogStoreWrite, MineLoadChunk, SealAnswer, SealTask,
 };
 use crate::{try_option, ZgsKeyValueDB};
 use anyhow::{anyhow, bail, Result};
@@ -11,6 +14,7 @@ use ethereum_types::H256;
 use kvdb_rocksdb::{Database, DatabaseConfig};
 use merkle_light::merkle::{log2_pow2, MerkleTree};
 use merkle_tree::RawLeafSha3Algorithm;
+use once_cell::sync::Lazy;
 use parking_lot::RwLock;
 use rayon::iter::ParallelIterator;
 use rayon::prelude::ParallelSlice;
@@ -19,46 +23,57 @@ use shared_types::{
     ChunkArrayWithProof, ChunkWithProof, DataRoot, FlowProof, FlowRangeProof, Merkle, Transaction,
 };
 use std::cmp::Ordering;
-use std::collections::BTreeMap;
+
 use std::path::Path;
-use std::sync::mpsc;
 use std::sync::Arc;
+use std::time::{Duration, Instant};
+
 use tracing::{debug, error, info, instrument, trace, warn};
 
-use super::tx_store::BlockHashAndSubmissionIndex;
-use super::{FlowSeal, MineLoadChunk, SealAnswer, SealTask};
+use crate::log_store::metrics;
 
 /// 256 Bytes
 pub const ENTRY_SIZE: usize = 256;
 /// 1024 Entries.
 pub const PORA_CHUNK_SIZE: usize = 1024;
 
-pub const COL_TX: u32 = 0;
-pub const COL_ENTRY_BATCH: u32 = 1;
-pub const COL_TX_DATA_ROOT_INDEX: u32 = 2;
-pub const COL_ENTRY_BATCH_ROOT: u32 = 3;
-pub const COL_TX_COMPLETED: u32 = 4;
-pub const COL_MISC: u32 = 5;
-pub const COL_SEAL_CONTEXT: u32 = 6;
-pub const COL_FLOW_MPT_NODES: u32 = 7;
-pub const COL_BLOCK_PROGRESS: u32 = 8;
+pub const COL_TX: u32 = 0; // flow db
+pub const COL_ENTRY_BATCH: u32 = 1; // data db
+pub const COL_TX_DATA_ROOT_INDEX: u32 = 2; // flow db
+pub const COL_TX_COMPLETED: u32 = 3; // data db
+pub const COL_MISC: u32 = 4; // flow db & data db
+pub const COL_FLOW_MPT_NODES: u32 = 5; // flow db
+pub const COL_BLOCK_PROGRESS: u32 = 6; // flow db
+pub const COL_PAD_DATA_LIST: u32 = 7; // flow db
+pub const COL_PAD_DATA_SYNC_HEIGH: u32 = 8; // data db
 pub const COL_NUM: u32 = 9;
 
+pub const DATA_DB_KEY: &str = "data_db";
+pub const FLOW_DB_KEY: &str = "flow_db";
+const PAD_DELAY: Duration = Duration::from_secs(2);
+
 // Process at most 1M entries (256MB) pad data at a time.
 const PAD_MAX_SIZE: usize = 1 << 20;
 
+static PAD_SEGMENT_ROOT: Lazy<H256> = Lazy::new(|| {
+    Merkle::new(
+        data_to_merkle_leaves(&[0; ENTRY_SIZE * PORA_CHUNK_SIZE]).unwrap(),
+        0,
+        None,
+    )
+    .root()
+});
 pub struct UpdateFlowMessage {
-    pub root_map: BTreeMap<usize, (H256, usize)>,
     pub pad_data: usize,
     pub tx_start_flow_index: u64,
 }
 
 pub struct LogManager {
-    pub(crate) db: Arc<dyn ZgsKeyValueDB>,
+    pub(crate) flow_db: Arc<dyn ZgsKeyValueDB>,
+    pub(crate) data_db: Arc<dyn ZgsKeyValueDB>,
     tx_store: TransactionStore,
     flow_store: Arc<FlowStore>,
     merkle: RwLock<MerkleManager>,
-    sender: mpsc::Sender<UpdateFlowMessage>,
 }
 
 struct MerkleManager {
@@ -94,6 +109,7 @@ impl MerkleManager {
     }
 
     fn revert_merkle_tree(&mut self, tx_seq: u64, tx_store: &TransactionStore) -> Result<()> {
+        debug!("revert merkle tree {}", tx_seq);
         // Special case for reverting tx_seq == 0
         if tx_seq == u64::MAX {
             self.pora_chunks_merkle.reset();
@@ -116,7 +132,7 @@ impl MerkleManager {
         if self.pora_chunks_merkle.leaves() == 0 && self.last_chunk_merkle.leaves() == 0 {
             self.last_chunk_merkle.append(H256::zero());
             self.pora_chunks_merkle
-                .update_last(*self.last_chunk_merkle.root());
+                .update_last(self.last_chunk_merkle.root());
         } else if self.last_chunk_merkle.leaves() != 0 {
             let last_chunk_start_index = self.last_chunk_start_index();
             let last_chunk_data = flow_store.get_available_entries(
@@ -180,6 +196,7 @@ impl LogStoreChunkWrite for LogManager {
         chunks: ChunkArray,
         maybe_file_proof: Option<FlowProof>,
     ) -> Result<bool> {
+        let start_time = Instant::now();
         let mut merkle = self.merkle.write();
         let tx = self
             .tx_store
@@ -205,13 +222,13 @@ impl LogStoreChunkWrite for LogManager {
         self.append_entries(flow_entry_array, &mut merkle)?;
 
         if let Some(file_proof) = maybe_file_proof {
-            let updated_node_list = merkle.pora_chunks_merkle.fill_with_file_proof(
+            merkle.pora_chunks_merkle.fill_with_file_proof(
                 file_proof,
                 tx.merkle_nodes,
                 tx.start_entry_index,
             )?;
-            self.flow_store.put_mpt_node_list(updated_node_list)?;
         }
+        metrics::PUT_CHUNKS.update_since(start_time);
         Ok(true)
     }
 
@@ -242,6 +259,7 @@ impl LogStoreWrite for LogManager {
     /// `put_tx` for the last tx when we restart the node to ensure that it succeeds.
     ///
     fn put_tx(&self, tx: Transaction) -> Result<()> {
+        let start_time = Instant::now();
         let mut merkle = self.merkle.write();
         debug!("put_tx: tx={:?}", tx);
         let expected_seq = self.tx_store.next_tx_seq();
@@ -257,7 +275,12 @@ impl LogStoreWrite for LogManager {
         }
         let maybe_same_data_tx_seq = self.tx_store.put_tx(tx.clone())?.first().cloned();
         // TODO(zz): Should we validate received tx?
-        self.append_subtree_list(tx.start_entry_index, tx.merkle_nodes.clone(), &mut merkle)?;
+        self.append_subtree_list(
+            tx.seq,
+            tx.start_entry_index,
+            tx.merkle_nodes.clone(),
+            &mut merkle,
+        )?;
         merkle.commit_merkle(tx.seq)?;
         debug!(
             "commit flow root: root={:?}",
@@ -271,6 +294,7 @@ impl LogStoreWrite for LogManager {
                 self.copy_tx_and_finalize(old_tx_seq, vec![tx.seq])?;
             }
         }
+        metrics::PUT_TX.update_since(start_time);
         Ok(())
     }
 
@@ -301,6 +325,7 @@ impl LogStoreWrite for LogManager {
     }
 
     fn finalize_tx_with_hash(&self, tx_seq: u64, tx_hash: H256) -> crate::error::Result<bool> {
+        let start_time = Instant::now();
         trace!(
             "finalize_tx_with_hash: tx_seq={} tx_hash={:?}",
             tx_seq,
@@ -329,6 +354,7 @@ impl LogStoreWrite for LogManager {
             if same_root_seq_list.first() == Some(&tx_seq) {
                 self.copy_tx_and_finalize(tx_seq, same_root_seq_list[1..].to_vec())?;
             }
+            metrics::FINALIZE_TX_WITH_HASH.update_since(start_time);
             Ok(true)
         } else {
             bail!("finalize tx hash with data missing: tx_seq={}", tx_seq)
@@ -355,7 +381,7 @@ impl LogStoreWrite for LogManager {
         merkle.revert_merkle_tree(tx_seq, &self.tx_store)?;
         merkle.try_initialize(&self.flow_store)?;
         assert_eq!(
-            Some(*merkle.last_chunk_merkle.root()),
+            Some(merkle.last_chunk_merkle.root()),
             merkle
                 .pora_chunks_merkle
                 .leaf_at(merkle.pora_chunks_merkle.leaves() - 1)?
@@ -376,10 +402,9 @@ impl LogStoreWrite for LogManager {
         // `merkle` is used in `validate_range_proof`.
         let mut merkle = self.merkle.write();
         if valid {
-            let updated_nodes = merkle
+            merkle
                 .pora_chunks_merkle
                 .fill_with_range_proof(data.proof.clone())?;
-            self.flow_store.put_mpt_node_list(updated_nodes)?;
         }
         Ok(valid)
     }
@@ -395,6 +420,42 @@ impl LogStoreWrite for LogManager {
     fn submit_seal_result(&self, answers: Vec<SealAnswer>) -> Result<()> {
         self.flow_store.submit_seal_result(answers)
     }
+
+    fn start_padding(&self, executor: &task_executor::TaskExecutor) {
+        let store = self.flow_store.clone();
+        executor.spawn(
+            async move {
+                let current_height = store.get_pad_data_sync_height().unwrap();
+                let mut start_index = current_height.unwrap_or(0);
+                loop {
+                    match store.get_pad_data(start_index) {
+                        std::result::Result::Ok(data) => {
+                            // Update the flow database.
+                            // This should be called before `complete_last_chunk_merkle` so that we do not save
+                            // subtrees with data known.
+                            if let Some(data) = data {
+                                for pad in data {
+                                    store
+                                        .append_entries(ChunkArray {
+                                            data: vec![0; pad.data_size as usize],
+                                            start_index: pad.start_index,
+                                        })
+                                        .unwrap();
+                                }
+                            };
+                            store.put_pad_data_sync_height(start_index).unwrap();
+                            start_index += 1;
+                        }
+                        std::result::Result::Err(_) => {
+                            debug!("Unable to get pad data, start_index={}", start_index);
+                            tokio::time::sleep(PAD_DELAY).await;
+                        }
+                    };
+                }
+            },
+            "pad_tx",
+        );
+    }
 }
 
 impl LogStoreChunkRead for LogManager {
@@ -476,7 +537,15 @@ impl LogStoreRead for LogManager {
     }
 
     fn get_tx_seq_by_data_root(&self, data_root: &DataRoot) -> crate::error::Result<Option<u64>> {
-        self.tx_store.get_first_tx_seq_by_data_root(data_root)
+        let seq_list = self.tx_store.get_tx_seq_list_by_data_root(data_root)?;
+        for tx_seq in &seq_list {
+            if self.tx_store.check_tx_completed(*tx_seq)? {
+                // Return the first finalized tx if possible.
+                return Ok(Some(*tx_seq));
+            }
+        }
+        // No tx is finalized, return the first one.
+        Ok(seq_list.first().cloned())
     }
 
     fn get_chunk_with_proof_by_tx_and_index(
@@ -520,6 +589,10 @@ impl LogStoreRead for LogManager {
         }))
     }
 
+    fn get_tx_status(&self, tx_seq: u64) -> Result<Option<TxStatus>> {
+        self.tx_store.get_tx_status(tx_seq)
+    }
+
     fn check_tx_completed(&self, tx_seq: u64) -> crate::error::Result<bool> {
         self.tx_store.check_tx_completed(tx_seq)
     }
@@ -577,7 +650,7 @@ impl LogStoreRead for LogManager {
     fn get_context(&self) -> crate::error::Result<(DataRoot, u64)> {
         let merkle = self.merkle.read_recursive();
         Ok((
-            *merkle.pora_chunks_merkle.root(),
+            merkle.pora_chunks_merkle.root(),
             merkle.last_chunk_start_index() + merkle.last_chunk_merkle.leaves() as u64,
         ))
     }
@@ -606,33 +679,37 @@ impl LogStoreRead for LogManager {
 impl LogManager {
     pub fn rocksdb(
         config: LogConfig,
-        path: impl AsRef<Path>,
-        executor: task_executor::TaskExecutor,
+        flow_path: impl AsRef<Path>,
+        data_path: impl AsRef<Path>,
     ) -> Result<Self> {
         let mut db_config = DatabaseConfig::with_columns(COL_NUM);
         db_config.enable_statistics = true;
-        let db = Arc::new(Database::open(&db_config, path)?);
-        Self::new(db, config, executor)
+        let flow_db_source = Arc::new(Database::open(&db_config, flow_path)?);
+        let data_db_source = Arc::new(Database::open(&db_config, data_path)?);
+        Self::new(flow_db_source, data_db_source, config)
     }
 
-    pub fn memorydb(config: LogConfig, executor: task_executor::TaskExecutor) -> Result<Self> {
-        let db = Arc::new(kvdb_memorydb::create(COL_NUM));
-        Self::new(db, config, executor)
+    pub fn memorydb(config: LogConfig) -> Result<Self> {
+        let flow_db = Arc::new(kvdb_memorydb::create(COL_NUM));
+        let data_db = Arc::new(kvdb_memorydb::create(COL_NUM));
+        Self::new(flow_db, data_db, config)
     }
 
     fn new(
-        db: Arc<dyn ZgsKeyValueDB>,
+        flow_db_source: Arc<dyn ZgsKeyValueDB>,
+        data_db_source: Arc<dyn ZgsKeyValueDB>,
         config: LogConfig,
-        executor: task_executor::TaskExecutor,
     ) -> Result<Self> {
-        let tx_store = TransactionStore::new(db.clone())?;
-        let flow_store = Arc::new(FlowStore::new(db.clone(), config.flow));
-        let mut initial_data = flow_store.get_chunk_root_list()?;
-        // If the last tx `put_tx` does not complete, we will revert it in `initial_data.subtree_list`
-        // first and call `put_tx` later. The known leaves in its data will be saved in `extra_leaves`
-        // and inserted later.
-        let mut extra_leaves = Vec::new();
-
+        let tx_store = TransactionStore::new(flow_db_source.clone(), data_db_source.clone())?;
+        let flow_db = Arc::new(FlowDBStore::new(flow_db_source.clone()));
+        let data_db = Arc::new(FlowDBStore::new(data_db_source.clone()));
+        let flow_store = Arc::new(FlowStore::new(
+            flow_db.clone(),
+            data_db.clone(),
+            config.flow.clone(),
+        ));
+        // If the last tx `put_tx` does not complete, we will revert it in `pora_chunks_merkle`
+        // first and call `put_tx` later.
         let next_tx_seq = tx_store.next_tx_seq();
         let mut start_tx_seq = if next_tx_seq > 0 {
             Some(next_tx_seq - 1)
@@ -640,15 +717,25 @@ impl LogManager {
             None
         };
         let mut last_tx_to_insert = None;
+
+        let mut pora_chunks_merkle = Merkle::new_with_subtrees(
+            flow_db,
+            config.flow.merkle_node_cache_capacity,
+            log2_pow2(PORA_CHUNK_SIZE),
+        )?;
         if let Some(last_tx_seq) = start_tx_seq {
             if !tx_store.check_tx_completed(last_tx_seq)? {
                 // Last tx not finalized, we need to check if its `put_tx` is completed.
                 let last_tx = tx_store
                     .get_tx_by_seq_number(last_tx_seq)?
                     .expect("tx missing");
-                let mut current_len = initial_data.leaves();
-                let expected_len =
-                    sector_to_segment(last_tx.start_entry_index + last_tx.num_entries() as u64);
+                let current_len = pora_chunks_merkle.leaves();
+                let expected_len = sector_to_segment(
+                    last_tx.start_entry_index
+                        + last_tx.num_entries() as u64
+                        + PORA_CHUNK_SIZE as u64
+                        - 1,
+                );
                 match expected_len.cmp(&(current_len)) {
                     Ordering::Less => {
                         bail!(
@@ -676,43 +763,33 @@ impl LogManager {
                                 previous_tx.start_entry_index + previous_tx.num_entries() as u64,
                             );
                             if current_len > expected_len {
-                                while let Some((subtree_depth, _)) = initial_data.subtree_list.pop()
-                                {
-                                    current_len -= 1 << (subtree_depth - 1);
-                                    if current_len == expected_len {
-                                        break;
-                                    }
-                                }
+                                pora_chunks_merkle.revert_to_leaves(expected_len)?;
                             } else {
-                                warn!(
-                                    "revert last tx with no-op: {} {}",
-                                    current_len, expected_len
-                                );
-                            }
                                 assert_eq!(current_len, expected_len);
-                            while let Some((index, h)) = initial_data.known_leaves.pop() {
-                                if index < current_len {
-                                    initial_data.known_leaves.push((index, h));
-                                    break;
-                                } else {
-                                    extra_leaves.push((index, h));
                             }
-                            }
-                            start_tx_seq = Some(last_tx_seq - 1);
+                            start_tx_seq = Some(previous_tx.seq);
                         };
                     }
                 }
             }
         }
 
-        let mut pora_chunks_merkle =
-            Merkle::new_with_subtrees(initial_data, log2_pow2(PORA_CHUNK_SIZE), start_tx_seq)?;
         let last_chunk_merkle = match start_tx_seq {
             Some(tx_seq) => {
-                tx_store.rebuild_last_chunk_merkle(pora_chunks_merkle.leaves(), tx_seq)?
+                let tx = tx_store.get_tx_by_seq_number(tx_seq)?.expect("tx missing");
+                if (tx.start_entry_index() + tx.num_entries() as u64) % PORA_CHUNK_SIZE as u64 == 0
+                {
+                    // The last chunk should be aligned, so it's empty.
+                    Merkle::new_with_depth(vec![], log2_pow2(PORA_CHUNK_SIZE) + 1, None)
+                } else {
+                    tx_store.rebuild_last_chunk_merkle(pora_chunks_merkle.leaves() - 1, tx_seq)?
+                }
             }
             // Initialize
-            None => Merkle::new_with_depth(vec![], log2_pow2(PORA_CHUNK_SIZE) + 1, None),
+            None => {
+                pora_chunks_merkle.reset();
+                Merkle::new_with_depth(vec![], 1, None)
+            }
         };
 
         debug!(
@@ -722,81 +799,35 @@ impl LogManager {
             last_chunk_merkle.leaves(),
         );
         if last_chunk_merkle.leaves() != 0 {
-            pora_chunks_merkle.append(*last_chunk_merkle.root());
+            pora_chunks_merkle.update_last(last_chunk_merkle.root());
+        }
         // update the merkle root
         pora_chunks_merkle.commit(start_tx_seq);
-        }
         let merkle = RwLock::new(MerkleManager {
             pora_chunks_merkle,
             last_chunk_merkle,
         });
 
-        let (sender, receiver) = mpsc::channel();
-
-        let mut log_manager = Self {
-            db,
+        let log_manager = Self {
+            flow_db: flow_db_source,
+            data_db: data_db_source,
             tx_store,
             flow_store,
             merkle,
-            sender,
         };
 
-        log_manager.start_receiver(receiver, executor);
-
         if let Some(tx) = last_tx_to_insert {
-            log_manager.revert_to(tx.seq - 1)?;
             log_manager.put_tx(tx)?;
-            let mut merkle = log_manager.merkle.write();
-            for (index, h) in extra_leaves {
-                if index < merkle.pora_chunks_merkle.leaves() {
-                    merkle.pora_chunks_merkle.fill_leaf(index, h);
-                } else {
-                    error!("out of range extra leaf: index={} hash={:?}", index, h);
-                }
-            }
-        } else {
-            assert!(extra_leaves.is_empty());
         }
         log_manager
             .merkle
             .write()
             .try_initialize(&log_manager.flow_store)?;
-        Ok(log_manager)
-    }
-
-    fn start_receiver(
-        &mut self,
-        rx: mpsc::Receiver<UpdateFlowMessage>,
-        executor: task_executor::TaskExecutor,
-    ) {
-        let flow_store = self.flow_store.clone();
-        executor.spawn(
-            async move {
-                loop {
-                    match rx.recv() {
-                        std::result::Result::Ok(data) => {
-                            // Update the root index.
-                            flow_store.put_batch_root_list(data.root_map).unwrap();
-                            // Update the flow database.
-                            // This should be called before `complete_last_chunk_merkle` so that we do not save
-                            // subtrees with data known.
-                            flow_store
-                                .append_entries(ChunkArray {
-                                    data: vec![0; data.pad_data],
-                                    start_index: data.tx_start_flow_index,
-                                })
-                                .unwrap();
-                        }
-                        std::result::Result::Err(_) => {
-                            error!("Receiver error");
-                        }
-                    };
-                }
-            },
-            "pad_tx",
+        info!(
+            "Log manager initialized, state={:?}",
+            log_manager.get_context()?
         );
-        // Wait for the spawned thread to finish
-        // let _ = handle.join().expect("Thread panicked");
+        Ok(log_manager)
     }
 
     fn gen_proof(&self, flow_index: u64, maybe_root: Option<DataRoot>) -> Result<FlowProof> {
@@ -848,26 +879,13 @@ impl LogManager {
                     .gen_proof(flow_index as usize % PORA_CHUNK_SIZE)?,
             }
         };
-        let r = entry_proof(&top_proof, &sub_proof);
-        if r.is_err() {
-            let raw_batch = self.flow_store.get_raw_batch(seg_index as u64)?.unwrap();
-            let db_root = self.flow_store.get_batch_root(seg_index as u64)?;
-            error!(
-                ?r,
-                ?db_root,
-                ?seg_index,
-                "gen proof error: top_leaves={}, last={}, raw_batch={}",
-                merkle.pora_chunks_merkle.leaves(),
-                merkle.last_chunk_merkle.leaves(),
-                serde_json::to_string(&raw_batch).unwrap(),
-            );
-        }
-        r
+        entry_proof(&top_proof, &sub_proof)
     }
 
     #[instrument(skip(self, merkle))]
     fn append_subtree_list(
         &self,
+        tx_seq: u64,
         tx_start_index: u64,
         merkle_list: Vec<(usize, DataRoot)>,
         merkle: &mut MerkleManager,
@@ -875,10 +893,10 @@ impl LogManager {
         if merkle_list.is_empty() {
             return Ok(());
         }
+        let start_time = Instant::now();
 
-        self.pad_tx(tx_start_index, &mut *merkle)?;
+        self.pad_tx(tx_seq, tx_start_index, &mut *merkle)?;
 
-        let mut batch_root_map = BTreeMap::new();
         for (subtree_depth, subtree_root) in merkle_list {
             let subtree_size = 1 << (subtree_depth - 1);
             if merkle.last_chunk_merkle.leaves() + subtree_size <= PORA_CHUNK_SIZE {
@@ -889,17 +907,13 @@ impl LogManager {
                     // `last_chunk_merkle` was empty, so this is a new leaf in the top_tree.
                     merkle
                         .pora_chunks_merkle
-                        .append_subtree(1, *merkle.last_chunk_merkle.root())?;
+                        .append_subtree(1, merkle.last_chunk_merkle.root())?;
                 } else {
                     merkle
                         .pora_chunks_merkle
-                        .update_last(*merkle.last_chunk_merkle.root());
+                        .update_last(merkle.last_chunk_merkle.root());
                 }
                 if merkle.last_chunk_merkle.leaves() == PORA_CHUNK_SIZE {
-                    batch_root_map.insert(
-                        merkle.pora_chunks_merkle.leaves() - 1,
-                        (*merkle.last_chunk_merkle.root(), 1),
-                    );
                     self.complete_last_chunk_merkle(
                         merkle.pora_chunks_merkle.leaves() - 1,
                         &mut *merkle,
@@ -910,22 +924,20 @@ impl LogManager {
                 // the chunks boundary.
                 assert_eq!(merkle.last_chunk_merkle.leaves(), 0);
                 assert!(subtree_size >= PORA_CHUNK_SIZE);
-                batch_root_map.insert(
-                    merkle.pora_chunks_merkle.leaves(),
-                    (subtree_root, subtree_depth - log2_pow2(PORA_CHUNK_SIZE)),
-                );
                 merkle
                     .pora_chunks_merkle
                     .append_subtree(subtree_depth - log2_pow2(PORA_CHUNK_SIZE), subtree_root)?;
             }
         }
-        self.flow_store.put_batch_root_list(batch_root_map)?;
+
+        metrics::APPEND_SUBTREE_LIST.update_since(start_time);
         Ok(())
     }
 
     #[instrument(skip(self, merkle))]
-    fn pad_tx(&self, tx_start_index: u64, merkle: &mut MerkleManager) -> Result<()> {
+    fn pad_tx(&self, tx_seq: u64, tx_start_index: u64, merkle: &mut MerkleManager) -> Result<()> {
         // Check if we need to pad the flow.
+        let start_time = Instant::now();
         let mut tx_start_flow_index =
             merkle.last_chunk_start_index() + merkle.last_chunk_merkle.leaves() as u64;
         let pad_size = tx_start_index - tx_start_flow_index;
@@ -934,10 +946,10 @@ impl LogManager {
             merkle.pora_chunks_merkle.leaves(),
             merkle.last_chunk_merkle.leaves()
         );
+        let mut pad_list = vec![];
         if pad_size != 0 {
             for pad_data in Self::padding(pad_size as usize) {
                 let mut is_full_empty = true;
-                let mut root_map = BTreeMap::new();
 
                 // Update the in-memory merkle tree.
                 let last_chunk_pad = if merkle.last_chunk_merkle.leaves() == 0 {
@@ -954,7 +966,7 @@ impl LogManager {
                         .append_list(data_to_merkle_leaves(&pad_data)?);
                     merkle
                         .pora_chunks_merkle
-                        .update_last(*merkle.last_chunk_merkle.root());
+                        .update_last(merkle.last_chunk_merkle.root());
                 } else {
                     if last_chunk_pad != 0 {
                         is_full_empty = false;
@@ -964,23 +976,14 @@ impl LogManager {
                             .append_list(data_to_merkle_leaves(&pad_data[..last_chunk_pad])?);
                         merkle
                             .pora_chunks_merkle
-                            .update_last(*merkle.last_chunk_merkle.root());
-                        root_map.insert(
-                            merkle.pora_chunks_merkle.leaves() - 1,
-                            (*merkle.last_chunk_merkle.root(), 1),
-                        );
+                            .update_last(merkle.last_chunk_merkle.root());
                         completed_chunk_index = Some(merkle.pora_chunks_merkle.leaves() - 1);
                     }
 
                     // Pad with more complete chunks.
                     let mut start_index = last_chunk_pad / ENTRY_SIZE;
                     while pad_data.len() >= (start_index + PORA_CHUNK_SIZE) * ENTRY_SIZE {
-                        let data = pad_data[start_index * ENTRY_SIZE
-                            ..(start_index + PORA_CHUNK_SIZE) * ENTRY_SIZE]
-                            .to_vec();
-                        let root = *Merkle::new(data_to_merkle_leaves(&data)?, 0, None).root();
-                        merkle.pora_chunks_merkle.append(root);
-                        root_map.insert(merkle.pora_chunks_merkle.leaves() - 1, (root, 1));
+                        merkle.pora_chunks_merkle.append(*PAD_SEGMENT_ROOT);
                         start_index += PORA_CHUNK_SIZE;
                     }
                     assert_eq!(pad_data.len(), start_index * ENTRY_SIZE);
@@ -988,13 +991,11 @@ impl LogManager {
 
                 let data_size = pad_data.len() / ENTRY_SIZE;
                 if is_full_empty {
-                    self.sender.send(UpdateFlowMessage {
-                        root_map,
-                        pad_data: pad_data.len(),
-                        tx_start_flow_index,
-                    })?;
+                    pad_list.push(PadPair {
+                        data_size: pad_data.len() as u64,
+                        start_index: tx_start_flow_index,
+                    });
                 } else {
-                    self.flow_store.put_batch_root_list(root_map).unwrap();
                     // Update the flow database.
                     // This should be called before `complete_last_chunk_merkle` so that we do not save
                     // subtrees with data known.
@@ -1015,6 +1016,10 @@ impl LogManager {
             merkle.pora_chunks_merkle.leaves(),
             merkle.last_chunk_merkle.leaves()
         );
+
+        self.flow_store.put_pad_data(&pad_list, tx_seq)?;
+
+        metrics::PAD_TX.update_since(start_time);
         Ok(())
     }
 
@@ -1056,7 +1061,7 @@ impl LogManager {
             }
             merkle
                 .pora_chunks_merkle
-                .update_last(*merkle.last_chunk_merkle.root());
+                .update_last(merkle.last_chunk_merkle.root());
         }
         let chunk_roots = self.flow_store.append_entries(flow_entry_array)?;
         for (chunk_index, chunk_root) in chunk_roots {
@@ -1143,6 +1148,8 @@ impl LogManager {
     }
 
     fn copy_tx_and_finalize(&self, from_tx_seq: u64, to_tx_seq_list: Vec<u64>) -> Result<()> {
+        let start_time = Instant::now();
+
         let mut merkle = self.merkle.write();
         let shard_config = self.flow_store.get_shard_config();
         // We have all the data need for this tx, so just copy them.
@@ -1191,6 +1198,8 @@ impl LogManager {
         for (seq, _) in to_tx_offset_list {
             self.tx_store.finalize_tx(seq)?;
         }
+
+        metrics::COPY_TX_AND_FINALIZE.update_since(start_time);
         Ok(())
     }
 
@@ -1263,6 +1272,7 @@ pub fn sub_merkle_tree(leaf_data: &[u8]) -> Result<FileMerkleTree> {
 }
 
 pub fn data_to_merkle_leaves(leaf_data: &[u8]) -> Result<Vec<H256>> {
+    let start_time = Instant::now();
     if leaf_data.len() % ENTRY_SIZE != 0 {
         bail!("merkle_tree: mismatched data size");
     }
@@ -1278,6 +1288,9 @@ pub fn data_to_merkle_leaves(leaf_data: &[u8]) -> Result<Vec<H256>> {
             .map(Sha3Algorithm::leaf)
             .collect()
     };
+
+    metrics::DATA_TO_MERKLE_LEAVES_SIZE.update(leaf_data.len());
+    metrics::DATA_TO_MERKLE_LEAVES.update_since(start_time);
     Ok(r)
 }
 

node/storage/src/log_store/metrics.rs

@@ -0,0 +1,43 @@
+use std::sync::Arc;
+
+use metrics::{register_timer, Gauge, GaugeUsize, Timer};
+
+lazy_static::lazy_static! {
+    pub static ref PUT_TX: Arc<dyn Timer> = register_timer("log_store_put_tx");
+
+    pub static ref PUT_CHUNKS: Arc<dyn Timer> = register_timer("log_store_put_chunks");
+
+    pub static ref TX_STORE_PUT: Arc<dyn Timer> = register_timer("log_store_tx_store_put_tx");
+
+    pub static ref CHECK_TX_COMPLETED: Arc<dyn Timer> =
+        register_timer("log_store_log_manager_check_tx_completed");
+
+    pub static ref APPEND_SUBTREE_LIST: Arc<dyn Timer> =
+        register_timer("log_store_log_manager_append_subtree_list");
+
+    pub static ref DATA_TO_MERKLE_LEAVES: Arc<dyn Timer> =
+        register_timer("log_store_log_manager_data_to_merkle_leaves");
+
+    pub static ref COPY_TX_AND_FINALIZE: Arc<dyn Timer> =
+        register_timer("log_store_log_manager_copy_tx_and_finalize");
+
+    pub static ref PAD_TX: Arc<dyn Timer> = register_timer("log_store_log_manager_pad_tx");
+
+    pub static ref PUT_BATCH_ROOT_LIST: Arc<dyn Timer> = register_timer("log_store_flow_store_put_batch_root_list");
+
+    pub static ref INSERT_SUBTREE_LIST: Arc<dyn Timer> =
+        register_timer("log_store_flow_store_insert_subtree_list");
+
+    pub static ref PUT_MPT_NODE: Arc<dyn Timer> = register_timer("log_store_flow_store_put_mpt_node");
+
+    pub static ref PUT_ENTRY_BATCH_LIST: Arc<dyn Timer> =
+        register_timer("log_store_flow_store_put_entry_batch_list");
+
+    pub static ref APPEND_ENTRIES: Arc<dyn Timer> = register_timer("log_store_flow_store_append_entries");
+
+    pub static ref FINALIZE_TX_WITH_HASH: Arc<dyn Timer> = register_timer("log_store_log_manager_finalize_tx_with_hash");
+
+    pub static ref DATA_TO_MERKLE_LEAVES_SIZE: Arc<dyn Gauge<usize>> = GaugeUsize::register("log_store_data_to_merkle_leaves_size");
+
+    pub static ref TX_BY_SEQ_NUMBER: Arc<dyn Timer> = register_timer("log_store_tx_store_get_tx_by_seq_number");
+}

node/storage/src/log_store/mod.rs

@@ -1,6 +1,7 @@
 use crate::config::ShardConfig;
-use append_merkle::MerkleTreeInitialData;
+
 use ethereum_types::H256;
+use flow_store::PadPair;
 use shared_types::{
     Chunk, ChunkArray, ChunkArrayWithProof, ChunkWithProof, DataRoot, FlowProof, FlowRangeProof,
     Transaction,
@@ -9,12 +10,13 @@ use zgs_spec::{BYTES_PER_SEAL, SEALS_PER_LOAD};
 
 use crate::error::Result;
 
-use self::tx_store::BlockHashAndSubmissionIndex;
+use self::tx_store::{BlockHashAndSubmissionIndex, TxStatus};
 
 pub mod config;
 mod flow_store;
-mod load_chunk;
+pub mod load_chunk;
 pub mod log_manager;
+mod metrics;
 mod seal_task_manager;
 #[cfg(test)]
 mod tests;
@@ -29,8 +31,12 @@ pub trait LogStoreRead: LogStoreChunkRead {
     fn get_tx_by_seq_number(&self, seq: u64) -> Result<Option<Transaction>>;
 
     /// Get a transaction by the data root of its data.
+    /// If all txs are not finalized, return the first one.
+    /// Otherwise, return the first finalized tx.
     fn get_tx_seq_by_data_root(&self, data_root: &DataRoot) -> Result<Option<u64>>;
 
+    /// If all txs are not finalized, return the first one.
+    /// Otherwise, return the first finalized tx.
     fn get_tx_by_data_root(&self, data_root: &DataRoot) -> Result<Option<Transaction>> {
         match self.get_tx_seq_by_data_root(data_root)? {
             Some(seq) => self.get_tx_by_seq_number(seq),
@@ -56,6 +62,8 @@ pub trait LogStoreRead: LogStoreChunkRead {
 
     fn check_tx_pruned(&self, tx_seq: u64) -> Result<bool>;
 
+    fn get_tx_status(&self, tx_seq: u64) -> Result<Option<TxStatus>>;
+
     fn next_tx_seq(&self) -> u64;
 
     fn get_sync_progress(&self) -> Result<Option<(u64, H256)>>;
@@ -158,6 +166,8 @@ pub trait LogStoreWrite: LogStoreChunkWrite {
     fn update_shard_config(&self, shard_config: ShardConfig);
 
     fn submit_seal_result(&self, answers: Vec<SealAnswer>) -> Result<()>;
+
+    fn start_padding(&self, executor: &task_executor::TaskExecutor);
 }
 
 pub trait LogStoreChunkWrite {
@@ -211,14 +221,16 @@ pub trait FlowRead {
     /// For simplicity, `index_start` and `index_end` must be at the batch boundaries.
     fn get_available_entries(&self, index_start: u64, index_end: u64) -> Result<Vec<ChunkArray>>;
 
-    fn get_chunk_root_list(&self) -> Result<MerkleTreeInitialData<DataRoot>>;
-
     fn load_sealed_data(&self, chunk_index: u64) -> Result<Option<MineLoadChunk>>;
 
     // An estimation of the number of entries in the flow db.
     fn get_num_entries(&self) -> Result<u64>;
 
     fn get_shard_config(&self) -> ShardConfig;
+
+    fn get_pad_data(&self, start_index: u64) -> Result<Option<Vec<PadPair>>>;
+
+    fn get_pad_data_sync_height(&self) -> Result<Option<u64>>;
 }
 
 pub trait FlowWrite {
@@ -233,6 +245,10 @@ pub trait FlowWrite {
 
     /// Update the shard config.
     fn update_shard_config(&self, shard_config: ShardConfig);
+
+    fn put_pad_data(&self, data_sizes: &[PadPair], tx_seq: u64) -> Result<()>;
+
+    fn put_pad_data_sync_height(&self, tx_seq: u64) -> Result<()>;
 }
 
 pub struct SealTask {
@@ -271,3 +287,23 @@ pub trait FlowSeal {
 
 pub trait Flow: FlowRead + FlowWrite + FlowSeal {}
 impl<T: FlowRead + FlowWrite + FlowSeal> Flow for T {}
+
+pub trait PadDataStoreRead {
+    fn get_pad_data(&self, start_index: u64) -> Result<Option<Vec<PadPair>>>;
+    fn get_pad_data_sync_height(&self) -> Result<Option<u64>>;
+}
+
+pub trait PadDataStoreWrite {
+    fn put_pad_data(&self, data_sizes: &[PadPair], tx_seq: u64) -> Result<()>;
+    fn put_pad_data_sync_height(&self, tx_seq: u64) -> Result<()>;
+    fn start_padding(&mut self, executor: &task_executor::TaskExecutor);
+}
+
+pub trait PadDataStore:
+    PadDataStoreRead + PadDataStoreWrite + config::Configurable + Send + Sync + 'static
+{
+}
+impl<T: PadDataStoreRead + PadDataStoreWrite + config::Configurable + Send + Sync + 'static>
+    PadDataStore for T
+{
+}

node/storage/src/log_store/tests.rs

@@ -8,15 +8,11 @@ use ethereum_types::H256;
 use rand::random;
 use shared_types::{compute_padded_chunk_size, ChunkArray, Transaction, CHUNK_SIZE};
 use std::cmp;
-use task_executor::test_utils::TestRuntime;
 
 #[test]
 fn test_put_get() {
     let config = LogConfig::default();
-    let runtime = TestRuntime::default();
-
-    let executor = runtime.task_executor.clone();
-    let store = LogManager::memorydb(config.clone(), executor).unwrap();
+    let store = LogManager::memorydb(config.clone()).unwrap();
     let chunk_count = config.flow.batch_size + config.flow.batch_size / 2 - 1;
     // Aligned with size.
     let start_offset = 1024;
@@ -173,10 +169,7 @@ fn test_put_tx() {
 
 fn create_store() -> LogManager {
     let config = LogConfig::default();
-    let runtime = TestRuntime::default();
-    let executor = runtime.task_executor.clone();
-
-    LogManager::memorydb(config, executor).unwrap()
+    LogManager::memorydb(config).unwrap()
 }
 
 fn put_tx(store: &mut LogManager, chunk_count: usize, seq: u64) {

node/storage/src/log_store/tx_store.rs

@@ -3,6 +3,7 @@ use crate::log_store::log_manager::{
     data_to_merkle_leaves, sub_merkle_tree, COL_BLOCK_PROGRESS, COL_MISC, COL_TX, COL_TX_COMPLETED,
     COL_TX_DATA_ROOT_INDEX, ENTRY_SIZE, PORA_CHUNK_SIZE,
 };
+use crate::log_store::metrics;
 use crate::{try_option, LogManager, ZgsKeyValueDB};
 use anyhow::{anyhow, Result};
 use append_merkle::{AppendMerkleTree, MerkleTreeRead, Sha3Algorithm};
@@ -15,14 +16,39 @@ use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
+use std::time::Instant;
 use tracing::{error, instrument};
 
 const LOG_SYNC_PROGRESS_KEY: &str = "log_sync_progress";
 const NEXT_TX_KEY: &str = "next_tx_seq";
 const LOG_LATEST_BLOCK_NUMBER_KEY: &str = "log_latest_block_number_key";
 
-const TX_STATUS_FINALIZED: u8 = 0;
-const TX_STATUS_PRUNED: u8 = 1;
+#[derive(Debug)]
+pub enum TxStatus {
+    Finalized,
+    Pruned,
+}
+
+impl From<TxStatus> for u8 {
+    fn from(value: TxStatus) -> Self {
+        match value {
+            TxStatus::Finalized => 0,
+            TxStatus::Pruned => 1,
+        }
+    }
+}
+
+impl TryFrom<u8> for TxStatus {
+    type Error = anyhow::Error;
+
+    fn try_from(value: u8) -> std::result::Result<Self, Self::Error> {
+        match value {
+            0 => Ok(TxStatus::Finalized),
+            1 => Ok(TxStatus::Pruned),
+            _ => Err(anyhow!("invalid value for tx status {}", value)),
+        }
+    }
+}
 
 #[derive(Clone, Debug)]
 pub struct BlockHashAndSubmissionIndex {
@@ -31,19 +57,24 @@ pub struct BlockHashAndSubmissionIndex {
 }
 
 pub struct TransactionStore {
-    kvdb: Arc<dyn ZgsKeyValueDB>,
+    flow_kvdb: Arc<dyn ZgsKeyValueDB>,
+    data_kvdb: Arc<dyn ZgsKeyValueDB>,
     /// This is always updated before writing the database to ensure no intermediate states.
     next_tx_seq: AtomicU64,
 }
 
 impl TransactionStore {
-    pub fn new(kvdb: Arc<dyn ZgsKeyValueDB>) -> Result<Self> {
-        let next_tx_seq = kvdb
+    pub fn new(
+        flow_kvdb: Arc<dyn ZgsKeyValueDB>,
+        data_kvdb: Arc<dyn ZgsKeyValueDB>,
+    ) -> Result<Self> {
+        let next_tx_seq = flow_kvdb
             .get(COL_TX, NEXT_TX_KEY.as_bytes())?
             .map(|a| decode_tx_seq(&a))
             .unwrap_or(Ok(0))?;
         Ok(Self {
-            kvdb,
+            flow_kvdb,
+            data_kvdb,
             next_tx_seq: AtomicU64::new(next_tx_seq),
         })
     }
@@ -51,6 +82,8 @@ impl TransactionStore {
     #[instrument(skip(self))]
     /// Return `Ok(Some(tx_seq))` if a previous transaction has the same tx root.
     pub fn put_tx(&self, mut tx: Transaction) -> Result<Vec<u64>> {
+        let start_time = Instant::now();
+
         let old_tx_seq_list = self.get_tx_seq_list_by_data_root(&tx.data_merkle_root)?;
         if old_tx_seq_list.last().is_some_and(|seq| *seq == tx.seq) {
             // The last tx is inserted again, so no need to process it.
@@ -58,7 +91,7 @@ impl TransactionStore {
             return Ok(old_tx_seq_list);
         }
 
-        let mut db_tx = self.kvdb.transaction();
+        let mut db_tx = self.flow_kvdb.transaction();
         if !tx.data.is_empty() {
             tx.size = tx.data.len() as u64;
             let mut padded_data = tx.data.clone();
@@ -85,31 +118,35 @@ impl TransactionStore {
             &new_tx_seq_list.as_ssz_bytes(),
         );
         self.next_tx_seq.store(tx.seq + 1, Ordering::SeqCst);
-        self.kvdb.write(db_tx)?;
+        self.flow_kvdb.write(db_tx)?;
+        metrics::TX_STORE_PUT.update_since(start_time);
         Ok(old_tx_seq_list)
     }
 
     pub fn get_tx_by_seq_number(&self, seq: u64) -> Result<Option<Transaction>> {
+        let start_time = Instant::now();
         if seq >= self.next_tx_seq() {
             return Ok(None);
         }
-        let value = try_option!(self.kvdb.get(COL_TX, &seq.to_be_bytes())?);
+        let value = try_option!(self.flow_kvdb.get(COL_TX, &seq.to_be_bytes())?);
         let tx = Transaction::from_ssz_bytes(&value).map_err(Error::from)?;
+        metrics::TX_BY_SEQ_NUMBER.update_since(start_time);
         Ok(Some(tx))
     }
 
     pub fn remove_tx_after(&self, min_seq: u64) -> Result<Vec<Transaction>> {
         let mut removed_txs = Vec::new();
         let max_seq = self.next_tx_seq();
-        let mut db_tx = self.kvdb.transaction();
+        let mut flow_db_tx = self.flow_kvdb.transaction();
+        let mut data_db_tx = self.data_kvdb.transaction();
         let mut modified_merkle_root_map = HashMap::new();
         for seq in min_seq..max_seq {
             let Some(tx) = self.get_tx_by_seq_number(seq)? else {
                 error!(?seq, ?max_seq, "Transaction missing before the end");
                 break;
             };
-            db_tx.delete(COL_TX, &seq.to_be_bytes());
-            db_tx.delete(COL_TX_COMPLETED, &seq.to_be_bytes());
+            flow_db_tx.delete(COL_TX, &seq.to_be_bytes());
+            data_db_tx.delete(COL_TX_COMPLETED, &seq.to_be_bytes());
             // We only remove tx when the blockchain reorgs.
             // If a tx is reverted, all data after it will also be reverted, so we call remove
             // all indices after it.
@@ -124,24 +161,25 @@ impl TransactionStore {
         }
         for (merkle_root, tx_seq_list) in modified_merkle_root_map {
             if tx_seq_list.is_empty() {
-                db_tx.delete(COL_TX_DATA_ROOT_INDEX, merkle_root.as_bytes());
+                flow_db_tx.delete(COL_TX_DATA_ROOT_INDEX, merkle_root.as_bytes());
             } else {
-                db_tx.put(
+                flow_db_tx.put(
                     COL_TX_DATA_ROOT_INDEX,
                     merkle_root.as_bytes(),
                     &tx_seq_list.as_ssz_bytes(),
                 );
             }
         }
-        db_tx.put(COL_TX, NEXT_TX_KEY.as_bytes(), &min_seq.to_be_bytes());
+        flow_db_tx.put(COL_TX, NEXT_TX_KEY.as_bytes(), &min_seq.to_be_bytes());
         self.next_tx_seq.store(min_seq, Ordering::SeqCst);
-        self.kvdb.write(db_tx)?;
+        self.data_kvdb.write(data_db_tx)?;
+        self.flow_kvdb.write(flow_db_tx)?;
         Ok(removed_txs)
     }
 
     pub fn get_tx_seq_list_by_data_root(&self, data_root: &DataRoot) -> Result<Vec<u64>> {
         let value = match self
-            .kvdb
+            .flow_kvdb
             .get(COL_TX_DATA_ROOT_INDEX, data_root.as_bytes())?
         {
             Some(v) => v,
@@ -150,37 +188,45 @@ impl TransactionStore {
         Ok(Vec::<u64>::from_ssz_bytes(&value).map_err(Error::from)?)
     }
 
-    pub fn get_first_tx_seq_by_data_root(&self, data_root: &DataRoot) -> Result<Option<u64>> {
-        let value = try_option!(self
-            .kvdb
-            .get(COL_TX_DATA_ROOT_INDEX, data_root.as_bytes())?);
-        let seq_list = Vec::<u64>::from_ssz_bytes(&value).map_err(Error::from)?;
-        Ok(seq_list.first().cloned())
-    }
-
     #[instrument(skip(self))]
     pub fn finalize_tx(&self, tx_seq: u64) -> Result<()> {
-        Ok(self.kvdb.put(
+        Ok(self.data_kvdb.put(
             COL_TX_COMPLETED,
             &tx_seq.to_be_bytes(),
-            &[TX_STATUS_FINALIZED],
+            &[TxStatus::Finalized.into()],
         )?)
     }
 
     #[instrument(skip(self))]
     pub fn prune_tx(&self, tx_seq: u64) -> Result<()> {
-        Ok(self
-            .kvdb
-            .put(COL_TX_COMPLETED, &tx_seq.to_be_bytes(), &[TX_STATUS_PRUNED])?)
+        Ok(self.data_kvdb.put(
+            COL_TX_COMPLETED,
+            &tx_seq.to_be_bytes(),
+            &[TxStatus::Pruned.into()],
+        )?)
+    }
+
+    pub fn get_tx_status(&self, tx_seq: u64) -> Result<Option<TxStatus>> {
+        let value = try_option!(self
+            .data_kvdb
+            .get(COL_TX_COMPLETED, &tx_seq.to_be_bytes())?);
+        match value.first() {
+            Some(v) => Ok(Some(TxStatus::try_from(*v)?)),
+            None => Ok(None),
+        }
     }
 
     pub fn check_tx_completed(&self, tx_seq: u64) -> Result<bool> {
-        Ok(self.kvdb.get(COL_TX_COMPLETED, &tx_seq.to_be_bytes())?
-            == Some(vec![TX_STATUS_FINALIZED]))
+        let start_time = Instant::now();
+        let status = self.get_tx_status(tx_seq)?;
+
+        metrics::CHECK_TX_COMPLETED.update_since(start_time);
+        Ok(matches!(status, Some(TxStatus::Finalized)))
     }
 
     pub fn check_tx_pruned(&self, tx_seq: u64) -> Result<bool> {
-        Ok(self.kvdb.get(COL_TX_COMPLETED, &tx_seq.to_be_bytes())? == Some(vec![TX_STATUS_PRUNED]))
+        let status = self.get_tx_status(tx_seq)?;
+        Ok(matches!(status, Some(TxStatus::Pruned)))
     }
 
     pub fn next_tx_seq(&self) -> u64 {
@@ -202,14 +248,14 @@ impl TransactionStore {
                 (progress.1, p).as_ssz_bytes(),
             ));
         }
-        Ok(self.kvdb.puts(items)?)
+        Ok(self.flow_kvdb.puts(items)?)
     }
 
     #[instrument(skip(self))]
     pub fn get_progress(&self) -> Result<Option<(u64, H256)>> {
         Ok(Some(
             <(u64, H256)>::from_ssz_bytes(&try_option!(self
-                .kvdb
+                .flow_kvdb
                 .get(COL_MISC, LOG_SYNC_PROGRESS_KEY.as_bytes())?))
             .map_err(Error::from)?,
         ))
@@ -217,7 +263,7 @@ impl TransactionStore {
 
     #[instrument(skip(self))]
     pub fn put_log_latest_block_number(&self, block_number: u64) -> Result<()> {
-        Ok(self.kvdb.put(
+        Ok(self.flow_kvdb.put(
             COL_MISC,
             LOG_LATEST_BLOCK_NUMBER_KEY.as_bytes(),
             &block_number.as_ssz_bytes(),
@@ -228,7 +274,7 @@ impl TransactionStore {
     pub fn get_log_latest_block_number(&self) -> Result<Option<u64>> {
         Ok(Some(
             <u64>::from_ssz_bytes(&try_option!(self
-                .kvdb
+                .flow_kvdb
                 .get(COL_MISC, LOG_LATEST_BLOCK_NUMBER_KEY.as_bytes())?))
             .map_err(Error::from)?,
         ))
@@ -240,7 +286,7 @@ impl TransactionStore {
     ) -> Result<Option<(H256, Option<u64>)>> {
         Ok(Some(
             <(H256, Option<u64>)>::from_ssz_bytes(&try_option!(self
-                .kvdb
+                .flow_kvdb
                 .get(COL_BLOCK_PROGRESS, &block_number.to_be_bytes())?))
             .map_err(Error::from)?,
         ))
@@ -248,7 +294,7 @@ impl TransactionStore {
 
     pub fn get_block_hashes(&self) -> Result<Vec<(u64, BlockHashAndSubmissionIndex)>> {
         let mut block_numbers = vec![];
-        for r in self.kvdb.iter(COL_BLOCK_PROGRESS) {
+        for r in self.flow_kvdb.iter(COL_BLOCK_PROGRESS) {
             let (key, val) = r?;
             let block_number =
                 u64::from_be_bytes(key.as_ref().try_into().map_err(|e| anyhow!("{:?}", e))?);
@@ -268,7 +314,7 @@ impl TransactionStore {
 
     pub fn delete_block_hash_by_number(&self, block_number: u64) -> Result<()> {
         Ok(self
-            .kvdb
+            .flow_kvdb
             .delete(COL_BLOCK_PROGRESS, &block_number.to_be_bytes())?)
     }
 
@@ -292,6 +338,9 @@ impl TransactionStore {
             match tx.start_entry_index.cmp(&last_chunk_start_index) {
                 cmp::Ordering::Greater => {
                     tx_list.push((tx_seq, tx.merkle_nodes));
+                    if tx.start_entry_index >= last_chunk_start_index + PORA_CHUNK_SIZE as u64 {
+                        break;
+                    }
                 }
                 cmp::Ordering::Equal => {
                     tx_list.push((tx_seq, tx.merkle_nodes));
@@ -335,11 +384,7 @@ impl TransactionStore {
         }
         let mut merkle = if last_chunk_start_index == 0 {
             // The first entry hash is initialized as zero.
-            AppendMerkleTree::<H256, Sha3Algorithm>::new_with_depth(
-                vec![H256::zero()],
-                log2_pow2(PORA_CHUNK_SIZE) + 1,
-                None,
-            )
+            AppendMerkleTree::<H256, Sha3Algorithm>::new_with_depth(vec![H256::zero()], 1, None)
         } else {
             AppendMerkleTree::<H256, Sha3Algorithm>::new_with_depth(
                 vec![],

node/sync/src/auto_sync/batcher.rs

@@ -84,14 +84,12 @@ impl Batcher {
     }
 
     async fn poll_tx(&self, tx_seq: u64) -> Result<Option<SyncResult>> {
-        // file already exists
-        if self.store.check_tx_completed(tx_seq).await?
-            || self.store.check_tx_pruned(tx_seq).await?
-        {
-            // File may be finalized during file sync, e.g. user uploaded file via RPC.
-            // In this case, just terminate the file sync.
-            let num_terminated = self.terminate_file_sync(tx_seq, false).await;
-            info!(%tx_seq, %num_terminated, "Terminate file sync due to file already finalized in db");
+        // file already finalized or even pruned
+        if let Some(tx_status) = self.store.get_store().get_tx_status(tx_seq)? {
+            let num_terminated: usize = self.terminate_file_sync(tx_seq, false).await;
+            if num_terminated > 0 {
+                info!(%tx_seq, %num_terminated, ?tx_status, "Terminate file sync due to file already completed in db");
+            }
             return Ok(Some(SyncResult::Completed));
         }
 

node/sync/src/auto_sync/batcher_random.rs

@@ -15,6 +15,7 @@ use tokio::time::sleep;
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct RandomBatcherState {
+    pub name: String,
     pub tasks: Vec<u64>,
     pub pending_txs: usize,
     pub ready_txs: usize,
@@ -22,6 +23,7 @@ pub struct RandomBatcherState {
 
 #[derive(Clone)]
 pub struct RandomBatcher {
+    name: String,
     config: Config,
     batcher: Batcher,
     sync_store: Arc<SyncStore>,
@@ -29,12 +31,14 @@ pub struct RandomBatcher {
 
 impl RandomBatcher {
     pub fn new(
+        name: String,
         config: Config,
         store: Store,
         sync_send: SyncSender,
         sync_store: Arc<SyncStore>,
     ) -> Self {
         Self {
+            name,
             config,
             batcher: Batcher::new(
                 config.max_random_workers,
@@ -50,6 +54,7 @@ impl RandomBatcher {
         let (pending_txs, ready_txs) = self.sync_store.stat().await?;
 
         Ok(RandomBatcherState {
+            name: self.name.clone(),
             tasks: self.batcher.tasks().await,
             pending_txs,
             ready_txs,
@@ -57,21 +62,20 @@ impl RandomBatcher {
     }
 
     pub async fn start(mut self, catched_up: Arc<AtomicBool>) {
-        info!("Start to sync files");
+        info!("Start to sync files, state = {:?}", self.get_state().await);
 
-        loop {
-            // disable file sync until catched up
-            if !catched_up.load(Ordering::Relaxed) {
+        // wait for log entry sync catched up
+        while !catched_up.load(Ordering::Relaxed) {
             trace!("Cannot sync file in catch-up phase");
             sleep(self.config.auto_sync_idle_interval).await;
-                continue;
         }
 
-            if let Ok(state) = self.get_state().await {
-                metrics::RANDOM_STATE_TXS_SYNCING.update(state.tasks.len() as u64);
-                metrics::RANDOM_STATE_TXS_READY.update(state.ready_txs as u64);
-                metrics::RANDOM_STATE_TXS_PENDING.update(state.pending_txs as u64);
-            }
+        loop {
+            // if let Ok(state) = self.get_state().await {
+            //     metrics::RANDOM_STATE_TXS_SYNCING.update(state.tasks.len() as u64);
+            //     metrics::RANDOM_STATE_TXS_READY.update(state.ready_txs as u64);
+            //     metrics::RANDOM_STATE_TXS_PENDING.update(state.pending_txs as u64);
+            // }
 
             match self.sync_once().await {
                 Ok(true) => {}

node/sync/src/auto_sync/historical_tx_writer.rs

@@ -0,0 +1,103 @@
+use std::sync::{
+    atomic::{AtomicU64, Ordering},
+    Arc,
+};
+
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+use storage::log_store::log_manager::DATA_DB_KEY;
+use storage_async::Store;
+use tokio::time::sleep;
+
+use crate::Config;
+
+use super::sync_store::{Queue, SyncStore};
+
+const KEY_NEXT_TX_SEQ: &str = "sync.manager.historical.next_tx_seq";
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HistoricalTxWriterState {
+    pub next_tx_seq: u64,
+    pub pending_txs: usize,
+    pub ready_txs: usize,
+}
+
+pub struct HistoricalTxWriter {
+    config: Config,
+    store: Store,
+    sync_store: Arc<SyncStore>,
+    next_tx_seq: Arc<AtomicU64>,
+}
+
+impl HistoricalTxWriter {
+    pub async fn new(config: Config, store: Store, sync_store: Arc<SyncStore>) -> Result<Self> {
+        let next_tx_seq = store
+            .get_config_decoded(&KEY_NEXT_TX_SEQ, DATA_DB_KEY)
+            .await?;
+
+        Ok(Self {
+            config,
+            store,
+            sync_store,
+            next_tx_seq: Arc::new(AtomicU64::new(next_tx_seq.unwrap_or(0))),
+        })
+    }
+
+    pub async fn get_state(&self) -> Result<HistoricalTxWriterState> {
+        let (pending_txs, ready_txs) = self.sync_store.stat().await?;
+
+        Ok(HistoricalTxWriterState {
+            next_tx_seq: self.next_tx_seq.load(Ordering::Relaxed),
+            pending_txs,
+            ready_txs,
+        })
+    }
+
+    pub async fn start(mut self) {
+        info!(
+            "Start to write historical files into sync store, state = {:?}",
+            self.get_state().await
+        );
+
+        loop {
+            match self.write_once().await {
+                Ok(true) => {}
+                Ok(false) => {
+                    trace!(
+                        "There is no tx to write in sync store, state = {:?}",
+                        self.get_state().await
+                    );
+                    sleep(self.config.auto_sync_idle_interval).await;
+                }
+                Err(err) => {
+                    warn!(%err, "Failed to write tx once, state = {:?}", self.get_state().await);
+                    sleep(self.config.auto_sync_error_interval).await;
+                }
+            }
+        }
+    }
+
+    async fn write_once(&mut self) -> Result<bool> {
+        let mut next_tx_seq = self.next_tx_seq.load(Ordering::Relaxed);
+
+        // no tx to write in sync store
+        if next_tx_seq >= self.store.get_store().next_tx_seq() {
+            return Ok(false);
+        }
+
+        // write tx in sync store if not finalized or pruned
+        if self.store.get_store().get_tx_status(next_tx_seq)?.is_none() {
+            self.sync_store.insert(next_tx_seq, Queue::Ready).await?;
+        }
+
+        // move forward
+        next_tx_seq += 1;
+        self.store
+            .set_config_encoded(&KEY_NEXT_TX_SEQ, &next_tx_seq, DATA_DB_KEY)
+            .await?;
+        self.next_tx_seq.store(next_tx_seq, Ordering::Relaxed);
+
+        Ok(true)
+    }
+}

node/sync/src/auto_sync/manager.rs

@@ -9,18 +9,24 @@ use storage_async::Store;
 use task_executor::TaskExecutor;
 use tokio::sync::{
     broadcast,
-    mpsc::{unbounded_channel, UnboundedSender},
+    mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
     oneshot,
 };
 
 use crate::{Config, SyncSender};
 
-use super::{batcher_random::RandomBatcher, batcher_serial::SerialBatcher, sync_store::SyncStore};
+use super::{
+    batcher_random::RandomBatcher,
+    batcher_serial::SerialBatcher,
+    historical_tx_writer::HistoricalTxWriter,
+    sync_store::{Queue, SyncStore},
+};
 
 pub struct AutoSyncManager {
-    pub serial: SerialBatcher,
+    pub serial: Option<SerialBatcher>,
     pub random: RandomBatcher,
     pub file_announcement_send: UnboundedSender<u64>,
+    pub new_file_send: UnboundedSender<u64>,
     pub catched_up: Arc<AtomicBool>,
 }
 
@@ -33,42 +39,112 @@ impl AutoSyncManager {
         log_sync_recv: broadcast::Receiver<LogSyncEvent>,
         catch_up_end_recv: oneshot::Receiver<()>,
     ) -> Result<Self> {
-        let (send, recv) = unbounded_channel();
-        let sync_store = Arc::new(SyncStore::new(store.clone()));
+        let (file_announcement_send, file_announcement_recv) = unbounded_channel();
+        let (new_file_send, new_file_recv) = unbounded_channel();
+        let sync_store = if config.neighbors_only {
+            // use v2 db to avoid reading v1 files that announced from the whole network instead of neighbors
+            Arc::new(SyncStore::new_with_name(
+                store.clone(),
+                "pendingv2",
+                "readyv2",
+            ))
+        } else {
+            Arc::new(SyncStore::new(store.clone()))
+        };
         let catched_up = Arc::new(AtomicBool::new(false));
 
+        // handle new file
+        executor.spawn(
+            Self::handle_new_file(new_file_recv, sync_store.clone()),
+            "auto_sync_handle_new_file",
+        );
+
         // sync in sequence
+        let serial = if config.neighbors_only {
+            None
+        } else {
             let serial =
                 SerialBatcher::new(config, store.clone(), sync_send.clone(), sync_store.clone())
                     .await?;
             executor.spawn(
                 serial
                     .clone()
-                .start(recv, log_sync_recv, catched_up.clone()),
+                    .start(file_announcement_recv, log_sync_recv, catched_up.clone()),
                 "auto_sync_serial",
             );
 
+            Some(serial)
+        };
+
         // sync randomly
-        let random = RandomBatcher::new(config, store, sync_send, sync_store);
+        let random = RandomBatcher::new(
+            "random".into(),
+            config,
+            store.clone(),
+            sync_send.clone(),
+            sync_store,
+        );
         executor.spawn(random.clone().start(catched_up.clone()), "auto_sync_random");
 
         // handle on catched up notification
-        let catched_up_cloned = catched_up.clone();
         executor.spawn(
-            async move {
-                if catch_up_end_recv.await.is_ok() {
-                    info!("log entry catched up");
-                    catched_up_cloned.store(true, Ordering::Relaxed);
-                }
-            },
+            Self::listen_catch_up(catch_up_end_recv, catched_up.clone()),
             "auto_sync_wait_for_catchup",
         );
 
+        // sync randomly for files without NewFile announcement
+        if config.neighbors_only {
+            let historical_sync_store = Arc::new(SyncStore::new_with_name(
+                store.clone(),
+                "pendingv2_historical",
+                "readyv2_historical",
+            ));
+
+            let writer =
+                HistoricalTxWriter::new(config, store.clone(), historical_sync_store.clone())
+                    .await?;
+            executor.spawn(writer.start(), "auto_sync_historical_writer");
+
+            let random_historical = RandomBatcher::new(
+                "random_historical".into(),
+                config,
+                store,
+                sync_send,
+                historical_sync_store,
+            );
+            executor.spawn(
+                random_historical.start(catched_up.clone()),
+                "auto_sync_random_historical",
+            );
+        }
+
         Ok(Self {
             serial,
             random,
-            file_announcement_send: send,
+            file_announcement_send,
+            new_file_send,
             catched_up,
         })
     }
+
+    async fn handle_new_file(
+        mut new_file_recv: UnboundedReceiver<u64>,
+        sync_store: Arc<SyncStore>,
+    ) {
+        while let Some(tx_seq) = new_file_recv.recv().await {
+            if let Err(err) = sync_store.insert(tx_seq, Queue::Ready).await {
+                warn!(?err, %tx_seq, "Failed to insert new file to ready queue");
+            }
+        }
+    }
+
+    async fn listen_catch_up(
+        catch_up_end_recv: oneshot::Receiver<()>,
+        catched_up: Arc<AtomicBool>,
+    ) {
+        if catch_up_end_recv.await.is_ok() {
+            info!("log entry catched up");
+            catched_up.store(true, Ordering::Relaxed);
+        }
+    }
 }

node/sync/src/auto_sync/metrics.rs

@@ -14,9 +14,9 @@ lazy_static::lazy_static! {
     pub static ref SEQUENTIAL_SYNC_RESULT_TIMEOUT: Arc<dyn Counter<usize>> = CounterUsize::register("sync_auto_sequential_sync_result_timeout");
 
     // random auto sync
-    pub static ref RANDOM_STATE_TXS_SYNCING: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register("sync_auto_random_state_txs_syncing", 1024);
-    pub static ref RANDOM_STATE_TXS_READY: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register("sync_auto_random_state_txs_ready", 1024);
-    pub static ref RANDOM_STATE_TXS_PENDING: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register("sync_auto_random_state_txs_pending", 1024);
+    // pub static ref RANDOM_STATE_TXS_SYNCING: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register("sync_auto_random_state_txs_syncing", 1024);
+    // pub static ref RANDOM_STATE_TXS_READY: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register("sync_auto_random_state_txs_ready", 1024);
+    // pub static ref RANDOM_STATE_TXS_PENDING: Arc<dyn Histogram> = Sample::ExpDecay(0.015).register("sync_auto_random_state_txs_pending", 1024);
 
     pub static ref RANDOM_SYNC_RESULT_COMPLETED: Arc<dyn Meter> = register_meter("sync_auto_random_sync_result_completed");
     pub static ref RANDOM_SYNC_RESULT_FAILED: Arc<dyn Counter<usize>> = CounterUsize::register("sync_auto_random_sync_result_failed");

node/sync/src/auto_sync/mod.rs

@@ -1,6 +1,7 @@
 mod batcher;
 pub mod batcher_random;
 pub mod batcher_serial;
+mod historical_tx_writer;
 pub mod manager;
 mod metrics;
 pub mod sync_store;

node/sync/src/auto_sync/sync_store.rs

@@ -1,7 +1,10 @@
 use super::tx_store::TxStore;
 use anyhow::Result;
 use std::sync::Arc;
-use storage::log_store::config::{ConfigTx, ConfigurableExt};
+use storage::log_store::{
+    config::{ConfigTx, ConfigurableExt},
+    log_manager::DATA_DB_KEY,
+};
 use storage_async::Store;
 use tokio::sync::RwLock;
 
@@ -42,6 +45,14 @@ impl SyncStore {
         }
     }
 
+    pub fn new_with_name(store: Store, pending: &'static str, ready: &'static str) -> Self {
+        Self {
+            store: Arc::new(RwLock::new(store)),
+            pending_txs: TxStore::new(pending),
+            ready_txs: TxStore::new(ready),
+        }
+    }
+
     /// Returns the number of pending txs and ready txs.
     pub async fn stat(&self) -> Result<(usize, usize)> {
         let async_store = self.store.read().await;
@@ -58,10 +69,10 @@ impl SyncStore {
         let store = async_store.get_store();
 
         // load next_tx_seq
-        let next_tx_seq = store.get_config_decoded(&KEY_NEXT_TX_SEQ)?;
+        let next_tx_seq = store.get_config_decoded(&KEY_NEXT_TX_SEQ, DATA_DB_KEY)?;
 
         // load max_tx_seq
-        let max_tx_seq = store.get_config_decoded(&KEY_MAX_TX_SEQ)?;
+        let max_tx_seq = store.get_config_decoded(&KEY_MAX_TX_SEQ, DATA_DB_KEY)?;
 
         Ok((next_tx_seq, max_tx_seq))
     }
@@ -69,13 +80,13 @@ impl SyncStore {
     pub async fn set_next_tx_seq(&self, tx_seq: u64) -> Result<()> {
         let async_store = self.store.write().await;
         let store = async_store.get_store();
-        store.set_config_encoded(&KEY_NEXT_TX_SEQ, &tx_seq)
+        store.set_config_encoded(&KEY_NEXT_TX_SEQ, &tx_seq, DATA_DB_KEY)
     }
 
     pub async fn set_max_tx_seq(&self, tx_seq: u64) -> Result<()> {
         let async_store = self.store.write().await;
         let store = async_store.get_store();
-        store.set_config_encoded(&KEY_MAX_TX_SEQ, &tx_seq)
+        store.set_config_encoded(&KEY_MAX_TX_SEQ, &tx_seq, DATA_DB_KEY)
     }
 
     pub async fn contains(&self, tx_seq: u64) -> Result<Option<Queue>> {
@@ -106,7 +117,7 @@ impl SyncStore {
                 }
 
                 let removed = self.pending_txs.remove(store, Some(&mut tx), tx_seq)?;
-                store.exec_configs(tx)?;
+                store.exec_configs(tx, DATA_DB_KEY)?;
 
                 if removed {
                     Ok(InsertResult::Upgraded)
@@ -120,7 +131,7 @@ impl SyncStore {
                 }
 
                 let removed = self.ready_txs.remove(store, Some(&mut tx), tx_seq)?;
-                store.exec_configs(tx)?;
+                store.exec_configs(tx, DATA_DB_KEY)?;
 
                 if removed {
                     Ok(InsertResult::Downgraded)
@@ -143,7 +154,7 @@ impl SyncStore {
 
         let added = self.ready_txs.add(store, Some(&mut tx), tx_seq)?;
 
-        store.exec_configs(tx)?;
+        store.exec_configs(tx, DATA_DB_KEY)?;
 
         Ok(added)
     }

node/sync/src/auto_sync/tx_store.rs

@@ -1,6 +1,7 @@
 use anyhow::Result;
 use rand::Rng;
 use storage::log_store::config::{ConfigTx, ConfigurableExt};
+use storage::log_store::log_manager::DATA_DB_KEY;
 use storage::log_store::Store;
 
 /// TxStore is used to store pending transactions that to be synchronized in advance.
@@ -32,11 +33,11 @@ impl TxStore {
     }
 
     fn index_of(&self, store: &dyn Store, tx_seq: u64) -> Result<Option<usize>> {
-        store.get_config_decoded(&self.key_seq_to_index(tx_seq))
+        store.get_config_decoded(&self.key_seq_to_index(tx_seq), DATA_DB_KEY)
     }
 
     fn at(&self, store: &dyn Store, index: usize) -> Result<Option<u64>> {
-        store.get_config_decoded(&self.key_index_to_seq(index))
+        store.get_config_decoded(&self.key_index_to_seq(index), DATA_DB_KEY)
     }
 
     pub fn has(&self, store: &dyn Store, tx_seq: u64) -> Result<bool> {
@@ -45,7 +46,7 @@ impl TxStore {
 
     pub fn count(&self, store: &dyn Store) -> Result<usize> {
         store
-            .get_config_decoded(&self.key_count)
+            .get_config_decoded(&self.key_count, DATA_DB_KEY)
             .map(|x| x.unwrap_or(0))
     }
 
@@ -70,7 +71,7 @@ impl TxStore {
         if let Some(db_tx) = db_tx {
             db_tx.append(&mut tx);
         } else {
-            store.exec_configs(tx)?;
+            store.exec_configs(tx, DATA_DB_KEY)?;
         }
 
         Ok(true)
@@ -130,7 +131,7 @@ impl TxStore {
         if let Some(db_tx) = db_tx {
             db_tx.append(&mut tx);
         } else {
-            store.exec_configs(tx)?;
+            store.exec_configs(tx, DATA_DB_KEY)?;
         }
 
         Ok(true)

node/sync/src/context.rs

@@ -1,12 +1,11 @@
-use network::{NetworkMessage, PeerAction, PeerId, PubsubMessage, ReportSource};
-use tokio::sync::mpsc;
+use network::{NetworkMessage, NetworkSender, PeerAction, PeerId, PubsubMessage, ReportSource};
 
 pub struct SyncNetworkContext {
-    network_send: mpsc::UnboundedSender<NetworkMessage>,
+    network_send: NetworkSender,
 }
 
 impl SyncNetworkContext {
-    pub fn new(network_send: mpsc::UnboundedSender<NetworkMessage>) -> Self {
+    pub fn new(network_send: NetworkSender) -> Self {
         Self { network_send }
     }
 

node/sync/src/controllers/serial.rs

@@ -14,12 +14,13 @@ use shared_types::{timestamp_now, ChunkArrayWithProof, TxID, CHUNK_SIZE};
 use ssz::Encode;
 use std::{sync::Arc, time::Instant};
 use storage::log_store::log_manager::{sector_to_segment, segment_to_sector, PORA_CHUNK_SIZE};
-use storage_async::Store;
+use storage_async::{ShardConfig, Store};
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum FailureReason {
     DBError(String),
     TxReverted(TxID),
+    TimeoutFindFile,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -159,11 +160,14 @@ impl SerialSyncController {
 
     /// Find more peers to sync chunks. Return whether `FindFile` pubsub message published,
     fn try_find_peers(&mut self) {
-        let (published, num_new_peers) = if self.goal.is_all_chunks() {
-            self.publish_find_file()
-        } else {
+        let (published, num_new_peers) = if !self.goal.is_all_chunks() {
             self.publish_find_chunks();
             (true, 0)
+        } else if self.config.neighbors_only {
+            self.do_publish_find_file();
+            (true, 0)
+        } else {
+            self.publish_find_file()
         };
 
         info!(%self.tx_seq, %published, %num_new_peers, "Finding peers");
@@ -199,14 +203,23 @@ impl SerialSyncController {
             return (false, num_new_peers);
         }
 
-        self.ctx.publish(PubsubMessage::FindFile(FindFile {
-            tx_id: self.tx_id,
-            timestamp: timestamp_now(),
-        }));
+        self.do_publish_find_file();
 
         (true, num_new_peers)
     }
 
+    fn do_publish_find_file(&self) {
+        let shard_config = self.store.get_store().get_shard_config();
+
+        self.ctx.publish(PubsubMessage::FindFile(FindFile {
+            tx_id: self.tx_id,
+            num_shard: shard_config.num_shard,
+            shard_id: shard_config.shard_id,
+            neighbors_only: self.config.neighbors_only,
+            timestamp: timestamp_now(),
+        }));
+    }
+
     fn publish_find_chunks(&self) {
         self.ctx.publish(PubsubMessage::FindChunks(FindChunks {
             tx_id: self.tx_id,
@@ -337,6 +350,14 @@ impl SerialSyncController {
         }
     }
 
+    /// Triggered when any peer (TCP connected) announced file via RPC message.
+    pub fn on_peer_announced(&mut self, peer_id: PeerId, shard_config: ShardConfig) {
+        self.peers
+            .add_new_peer_with_config(peer_id, Multiaddr::empty(), shard_config);
+        self.peers
+            .update_state_force(&peer_id, PeerState::Connected);
+    }
+
     pub fn on_dail_failed(&mut self, peer_id: PeerId, err: &DialError) {
         match err {
             DialError::ConnectionLimit(_) => {
@@ -545,6 +566,9 @@ impl SerialSyncController {
                 info!(%self.tx_seq, "Succeeded to finalize file");
                 self.state = SyncState::Completed;
                 metrics::SERIAL_SYNC_FILE_COMPLETED.update_since(self.since.0);
+                // notify neighbor nodes about new file completed to sync
+                self.ctx
+                    .send(NetworkMessage::AnnounceLocalFile { tx_id: self.tx_id });
             }
             Ok(false) => {
                 warn!(?self.tx_id, %self.tx_seq, "Transaction reverted during finalize_tx");
@@ -636,14 +660,21 @@ impl SerialSyncController {
                         .all_shards_available(vec![Found, Connecting, Connected])
                     {
                         self.state = SyncState::FoundPeers;
+                    } else {
+                        // FindFile timeout
+                        if since.elapsed() >= self.config.peer_find_timeout {
+                            if self.config.neighbors_only {
+                                self.state = SyncState::Failed {
+                                    reason: FailureReason::TimeoutFindFile,
+                                };
                             } else {
                                 // storage node may not have the specific file when `FindFile`
                                 // gossip message received. In this case, just broadcast the
                                 // `FindFile` message again.
-                        if since.elapsed() >= self.config.peer_find_timeout {
                                 debug!(%self.tx_seq, "Finding peer timeout and try to find peers again");
                                 self.try_find_peers();
                             }
+                        }
 
                         completed = true;
                     }
@@ -719,13 +750,13 @@ mod tests {
     use crate::test_util::create_2_store;
     use crate::test_util::tests::create_file_location_cache;
     use libp2p::identity;
+    use network::{new_network_channel, NetworkReceiver};
     use network::{ReportSource, Request};
     use storage::log_store::log_manager::LogConfig;
     use storage::log_store::log_manager::LogManager;
     use storage::log_store::LogStoreRead;
     use storage::H256;
     use task_executor::{test_utils::TestRuntime, TaskExecutor};
-    use tokio::sync::mpsc::{self, UnboundedReceiver};
 
     #[test]
     fn test_status() {
@@ -1513,6 +1544,10 @@ mod tests {
 
         controller.on_response(peer_id, chunks).await;
         assert_eq!(*controller.get_status(), SyncState::Completed);
+        assert!(matches!(
+            network_recv.try_recv().unwrap(),
+            NetworkMessage::AnnounceLocalFile { .. }
+        ));
         assert!(network_recv.try_recv().is_err());
     }
 
@@ -1614,7 +1649,7 @@ mod tests {
     fn create_default_controller(
         task_executor: TaskExecutor,
         peer_id: Option<PeerId>,
-    ) -> (SerialSyncController, UnboundedReceiver<NetworkMessage>) {
+    ) -> (SerialSyncController, NetworkReceiver) {
         let tx_id = TxID {
             seq: 0,
             hash: H256::random(),
@@ -1622,7 +1657,7 @@ mod tests {
         let num_chunks = 123;
 
         let config = LogConfig::default();
-        let store = Arc::new(LogManager::memorydb(config, task_executor.clone()).unwrap());
+        let store = Arc::new(LogManager::memorydb(config).unwrap());
 
         create_controller(task_executor, peer_id, store, tx_id, num_chunks)
     }
@@ -1633,8 +1668,8 @@ mod tests {
         store: Arc<LogManager>,
         tx_id: TxID,
         num_chunks: usize,
-    ) -> (SerialSyncController, UnboundedReceiver<NetworkMessage>) {
-        let (network_send, network_recv) = mpsc::unbounded_channel::<NetworkMessage>();
+    ) -> (SerialSyncController, NetworkReceiver) {
+        let (network_send, network_recv) = new_network_channel();
         let ctx = Arc::new(SyncNetworkContext::new(network_send));
 
         let peer_id = match peer_id {

node/sync/src/lib.rs

@@ -21,6 +21,10 @@ use std::{
 #[serde(default)]
 pub struct Config {
     // sync service config
+    /// Indicates whether to sync file from neighbor nodes only.
+    /// This is to avoid flooding file announcements in the whole network,
+    /// which leads to high latency or even timeout to sync files.
+    pub neighbors_only: bool,
     #[serde(deserialize_with = "deserialize_duration")]
     pub heartbeat_interval: Duration,
     pub auto_sync_enabled: bool,
@@ -64,6 +68,7 @@ impl Default for Config {
     fn default() -> Self {
         Self {
             // sync service config
+            neighbors_only: false,
             heartbeat_interval: Duration::from_secs(5),
             auto_sync_enabled: false,
             max_sync_files: 8,

node/sync/src/service.rs

@@ -8,11 +8,11 @@ use anyhow::{anyhow, bail, Result};
 use file_location_cache::FileLocationCache;
 use libp2p::swarm::DialError;
 use log_entry_sync::LogSyncEvent;
-use network::types::{AnnounceChunks, FindFile};
-use network::PubsubMessage;
+use network::rpc::methods::FileAnnouncement;
+use network::types::{AnnounceChunks, FindFile, NewFile};
 use network::{
-    rpc::GetChunksRequest, rpc::RPCResponseErrorCode, Multiaddr, NetworkMessage, PeerId,
-    PeerRequestId, SyncId as RequestId,
+    rpc::GetChunksRequest, rpc::RPCResponseErrorCode, Multiaddr, NetworkMessage, NetworkSender,
+    PeerId, PeerRequestId, PubsubMessage, SyncId as RequestId,
 };
 use shared_types::{bytes_to_chunks, timestamp_now, ChunkArrayWithProof, Transaction, TxID};
 use std::sync::atomic::Ordering;
@@ -26,7 +26,7 @@ use storage::error::Result as StorageResult;
 use storage::log_store::log_manager::{sector_to_segment, segment_to_sector, PORA_CHUNK_SIZE};
 use storage::log_store::Store as LogStore;
 use storage_async::Store;
-use tokio::sync::{broadcast, mpsc, oneshot};
+use tokio::sync::{broadcast, oneshot};
 
 pub type SyncSender = channel::Sender<SyncMessage, SyncRequest, SyncResponse>;
 pub type SyncReceiver = channel::Receiver<SyncMessage, SyncRequest, SyncResponse>;
@@ -70,6 +70,15 @@ pub enum SyncMessage {
     AnnounceChunksGossip {
         msg: AnnounceChunks,
     },
+    NewFile {
+        from: PeerId,
+        msg: NewFile,
+    },
+    AnnounceFile {
+        peer_id: PeerId,
+        request_id: PeerRequestId,
+        announcement: FileAnnouncement,
+    },
 }
 
 #[derive(Debug)]
@@ -132,7 +141,7 @@ pub struct SyncService {
 impl SyncService {
     pub async fn spawn(
         executor: task_executor::TaskExecutor,
-        network_send: mpsc::UnboundedSender<NetworkMessage>,
+        network_send: NetworkSender,
         store: Arc<dyn LogStore>,
         file_location_cache: Arc<FileLocationCache>,
         event_recv: broadcast::Receiver<LogSyncEvent>,
@@ -153,7 +162,7 @@ impl SyncService {
     pub async fn spawn_with_config(
         config: Config,
         executor: task_executor::TaskExecutor,
-        network_send: mpsc::UnboundedSender<NetworkMessage>,
+        network_send: NetworkSender,
         store: Arc<dyn LogStore>,
         file_location_cache: Arc<FileLocationCache>,
         event_recv: broadcast::Receiver<LogSyncEvent>,
@@ -265,6 +274,12 @@ impl SyncService {
             SyncMessage::AnnounceShardConfig { .. } => {
                 // FIXME: Check if controllers need to be reset?
             }
+            SyncMessage::NewFile { from, msg } => self.on_new_file_gossip(from, msg).await,
+            SyncMessage::AnnounceFile {
+                peer_id,
+                announcement,
+                ..
+            } => self.on_announce_file(peer_id, announcement).await,
         }
     }
 
@@ -279,7 +294,10 @@ impl SyncService {
                     Some(manager) => SyncServiceState {
                         num_syncing: self.controllers.len(),
                         catched_up: Some(manager.catched_up.load(Ordering::Relaxed)),
-                        auto_sync_serial: Some(manager.serial.get_state().await),
+                        auto_sync_serial: match &manager.serial {
+                            Some(v) => Some(v.get_state().await),
+                            None => None,
+                        },
                         auto_sync_random: manager.random.get_state().await.ok(),
                     },
                     None => SyncServiceState {
@@ -577,8 +595,12 @@ impl SyncService {
             Some(tx) => tx,
             None => bail!("Transaction not found"),
         };
+        let shard_config = self.store.get_store().get_shard_config();
         self.ctx.publish(PubsubMessage::FindFile(FindFile {
             tx_id: tx.id(),
+            num_shard: shard_config.num_shard,
+            shard_id: shard_config.shard_id,
+            neighbors_only: false,
             timestamp: timestamp_now(),
         }));
         Ok(())
@@ -642,7 +664,10 @@ impl SyncService {
                             Some(s) => s,
                             None => {
                                 debug!(%tx.seq, "No more data needed");
-                                self.store.finalize_tx_with_hash(tx.seq, tx.hash()).await?;
+                                if self.store.finalize_tx_with_hash(tx.seq, tx.hash()).await? {
+                                    self.ctx
+                                        .send(NetworkMessage::AnnounceLocalFile { tx_id: tx.id() });
+                                }
                                 return Ok(());
                             }
                         };
@@ -748,6 +773,34 @@ impl SyncService {
         }
     }
 
+    /// Handle on `NewFile` gossip message received.
+    async fn on_new_file_gossip(&mut self, from: PeerId, msg: NewFile) {
+        debug!(%from, ?msg, "Received NewFile gossip");
+
+        if let Some(controller) = self.controllers.get_mut(&msg.tx_id.seq) {
+            // Notify new peer announced if file already in sync
+            if let Ok(shard_config) = ShardConfig::new(msg.shard_id, msg.num_shard) {
+                controller.on_peer_announced(from, shard_config);
+                controller.transition();
+            }
+        } else if let Some(manager) = &self.auto_sync_manager {
+            let _ = manager.new_file_send.send(msg.tx_id.seq);
+        }
+    }
+
+    /// Handle on `AnnounceFile` RPC message received.
+    async fn on_announce_file(&mut self, from: PeerId, announcement: FileAnnouncement) {
+        // Notify new peer announced if file already in sync
+        if let Some(controller) = self.controllers.get_mut(&announcement.tx_id.seq) {
+            if let Ok(shard_config) =
+                ShardConfig::new(announcement.shard_id, announcement.num_shard)
+            {
+                controller.on_peer_announced(from, shard_config);
+                controller.transition();
+            }
+        }
+    }
+
     /// Terminate file sync of `min_tx_seq`.
     /// If `is_reverted` is `true` (means confirmed transactions reverted),
     /// also terminate `tx_seq` greater than `min_tx_seq`
@@ -843,7 +896,9 @@ mod tests {
     use crate::test_util::tests::create_file_location_cache;
     use libp2p::identity;
     use network::discovery::ConnectionId;
+    use network::new_network_channel;
     use network::rpc::SubstreamId;
+    use network::NetworkReceiver;
     use network::ReportSource;
     use shared_types::ChunkArray;
     use shared_types::Transaction;
@@ -855,8 +910,6 @@ mod tests {
     use storage::log_store::LogStoreRead;
     use storage::H256;
     use task_executor::test_utils::TestRuntime;
-    use tokio::sync::mpsc::UnboundedReceiver;
-    use tokio::sync::mpsc::UnboundedSender;
 
     struct TestSyncRuntime {
         runtime: TestRuntime,
@@ -870,8 +923,8 @@ mod tests {
         init_peer_id: PeerId,
         file_location_cache: Arc<FileLocationCache>,
 
-        network_send: UnboundedSender<NetworkMessage>,
-        network_recv: UnboundedReceiver<NetworkMessage>,
+        network_send: NetworkSender,
+        network_recv: NetworkReceiver,
         event_send: broadcast::Sender<LogSyncEvent>,
         catch_up_end_recv: Option<oneshot::Receiver<()>>,
     }
@@ -888,7 +941,7 @@ mod tests {
             let (store, peer_store, txs, data) = create_2_store(chunk_counts);
             let init_data = data[0].clone();
             let init_peer_id = identity::Keypair::generate_ed25519().public().to_peer_id();
-            let (network_send, network_recv) = mpsc::unbounded_channel::<NetworkMessage>();
+            let (network_send, network_recv) = new_network_channel();
             let (event_send, _) = broadcast::channel(16);
             let (_, catch_up_end_recv) = oneshot::channel();
 
@@ -952,7 +1005,7 @@ mod tests {
         let file_location_cache: Arc<FileLocationCache> =
             create_file_location_cache(init_peer_id, vec![txs[0].id()]);
 
-        let (network_send, mut network_recv) = mpsc::unbounded_channel::<NetworkMessage>();
+        let (network_send, mut network_recv) = new_network_channel();
         let (_, sync_recv) = channel::Channel::unbounded("test");
 
         let mut sync = SyncService {
@@ -981,7 +1034,7 @@ mod tests {
         let file_location_cache: Arc<FileLocationCache> =
             create_file_location_cache(init_peer_id, vec![txs[0].id()]);
 
-        let (network_send, mut network_recv) = mpsc::unbounded_channel::<NetworkMessage>();
+        let (network_send, mut network_recv) = new_network_channel();
         let (_, sync_recv) = channel::Channel::unbounded("test");
 
         let mut sync = SyncService {
@@ -1294,15 +1347,13 @@ mod tests {
 
         let config = LogConfig::default();
 
-        let executor = runtime.task_executor.clone();
-
-        let store = Arc::new(LogManager::memorydb(config.clone(), executor).unwrap());
+        let store = Arc::new(LogManager::memorydb(config.clone()).unwrap());
 
         let init_peer_id = identity::Keypair::generate_ed25519().public().to_peer_id();
         let file_location_cache: Arc<FileLocationCache> =
             create_file_location_cache(init_peer_id, vec![]);
 
-        let (network_send, mut network_recv) = mpsc::unbounded_channel::<NetworkMessage>();
+        let (network_send, mut network_recv) = new_network_channel();
         let (_event_send, event_recv) = broadcast::channel(16);
         let (_, catch_up_end_recv) = oneshot::channel();
         let sync_send = SyncService::spawn_with_config(
@@ -1504,6 +1555,10 @@ mod tests {
         .await;
 
         wait_for_tx_finalized(runtime.store.clone(), tx_seq).await;
+        assert!(matches!(
+            runtime.network_recv.try_recv().unwrap(),
+            NetworkMessage::AnnounceLocalFile { .. }
+        ));
 
         assert!(!runtime.store.check_tx_completed(0).unwrap());
 
@@ -1528,6 +1583,10 @@ mod tests {
         .await;
 
         wait_for_tx_finalized(runtime.store, tx_seq).await;
+        assert!(matches!(
+            runtime.network_recv.try_recv().unwrap(),
+            NetworkMessage::AnnounceLocalFile { .. }
+        ));
 
         sync_send
             .notify(SyncMessage::PeerDisconnected {
@@ -1721,7 +1780,7 @@ mod tests {
     }
 
     async fn receive_chunk_request(
-        network_recv: &mut UnboundedReceiver<NetworkMessage>,
+        network_recv: &mut NetworkReceiver,
         sync_send: &SyncSender,
         peer_store: Arc<LogManager>,
         init_peer_id: PeerId,

node/sync/src/test_util.rs

@@ -9,8 +9,6 @@ use storage::{
     LogManager,
 };
 
-use task_executor::test_utils::TestRuntime;
-
 /// Creates stores for local node and peers with initialized transaction of specified chunk count.
 /// The first store is for local node, and data not stored. The second store is for peers, and all
 /// transactions are finalized for file sync.
@@ -24,11 +22,8 @@ pub fn create_2_store(
     Vec<Vec<u8>>,
 ) {
     let config = LogConfig::default();
-    let runtime = TestRuntime::default();
-
-    let executor = runtime.task_executor.clone();
-    let mut store = LogManager::memorydb(config.clone(), executor.clone()).unwrap();
-    let mut peer_store = LogManager::memorydb(config, executor).unwrap();
+    let mut store = LogManager::memorydb(config.clone()).unwrap();
+    let mut peer_store = LogManager::memorydb(config).unwrap();
 
     let mut offset = 1;
     let mut txs = vec![];
@@ -120,10 +115,7 @@ pub mod tests {
 
     impl TestStoreRuntime {
         pub fn new_store() -> impl LogStore {
-            let runtime = TestRuntime::default();
-
-            let executor = runtime.task_executor.clone();
-            LogManager::memorydb(LogConfig::default(), executor).unwrap()
+            LogManager::memorydb(LogConfig::default()).unwrap()
         }
 
         pub fn new(store: Arc<dyn LogStore>) -> TestStoreRuntime {

requirements.txt

@@ -1,10 +1,10 @@
 jsonrpcclient==4.0.3
 pyyaml==6.0.1
-pysha3==1.0.2
-coincurve==18.0.0
-eth-utils==3.0.0
+safe-pysha3==1.0.4
+coincurve==20.0.0
+eth-utils==5.1.0
 py-ecc==7.0.0
-web3==6.14.0
+web3==7.5.0
 eth_tester
 cffi==1.16.0
-rtoml==0.10.0
+rtoml==0.11.0

run/config-testnet-standard.toml

@@ -176,7 +176,7 @@ mine_contract_address = "0x1785c8683b3c527618eFfF78d876d9dCB4b70285"
 # If this limit is reached, the node will update its `shard_position`
 # and store only half data.
 #
-db_max_num_sectors = 1000000000
+db_max_num_sectors = 4000000000
 
 # The format is <shard_id>/<shard_number>, where the shard number is 2^n.
 # This only applies if there is no stored shard config in db.
@@ -232,6 +232,10 @@ batcher_announcement_capacity = 100
 # all files, and sufficient disk space is required.
 auto_sync_enabled = true
 
+# Indicates whether to sync file from neighbor nodes only. This is to avoid flooding file
+# announcements in the whole network, which leads to high latency or even timeout to sync files.
+neighbors_only = true
+
 #  Maximum number of files in sync from other peers simultaneously.
 # max_sync_files = 8
 
@@ -324,7 +328,7 @@ auto_sync_enabled = true
 # enabled = false
 
 # Interval to output metrics periodically, e.g. "10s", "30s" or "60s".
-# report_interval = ""
+# report_interval = "10s"
 
 # File name to output metrics periodically.
 # file_report_output = ""

run/config-testnet-turbo.toml

@@ -188,7 +188,7 @@ mine_contract_address = "0x6815F41019255e00D6F34aAB8397a6Af5b6D806f"
 # If this limit is reached, the node will update its `shard_position`
 # and store only half data.
 #
-db_max_num_sectors = 1000000000
+db_max_num_sectors = 4000000000
 
 # The format is <shard_id>/<shard_number>, where the shard number is 2^n.
 # This only applies if there is no stored shard config in db.
@@ -244,6 +244,10 @@ batcher_announcement_capacity = 100
 # all files, and sufficient disk space is required.
 auto_sync_enabled = true
 
+# Indicates whether to sync file from neighbor nodes only. This is to avoid flooding file
+# announcements in the whole network, which leads to high latency or even timeout to sync files.
+neighbors_only = true
+
 #  Maximum number of files in sync from other peers simultaneously.
 # max_sync_files = 8
 
@@ -336,7 +340,7 @@ auto_sync_enabled = true
 # enabled = false
 
 # Interval to output metrics periodically, e.g. "10s", "30s" or "60s".
-# report_interval = ""
+# report_interval = "10s"
 
 # File name to output metrics periodically.
 # file_report_output = ""

run/config.toml

@@ -246,6 +246,10 @@
 # all files, and sufficient disk space is required.
 # auto_sync_enabled = false
 
+# Indicates whether to sync file from neighbor nodes only. This is to avoid flooding file
+# announcements in the whole network, which leads to high latency or even timeout to sync files.
+neighbors_only = true
+
 #  Maximum number of files in sync from other peers simultaneously.
 # max_sync_files = 8
 
@@ -338,7 +342,7 @@
 # enabled = false
 
 # Interval to output metrics periodically, e.g. "10s", "30s" or "60s".
-# report_interval = ""
+# report_interval = "10s"
 
 # File name to output metrics periodically.
 # file_report_output = ""

storage-contracts-abis/0g-storage-contracts-rev

@@ -1 +1 @@
-75c251804a29ab22adced50d92478cf0baf834bc
+bea58429e436e4952ae69235d9079cfc4ac5f3b3

storage-contracts-abis/DummyMarket.json

@@ -40,8 +40,8 @@
       "type": "function"
     }
   ],
-  "bytecode": "0x608060405234801561001057600080fd5b5060be8061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c806361ec5082146037578063da6eb36a14604b575b600080fd5b600060405190815260200160405180910390f35b605b6056366004605d565b505050565b005b600080600060608486031215607157600080fd5b50508135936020830135935060409092013591905056fea264697066735822122044ebf96fcad90f0bbc521513843d64fbc182c5c913a8210a4d638393793be63064736f6c63430008100033",
-  "deployedBytecode": "0x6080604052348015600f57600080fd5b506004361060325760003560e01c806361ec5082146037578063da6eb36a14604b575b600080fd5b600060405190815260200160405180910390f35b605b6056366004605d565b505050565b005b600080600060608486031215607157600080fd5b50508135936020830135935060409092013591905056fea264697066735822122044ebf96fcad90f0bbc521513843d64fbc182c5c913a8210a4d638393793be63064736f6c63430008100033",
+  "bytecode": "0x608060405234801561001057600080fd5b5060be8061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c806361ec5082146037578063da6eb36a14604b575b600080fd5b600060405190815260200160405180910390f35b605b6056366004605d565b505050565b005b600080600060608486031215607157600080fd5b50508135936020830135935060409092013591905056fea264697066735822122080db0b00f4b93cc320a2df449a74e503451a2675da518eff0fc5b7cf0ae8c90c64736f6c63430008100033",
+  "deployedBytecode": "0x6080604052348015600f57600080fd5b506004361060325760003560e01c806361ec5082146037578063da6eb36a14604b575b600080fd5b600060405190815260200160405180910390f35b605b6056366004605d565b505050565b005b600080600060608486031215607157600080fd5b50508135936020830135935060409092013591905056fea264697066735822122080db0b00f4b93cc320a2df449a74e503451a2675da518eff0fc5b7cf0ae8c90c64736f6c63430008100033",
   "linkReferences": {},
   "deployedLinkReferences": {}
 }

storage-contracts-abis/DummyReward.json

@@ -70,8 +70,8 @@
       "type": "function"
     }
   ],
-  "bytecode": "0x608060405234801561001057600080fd5b5060f18061001f6000396000f3fe60806040526004361060265760003560e01c806359e9670014602b578063b7a3c04c14603c575b600080fd5b603a60363660046058565b5050565b005b348015604757600080fd5b50603a60533660046079565b505050565b60008060408385031215606a57600080fd5b50508035926020909101359150565b600080600060608486031215608d57600080fd5b8335925060208401356001600160a01b038116811460aa57600080fd5b92959294505050604091909101359056fea2646970667358221220ce57385afc7714a4000e530d1e1154d214fc1c0e2392abde201018635be1a2ab64736f6c63430008100033",
-  "deployedBytecode": "0x60806040526004361060265760003560e01c806359e9670014602b578063b7a3c04c14603c575b600080fd5b603a60363660046058565b5050565b005b348015604757600080fd5b50603a60533660046079565b505050565b60008060408385031215606a57600080fd5b50508035926020909101359150565b600080600060608486031215608d57600080fd5b8335925060208401356001600160a01b038116811460aa57600080fd5b92959294505050604091909101359056fea2646970667358221220ce57385afc7714a4000e530d1e1154d214fc1c0e2392abde201018635be1a2ab64736f6c63430008100033",
+  "bytecode": "0x608060405234801561001057600080fd5b5060f18061001f6000396000f3fe60806040526004361060265760003560e01c806359e9670014602b578063b7a3c04c14603c575b600080fd5b603a60363660046058565b5050565b005b348015604757600080fd5b50603a60533660046079565b505050565b60008060408385031215606a57600080fd5b50508035926020909101359150565b600080600060608486031215608d57600080fd5b8335925060208401356001600160a01b038116811460aa57600080fd5b92959294505050604091909101359056fea2646970667358221220d2f22ec6a41724281bad8a768c241562927a5fcc8ba600f3b3784f584a68c65864736f6c63430008100033",
+  "deployedBytecode": "0x60806040526004361060265760003560e01c806359e9670014602b578063b7a3c04c14603c575b600080fd5b603a60363660046058565b5050565b005b348015604757600080fd5b50603a60533660046079565b505050565b60008060408385031215606a57600080fd5b50508035926020909101359150565b600080600060608486031215608d57600080fd5b8335925060208401356001600160a01b038116811460aa57600080fd5b92959294505050604091909101359056fea2646970667358221220d2f22ec6a41724281bad8a768c241562927a5fcc8ba600f3b3784f584a68c65864736f6c63430008100033",
   "linkReferences": {},
   "deployedLinkReferences": {}
 }