Merge 93885bb25c into 5849e9c2ba

* fix: finalize file does not need to save

* fix: rust fmt

node/log_entry_sync/src/sync_manager/mod.rs

@@ -6,12 +6,13 @@ use ethereum_types::H256;
 use ethers::{prelude::Middleware, types::BlockNumber};
 use futures::FutureExt;
 use jsonrpsee::tracing::{debug, error, warn};
-use shared_types::{ChunkArray, Transaction};
+use shared_types::{bytes_to_chunks, ChunkArray, Transaction};
 use std::collections::BTreeMap;
 use std::fmt::Debug;
 use std::future::Future;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
+use storage::log_store::log_manager::sector_to_segment;
 use storage::log_store::{tx_store::BlockHashAndSubmissionIndex, Store};
 use task_executor::{ShutdownReason, TaskExecutor};
 use thiserror::Error;
@@ -477,6 +478,17 @@ impl LogSyncManager {
                     error!("put_tx data error: e={:?}", e);
                     return false;
                 }
+            } else {
+                let store = self.store.clone();
+                let shard_config = store.flow().get_shard_config();
+                if sector_to_segment(bytes_to_chunks(tx.size as usize) as u64)
+                    < shard_config.shard_id
+                {
+                    if let Err(e) = store.finalize_tx_with_hash(tx.seq, tx.hash()) {
+                        error!("finalize file that does not need to store: e={:?}", e);
+                        return false;
+                    }
+                }
             }
             self.data_cache.garbage_collect(self.next_tx_seq);
             self.next_tx_seq += 1;

node/pruner/src/lib.rs

@@ -198,11 +198,11 @@ impl Pruner {
             ))),
             Ordering::Equal => Ok(None),
             Ordering::Greater => {
-                bail!(
+                error!(
                     "Unexpected first_rewardable_chunk revert: old={} new={}",
-                    self.first_rewardable_chunk,
-                    new_first_rewardable
+                    self.first_rewardable_chunk, new_first_rewardable
                 );
+                Ok(None)
             }
         }
     }

node/storage/src/config.rs

@@ -261,30 +261,30 @@ mod tests {
     #[test]
     fn test_shard_intersect() {
         // 1 shard
-        assert_eq!(new_config(0, 1).intersect(&new_config(0, 1)), true);
+        assert!(new_config(0, 1).intersect(&new_config(0, 1)));
 
         // either is 1 shard
-        assert_eq!(new_config(0, 1).intersect(&new_config(0, 2)), true);
-        assert_eq!(new_config(0, 1).intersect(&new_config(1, 2)), true);
-        assert_eq!(new_config(0, 2).intersect(&new_config(0, 1)), true);
-        assert_eq!(new_config(1, 2).intersect(&new_config(0, 1)), true);
+        assert!(new_config(0, 1).intersect(&new_config(0, 2)));
+        assert!(new_config(0, 1).intersect(&new_config(1, 2)));
+        assert!(new_config(0, 2).intersect(&new_config(0, 1)));
+        assert!(new_config(1, 2).intersect(&new_config(0, 1)));
 
         // same shards
-        assert_eq!(new_config(1, 4).intersect(&new_config(0, 4)), false);
-        assert_eq!(new_config(1, 4).intersect(&new_config(1, 4)), true);
-        assert_eq!(new_config(1, 4).intersect(&new_config(2, 4)), false);
-        assert_eq!(new_config(1, 4).intersect(&new_config(3, 4)), false);
+        assert!(!new_config(1, 4).intersect(&new_config(0, 4)));
+        assert!(new_config(1, 4).intersect(&new_config(1, 4)));
+        assert!(!new_config(1, 4).intersect(&new_config(2, 4)));
+        assert!(!new_config(1, 4).intersect(&new_config(3, 4)));
 
         // left shards is less
-        assert_eq!(new_config(1, 2).intersect(&new_config(0, 4)), false);
-        assert_eq!(new_config(1, 2).intersect(&new_config(1, 4)), false);
-        assert_eq!(new_config(1, 2).intersect(&new_config(2, 4)), true);
-        assert_eq!(new_config(1, 2).intersect(&new_config(3, 4)), true);
+        assert!(!new_config(1, 2).intersect(&new_config(0, 4)));
+        assert!(!new_config(1, 2).intersect(&new_config(1, 4)));
+        assert!(new_config(1, 2).intersect(&new_config(2, 4)));
+        assert!(new_config(1, 2).intersect(&new_config(3, 4)));
 
         // right shards is less
-        assert_eq!(new_config(1, 4).intersect(&new_config(0, 2)), true);
-        assert_eq!(new_config(1, 4).intersect(&new_config(1, 2)), false);
-        assert_eq!(new_config(2, 4).intersect(&new_config(0, 2)), false);
-        assert_eq!(new_config(2, 4).intersect(&new_config(1, 2)), true);
+        assert!(new_config(1, 4).intersect(&new_config(0, 2)));
+        assert!(!new_config(1, 4).intersect(&new_config(1, 2)));
+        assert!(!new_config(2, 4).intersect(&new_config(0, 2)));
+        assert!(new_config(2, 4).intersect(&new_config(1, 2)));
     }
 }

tests/config/0gchain-init-genesis-mac.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+set -e
+
+BINARY=$(cd $(dirname ${BASH_SOURCE[0]})/../tmp; pwd)/0gchaind
+ROOT_DIR=${1:-.}
+NUM_NODES=${2:-3}
+P2P_PORT_START=${3:-26656}
+CHAIN_ID=zgchainpy_9000-777
+
+# install jq if not unavailable
+jq --version >/dev/null 2>&1 || sudo snap install jq -y
+
+mkdir -p $ROOT_DIR
+
+# Init configs
+for ((i=0; i<$NUM_NODES; i++)) do
+	$BINARY init node$i --home $ROOT_DIR/node$i --chain-id $CHAIN_ID
+	
+	# Change genesis.json
+	GENESIS=$ROOT_DIR/node$i/config/genesis.json
+	TMP_GENESIS=$ROOT_DIR/node$i/config/tmp_genesis.json
+
+	# Replace stake with neuron
+	sed -in-place='' 's/stake/ua0gi/g' "$GENESIS"
+
+	# Replace the default evm denom of aphoton with neuron
+	sed -in-place='' 's/aphoton/neuron/g' "$GENESIS"
+
+	cat $GENESIS | jq '.consensus_params.block.max_gas = "25000000"' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+
+	# Zero out the total supply so it gets recalculated during InitGenesis
+	cat $GENESIS | jq '.app_state.bank.supply = []' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+
+	# Disable fee market
+	cat $GENESIS | jq '.app_state.feemarket.params.no_base_fee = true' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+
+	# Disable london fork
+	cat $GENESIS | jq '.app_state.evm.params.chain_config.london_block = null' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+	cat $GENESIS | jq '.app_state.evm.params.chain_config.arrow_glacier_block = null' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+	cat $GENESIS | jq '.app_state.evm.params.chain_config.gray_glacier_block = null' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+	cat $GENESIS | jq '.app_state.evm.params.chain_config.merge_netsplit_block = null' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+	cat $GENESIS | jq '.app_state.evm.params.chain_config.shanghai_block = null' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+	cat $GENESIS | jq '.app_state.evm.params.chain_config.cancun_block = null' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+
+	# cat $GENESIS | jq '.app_state["staking"]["params"]["bond_denom"]="a0gi"' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+	# cat $GENESIS | jq '.app_state["gov"]["params"]["min_deposit"][0]["denom"]="a0gi"' >$TMP_GENESIS && mv $TMP_GENESIS $GENESIS
+
+	cat "$GENESIS" | jq '.app_state["staking"]["params"]["max_validators"]=125' >"$TMP_GENESIS" && mv "$TMP_GENESIS" "$GENESIS"
+	cat "$GENESIS" | jq '.app_state["slashing"]["params"]["signed_blocks_window"]="1000"' >"$TMP_GENESIS" && mv "$TMP_GENESIS" "$GENESIS"
+
+	cat "$GENESIS" | jq '.app_state["consensus_params"]["block"]["time_iota_ms"]="3000"' >"$TMP_GENESIS" && mv "$TMP_GENESIS" "$GENESIS"
+
+	# Change app.toml
+	APP_TOML=$ROOT_DIR/node$i/config/app.toml
+	sed -i '' 's/minimum-gas-prices = "0ua0gi"/minimum-gas-prices = "1000000000neuron"/' $APP_TOML
+	sed -i '' '/\[grpc\]/,/^\[/ s/enable = true/enable = false/' $APP_TOML
+	sed -i '' '/\[grpc-web\]/,/^\[/ s/enable = true/enable = false/' $APP_TOML
+	sed -i '' '/\[json-rpc\]/,/^\[/ s/enable = false/enable = true/' $APP_TOML
+
+	# Change config.toml
+	CONFIG_TOML=$ROOT_DIR/node$i/config/config.toml
+	sed -i '' '/seeds = /c\ 
+	seeds = ""' $CONFIG_TOML
+	sed -i '' 's/addr_book_strict = true/addr_book_strict = false/' $CONFIG_TOML
+done
+
+# Update persistent_peers in config.toml
+for ((i=1; i<$NUM_NODES; i++)) do
+	PERSISTENT_NODES=""
+	for ((j=0; j<$i; j++)) do
+		if [[ $j -gt 0 ]]; then PERSISTENT_NODES=$PERSISTENT_NODES,; fi
+		NODE_ID=`$BINARY tendermint show-node-id --home $ROOT_DIR/node$j`
+		P2P_PORT=$(($P2P_PORT_START+$j))
+		PERSISTENT_NODES=$PERSISTENT_NODES$NODE_ID@127.0.0.1:$P2P_PORT
+	done
+	sed -i '' "/persistent_peers = /c\ 
+	persistent_peers = \"$PERSISTENT_NODES\"" $ROOT_DIR/node$i/config/config.toml
+done
+
+# Create genesis with a single validator
+$BINARY keys add val0 --keyring-backend test --home $ROOT_DIR/node0
+$BINARY add-genesis-account val0 15000000000000000000ua0gi --keyring-backend test --home $ROOT_DIR/node0
+
+# add genesis account for tests, see GENESIS_PRIV_KEY and GENESIS_PRIV_KEY1 in node_config.py
+$BINARY add-genesis-account 0g1l0j9dqdvd3fatfqywhm4y6avrln4jracmt6ztf 40000000000000000000ua0gi --home $ROOT_DIR/node0
+$BINARY add-genesis-account 0g1pemg6y3etj9tlhkl0vdwkrw36f74u2nl8sjw7g 40000000000000000000ua0gi --home $ROOT_DIR/node0
+
+mkdir -p $ROOT_DIR/gentxs
+$BINARY gentx val0 10000000000000000000ua0gi --keyring-backend test --home $ROOT_DIR/node0 --output-document $ROOT_DIR/gentxs/node0.json
+$BINARY collect-gentxs --home $ROOT_DIR/node0 --gentx-dir $ROOT_DIR/gentxs
+$BINARY validate-genesis --home $ROOT_DIR/node0
+for ((i=1; i<$NUM_NODES; i++)) do
+	cp $ROOT_DIR/node0/config/genesis.json $ROOT_DIR/node$i/config/genesis.json
+done

tests/test_framework/zg_node.py

@@ -1,6 +1,7 @@
 import os
 import subprocess
 import tempfile
+import platform
 
 from test_framework.blockchain_node import BlockChainNodeType, BlockchainNode
 from utility.utils import blockchain_rpc_port, arrange_port
@@ -21,6 +22,13 @@ def zg_node_init_genesis(binary: str, root_dir: str, num_nodes: int):
         os.path.dirname(os.path.realpath(__file__)), # test_framework folder
         "..", "config", "0gchain-init-genesis.sh"
     )
+    sys = platform.system().lower()
+    if sys == "darwin":
+        shell_script = os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), # test_framework folder
+            "..", "config", "0gchain-init-genesis-mac.sh"
+        )
+
 
     zgchaind_dir = os.path.join(root_dir, "0gchaind")
     os.mkdir(zgchaind_dir)