From 4eb2a50b0e67789ee7ee5c3cd8555acc0049de70 Mon Sep 17 00:00:00 2001 From: peilun-conflux <48905552+peilun-conflux@users.noreply.github.com> Date: Sat, 29 Jun 2024 17:08:02 +0800 Subject: [PATCH] Use inner lock in storage and use async lock. (#92) * Use inner lock in storage. * Remove mut. * Remove async lock for storage. * Fix tests and warnings. * Use spawn_blocking for storage task. * Fix clippy. * Finalize the new tx at last. * Revert "Finalize the new tx at last." This reverts commit b56ad5582daa2863e9052996091af20758f9e416. * Wait for old same-root txs to finalize. * Use async storage in miner. * Update rust version to 1.79.0. * Use Vec to avoid stack overflow. * Fix unused warning. * Fix clippy. * Fix test warning. * Fix test. * fmt. * Use async storage in pruner. * nit. --- .github/workflows/rust.yml | 2 + Cargo.lock | 853 +++++++++--------- common/append_merkle/src/lib.rs | 6 +- common/merkle_light/src/lib.rs | 1 - common/merkle_light/src/test_cmh.rs | 20 +- common/merkle_light/src/test_item.rs | 4 +- node/chunk_pool/src/handler.rs | 4 +- node/chunk_pool/src/lib.rs | 2 +- .../src/mem_pool/chunk_pool_inner.rs | 6 +- node/log_entry_sync/src/lib.rs | 5 +- node/log_entry_sync/src/rpc_proxy/eth.rs | 30 - node/log_entry_sync/src/rpc_proxy/mod.rs | 38 - .../log_entry_sync/src/sync_manager/config.rs | 2 +- .../src/sync_manager/log_entry_fetcher.rs | 12 +- node/log_entry_sync/src/sync_manager/mod.rs | 22 +- node/miner/Cargo.toml | 3 +- node/miner/src/loader.rs | 10 +- node/miner/src/miner_id.rs | 22 +- node/miner/src/sealer.rs | 24 +- node/miner/src/service.rs | 10 +- node/miner/src/submitter.rs | 21 +- node/network/tests/common/mod.rs | 1 + node/pruner/Cargo.toml | 1 + node/pruner/src/lib.rs | 36 +- node/router/src/libp2p_event_handler.rs | 24 +- node/router/src/service.rs | 2 +- node/rpc/src/lib.rs | 2 +- node/rpc/src/zgs/impl.rs | 11 +- node/src/client/builder.rs | 28 +- node/src/client/environment.rs | 7 +- node/storage-async/Cargo.toml | 3 +- node/storage-async/src/lib.rs | 83 +- node/storage/Cargo.toml | 1 + node/storage/benches/benchmark.rs | 8 +- node/storage/src/log_store/flow_store.rs | 45 +- node/storage/src/log_store/log_manager.rs | 377 ++++---- node/storage/src/log_store/mod.rs | 30 +- node/storage/src/log_store/tests.rs | 2 +- node/sync/src/auto_sync/sync_store.rs | 44 +- node/sync/src/controllers/serial.rs | 30 +- node/sync/src/service.rs | 86 +- node/sync/src/test_util.rs | 17 +- tests/same_root_test.py | 6 +- .../discv5/src/socket/filter/cache.rs | 1 + 44 files changed, 934 insertions(+), 1008 deletions(-) delete mode 100644 node/log_entry_sync/src/rpc_proxy/eth.rs delete mode 100644 node/log_entry_sync/src/rpc_proxy/mod.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 311d6cc..c3cd670 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -67,4 +67,6 @@ jobs: uses: actions-rs/cargo@v1 with: command: clippy + # blocks_in_conditions is triggered for tracing::instrument. + # This can be removed after the fix is released. args: -- -D warnings \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 84d2561..44348de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,9 +14,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -104,7 +104,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "android-tzdata" @@ -221,27 +221,25 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener 5.2.0", - "event-listener-strategy 0.5.1", + "event-listener-strategy", "futures-core", "pin-project-lite 0.2.14", ] [[package]] name = "async-executor" -version = "1.9.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b3e585719c2358d2660232671ca8ca4ddb4be4ce8a1842d6c2dc8685303316" +checksum = "c8828ec6e544c02b0d6691d21ed9f9218d0384a82542855073c2a3f58304aaf0" dependencies = [ - "async-lock 3.3.0", "async-task", "concurrent-queue", - "fastrand 2.0.2", + "fastrand 2.1.0", "futures-lite 2.3.0", "slab", ] @@ -252,10 +250,10 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel 2.2.0", + "async-channel 2.3.1", "async-executor", - "async-io 2.3.2", - "async-lock 3.3.0", + "async-io 2.3.3", + "async-lock 3.4.0", "blocking", "futures-lite 2.3.0", "once_cell", @@ -283,18 +281,18 @@ dependencies = [ [[package]] name = "async-io" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +checksum = "0d6baa8f0178795da0e71bc42c9e5d13261aac7ee549853162e66a241ba17964" dependencies = [ - "async-lock 3.3.0", + "async-lock 3.4.0", "cfg-if", "concurrent-queue", "futures-io", "futures-lite 2.3.0", "parking", - "polling 3.6.0", - "rustix 0.38.32", + "polling 3.7.2", + "rustix 0.38.34", "slab", "tracing", "windows-sys 0.52.0", @@ -311,12 +309,12 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 4.0.3", - "event-listener-strategy 0.4.0", + "event-listener 5.3.1", + "event-listener-strategy", "pin-project-lite 0.2.14", ] @@ -333,26 +331,26 @@ dependencies = [ "cfg-if", "event-listener 3.1.0", "futures-lite 1.13.0", - "rustix 0.38.32", + "rustix 0.38.34", "windows-sys 0.48.0", ] [[package]] name = "async-signal" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" +checksum = "794f185324c2f00e771cd9f1ae8b5ac68be2ca7abb129a87afd6e86d228bc54d" dependencies = [ - "async-io 2.3.2", - "async-lock 2.8.0", + "async-io 2.3.3", + "async-lock 3.4.0", "atomic-waker", "cfg-if", "futures-core", "futures-io", - "rustix 0.38.32", + "rustix 0.38.34", "signal-hook-registry", "slab", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -399,19 +397,19 @@ dependencies = [ [[package]] name = "async-task" -version = "4.7.0" +version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.79" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -475,20 +473,20 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] name = "autocfg" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -568,7 +566,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -594,9 +592,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "bitmaps" @@ -657,18 +655,15 @@ dependencies = [ [[package]] name = "blocking" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" dependencies = [ - "async-channel 2.2.0", - "async-lock 3.3.0", + "async-channel 2.3.1", "async-task", - "fastrand 2.0.2", "futures-io", "futures-lite 2.3.0", "piper", - "tracing", ] [[package]] @@ -699,9 +694,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.4" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byte-slice-cast" @@ -747,9 +742,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" dependencies = [ "serde", ] @@ -785,12 +780,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.90" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -847,16 +843,16 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -923,9 +919,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -1020,9 +1016,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ "crossbeam-utils", ] @@ -1048,9 +1044,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.11.3" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ba00838774b4ab0233e355d26710fbfc8327a05c017f6dc4873f876d1f79f78" +checksum = "94fb8a24a26d37e1ffd45343323dc9fe6654ceea44c12f2fcb3d7ac29e610bc6" dependencies = [ "cfg-if", "cpufeatures 0.2.12", @@ -1124,9 +1120,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1169,9 +1165,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -1197,9 +1193,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crunchy" @@ -1306,15 +1302,14 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.2" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", "cpufeatures 0.2.12", "curve25519-dalek-derive", "fiat-crypto", - "platforms", "rustc_version", "subtle", "zeroize", @@ -1328,7 +1323,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -1368,9 +1363,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "der" @@ -1403,13 +1398,13 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.17" +version = "0.99.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.68", ] [[package]] @@ -1623,9 +1618,9 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "elliptic-curve" @@ -1668,18 +1663,18 @@ dependencies = [ [[package]] name = "ena" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c533630cf40e9caa44bd91aadc88a75d75a4c3a12b4cfde353cbed41daa1e1f1" +checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" dependencies = [ "log", ] [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] @@ -1740,9 +1735,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1978,8 +1973,8 @@ dependencies = [ "reqwest", "serde", "serde_json", - "syn 2.0.58", - "toml 0.8.12", + "syn 2.0.68", + "toml 0.8.14", "walkdir", ] @@ -1996,7 +1991,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -2021,8 +2016,8 @@ dependencies = [ "rlp", "serde", "serde_json", - "strum 0.26.2", - "syn 2.0.58", + "strum 0.26.3", + "syn 2.0.68", "tempfile", "thiserror", "tiny-keccak", @@ -2180,20 +2175,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "4.0.3" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite 0.2.14", -] - -[[package]] -name = "event-listener" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b5fb89194fa3cad959b833185b3063ba881dbfc7030680b314250779fb4cc91" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" dependencies = [ "concurrent-queue", "parking", @@ -2202,21 +2186,11 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.4.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener 4.0.3", - "pin-project-lite 0.2.14", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332f51cb23d20b0de8458b86580878211da09bcd4503cb579c225b3d124cabb3" -dependencies = [ - "event-listener 5.2.0", + "event-listener 5.3.1", "pin-project-lite 0.2.14", ] @@ -2250,9 +2224,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "ff" @@ -2276,9 +2250,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c007b1ae3abe1cb6f85a16305acd418b7ca6343b953633fee2b76d8f108b830f" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "file_location_cache" @@ -2286,7 +2260,7 @@ version = "0.1.0" dependencies = [ "hashlink 0.8.4", "network", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "priority-queue", "rand 0.8.5", "shared_types", @@ -2326,9 +2300,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "libz-sys", @@ -2448,7 +2422,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.0.2", + "fastrand 2.1.0", "futures-core", "futures-io", "parking", @@ -2473,7 +2447,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -2560,9 +2534,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -2591,9 +2565,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "git-version" @@ -2612,7 +2586,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -2630,8 +2604,8 @@ dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -2703,9 +2677,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", @@ -2716,15 +2690,15 @@ dependencies = [ "indexmap 2.2.6", "slab", "tokio", - "tokio-util 0.7.10", + "tokio-util 0.7.11", "tracing", ] [[package]] name = "half" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if", "crunchy", @@ -2750,9 +2724,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.11", "allocator-api2", @@ -2782,7 +2756,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -2809,6 +2783,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -2824,6 +2804,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2925,9 +2911,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -2937,9 +2923,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", @@ -2952,7 +2938,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite 0.2.14", - "socket2 0.5.6", + "socket2 0.5.7", "tokio", "tower-service", "tracing", @@ -2984,7 +2970,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.10", + "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", ] @@ -3105,7 +3091,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" dependencies = [ - "parity-scale-codec 3.6.9", + "parity-scale-codec 3.6.12", ] [[package]] @@ -3169,7 +3155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -3183,9 +3169,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] @@ -3207,7 +3193,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.6", + "socket2 0.5.7", "widestring", "windows-sys 0.48.0", "winreg", @@ -3254,9 +3240,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.28" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] @@ -3319,7 +3305,7 @@ dependencies = [ "thiserror", "tokio", "tokio-rustls 0.23.4", - "tokio-util 0.7.10", + "tokio-util 0.7.11", "tracing", "webpki-roots 0.22.6", ] @@ -3342,7 +3328,7 @@ dependencies = [ "hyper", "jsonrpsee-types", "lazy_static", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "rand 0.8.5", "rustc-hash", "serde", @@ -3453,7 +3439,7 @@ dependencies = [ "soketto", "tokio", "tokio-stream", - "tokio-util 0.7.10", + "tokio-util 0.7.11", "tracing", ] @@ -3531,7 +3517,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf7a85fe66f9ff9cd74e169fdd2c94c6e1e74c412c99a73b4df3200b5d3760b2" dependencies = [ "kvdb", - "parking_lot 0.12.1", + "parking_lot 0.12.3", ] [[package]] @@ -3542,7 +3528,7 @@ checksum = "b644c70b92285f66bfc2032922a79000ea30af7bc2ab31902992a5dcb9b434f6" dependencies = [ "kvdb", "num_cpus", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "regex", "rocksdb", "smallvec", @@ -3561,7 +3547,7 @@ dependencies = [ "lalrpop-util", "petgraph", "regex", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", "string_cache", "term", "tiny-keccak", @@ -3575,14 +3561,14 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" dependencies = [ - "regex-automata 0.4.6", + "regex-automata 0.4.7", ] [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" @@ -3592,18 +3578,18 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -3621,7 +3607,7 @@ dependencies = [ "bytes", "futures", "futures-timer", - "getrandom 0.2.12", + "getrandom 0.2.15", "instant", "lazy_static", "libp2p-autonat", @@ -3650,7 +3636,7 @@ dependencies = [ "libp2p-websocket", "libp2p-yamux", "multiaddr 0.14.0", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "pin-project 1.1.5", "rand 0.7.3", "smallvec", @@ -3729,7 +3715,7 @@ dependencies = [ "multiaddr 0.14.0", "multihash 0.16.3", "multistream-select 0.11.0", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "pin-project 1.1.5", "prost 0.9.0", "prost-build 0.9.0", @@ -3764,7 +3750,7 @@ dependencies = [ "multiaddr 0.14.0", "multihash 0.16.3", "multistream-select 0.11.0", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "pin-project 1.1.5", "prost 0.10.4", "prost-build 0.10.4", @@ -3800,7 +3786,7 @@ dependencies = [ "futures", "libp2p-core 0.33.0", "log", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "smallvec", "trust-dns-resolver", ] @@ -3949,7 +3935,7 @@ dependencies = [ "libp2p-core 0.33.0", "log", "nohash-hasher", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "rand 0.7.3", "smallvec", "unsigned-varint", @@ -4177,7 +4163,7 @@ dependencies = [ "futures-rustls", "libp2p-core 0.33.0", "log", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "quicksink", "rw-stream-sink 0.3.0", "soketto", @@ -4193,7 +4179,7 @@ checksum = "8fe653639ad74877c759720febb0cbcbf4caa221adde4eed2d3126ce5c6f381f" dependencies = [ "futures", "libp2p-core 0.33.0", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "thiserror", "yamux", ] @@ -4204,7 +4190,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "libc", ] @@ -4273,9 +4259,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.16" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" dependencies = [ "cc", "pkg-config", @@ -4304,15 +4290,15 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -4400,9 +4386,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "merkle_light" @@ -4441,6 +4427,7 @@ dependencies = [ "rand 0.8.5", "shared_types", "storage", + "storage-async", "task_executor", "tiny-keccak", "tokio", @@ -4457,9 +4444,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] @@ -4688,7 +4675,7 @@ dependencies = [ "libp2p", "lighthouse_metrics", "lru", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "prometheus-client", "rand 0.8.5", "regex", @@ -4737,7 +4724,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "cfg_aliases", "libc", @@ -4771,11 +4758,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -4797,9 +4783,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -4833,14 +4819,14 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] name = "object" -version = "0.32.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] @@ -4947,15 +4933,15 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.6.9" +version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881331e34fa842a2fb61cc2db9643a8fedc615e47cfcc52597d1af0db9a7e8fe" +checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" dependencies = [ "arrayvec", "bitvec 1.0.1", "byte-slice-cast", "impl-trait-for-tuples", - "parity-scale-codec-derive 3.6.9", + "parity-scale-codec-derive 3.6.12", "serde", ] @@ -4973,11 +4959,11 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.6.9" +version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be30eaf4b0a9fba5336683b38de57bb86d179a35862ba6bfcf57625d006bde5b" +checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" dependencies = [ - "proc-macro-crate 2.0.0", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", "syn 1.0.109", @@ -5008,12 +4994,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core 0.9.9", + "parking_lot_core 0.9.10", ] [[package]] @@ -5032,15 +5018,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall 0.5.2", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] @@ -5056,9 +5042,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "path-slash" @@ -5117,9 +5103,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" dependencies = [ "memchr", "thiserror", @@ -5128,9 +5114,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73541b156d32197eecda1a4014d7f868fd2bcb3c550d5386087cfba442bf69c" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" dependencies = [ "pest", "pest_generator", @@ -5138,22 +5124,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] name = "pest_meta" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" dependencies = [ "once_cell", "pest", @@ -5162,9 +5148,9 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", "indexmap 2.2.6", @@ -5210,7 +5196,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -5268,7 +5254,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -5291,12 +5277,12 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" dependencies = [ "atomic-waker", - "fastrand 2.0.2", + "fastrand 2.1.0", "futures-io", ] @@ -5326,17 +5312,11 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" -[[package]] -name = "platforms" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7" - [[package]] name = "plotters" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" dependencies = [ "num-traits", "plotters-backend", @@ -5347,15 +5327,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" +checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" [[package]] name = "plotters-svg" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" dependencies = [ "plotters-backend", ] @@ -5378,15 +5358,15 @@ dependencies = [ [[package]] name = "polling" -version = "3.6.0" +version = "3.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" +checksum = "a3ed00ed3fbf728b5816498ecd316d1716eecaced9c0c8d2c5a6740ca214985b" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi 0.3.9", + "hermit-abi 0.4.0", "pin-project-lite 0.2.14", - "rustix 0.38.32", + "rustix 0.38.34", "tracing", "windows-sys 0.52.0", ] @@ -5446,12 +5426,12 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -5501,15 +5481,6 @@ dependencies = [ "toml 0.5.11", ] -[[package]] -name = "proc-macro-crate" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" -dependencies = [ - "toml_edit 0.20.7", -] - [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -5545,24 +5516,24 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] [[package]] name = "prometheus" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ "cfg-if", "fnv", "lazy_static", "memchr", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "protobuf", "thiserror", ] @@ -5592,17 +5563,17 @@ dependencies = [ [[package]] name = "proptest" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "lazy_static", "num-traits", "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", "unarray", ] @@ -5741,6 +5712,7 @@ dependencies = [ "miner", "rand 0.8.5", "storage", + "storage-async", "task_executor", "tokio", "tracing", @@ -5765,9 +5737,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -5871,7 +5843,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.15", ] [[package]] @@ -5932,11 +5904,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", ] [[package]] @@ -5945,21 +5917,21 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.15", "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -5973,13 +5945,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] @@ -5990,9 +5962,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "remove_dir_all" @@ -6026,7 +5998,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite 0.2.14", - "rustls 0.21.10", + "rustls 0.21.12", "rustls-pemfile", "serde", "serde_json", @@ -6098,7 +6070,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.12", + "getrandom 0.2.15", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -6243,9 +6215,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -6284,14 +6256,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", - "linux-raw-sys 0.4.13", + "linux-raw-sys 0.4.14", "windows-sys 0.52.0", ] @@ -6309,9 +6281,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.8", @@ -6352,9 +6324,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "rw-stream-sink" @@ -6380,9 +6352,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "salsa20" @@ -6404,23 +6376,23 @@ dependencies = [ [[package]] name = "scale-info" -version = "2.11.1" +version = "2.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "788745a868b0e751750388f4e6546eb921ef714a4317fa6954f7cde114eb2eb7" +checksum = "eca070c12893629e2cc820a9761bedf6ce1dcddc9852984d1dc734b8bd9bd024" dependencies = [ "cfg-if", "derive_more", - "parity-scale-codec 3.6.9", + "parity-scale-codec 3.6.12", "scale-info-derive", ] [[package]] name = "scale-info-derive" -version = "2.11.1" +version = "2.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dc2f4e8bc344b9fc3d5f74f72c2e55bfc38d28dc2ebc69c194a3df424e4d9ac" +checksum = "2d35494501194174bda522a32605929eefc9ecf7e0a326c26db1fdd85881eb62" dependencies = [ - "proc-macro-crate 1.1.3", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", "syn 1.0.109", @@ -6493,11 +6465,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -6506,9 +6478,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -6516,9 +6488,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ "serde", ] @@ -6537,29 +6509,29 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.197" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] name = "serde_json" -version = "1.0.115" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" +checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" dependencies = [ "itoa", "ryu", @@ -6568,9 +6540,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -6682,9 +6654,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -6763,7 +6735,7 @@ dependencies = [ "aes-gcm 0.10.3", "blake2", "chacha20poly1305", - "curve25519-dalek 4.1.2", + "curve25519-dalek 4.1.3", "rand_core 0.6.4", "ring 0.17.8", "rustc_version", @@ -6783,9 +6755,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -6884,6 +6856,7 @@ dependencies = [ "kvdb-rocksdb", "merkle_light", "merkle_tree", + "parking_lot 0.12.3", "rand 0.8.5", "rayon", "serde", @@ -6902,6 +6875,7 @@ name = "storage-async" version = "0.1.0" dependencies = [ "anyhow", + "eth2_ssz", "shared_types", "storage", "task_executor", @@ -6917,7 +6891,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "phf_shared 0.10.0", "precomputed-hash", ] @@ -6939,11 +6913,11 @@ dependencies = [ [[package]] name = "strum" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ - "strum_macros 0.26.2", + "strum_macros 0.26.4", ] [[package]] @@ -6961,15 +6935,15 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -7011,9 +6985,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.58" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote", @@ -7124,8 +7098,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.0.2", - "rustix 0.38.32", + "fastrand 2.1.0", + "rustix 0.38.34", "windows-sys 0.52.0", ] @@ -7157,22 +7131,22 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -7197,9 +7171,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -7218,9 +7192,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -7247,9 +7221,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" dependencies = [ "tinyvec_macros", ] @@ -7262,19 +7236,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", "libc", "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "pin-project-lite 0.2.14", "signal-hook-registry", - "socket2 0.5.6", + "socket2 0.5.7", "tokio-macros", "windows-sys 0.48.0", ] @@ -7291,13 +7265,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -7317,7 +7291,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.10", + "rustls 0.21.12", "tokio", ] @@ -7330,7 +7304,7 @@ dependencies = [ "futures-core", "pin-project-lite 0.2.14", "tokio", - "tokio-util 0.7.10", + "tokio-util 0.7.11", ] [[package]] @@ -7341,7 +7315,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.10", + "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", "tungstenite", @@ -7366,9 +7340,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", @@ -7376,7 +7350,6 @@ dependencies = [ "futures-sink", "pin-project-lite 0.2.14", "tokio", - "tracing", ] [[package]] @@ -7390,36 +7363,25 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.9", + "toml_edit 0.22.14", ] [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.20.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.21.1" @@ -7433,15 +7395,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.9" +version = "0.22.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" dependencies = [ "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.5", + "winnow 0.6.13", ] [[package]] @@ -7482,7 +7444,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -7536,11 +7498,10 @@ dependencies = [ [[package]] name = "tracing-test" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4" +checksum = "557b891436fe0d5e0e363427fc7f217abf9ccd510d5136549847bdcbcd011d68" dependencies = [ - "lazy_static", "tracing-core", "tracing-subscriber", "tracing-test-macro", @@ -7548,13 +7509,12 @@ dependencies = [ [[package]] name = "tracing-test-macro" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" +checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ - "lazy_static", "quote", - "syn 1.0.109", + "syn 2.0.68", ] [[package]] @@ -7605,7 +7565,7 @@ dependencies = [ "lazy_static", "log", "lru-cache", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "resolv-conf", "smallvec", "thiserror", @@ -7632,7 +7592,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.21.10", + "rustls 0.21.12", "sha1", "thiserror", "url", @@ -7762,9 +7722,9 @@ version = "0.1.0" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna 0.5.0", @@ -7783,7 +7743,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.15", "serde", ] @@ -7795,9 +7755,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74797339c3b98616c009c7c3eb53a0ce41e85c8ec66bd3db96ed132d20cfdee8" +checksum = "5a84c137d37ab0142f0f2ddfe332651fdbf252e7b7dbb4e67b6c1f1b2e925101" [[package]] name = "vcpkg" @@ -7819,9 +7779,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "waker-fn" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" [[package]] name = "walkdir" @@ -7875,7 +7835,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", "wasm-bindgen-shared", ] @@ -7909,7 +7869,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7979,14 +7939,14 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.32", + "rustix 0.38.34", ] [[package]] name = "widestring" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" [[package]] name = "wildmatch" @@ -8012,11 +7972,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -8044,7 +8004,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -8062,7 +8022,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -8082,17 +8042,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -8103,9 +8064,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -8121,9 +8082,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -8139,9 +8100,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -8157,9 +8124,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -8175,9 +8142,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -8187,9 +8154,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -8205,9 +8172,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" @@ -8220,9 +8187,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.5" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" dependencies = [ "memchr", ] @@ -8315,7 +8282,7 @@ dependencies = [ "futures", "log", "nohash-hasher", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "rand 0.8.5", "static_assertions", ] @@ -8328,29 +8295,29 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] @@ -8363,7 +8330,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.68", ] [[package]] @@ -8468,9 +8435,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.11+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" dependencies = [ "cc", "pkg-config", diff --git a/common/append_merkle/src/lib.rs b/common/append_merkle/src/lib.rs index a6ec839..fbe278d 100644 --- a/common/append_merkle/src/lib.rs +++ b/common/append_merkle/src/lib.rs @@ -724,10 +724,10 @@ mod tests { } } - fn verify(data: &Vec, merkle: &mut AppendMerkleTree) { - for i in 0..data.len() { + fn verify(data: &[H256], merkle: &mut AppendMerkleTree) { + for (i, item) in data.iter().enumerate() { let proof = merkle.gen_proof(i + 1).unwrap(); - let r = merkle.validate(&proof, &data[i], i + 1); + let r = merkle.validate(&proof, item, i + 1); assert!(matches!(r, Ok(true)), "{:?}", r); } for i in (0..data.len()).step_by(6) { diff --git a/common/merkle_light/src/lib.rs b/common/merkle_light/src/lib.rs index 1147981..e2e0266 100644 --- a/common/merkle_light/src/lib.rs +++ b/common/merkle_light/src/lib.rs @@ -148,7 +148,6 @@ unused_qualifications, missing_debug_implementations, missing_copy_implementations, - trivial_casts, trivial_numeric_casts, unsafe_code, unstable_features, diff --git a/common/merkle_light/src/test_cmh.rs b/common/merkle_light/src/test_cmh.rs index 925662d..7cc7028 100644 --- a/common/merkle_light/src/test_cmh.rs +++ b/common/merkle_light/src/test_cmh.rs @@ -1,5 +1,3 @@ -#![cfg(test)] - use crate::hash::{Algorithm, Hashable}; use crate::merkle::MerkleTree; use crate::test_item::Item; @@ -9,15 +7,15 @@ use std::iter::FromIterator; /// Custom merkle hash util test #[derive(Debug, Clone, Default)] -struct CMH(DefaultHasher); +struct Cmh(DefaultHasher); -impl CMH { - pub fn new() -> CMH { - CMH(DefaultHasher::new()) +impl Cmh { + pub fn new() -> Cmh { + Cmh(DefaultHasher::new()) } } -impl Hasher for CMH { +impl Hasher for Cmh { #[inline] fn write(&mut self, msg: &[u8]) { self.0.write(msg) @@ -29,7 +27,7 @@ impl Hasher for CMH { } } -impl Algorithm for CMH { +impl Algorithm for Cmh { #[inline] fn hash(&mut self) -> Item { Item(self.finish()) @@ -37,7 +35,7 @@ impl Algorithm for CMH { #[inline] fn reset(&mut self) { - *self = CMH::default() + *self = Cmh::default() } #[inline] @@ -57,8 +55,8 @@ impl Algorithm for CMH { #[test] fn test_custom_merkle_hasher() { - let mut a = CMH::new(); - let mt: MerkleTree = MerkleTree::from_iter([1, 2, 3, 4, 5].iter().map(|x| { + let mut a = Cmh::new(); + let mt: MerkleTree = MerkleTree::from_iter([1, 2, 3, 4, 5].iter().map(|x| { a.reset(); x.hash(&mut a); a.hash() diff --git a/common/merkle_light/src/test_item.rs b/common/merkle_light/src/test_item.rs index 8e8c62b..1a89368 100644 --- a/common/merkle_light/src/test_item.rs +++ b/common/merkle_light/src/test_item.rs @@ -1,8 +1,6 @@ -#![cfg(test)] #![allow(unsafe_code)] use crate::hash::{Algorithm, Hashable}; -use std::mem; use std::slice; #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default, Debug)] @@ -10,7 +8,7 @@ pub struct Item(pub u64); impl AsRef<[u8]> for Item { fn as_ref(&self) -> &[u8] { - unsafe { slice::from_raw_parts(mem::transmute(&self.0), 8) } + unsafe { slice::from_raw_parts(&self.0 as *const u64 as *const u8, 8) } } } diff --git a/node/chunk_pool/src/handler.rs b/node/chunk_pool/src/handler.rs index f11347a..20f9122 100644 --- a/node/chunk_pool/src/handler.rs +++ b/node/chunk_pool/src/handler.rs @@ -12,7 +12,7 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; pub struct ChunkPoolHandler { receiver: UnboundedReceiver, mem_pool: Arc, - log_store: Store, + log_store: Arc, sender: UnboundedSender, } @@ -20,7 +20,7 @@ impl ChunkPoolHandler { pub(crate) fn new( receiver: UnboundedReceiver, mem_pool: Arc, - log_store: Store, + log_store: Arc, sender: UnboundedSender, ) -> Self { ChunkPoolHandler { diff --git a/node/chunk_pool/src/lib.rs b/node/chunk_pool/src/lib.rs index 80929dd..401f319 100644 --- a/node/chunk_pool/src/lib.rs +++ b/node/chunk_pool/src/lib.rs @@ -28,7 +28,7 @@ impl Config { pub fn unbounded( config: Config, - log_store: storage_async::Store, + log_store: Arc, network_send: tokio::sync::mpsc::UnboundedSender, ) -> (Arc, ChunkPoolHandler) { let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); diff --git a/node/chunk_pool/src/mem_pool/chunk_pool_inner.rs b/node/chunk_pool/src/mem_pool/chunk_pool_inner.rs index e2a5581..8d5aec3 100644 --- a/node/chunk_pool/src/mem_pool/chunk_pool_inner.rs +++ b/node/chunk_pool/src/mem_pool/chunk_pool_inner.rs @@ -82,14 +82,14 @@ impl From for (ChunkArray, FileProof) { /// and data root verified on blockchain. pub struct MemoryChunkPool { inner: Mutex, - log_store: Store, + log_store: Arc, sender: UnboundedSender, } impl MemoryChunkPool { pub(crate) fn new( config: Config, - log_store: Store, + log_store: Arc, sender: UnboundedSender, ) -> Self { MemoryChunkPool { @@ -99,7 +99,7 @@ impl MemoryChunkPool { } } - pub fn validate_segment_size(&self, segment: &Vec) -> Result<()> { + pub fn validate_segment_size(&self, segment: &[u8]) -> Result<()> { if segment.is_empty() { bail!("data is empty"); } diff --git a/node/log_entry_sync/src/lib.rs b/node/log_entry_sync/src/lib.rs index 9f27bd2..c46b9c1 100644 --- a/node/log_entry_sync/src/lib.rs +++ b/node/log_entry_sync/src/lib.rs @@ -1,10 +1,11 @@ extern crate core; -pub(crate) mod rpc_proxy; mod sync_manager; -pub use rpc_proxy::ContractAddress; +use ethers::prelude::H160; pub use sync_manager::{ config::{CacheConfig, LogSyncConfig}, LogSyncEvent, LogSyncManager, }; + +pub type ContractAddress = H160; diff --git a/node/log_entry_sync/src/rpc_proxy/eth.rs b/node/log_entry_sync/src/rpc_proxy/eth.rs deleted file mode 100644 index 25566fb..0000000 --- a/node/log_entry_sync/src/rpc_proxy/eth.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::rpc_proxy::{ContractAddress, EvmRpcProxy, SubEvent, SubFilter}; -use async_trait::async_trait; -use ethers::prelude::{Bytes, Middleware, Provider}; -use ethers::providers::Ws; -use ethers::types::TransactionRequest; -use jsonrpsee::core::client::Subscription; - -pub struct EthClient { - client: Provider, -} - -impl EthClient { - #[allow(unused)] - pub async fn new(url: &str) -> anyhow::Result { - let client = Provider::new(Ws::connect(url).await?); - Ok(Self { client }) - } -} - -#[async_trait] -impl EvmRpcProxy for EthClient { - async fn call(&self, to: ContractAddress, data: Bytes) -> anyhow::Result { - let request = TransactionRequest::new().to(to).data(data); - Ok(self.client.call(&request.into(), None).await?) - } - - async fn sub_events(&self, _filter: SubFilter) -> Subscription { - todo!() - } -} diff --git a/node/log_entry_sync/src/rpc_proxy/mod.rs b/node/log_entry_sync/src/rpc_proxy/mod.rs deleted file mode 100644 index a1efe63..0000000 --- a/node/log_entry_sync/src/rpc_proxy/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -use anyhow::Result; -use async_trait::async_trait; -use ethereum_types::{H160, H256}; -use ethers::prelude::Bytes; -use jsonrpsee::core::client::Subscription; - -// TODO: Define accounts/filter/events as associated types? -// TODO: Define an abstraction suitable for other chains. -#[async_trait] -pub trait EvmRpcProxy { - async fn call(&self, to: ContractAddress, data: Bytes) -> Result; - - async fn sub_events(&self, filter: SubFilter) -> Subscription; -} - -pub type ContractAddress = H160; - -pub type Topic = H256; - -#[allow(unused)] -pub struct SubFilter { - to: Option, - topics: Vec, -} - -#[allow(unused)] -pub struct SubEvent { - /// Address - pub address: ContractAddress, - - /// Topics - pub topics: Vec, - - /// Data - pub data: Bytes, -} - -pub(crate) mod eth; diff --git a/node/log_entry_sync/src/sync_manager/config.rs b/node/log_entry_sync/src/sync_manager/config.rs index 766a926..15d540c 100644 --- a/node/log_entry_sync/src/sync_manager/config.rs +++ b/node/log_entry_sync/src/sync_manager/config.rs @@ -1,4 +1,4 @@ -use crate::rpc_proxy::ContractAddress; +use crate::ContractAddress; pub struct LogSyncConfig { pub rpc_endpoint_url: String, diff --git a/node/log_entry_sync/src/sync_manager/log_entry_fetcher.rs b/node/log_entry_sync/src/sync_manager/log_entry_fetcher.rs index a3ead52..3773b24 100644 --- a/node/log_entry_sync/src/sync_manager/log_entry_fetcher.rs +++ b/node/log_entry_sync/src/sync_manager/log_entry_fetcher.rs @@ -1,6 +1,6 @@ -use crate::rpc_proxy::ContractAddress; use crate::sync_manager::log_query::LogQuery; use crate::sync_manager::RETRY_WAIT_MS; +use crate::ContractAddress; use anyhow::{anyhow, bail, Result}; use append_merkle::{Algorithm, Sha3Algorithm}; use contract_interface::{SubmissionNode, SubmitFilter, ZgsFlow}; @@ -122,7 +122,7 @@ impl LogEntryFetcher { pub fn start_remove_finalized_block_task( &self, executor: &TaskExecutor, - store: Arc>, + store: Arc, block_hash_cache: Arc>>>, default_finalized_block_count: u64, remove_finalized_block_interval_minutes: u64, @@ -133,7 +133,7 @@ impl LogEntryFetcher { loop { debug!("processing finalized block"); - let processed_block_number = match store.read().await.get_sync_progress() { + let processed_block_number = match store.get_sync_progress() { Ok(Some((processed_block_number, _))) => Some(processed_block_number), Ok(None) => None, Err(e) => { @@ -176,9 +176,7 @@ impl LogEntryFetcher { } for key in pending_keys.into_iter() { - if let Err(e) = - store.write().await.delete_block_hash_by_number(key) - { + if let Err(e) = store.delete_block_hash_by_number(key) { error!( "remove block tx for number {} error: e={:?}", key, e @@ -599,7 +597,7 @@ fn submission_event_to_transaction(e: SubmitFilter) -> LogFetchProgress { }) } -fn nodes_to_root(node_list: &Vec) -> DataRoot { +fn nodes_to_root(node_list: &[SubmissionNode]) -> DataRoot { let mut root: DataRoot = node_list.last().expect("not empty").root.into(); for next_node in node_list[..node_list.len() - 1].iter().rev() { root = Sha3Algorithm::parent(&next_node.root.into(), &root); diff --git a/node/log_entry_sync/src/sync_manager/mod.rs b/node/log_entry_sync/src/sync_manager/mod.rs index fef288f..c074d92 100644 --- a/node/log_entry_sync/src/sync_manager/mod.rs +++ b/node/log_entry_sync/src/sync_manager/mod.rs @@ -33,7 +33,7 @@ pub enum LogSyncEvent { pub struct LogSyncManager { config: LogSyncConfig, log_fetcher: LogEntryFetcher, - store: Arc>, + store: Arc, data_cache: DataCache, next_tx_seq: u64, @@ -48,9 +48,9 @@ impl LogSyncManager { pub async fn spawn( config: LogSyncConfig, executor: TaskExecutor, - store: Arc>, + store: Arc, ) -> Result> { - let next_tx_seq = store.read().await.next_tx_seq(); + let next_tx_seq = store.next_tx_seq(); let executor_clone = executor.clone(); let mut shutdown_sender = executor.shutdown_sender(); @@ -81,8 +81,6 @@ impl LogSyncManager { let block_hash_cache = Arc::new(RwLock::new( store - .read() - .await .get_block_hashes()? .into_iter() .map(|(x, y)| (x, Some(y))) @@ -124,7 +122,7 @@ impl LogSyncManager { // TODO(zz): Handle reorg instead of return. let mut need_handle_reorg = false; let (mut start_block_number, mut start_block_hash) = - match log_sync_manager.store.read().await.get_sync_progress()? { + match log_sync_manager.store.get_sync_progress()? { // No previous progress, so just use config. None => { let block_number = log_sync_manager.config.start_block_number; @@ -176,7 +174,7 @@ impl LogSyncManager { log_sync_manager.handle_data(reorg_rx).await?; if let Some((block_number, block_hash)) = - log_sync_manager.store.read().await.get_sync_progress()? + log_sync_manager.store.get_sync_progress()? { start_block_number = block_number; start_block_hash = block_hash; @@ -301,7 +299,7 @@ impl LogSyncManager { async fn process_reverted(&mut self, tx_seq: u64) { warn!("revert for chain reorg: seq={}", tx_seq); { - let store = self.store.read().await; + let store = self.store.clone(); for seq in tx_seq..self.next_tx_seq { if matches!(store.check_tx_completed(seq), Ok(true)) { if let Ok(Some(tx)) = store.get_tx_by_seq_number(seq) { @@ -325,7 +323,7 @@ impl LogSyncManager { let _ = self.event_send.send(LogSyncEvent::ReorgDetected { tx_seq }); // TODO(zz): `wrapping_sub` here is a hack to handle the case of tx_seq=0. - if let Err(e) = self.store.write().await.revert_to(tx_seq.wrapping_sub(1)) { + if let Err(e) = self.store.revert_to(tx_seq.wrapping_sub(1)) { error!("revert_to fails: e={:?}", e); return; } @@ -353,7 +351,7 @@ impl LogSyncManager { ); } - self.store.write().await.put_sync_progress(( + self.store.put_sync_progress(( block_number, block_hash, first_submission_index, @@ -396,12 +394,12 @@ impl LogSyncManager { } async fn put_tx_inner(&mut self, tx: Transaction) -> bool { - if let Err(e) = self.store.write().await.put_tx(tx.clone()) { + if let Err(e) = self.store.put_tx(tx.clone()) { error!("put_tx error: e={:?}", e); false } else { if let Some(data) = self.data_cache.pop_data(&tx.data_merkle_root) { - let mut store = self.store.write().await; + let store = self.store.clone(); // We are holding a mutable reference of LogSyncManager, so no chain reorg is // possible after put_tx. if let Err(e) = store diff --git a/node/miner/Cargo.toml b/node/miner/Cargo.toml index 3fafda9..7f07aa9 100644 --- a/node/miner/Cargo.toml +++ b/node/miner/Cargo.toml @@ -20,4 +20,5 @@ ethers = "^2" lazy_static = "1.4" async-trait = "0.1.56" shared_types = { path = "../shared_types" } -hex = "0.4" \ No newline at end of file +hex = "0.4" +storage-async = { path = "../storage-async" } \ No newline at end of file diff --git a/node/miner/src/loader.rs b/node/miner/src/loader.rs index 92b7397..9ef81a3 100644 --- a/node/miner/src/loader.rs +++ b/node/miner/src/loader.rs @@ -1,7 +1,6 @@ use async_trait::async_trait; -use std::sync::Arc; -use storage::log_store::{MineLoadChunk, Store}; -use tokio::sync::RwLock; +use storage::log_store::MineLoadChunk; +use storage_async::Store; #[async_trait] pub trait PoraLoader: Send + Sync { @@ -9,10 +8,9 @@ pub trait PoraLoader: Send + Sync { } #[async_trait] -impl PoraLoader for Arc> { +impl PoraLoader for Store { async fn load_sealed_data(&self, chunk_index: u64) -> Option { - let store = &*self.read().await; - match store.flow().load_sealed_data(chunk_index) { + match self.load_sealed_data(chunk_index).await { Ok(Some(chunk)) => Some(chunk), _ => None, } diff --git a/node/miner/src/miner_id.rs b/node/miner/src/miner_id.rs index 7b50dcc..6deac4b 100644 --- a/node/miner/src/miner_id.rs +++ b/node/miner/src/miner_id.rs @@ -5,26 +5,26 @@ use ethereum_types::Address; use ethers::contract::ContractCall; use ethers::contract::EthEvent; use std::sync::Arc; -use storage::log_store::{config::ConfigurableExt, Store}; use storage::H256; -use tokio::sync::RwLock; +use storage_async::Store; const MINER_ID: &str = "mine.miner_id"; -pub fn load_miner_id(store: &dyn Store) -> storage::error::Result> { - store.get_config_decoded(&MINER_ID) +pub async fn load_miner_id(store: &Store) -> storage::error::Result> { + store.get_config_decoded(&MINER_ID).await } -fn set_miner_id(store: &dyn Store, miner_id: &H256) -> storage::error::Result<()> { - store.set_config_encoded(&MINER_ID, miner_id) +async fn set_miner_id(store: &Store, miner_id: &H256) -> storage::error::Result<()> { + store.set_config_encoded(&MINER_ID, miner_id).await } pub(crate) async fn check_and_request_miner_id( config: &MinerConfig, - store: &RwLock, + store: &Store, provider: &Arc, ) -> Result { - let db_miner_id = load_miner_id(&*store.read().await) + let db_miner_id = load_miner_id(store) + .await .map_err(|e| format!("miner_id on db corrupt: {:?}", e))?; let mine_contract = PoraMine::new(config.mine_address, provider.clone()); @@ -42,7 +42,8 @@ pub(crate) async fn check_and_request_miner_id( } (None, Some(c_id)) => { check_miner_id(&mine_contract, c_id).await?; - set_miner_id(&*store.write().await, &c_id) + set_miner_id(store, &c_id) + .await .map_err(|e| format!("set miner id on db corrupt: {:?}", e))?; Ok(c_id) } @@ -53,7 +54,8 @@ pub(crate) async fn check_and_request_miner_id( (None, None) => { let beneficiary = provider.address(); let id = request_miner_id(&mine_contract, beneficiary).await?; - set_miner_id(&*store.write().await, &id) + set_miner_id(store, &id) + .await .map_err(|e| format!("set miner id on db corrupt: {:?}", e))?; Ok(id) } diff --git a/node/miner/src/sealer.rs b/node/miner/src/sealer.rs index 59390d4..aed65a0 100644 --- a/node/miner/src/sealer.rs +++ b/node/miner/src/sealer.rs @@ -1,16 +1,14 @@ use std::{collections::BTreeMap, sync::Arc}; use ethereum_types::H256; -use tokio::{ - sync::RwLock, - time::{sleep, Duration, Instant}, -}; +use tokio::time::{sleep, Duration, Instant}; use contract_interface::{EpochRangeWithContextDigest, ZgsFlow}; use storage::{ error::Result, - log_store::{SealAnswer, SealTask, Store}, + log_store::{SealAnswer, SealTask}, }; +use storage_async::Store; use task_executor::TaskExecutor; use zgs_spec::SECTORS_PER_SEAL; @@ -22,7 +20,7 @@ const CHAIN_STATUS_QUERY_PERIOD: u64 = 5; pub struct Sealer { flow_contract: ZgsFlow, - store: Arc>, + store: Arc, context_cache: BTreeMap, last_context_flow_length: u64, miner_id: H256, @@ -32,7 +30,7 @@ impl Sealer { pub fn spawn( executor: TaskExecutor, provider: Arc, - store: Arc>, + store: Arc, config: &MinerConfig, miner_id: H256, ) { @@ -152,19 +150,11 @@ impl Sealer { async fn fetch_task(&self) -> Result>> { let seal_index_max = self.last_context_flow_length as usize / SECTORS_PER_SEAL; - self.store - .read() - .await - .flow() - .pull_seal_chunk(seal_index_max) + self.store.pull_seal_chunk(seal_index_max).await } async fn submit_answer(&self, answers: Vec) -> Result<()> { - self.store - .write() - .await - .flow_mut() - .submit_seal_result(answers) + self.store.submit_seal_result(answers).await } async fn seal_iteration(&mut self) -> Result { diff --git a/node/miner/src/service.rs b/node/miner/src/service.rs index d31ad88..485fe6c 100644 --- a/node/miner/src/service.rs +++ b/node/miner/src/service.rs @@ -5,9 +5,9 @@ use crate::{config::MinerConfig, mine::PoraService, watcher::MineContextWatcher} use network::NetworkMessage; use std::sync::Arc; use storage::config::ShardConfig; -use storage::log_store::Store; +use storage_async::Store; +use tokio::sync::broadcast; use tokio::sync::mpsc; -use tokio::sync::{broadcast, RwLock}; #[derive(Clone, Debug)] pub enum MinerMessage { @@ -29,13 +29,13 @@ impl MineService { executor: task_executor::TaskExecutor, _network_send: mpsc::UnboundedSender, config: MinerConfig, - store: Arc>, + store: Arc, ) -> Result, String> { let provider = Arc::new(config.make_provider().await?); let (msg_send, msg_recv) = broadcast::channel(1024); - let miner_id = check_and_request_miner_id(&config, &store, &provider).await?; + let miner_id = check_and_request_miner_id(&config, store.as_ref(), &provider).await?; debug!("miner id setting complete."); let mine_context_receiver = MineContextWatcher::spawn( @@ -49,7 +49,7 @@ impl MineService { executor.clone(), msg_recv.resubscribe(), mine_context_receiver, - Arc::new(store.clone()), + store.clone(), &config, miner_id, ); diff --git a/node/miner/src/submitter.rs b/node/miner/src/submitter.rs index ad76b5c..642bf20 100644 --- a/node/miner/src/submitter.rs +++ b/node/miner/src/submitter.rs @@ -6,14 +6,14 @@ use ethers::providers::PendingTransaction; use hex::ToHex; use shared_types::FlowRangeProof; use std::sync::Arc; -use storage::log_store::Store; +use storage_async::Store; use task_executor::TaskExecutor; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::mpsc; use crate::config::{MineServiceMiddleware, MinerConfig}; use crate::pora::AnswerWithoutProof; -use zgs_spec::SECTORS_PER_SEAL; +use zgs_spec::{BYTES_PER_SEAL, SECTORS_PER_SEAL}; const SUBMISSION_RETIES: usize = 3; @@ -22,7 +22,7 @@ pub struct Submitter { mine_contract: PoraMine, flow_contract: ZgsFlow, default_gas_limit: Option, - store: Arc>, + store: Arc, } impl Submitter { @@ -30,7 +30,7 @@ impl Submitter { executor: TaskExecutor, mine_answer_receiver: mpsc::UnboundedReceiver, provider: Arc, - store: Arc>, + store: Arc, config: &MinerConfig, ) { let mine_contract = PoraMine::new(config.mine_address, provider.clone()); @@ -80,13 +80,12 @@ impl Submitter { let flow_proof = self .store - .read() - .await .get_proof_at_root( - &mine_answer.context_flow_root, + mine_answer.context_flow_root, mine_answer.recall_position, SECTORS_PER_SEAL as u64, ) + .await .map_err(|e| e.to_string())?; let answer = PoraAnswer { @@ -97,7 +96,11 @@ impl Submitter { recall_position: mine_answer.recall_position.into(), seal_offset: mine_answer.seal_offset.into(), sealed_context_digest: sealed_context_digest.digest, - sealed_data: unsafe { std::mem::transmute(mine_answer.sealed_data) }, + sealed_data: unsafe { + std::mem::transmute::<[u8; BYTES_PER_SEAL], [[u8; 32]; BYTES_PER_SEAL / 32]>( + mine_answer.sealed_data, + ) + }, merkle_proof: flow_proof_to_pora_merkle_proof(flow_proof), }; trace!("submit_answer: answer={:?}", answer); diff --git a/node/network/tests/common/mod.rs b/node/network/tests/common/mod.rs index ac7f019..fc6ecc8 100644 --- a/node/network/tests/common/mod.rs +++ b/node/network/tests/common/mod.rs @@ -24,6 +24,7 @@ type ReqId = usize; use tempfile::Builder as TempBuilder; use tokio::sync::mpsc::unbounded_channel; +#[allow(unused)] pub struct Libp2pInstance(LibP2PService, exit_future::Signal); impl std::ops::Deref for Libp2pInstance { diff --git a/node/pruner/Cargo.toml b/node/pruner/Cargo.toml index 373641a..7fe0e2f 100644 --- a/node/pruner/Cargo.toml +++ b/node/pruner/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] storage = { path = "../storage" } +storage-async = { path = "../storage-async" } miner = { path = "../miner" } anyhow = "1.0.86" tokio = "1.37.0" diff --git a/node/pruner/src/lib.rs b/node/pruner/src/lib.rs index 8e1cf0d..4ceab42 100644 --- a/node/pruner/src/lib.rs +++ b/node/pruner/src/lib.rs @@ -5,11 +5,10 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use storage::config::{ShardConfig, SHARD_CONFIG_KEY}; -use storage::log_store::config::ConfigurableExt; -use storage::log_store::Store; +use storage_async::Store; use task_executor::TaskExecutor; -use tokio::sync::{broadcast, mpsc, RwLock}; -use tracing::debug; +use tokio::sync::{broadcast, mpsc}; +use tracing::{debug, info}; // Start pruning when the db directory size exceeds 0.9 * limit. const PRUNE_THRESHOLD: f32 = 0.9; @@ -32,7 +31,7 @@ impl PrunerConfig { pub struct Pruner { config: PrunerConfig, - store: Arc>, + store: Arc, sender: mpsc::UnboundedSender, miner_sender: Option>, @@ -42,10 +41,10 @@ impl Pruner { pub async fn spawn( executor: TaskExecutor, mut config: PrunerConfig, - store: Arc>, + store: Arc, miner_sender: Option>, ) -> Result> { - if let Some(shard_config) = get_shard_config(&store).await? { + if let Some(shard_config) = get_shard_config(store.as_ref()).await? { config.shard_config = shard_config; } let (tx, rx) = mpsc::unbounded_channel(); @@ -68,7 +67,7 @@ impl Pruner { pub async fn start(mut self) -> Result<()> { loop { if let Some(delete_list) = self.maybe_update().await? { - debug!(new_config = ?self.config.shard_config, "new shard config"); + info!(new_config = ?self.config.shard_config, "new shard config"); self.put_shard_config().await?; let mut batch = Vec::with_capacity(self.config.batch_size); let mut iter = delete_list.peekable(); @@ -76,7 +75,7 @@ impl Pruner { batch.push(index); if batch.len() == self.config.batch_size || iter.peek().is_none() { debug!(start = batch.first(), end = batch.last(), "prune batch"); - self.store.write().await.remove_chunks_batch(&batch)?; + self.store.remove_chunks_batch(&batch).await?; batch = Vec::with_capacity(self.config.batch_size); tokio::time::sleep(self.config.batch_wait_time).await; } @@ -87,7 +86,7 @@ impl Pruner { } async fn maybe_update(&mut self) -> Result>>> { - let current_size = self.store.read().await.flow().get_num_entries()?; + let current_size = self.store.get_num_entries().await?; debug!( current_size = current_size, config = ?self.config.shard_config, @@ -109,7 +108,7 @@ impl Pruner { config.num_shard *= 2; // Generate delete list - let flow_len = self.store.read().await.get_context()?.1; + let flow_len = self.store.get_context().await?.1; let start_index = old_shard_id + (!rand_bit) as usize * old_num_shard; return Ok(Some(Box::new( (start_index as u64..flow_len).step_by(config.num_shard), @@ -124,16 +123,17 @@ impl Pruner { } self.sender .send(PrunerMessage::ChangeShardConfig(self.config.shard_config))?; - let mut store = self.store.write().await; - store - .flow_mut() - .update_shard_config(self.config.shard_config); - store.set_config_encoded(&SHARD_CONFIG_KEY, &self.config.shard_config) + self.store + .update_shard_config(self.config.shard_config) + .await; + self.store + .set_config_encoded(&SHARD_CONFIG_KEY, &self.config.shard_config) + .await } } -async fn get_shard_config(store: &RwLock) -> Result> { - store.read().await.get_config_decoded(&SHARD_CONFIG_KEY) +async fn get_shard_config(store: &Store) -> Result> { + store.get_config_decoded(&SHARD_CONFIG_KEY).await } #[derive(Debug)] diff --git a/node/router/src/libp2p_event_handler.rs b/node/router/src/libp2p_event_handler.rs index 678578d..8f05a01 100644 --- a/node/router/src/libp2p_event_handler.rs +++ b/node/router/src/libp2p_event_handler.rs @@ -269,13 +269,7 @@ impl Libp2pEventHandler { }; let timestamp = timestamp_now(); - let shard_config = self - .store - .get_store() - .read() - .await - .flow() - .get_shard_config(); + let shard_config = self.store.get_store().flow().get_shard_config(); let msg = AnnounceFile { tx_id, @@ -597,8 +591,8 @@ mod tests { sync_send: SyncSender, sync_recv: SyncReceiver, chunk_pool_send: mpsc::UnboundedSender, - chunk_pool_recv: mpsc::UnboundedReceiver, - store: Arc>, + // chunk_pool_recv: mpsc::UnboundedReceiver, + store: Arc, file_location_cache: Arc, peers: Arc>, } @@ -609,7 +603,7 @@ mod tests { let (network_globals, keypair) = Context::new_network_globals(); let (network_send, network_recv) = mpsc::unbounded_channel(); let (sync_send, sync_recv) = channel::Channel::unbounded(); - let (chunk_pool_send, chunk_pool_recv) = mpsc::unbounded_channel(); + let (chunk_pool_send, _chunk_pool_recv) = mpsc::unbounded_channel(); let store = LogManager::memorydb(LogConfig::default()).unwrap(); Self { runtime, @@ -620,8 +614,8 @@ mod tests { sync_send, sync_recv, chunk_pool_send, - chunk_pool_recv, - store: Arc::new(RwLock::new(store)), + // chunk_pool_recv, + store: Arc::new(store), file_location_cache: Arc::new(FileLocationCache::default()), peers: Arc::new(RwLock::new(PeerManager::new(Config::default()))), } @@ -1003,7 +997,11 @@ mod tests { assert_eq!(peer_id, *ctx.network_globals.peer_id.read()); assert_eq!( addr, - *ctx.network_globals.listen_multiaddrs.read().get(0).unwrap() + *ctx.network_globals + .listen_multiaddrs + .read() + .first() + .unwrap() ); } Ok(_) => panic!("Unexpected sync message type received"), diff --git a/node/router/src/service.rs b/node/router/src/service.rs index 0d5f4b8..f1590a2 100644 --- a/node/router/src/service.rs +++ b/node/router/src/service.rs @@ -58,7 +58,7 @@ impl RouterService { _miner_send: Option>, chunk_pool_send: UnboundedSender, pruner_recv: Option>, - store: Arc>, + store: Arc, file_location_cache: Arc, local_keypair: Keypair, config: Config, diff --git a/node/rpc/src/lib.rs b/node/rpc/src/lib.rs index 7472839..3b8098a 100644 --- a/node/rpc/src/lib.rs +++ b/node/rpc/src/lib.rs @@ -44,7 +44,7 @@ pub struct Context { pub network_send: UnboundedSender, pub sync_send: SyncSender, pub chunk_pool: Arc, - pub log_store: Store, + pub log_store: Arc, pub shutdown_sender: Sender, pub mine_service_sender: Option>, } diff --git a/node/rpc/src/zgs/impl.rs b/node/rpc/src/zgs/impl.rs index 46d7b6d..36ba47e 100644 --- a/node/rpc/src/zgs/impl.rs +++ b/node/rpc/src/zgs/impl.rs @@ -23,8 +23,6 @@ impl RpcServer for RpcServerImpl { .ctx .log_store .get_store() - .read() - .await .get_sync_progress()? .unwrap_or_default(); @@ -156,14 +154,7 @@ impl RpcServer for RpcServerImpl { async fn get_shard_config(&self) -> RpcResult { debug!("zgs_getShardConfig"); - let shard_config = self - .ctx - .log_store - .get_store() - .read() - .await - .flow() - .get_shard_config(); + let shard_config = self.ctx.log_store.get_store().flow().get_shard_config(); Ok(shard_config) } } diff --git a/node/src/client/builder.rs b/node/src/client/builder.rs index 3933557..28dc596 100644 --- a/node/src/client/builder.rs +++ b/node/src/client/builder.rs @@ -15,7 +15,7 @@ use storage::log_store::log_manager::LogConfig; use storage::log_store::Store; use storage::{LogManager, StorageConfig}; use sync::{SyncSender, SyncService}; -use tokio::sync::{broadcast, mpsc, RwLock}; +use tokio::sync::{broadcast, mpsc}; macro_rules! require { ($component:expr, $self:ident, $e:ident) => { @@ -68,8 +68,8 @@ struct ChunkPoolComponents { #[derive(Default)] pub struct ClientBuilder { runtime_context: Option, - store: Option>>, - async_store: Option, + store: Option>, + async_store: Option>, file_location_cache: Option>, network: Option, sync: Option, @@ -89,15 +89,18 @@ impl ClientBuilder { /// Initializes in-memory storage. pub fn with_memory_store(mut self) -> Result { // TODO(zz): Set config. - let store = Arc::new(RwLock::new( + let store = Arc::new( LogManager::memorydb(LogConfig::default()) .map_err(|e| format!("Unable to start in-memory store: {:?}", e))?, - )); + ); self.store = Some(store.clone()); if let Some(ctx) = self.runtime_context.as_ref() { - self.async_store = Some(storage_async::Store::new(store, ctx.executor.clone())); + self.async_store = Some(Arc::new(storage_async::Store::new( + store, + ctx.executor.clone(), + ))); } Ok(self) @@ -105,15 +108,18 @@ impl ClientBuilder { /// Initializes RocksDB storage. pub fn with_rocksdb_store(mut self, config: &StorageConfig) -> Result { - let store = Arc::new(RwLock::new( + let store = Arc::new( LogManager::rocksdb(LogConfig::default(), &config.db_dir) .map_err(|e| format!("Unable to start RocksDB store: {:?}", e))?, - )); + ); self.store = Some(store.clone()); if let Some(ctx) = self.runtime_context.as_ref() { - self.async_store = Some(storage_async::Store::new(store, ctx.executor.clone())); + self.async_store = Some(Arc::new(storage_async::Store::new( + store, + ctx.executor.clone(), + ))); } Ok(self) @@ -177,7 +183,7 @@ impl ClientBuilder { if let Some(config) = config { let executor = require!("miner", self, runtime_context).clone().executor; let network_send = require!("miner", self, network).send.clone(); - let store = self.store.as_ref().unwrap().clone(); + let store = self.async_store.as_ref().unwrap().clone(); let send = MineService::spawn(executor, network_send, config, store).await?; self.miner = Some(MinerComponents { send }); @@ -189,7 +195,7 @@ impl ClientBuilder { pub async fn with_pruner(mut self, config: Option) -> Result { if let Some(config) = config { let miner_send = self.miner.as_ref().map(|miner| miner.send.clone()); - let store = require!("pruner", self, store).clone(); + let store = require!("pruner", self, async_store).clone(); let executor = require!("pruner", self, runtime_context).clone().executor; let recv = Pruner::spawn(executor, config, store, miner_send) .await diff --git a/node/src/client/environment.rs b/node/src/client/environment.rs index a484a98..5f7eec5 100644 --- a/node/src/client/environment.rs +++ b/node/src/client/environment.rs @@ -119,8 +119,7 @@ impl Environment { let inner_shutdown = async move { rx.next().await.ok_or("Internal shutdown channel exhausted") }; futures::pin_mut!(inner_shutdown); - - match self.runtime().block_on(async { + let shutdown = async { let mut handles = vec![]; // setup for handling SIGTERM @@ -151,7 +150,9 @@ impl Environment { } future::select(inner_shutdown, future::select_all(handles.into_iter())).await - }) { + }; + + match self.runtime().block_on(shutdown) { future::Either::Left((Ok(reason), _)) => { info!(reason = reason.message(), "Internal shutdown received"); Ok(reason) diff --git a/node/storage-async/Cargo.toml b/node/storage-async/Cargo.toml index 20ddbe5..f7e552c 100644 --- a/node/storage-async/Cargo.toml +++ b/node/storage-async/Cargo.toml @@ -9,4 +9,5 @@ shared_types = { path = "../shared_types" } storage = { path = "../storage" } task_executor = { path = "../../common/task_executor" } tokio = { version = "1.19.2", features = ["sync"] } -tracing = "0.1.35" \ No newline at end of file +tracing = "0.1.35" +eth2_ssz = "0.4.0" \ No newline at end of file diff --git a/node/storage-async/src/lib.rs b/node/storage-async/src/lib.rs index 012d819..419b380 100644 --- a/node/storage-async/src/lib.rs +++ b/node/storage-async/src/lib.rs @@ -2,13 +2,18 @@ extern crate tracing; use anyhow::bail; -use shared_types::{Chunk, ChunkArray, ChunkArrayWithProof, DataRoot, FlowProof, Transaction}; +use shared_types::{ + Chunk, ChunkArray, ChunkArrayWithProof, DataRoot, FlowProof, FlowRangeProof, Transaction, +}; +use ssz::{Decode, Encode}; use std::sync::Arc; use storage::{error, error::Result, log_store::Store as LogStore, H256}; use task_executor::TaskExecutor; -use tokio::sync::{oneshot, RwLock}; +use tokio::sync::oneshot; pub use storage::config::ShardConfig; +use storage::log_store::config::ConfigurableExt; +use storage::log_store::{MineLoadChunk, SealAnswer, SealTask}; /// The name of the worker tokio tasks. const WORKER_TASK_NAME: &str = "async_storage_worker"; @@ -28,14 +33,14 @@ macro_rules! delegate { #[derive(Clone)] pub struct Store { /// Log and transaction storage. - store: Arc>, + store: Arc, /// Tokio executor for spawning worker tasks. executor: TaskExecutor, } impl Store { - pub fn new(store: Arc>, executor: TaskExecutor) -> Self { + pub fn new(store: Arc, executor: TaskExecutor) -> Self { Store { store, executor } } @@ -49,6 +54,8 @@ impl Store { delegate!(fn get_chunk_by_flow_index(index: u64, length: u64) -> Result>); delegate!(fn finalize_tx(tx_seq: u64) -> Result<()>); delegate!(fn finalize_tx_with_hash(tx_seq: u64, tx_hash: H256) -> Result); + delegate!(fn get_proof_at_root(root: DataRoot, index: u64, length: u64) -> Result); + delegate!(fn get_context() -> Result<(DataRoot, u64)>); pub async fn get_tx_seq_by_data_root(&self, data_root: &DataRoot) -> Result> { let root = *data_root; @@ -62,18 +69,76 @@ impl Store { .await } + pub async fn get_config_decoded + Send + Sync, T: Decode + Send + 'static>( + &self, + key: &K, + ) -> Result> { + let key = key.as_ref().to_vec(); + self.spawn(move |store| store.get_config_decoded(&key)) + .await + } + + pub async fn set_config_encoded + Send + Sync, T: Encode + Send + Sync>( + &self, + key: &K, + value: &T, + ) -> anyhow::Result<()> { + let key = key.as_ref().to_vec(); + let value = value.as_ssz_bytes(); + self.spawn(move |store| store.set_config(&key, &value)) + .await + } + + pub async fn pull_seal_chunk( + &self, + seal_index_max: usize, + ) -> anyhow::Result>> { + self.spawn(move |store| store.flow().pull_seal_chunk(seal_index_max)) + .await + } + + pub async fn submit_seal_result(&self, answers: Vec) -> anyhow::Result<()> { + self.spawn(move |store| store.flow().submit_seal_result(answers)) + .await + } + + pub async fn load_sealed_data(&self, chunk_index: u64) -> Result> { + self.spawn(move |store| store.flow().load_sealed_data(chunk_index)) + .await + } + + pub async fn get_num_entries(&self) -> Result { + self.spawn(move |store| store.flow().get_num_entries()) + .await + } + + pub async fn remove_chunks_batch(&self, batch_list: &[u64]) -> Result<()> { + let batch_list = batch_list.to_vec(); + self.spawn(move |store| store.remove_chunks_batch(&batch_list)) + .await + } + + pub async fn update_shard_config(&self, shard_config: ShardConfig) { + self.spawn(move |store| { + store.flow().update_shard_config(shard_config); + Ok(()) + }) + .await + .expect("always ok") + } + async fn spawn(&self, f: F) -> Result where - F: FnOnce(&mut dyn LogStore) -> Result + Send + 'static, + F: FnOnce(&dyn LogStore) -> Result + Send + 'static, T: Send + 'static, { let store = self.store.clone(); let (tx, rx) = oneshot::channel(); - self.executor.spawn( - async move { + self.executor.spawn_blocking( + move || { // FIXME(zz): Not all functions need `write`. Refactor store usage. - let res = f(&mut *store.write().await); + let res = f(&*store); if tx.send(res).is_err() { error!("Unable to complete async storage operation: the receiver dropped"); @@ -87,7 +152,7 @@ impl Store { } // FIXME(zz): Refactor the lock and async call here. - pub fn get_store(&self) -> &RwLock { + pub fn get_store(&self) -> &dyn LogStore { self.store.as_ref() } } diff --git a/node/storage/Cargo.toml b/node/storage/Cargo.toml index 43547d0..9eff9e0 100644 --- a/node/storage/Cargo.toml +++ b/node/storage/Cargo.toml @@ -27,6 +27,7 @@ static_assertions = "1.1" tiny-keccak = "*" itertools = "0.13.0" serde = { version = "1.0.197", features = ["derive"] } +parking_lot = "0.12.3" [dev-dependencies] tempdir = "0.3.7" diff --git a/node/storage/benches/benchmark.rs b/node/storage/benches/benchmark.rs index 8aae9f7..5caa916 100644 --- a/node/storage/benches/benchmark.rs +++ b/node/storage/benches/benchmark.rs @@ -37,8 +37,8 @@ fn write_performance(c: &mut Criterion) { for _ in 0..5000 { let mut data = vec![0; data_size]; - for i in 0..data_size { - data[i] = random(); + for item in data.iter_mut().take(data_size) { + *item = random(); } let merkel_nodes = tx_subtree_root_list_padded(&data[..]); @@ -119,8 +119,8 @@ fn read_performance(c: &mut Criterion) { for seq in 0..tx_size { let mut data = vec![0; data_size]; - for i in 0..data_size { - data[i] = random(); + for item in data.iter_mut().take(data_size) { + *item = random(); } let merkel_nodes = tx_subtree_root_list_padded(&data[..]); diff --git a/node/storage/src/log_store/flow_store.rs b/node/storage/src/log_store/flow_store.rs index c4f63f0..64fbe20 100644 --- a/node/storage/src/log_store/flow_store.rs +++ b/node/storage/src/log_store/flow_store.rs @@ -10,6 +10,7 @@ use crate::{try_option, ZgsKeyValueDB}; use anyhow::{anyhow, bail, Result}; use append_merkle::{MerkleTreeInitialData, MerkleTreeRead}; use itertools::Itertools; +use parking_lot::RwLock; use shared_types::{ChunkArray, DataRoot, FlowProof}; use ssz::{Decode, Encode}; use ssz_derive::{Decode as DeriveDecode, Encode as DeriveEncode}; @@ -24,10 +25,10 @@ use zgs_spec::{BYTES_PER_SECTOR, SEALS_PER_LOAD, SECTORS_PER_LOAD, SECTORS_PER_S pub struct FlowStore { db: FlowDBStore, // TODO(kevin): This is an in-memory cache for recording which chunks are ready for sealing. It should be persisted on disk. - to_seal_set: BTreeMap, + to_seal_set: RwLock>, // Data sealing is an asynchronized process. // The sealing service uses the version number to distinguish if revert happens during sealing. - to_seal_version: usize, + to_seal_version: RwLock, config: FlowConfig, } @@ -36,7 +37,7 @@ impl FlowStore { Self { db: FlowDBStore::new(db), to_seal_set: Default::default(), - to_seal_version: 0, + to_seal_version: RwLock::new(0), config, } } @@ -86,7 +87,7 @@ impl FlowStore { #[derive(Clone, Debug)] pub struct FlowConfig { pub batch_size: usize, - pub shard_config: ShardConfig, + pub shard_config: Arc>, } impl Default for FlowConfig { @@ -207,14 +208,15 @@ impl FlowRead for FlowStore { } fn get_shard_config(&self) -> ShardConfig { - self.config.shard_config + *self.config.shard_config.read() } } impl FlowWrite for FlowStore { /// Return the roots of completed chunks. The order is guaranteed to be increasing /// by chunk index. - fn append_entries(&mut self, data: ChunkArray) -> Result> { + fn append_entries(&self, data: ChunkArray) -> Result> { + let mut to_seal_set = self.to_seal_set.write(); trace!("append_entries: {} {}", data.start_index, data.data.len()); if data.data.len() % BYTES_PER_SECTOR != 0 { bail!("append_entries: invalid data size, len={}", data.data.len()); @@ -231,7 +233,7 @@ impl FlowWrite for FlowStore { .expect("in range"); let chunk_index = chunk.start_index / self.config.batch_size as u64; - if !self.config.shard_config.in_range(chunk_index) { + if !self.config.shard_config.read().in_range(chunk_index) { // The data are in a shard range that we are not storing. continue; } @@ -246,9 +248,9 @@ impl FlowWrite for FlowStore { chunk.data, )?; completed_seals.into_iter().for_each(|x| { - self.to_seal_set.insert( + to_seal_set.insert( chunk_index as usize * SEALS_PER_LOAD + x as usize, - self.to_seal_version, + *self.to_seal_version.read(), ); }); @@ -257,27 +259,29 @@ impl FlowWrite for FlowStore { self.db.put_entry_batch_list(batch_list) } - fn truncate(&mut self, start_index: u64) -> crate::error::Result<()> { + fn truncate(&self, start_index: u64) -> crate::error::Result<()> { + let mut to_seal_set = self.to_seal_set.write(); + let mut to_seal_version = self.to_seal_version.write(); let to_reseal = self.db.truncate(start_index, self.config.batch_size)?; - self.to_seal_set - .split_off(&(start_index as usize / SECTORS_PER_SEAL)); - self.to_seal_version += 1; + to_seal_set.split_off(&(start_index as usize / SECTORS_PER_SEAL)); + *to_seal_version += 1; to_reseal.into_iter().for_each(|x| { - self.to_seal_set.insert(x, self.to_seal_version); + to_seal_set.insert(x, *to_seal_version); }); Ok(()) } - fn update_shard_config(&mut self, shard_config: ShardConfig) { - self.config.shard_config = shard_config; + fn update_shard_config(&self, shard_config: ShardConfig) { + *self.config.shard_config.write() = shard_config; } } impl FlowSeal for FlowStore { fn pull_seal_chunk(&self, seal_index_max: usize) -> Result>> { - let mut to_seal_iter = self.to_seal_set.iter(); + let to_seal_set = self.to_seal_set.read(); + let mut to_seal_iter = to_seal_set.iter(); let (&first_index, &first_version) = try_option!(to_seal_iter.next()); if first_index >= seal_index_max { return Ok(None); @@ -309,9 +313,10 @@ impl FlowSeal for FlowStore { Ok(Some(tasks)) } - fn submit_seal_result(&mut self, answers: Vec) -> Result<()> { + fn submit_seal_result(&self, answers: Vec) -> Result<()> { + let mut to_seal_set = self.to_seal_set.write(); let is_consistent = |answer: &SealAnswer| { - self.to_seal_set + to_seal_set .get(&(answer.seal_index as usize)) .map_or(false, |cur_ver| cur_ver == &answer.version) }; @@ -337,7 +342,7 @@ impl FlowSeal for FlowStore { debug!("Seal chunks: indices = {:?}", removed_seal_index); for idx in removed_seal_index.into_iter() { - self.to_seal_set.remove(&idx); + to_seal_set.remove(&idx); } self.db.put_entry_raw(updated_chunk)?; diff --git a/node/storage/src/log_store/log_manager.rs b/node/storage/src/log_store/log_manager.rs index 7d1b1c7..f780728 100644 --- a/node/storage/src/log_store/log_manager.rs +++ b/node/storage/src/log_store/log_manager.rs @@ -10,6 +10,7 @@ use ethereum_types::H256; use kvdb_rocksdb::{Database, DatabaseConfig}; use merkle_light::merkle::{log2_pow2, MerkleTree}; use merkle_tree::RawLeafSha3Algorithm; +use parking_lot::RwLock; use rayon::iter::ParallelIterator; use rayon::prelude::ParallelSlice; use shared_types::{ @@ -48,6 +49,10 @@ pub struct LogManager { pub(crate) db: Arc, tx_store: TransactionStore, flow_store: FlowStore, + merkle: RwLock, +} + +struct MerkleManager { // TODO(zz): Refactor the in-memory merkle and in-disk storage together. pora_chunks_merkle: Merkle, /// The in-memory structure of the sub merkle tree of the last chunk. @@ -55,6 +60,80 @@ pub struct LogManager { last_chunk_merkle: Merkle, } +impl MerkleManager { + fn last_chunk_start_index(&self) -> u64 { + if self.pora_chunks_merkle.leaves() == 0 { + 0 + } else { + PORA_CHUNK_SIZE as u64 + * if self.last_chunk_merkle.leaves() == 0 { + // The last chunk is empty and its root hash is not in `pora_chunk_merkle`, + // so all chunks in `pora_chunk_merkle` is complete. + self.pora_chunks_merkle.leaves() + } else { + // The last chunk has data, so we need to exclude it from `pora_chunks_merkle`. + self.pora_chunks_merkle.leaves() - 1 + } as u64 + } + } + + #[instrument(skip(self))] + fn commit_merkle(&mut self, tx_seq: u64) -> Result<()> { + self.pora_chunks_merkle.commit(Some(tx_seq)); + self.last_chunk_merkle.commit(Some(tx_seq)); + Ok(()) + } + + fn revert_merkle_tree(&mut self, tx_seq: u64, tx_store: &TransactionStore) -> Result<()> { + // Special case for reverting tx_seq == 0 + if tx_seq == u64::MAX { + self.pora_chunks_merkle.reset(); + self.last_chunk_merkle.reset(); + return Ok(()); + } + let old_leaves = self.pora_chunks_merkle.leaves(); + self.pora_chunks_merkle.revert_to(tx_seq)?; + if old_leaves == self.pora_chunks_merkle.leaves() { + self.last_chunk_merkle.revert_to(tx_seq)?; + } else { + // We are reverting to a position before the current last_chunk. + self.last_chunk_merkle = + tx_store.rebuild_last_chunk_merkle(self.pora_chunks_merkle.leaves() - 1, tx_seq)?; + } + Ok(()) + } + + fn try_initialize(&mut self, flow_store: &FlowStore) -> Result<()> { + if self.pora_chunks_merkle.leaves() == 0 && self.last_chunk_merkle.leaves() == 0 { + self.last_chunk_merkle.append(H256::zero()); + self.pora_chunks_merkle + .update_last(*self.last_chunk_merkle.root()); + } else if self.last_chunk_merkle.leaves() != 0 { + let last_chunk_start_index = self.last_chunk_start_index(); + let last_chunk_data = flow_store.get_available_entries( + last_chunk_start_index, + last_chunk_start_index + PORA_CHUNK_SIZE as u64, + )?; + for e in last_chunk_data { + let start_index = e.start_index - last_chunk_start_index; + for i in 0..e.data.len() / ENTRY_SIZE { + let index = i + start_index as usize; + if index >= self.last_chunk_merkle.leaves() { + // We revert the merkle tree before truncate the flow store, + // so last_chunk_data may include data that should have been truncated. + break; + } + self.last_chunk_merkle.fill_leaf( + index, + Sha3Algorithm::leaf(&e.data[i * ENTRY_SIZE..(i + 1) * ENTRY_SIZE]), + ); + } + } + } + Ok(()) + } +} + #[derive(Clone, Default)] pub struct LogConfig { pub flow: FlowConfig, @@ -71,7 +150,8 @@ impl LogStoreInner for LogManager { } impl LogStoreChunkWrite for LogManager { - fn put_chunks(&mut self, tx_seq: u64, chunks: ChunkArray) -> Result<()> { + fn put_chunks(&self, tx_seq: u64, chunks: ChunkArray) -> Result<()> { + let mut merkle = self.merkle.write(); let tx = self .tx_store .get_tx_by_seq_number(tx_seq)? @@ -90,17 +170,18 @@ impl LogStoreChunkWrite for LogManager { // TODO: Use another struct to avoid confusion. let mut flow_entry_array = chunks; flow_entry_array.start_index += tx.start_entry_index; - self.append_entries(flow_entry_array)?; + self.append_entries(flow_entry_array, &mut merkle)?; Ok(()) } fn put_chunks_with_tx_hash( - &mut self, + &self, tx_seq: u64, tx_hash: H256, chunks: ChunkArray, maybe_file_proof: Option, ) -> Result { + let mut merkle = self.merkle.write(); let tx = self .tx_store .get_tx_by_seq_number(tx_seq)? @@ -122,10 +203,10 @@ impl LogStoreChunkWrite for LogManager { // TODO: Use another struct to avoid confusion. let mut flow_entry_array = chunks; flow_entry_array.start_index += tx.start_entry_index; - self.append_entries(flow_entry_array)?; + self.append_entries(flow_entry_array, &mut merkle)?; if let Some(file_proof) = maybe_file_proof { - let updated_node_list = self.pora_chunks_merkle.fill_with_file_proof( + let updated_node_list = merkle.pora_chunks_merkle.fill_with_file_proof( file_proof, tx.merkle_nodes, tx.start_entry_index, @@ -161,7 +242,8 @@ impl LogStoreWrite for LogManager { /// Only the last tx may have this case, so we rerun /// `put_tx` for the last tx when we restart the node to ensure that it succeeds. /// - fn put_tx(&mut self, tx: Transaction) -> Result<()> { + fn put_tx(&self, tx: Transaction) -> Result<()> { + let mut merkle = self.merkle.write(); debug!("put_tx: tx={:?}", tx); let expected_seq = self.next_tx_seq(); if tx.seq != expected_seq { @@ -176,12 +258,14 @@ impl LogStoreWrite for LogManager { } let maybe_same_data_tx_seq = self.tx_store.put_tx(tx.clone())?.first().cloned(); // TODO(zz): Should we validate received tx? - self.append_subtree_list(tx.merkle_nodes.clone())?; - self.commit_merkle(tx.seq)?; + self.append_subtree_list(tx.merkle_nodes.clone(), &mut merkle)?; + merkle.commit_merkle(tx.seq)?; debug!( "commit flow root: root={:?}", - self.pora_chunks_merkle.root() + merkle.pora_chunks_merkle.root() ); + // Drop the lock because `copy_tx_data` will lock again. + drop(merkle); if let Some(old_tx_seq) = maybe_same_data_tx_seq { if self.check_tx_completed(old_tx_seq)? { @@ -192,7 +276,7 @@ impl LogStoreWrite for LogManager { Ok(()) } - fn finalize_tx(&mut self, tx_seq: u64) -> Result<()> { + fn finalize_tx(&self, tx_seq: u64) -> Result<()> { let tx = self .tx_store .get_tx_by_seq_number(tx_seq)? @@ -218,7 +302,7 @@ impl LogStoreWrite for LogManager { } } - fn finalize_tx_with_hash(&mut self, tx_seq: u64, tx_hash: H256) -> crate::error::Result { + fn finalize_tx_with_hash(&self, tx_seq: u64, tx_hash: H256) -> crate::error::Result { trace!( "finalize_tx_with_hash: tx_seq={} tx_hash={:?}", tx_seq, @@ -259,24 +343,34 @@ impl LogStoreWrite for LogManager { /// Return the reverted Transactions in order. /// `tx_seq == u64::MAX` is a special case for reverting all transactions. - fn revert_to(&mut self, tx_seq: u64) -> Result> { + fn revert_to(&self, tx_seq: u64) -> Result> { // FIXME(zz): If this revert is triggered by chain reorg after restarts, this will fail. - self.revert_merkle_tree(tx_seq)?; - let start_index = self.last_chunk_start_index() * PORA_CHUNK_SIZE as u64 - + self.last_chunk_merkle.leaves() as u64; + let mut merkle = self.merkle.write(); + merkle.revert_merkle_tree(tx_seq, &self.tx_store)?; + merkle.try_initialize(&self.flow_store)?; + assert_eq!( + Some(*merkle.last_chunk_merkle.root()), + merkle + .pora_chunks_merkle + .leaf_at(merkle.pora_chunks_merkle.leaves() - 1)? + ); + let start_index = merkle.last_chunk_start_index() * PORA_CHUNK_SIZE as u64 + + merkle.last_chunk_merkle.leaves() as u64; self.flow_store.truncate(start_index)?; let start = if tx_seq != u64::MAX { tx_seq + 1 } else { 0 }; self.tx_store.remove_tx_after(start) } fn validate_and_insert_range_proof( - &mut self, + &self, tx_seq: u64, data: &ChunkArrayWithProof, ) -> Result { let valid = self.validate_range_proof(tx_seq, data)?; + // `merkle` is used in `validate_range_proof`. + let mut merkle = self.merkle.write(); if valid { - let updated_nodes = self + let updated_nodes = merkle .pora_chunks_merkle .fill_with_range_proof(data.proof.clone())?; self.flow_store.put_mpt_node_list(updated_nodes)?; @@ -421,7 +515,11 @@ impl LogStoreRead for LogManager { &leaves, (data.chunks.start_index + tx.start_entry_index) as usize, )?; - Ok(self.pora_chunks_merkle.check_root(&data.proof.root())) + Ok(self + .merkle + .read_recursive() + .pora_chunks_merkle + .check_root(&data.proof.root())) } fn get_sync_progress(&self) -> Result> { @@ -442,12 +540,12 @@ impl LogStoreRead for LogManager { fn get_proof_at_root( &self, - root: &DataRoot, + root: DataRoot, index: u64, length: u64, ) -> crate::error::Result { - let left_proof = self.gen_proof(index, Some(*root))?; - let right_proof = self.gen_proof(index + length - 1, Some(*root))?; + let left_proof = self.gen_proof(index, Some(root))?; + let right_proof = self.gen_proof(index + length - 1, Some(root))?; Ok(FlowRangeProof { left_proof, right_proof, @@ -455,9 +553,10 @@ impl LogStoreRead for LogManager { } fn get_context(&self) -> crate::error::Result<(DataRoot, u64)> { + let merkle = self.merkle.read_recursive(); Ok(( - *self.pora_chunks_merkle.root(), - self.last_chunk_start_index() + self.last_chunk_merkle.leaves() as u64, + *merkle.pora_chunks_merkle.root(), + merkle.last_chunk_start_index() + merkle.last_chunk_merkle.leaves() as u64, )) } } @@ -578,19 +677,23 @@ impl LogManager { // update the merkle root pora_chunks_merkle.commit(start_tx_seq); } - let mut log_manager = Self { + let merkle = RwLock::new(MerkleManager { + pora_chunks_merkle, + last_chunk_merkle, + }); + let log_manager = Self { db, tx_store, flow_store, - pora_chunks_merkle, - last_chunk_merkle, + merkle, }; if let Some(tx) = last_tx_to_insert { log_manager.put_tx(tx)?; + let mut merkle = log_manager.merkle.write(); for (index, h) in extra_leaves { - if index < log_manager.pora_chunks_merkle.leaves() { - log_manager.pora_chunks_merkle.fill_leaf(index, h); + if index < merkle.pora_chunks_merkle.leaves() { + merkle.pora_chunks_merkle.fill_leaf(index, h); } else { error!("out of range extra leaf: index={} hash={:?}", index, h); } @@ -598,45 +701,19 @@ impl LogManager { } else { assert!(extra_leaves.is_empty()); } - log_manager.try_initialize()?; + log_manager + .merkle + .write() + .try_initialize(&log_manager.flow_store)?; Ok(log_manager) } - fn try_initialize(&mut self) -> Result<()> { - if self.pora_chunks_merkle.leaves() == 0 && self.last_chunk_merkle.leaves() == 0 { - self.last_chunk_merkle.append(H256::zero()); - self.pora_chunks_merkle - .update_last(*self.last_chunk_merkle.root()); - } else if self.last_chunk_merkle.leaves() != 0 { - let last_chunk_start_index = self.last_chunk_start_index(); - let last_chunk_data = self.flow_store.get_available_entries( - last_chunk_start_index, - last_chunk_start_index + PORA_CHUNK_SIZE as u64, - )?; - for e in last_chunk_data { - let start_index = e.start_index - last_chunk_start_index; - for i in 0..e.data.len() / ENTRY_SIZE { - let index = i + start_index as usize; - if index >= self.last_chunk_merkle.leaves() { - // We revert the merkle tree before truncate the flow store, - // so last_chunk_data may include data that should have been truncated. - break; - } - self.last_chunk_merkle.fill_leaf( - index, - Sha3Algorithm::leaf(&e.data[i * ENTRY_SIZE..(i + 1) * ENTRY_SIZE]), - ); - } - } - } - Ok(()) - } - fn gen_proof(&self, flow_index: u64, maybe_root: Option) -> Result { + let merkle = self.merkle.read_recursive(); let chunk_index = flow_index / PORA_CHUNK_SIZE as u64; let top_proof = match maybe_root { - None => self.pora_chunks_merkle.gen_proof(chunk_index as usize)?, - Some(root) => self + None => merkle.pora_chunks_merkle.gen_proof(chunk_index as usize)?, + Some(root) => merkle .pora_chunks_merkle .at_root_version(&root)? .gen_proof(chunk_index as usize)?, @@ -649,17 +726,17 @@ impl LogManager { // and `flow_index` must be within a complete PoRA chunk. For possible future usages, // we'll need to find the flow length at the given root and load a partial chunk // if `flow_index` is in the last chunk. - let sub_proof = if chunk_index as usize != self.pora_chunks_merkle.leaves() - 1 - || self.last_chunk_merkle.leaves() == 0 + let sub_proof = if chunk_index as usize != merkle.pora_chunks_merkle.leaves() - 1 + || merkle.last_chunk_merkle.leaves() == 0 { self.flow_store .gen_proof_in_batch(chunk_index as usize, flow_index as usize % PORA_CHUNK_SIZE)? } else { match maybe_root { - None => self + None => merkle .last_chunk_merkle .gen_proof(flow_index as usize % PORA_CHUNK_SIZE)?, - Some(root) => self + Some(root) => merkle .last_chunk_merkle .at_root_version(&root)? .gen_proof(flow_index as usize % PORA_CHUNK_SIZE)?, @@ -668,45 +745,56 @@ impl LogManager { entry_proof(&top_proof, &sub_proof) } - #[instrument(skip(self))] - fn append_subtree_list(&mut self, merkle_list: Vec<(usize, DataRoot)>) -> Result<()> { + #[instrument(skip(self, merkle))] + fn append_subtree_list( + &self, + merkle_list: Vec<(usize, DataRoot)>, + merkle: &mut MerkleManager, + ) -> Result<()> { if merkle_list.is_empty() { return Ok(()); } - self.pad_tx(1 << (merkle_list[0].0 - 1))?; + self.pad_tx(1 << (merkle_list[0].0 - 1), &mut *merkle)?; let mut batch_root_map = BTreeMap::new(); for (subtree_depth, subtree_root) in merkle_list { let subtree_size = 1 << (subtree_depth - 1); - if self.last_chunk_merkle.leaves() + subtree_size <= PORA_CHUNK_SIZE { - self.last_chunk_merkle + if merkle.last_chunk_merkle.leaves() + subtree_size <= PORA_CHUNK_SIZE { + merkle + .last_chunk_merkle .append_subtree(subtree_depth, subtree_root)?; - if self.last_chunk_merkle.leaves() == subtree_size { + if merkle.last_chunk_merkle.leaves() == subtree_size { // `last_chunk_merkle` was empty, so this is a new leaf in the top_tree. - self.pora_chunks_merkle - .append_subtree(1, *self.last_chunk_merkle.root())?; + merkle + .pora_chunks_merkle + .append_subtree(1, *merkle.last_chunk_merkle.root())?; } else { - self.pora_chunks_merkle - .update_last(*self.last_chunk_merkle.root()); + merkle + .pora_chunks_merkle + .update_last(*merkle.last_chunk_merkle.root()); } - if self.last_chunk_merkle.leaves() == PORA_CHUNK_SIZE { + if merkle.last_chunk_merkle.leaves() == PORA_CHUNK_SIZE { batch_root_map.insert( - self.pora_chunks_merkle.leaves() - 1, - (*self.last_chunk_merkle.root(), 1), + merkle.pora_chunks_merkle.leaves() - 1, + (*merkle.last_chunk_merkle.root(), 1), ); - self.complete_last_chunk_merkle(self.pora_chunks_merkle.leaves() - 1)?; + self.complete_last_chunk_merkle( + merkle.pora_chunks_merkle.leaves() - 1, + &mut *merkle, + )?; } } else { // `last_chunk_merkle` has been padded here, so a subtree should not be across // the chunks boundary. - assert_eq!(self.last_chunk_merkle.leaves(), 0); + assert_eq!(merkle.last_chunk_merkle.leaves(), 0); assert!(subtree_size >= PORA_CHUNK_SIZE); batch_root_map.insert( - self.pora_chunks_merkle.leaves(), + merkle.pora_chunks_merkle.leaves(), (subtree_root, subtree_depth - log2_pow2(PORA_CHUNK_SIZE)), ); - self.pora_chunks_merkle + merkle + .pora_chunks_merkle .append_subtree(subtree_depth - log2_pow2(PORA_CHUNK_SIZE), subtree_root)?; } } @@ -714,46 +802,50 @@ impl LogManager { Ok(()) } - #[instrument(skip(self))] - fn pad_tx(&mut self, first_subtree_size: u64) -> Result<()> { + #[instrument(skip(self, merkle))] + fn pad_tx(&self, first_subtree_size: u64, merkle: &mut MerkleManager) -> Result<()> { // Check if we need to pad the flow. let mut tx_start_flow_index = - self.last_chunk_start_index() + self.last_chunk_merkle.leaves() as u64; + merkle.last_chunk_start_index() + merkle.last_chunk_merkle.leaves() as u64; let extra = tx_start_flow_index % first_subtree_size; trace!( "before pad_tx {} {}", - self.pora_chunks_merkle.leaves(), - self.last_chunk_merkle.leaves() + merkle.pora_chunks_merkle.leaves(), + merkle.last_chunk_merkle.leaves() ); if extra != 0 { for pad_data in Self::padding((first_subtree_size - extra) as usize) { let mut root_map = BTreeMap::new(); // Update the in-memory merkle tree. - let last_chunk_pad = if self.last_chunk_merkle.leaves() == 0 { + let last_chunk_pad = if merkle.last_chunk_merkle.leaves() == 0 { 0 } else { - (PORA_CHUNK_SIZE - self.last_chunk_merkle.leaves()) * ENTRY_SIZE + (PORA_CHUNK_SIZE - merkle.last_chunk_merkle.leaves()) * ENTRY_SIZE }; let mut completed_chunk_index = None; if pad_data.len() < last_chunk_pad { - self.last_chunk_merkle + merkle + .last_chunk_merkle .append_list(data_to_merkle_leaves(&pad_data)?); - self.pora_chunks_merkle - .update_last(*self.last_chunk_merkle.root()); + merkle + .pora_chunks_merkle + .update_last(*merkle.last_chunk_merkle.root()); } else { if last_chunk_pad != 0 { // Pad the last chunk. - self.last_chunk_merkle + merkle + .last_chunk_merkle .append_list(data_to_merkle_leaves(&pad_data[..last_chunk_pad])?); - self.pora_chunks_merkle - .update_last(*self.last_chunk_merkle.root()); + merkle + .pora_chunks_merkle + .update_last(*merkle.last_chunk_merkle.root()); root_map.insert( - self.pora_chunks_merkle.leaves() - 1, - (*self.last_chunk_merkle.root(), 1), + merkle.pora_chunks_merkle.leaves() - 1, + (*merkle.last_chunk_merkle.root(), 1), ); - completed_chunk_index = Some(self.pora_chunks_merkle.leaves() - 1); + completed_chunk_index = Some(merkle.pora_chunks_merkle.leaves() - 1); } // Pad with more complete chunks. @@ -763,8 +855,8 @@ impl LogManager { ..(start_index + PORA_CHUNK_SIZE) * ENTRY_SIZE] .to_vec(); let root = *Merkle::new(data_to_merkle_leaves(&data)?, 0, None).root(); - self.pora_chunks_merkle.append(root); - root_map.insert(self.pora_chunks_merkle.leaves() - 1, (root, 1)); + merkle.pora_chunks_merkle.append(root); + root_map.insert(merkle.pora_chunks_merkle.leaves() - 1, (root, 1)); start_index += PORA_CHUNK_SIZE; } assert_eq!(pad_data.len(), start_index * ENTRY_SIZE); @@ -782,20 +874,24 @@ impl LogManager { })?; tx_start_flow_index += data_size as u64; if let Some(index) = completed_chunk_index { - self.complete_last_chunk_merkle(index)?; + self.complete_last_chunk_merkle(index, &mut *merkle)?; } } } trace!( "after pad_tx {} {}", - self.pora_chunks_merkle.leaves(), - self.last_chunk_merkle.leaves() + merkle.pora_chunks_merkle.leaves(), + merkle.last_chunk_merkle.leaves() ); Ok(()) } - fn append_entries(&mut self, flow_entry_array: ChunkArray) -> Result<()> { - let last_chunk_start_index = self.last_chunk_start_index(); + fn append_entries( + &self, + flow_entry_array: ChunkArray, + merkle: &mut MerkleManager, + ) -> Result<()> { + let last_chunk_start_index = merkle.last_chunk_start_index(); if flow_entry_array.start_index + bytes_to_chunks(flow_entry_array.data.len()) as u64 > last_chunk_start_index { @@ -822,14 +918,16 @@ impl LogManager { .chunks_exact(ENTRY_SIZE) .enumerate() { - self.last_chunk_merkle + merkle + .last_chunk_merkle .fill_leaf(chunk_start_index + local_index, Sha3Algorithm::leaf(entry)); } } let chunk_roots = self.flow_store.append_entries(flow_entry_array)?; for (chunk_index, chunk_root) in chunk_roots { - if chunk_index < self.pora_chunks_merkle.leaves() as u64 { - self.pora_chunks_merkle + if chunk_index < merkle.pora_chunks_merkle.leaves() as u64 { + merkle + .pora_chunks_merkle .fill_leaf(chunk_index as usize, chunk_root); } else { // TODO(zz): This assumption may be false in the future. @@ -857,62 +955,12 @@ impl LogManager { vec![0; len * ENTRY_SIZE] } - fn last_chunk_start_index(&self) -> u64 { - if self.pora_chunks_merkle.leaves() == 0 { - 0 - } else { - PORA_CHUNK_SIZE as u64 - * if self.last_chunk_merkle.leaves() == 0 { - // The last chunk is empty and its root hash is not in `pora_chunk_merkle`, - // so all chunks in `pora_chunk_merkle` is complete. - self.pora_chunks_merkle.leaves() - } else { - // The last chunk has data, so we need to exclude it from `pora_chunks_merkle`. - self.pora_chunks_merkle.leaves() - 1 - } as u64 - } - } - - #[instrument(skip(self))] - fn commit_merkle(&mut self, tx_seq: u64) -> Result<()> { - self.pora_chunks_merkle.commit(Some(tx_seq)); - self.last_chunk_merkle.commit(Some(tx_seq)); - Ok(()) - } - - fn revert_merkle_tree(&mut self, tx_seq: u64) -> Result<()> { - // Special case for reverting tx_seq == 0 - if tx_seq == u64::MAX { - self.pora_chunks_merkle.reset(); - self.last_chunk_merkle.reset(); - self.try_initialize()?; - return Ok(()); - } - let old_leaves = self.pora_chunks_merkle.leaves(); - self.pora_chunks_merkle.revert_to(tx_seq)?; - if old_leaves == self.pora_chunks_merkle.leaves() { - self.last_chunk_merkle.revert_to(tx_seq)?; - } else { - // We are reverting to a position before the current last_chunk. - self.last_chunk_merkle = self - .tx_store - .rebuild_last_chunk_merkle(self.pora_chunks_merkle.leaves() - 1, tx_seq)?; - self.try_initialize()?; - assert_eq!( - Some(*self.last_chunk_merkle.root()), - self.pora_chunks_merkle - .leaf_at(self.pora_chunks_merkle.leaves() - 1)? - ); - } - Ok(()) - } - #[cfg(test)] pub fn flow_store(&self) -> &FlowStore { &self.flow_store } - fn padding_rear_data(&mut self, tx: &Transaction) -> Result<()> { + fn padding_rear_data(&self, tx: &Transaction) -> Result<()> { let (chunks, _) = compute_padded_chunk_size(tx.size as usize); let (segments_for_proof, last_segment_size_for_proof) = compute_segment_size(chunks, PORA_CHUNK_SIZE); @@ -959,7 +1007,8 @@ impl LogManager { Ok(()) } - fn copy_tx_data(&mut self, from_tx_seq: u64, to_tx_seq_list: Vec) -> Result<()> { + fn copy_tx_data(&self, from_tx_seq: u64, to_tx_seq_list: Vec) -> Result<()> { + let mut merkle = self.merkle.write(); // We have all the data need for this tx, so just copy them. let old_tx = self .get_tx_by_seq_number(from_tx_seq)? @@ -992,7 +1041,7 @@ impl LogManager { for (_, offset) in &to_tx_offset_list { let mut data = batch_data.clone(); data.start_index += offset; - self.append_entries(data)?; + self.append_entries(data, &mut merkle)?; } } // num_entries() includes the rear padding data, so no need for more padding. @@ -1007,9 +1056,9 @@ impl LogManager { /// we can still provide proof for known data in it. /// Another choice is to insert these subtrees earlier in `put_tx`. To insert them here can /// batch them and avoid inserting for the subtrees with all data known. - fn complete_last_chunk_merkle(&mut self, index: usize) -> Result<()> { - let subtree_list = self.last_chunk_merkle.get_subtrees(); - self.last_chunk_merkle = + fn complete_last_chunk_merkle(&self, index: usize, merkle: &mut MerkleManager) -> Result<()> { + let subtree_list = merkle.last_chunk_merkle.get_subtrees(); + merkle.last_chunk_merkle = Merkle::new_with_depth(vec![], log2_pow2(PORA_CHUNK_SIZE) + 1, None); // Only insert non-leave subtrees. The leave data should have been available. diff --git a/node/storage/src/log_store/mod.rs b/node/storage/src/log_store/mod.rs index 0c5022d..04ef1a6 100644 --- a/node/storage/src/log_store/mod.rs +++ b/node/storage/src/log_store/mod.rs @@ -62,8 +62,7 @@ pub trait LogStoreRead: LogStoreChunkRead { fn validate_range_proof(&self, tx_seq: u64, data: &ChunkArrayWithProof) -> Result; - fn get_proof_at_root(&self, root: &DataRoot, index: u64, length: u64) - -> Result; + fn get_proof_at_root(&self, root: DataRoot, index: u64, length: u64) -> Result; /// Return flow root and length. fn get_context(&self) -> Result<(DataRoot, u64)>; @@ -103,7 +102,7 @@ pub trait LogStoreChunkRead { pub trait LogStoreWrite: LogStoreChunkWrite { /// Store a data entry metadata. - fn put_tx(&mut self, tx: Transaction) -> Result<()>; + fn put_tx(&self, tx: Transaction) -> Result<()>; /// Finalize a transaction storage. /// This will compute and the merkle tree, check the data root, and persist a part of the merkle @@ -111,8 +110,8 @@ pub trait LogStoreWrite: LogStoreChunkWrite { /// /// This will return error if not all chunks are stored. But since this check can be expensive, /// the caller is supposed to track chunk statuses and call this after storing all the chunks. - fn finalize_tx(&mut self, tx_seq: u64) -> Result<()>; - fn finalize_tx_with_hash(&mut self, tx_seq: u64, tx_hash: H256) -> Result; + fn finalize_tx(&self, tx_seq: u64) -> Result<()>; + fn finalize_tx_with_hash(&self, tx_seq: u64, tx_hash: H256) -> Result; /// Store the progress of synced block number and its hash. fn put_sync_progress(&self, progress: (u64, H256, Option>)) -> Result<()>; @@ -121,11 +120,11 @@ pub trait LogStoreWrite: LogStoreChunkWrite { /// This is needed when transactions are reverted because of chain reorg. /// /// Reverted transactions are returned in order. - fn revert_to(&mut self, tx_seq: u64) -> Result>; + fn revert_to(&self, tx_seq: u64) -> Result>; /// If the proof is valid, fill the tree nodes with the new data. fn validate_and_insert_range_proof( - &mut self, + &self, tx_seq: u64, data: &ChunkArrayWithProof, ) -> Result; @@ -135,10 +134,10 @@ pub trait LogStoreWrite: LogStoreChunkWrite { pub trait LogStoreChunkWrite { /// Store data chunks of a data entry. - fn put_chunks(&mut self, tx_seq: u64, chunks: ChunkArray) -> Result<()>; + fn put_chunks(&self, tx_seq: u64, chunks: ChunkArray) -> Result<()>; fn put_chunks_with_tx_hash( - &mut self, + &self, tx_seq: u64, tx_hash: H256, chunks: ChunkArray, @@ -169,14 +168,15 @@ pub trait LogStoreInner { } pub struct MineLoadChunk { - pub loaded_chunk: [[u8; BYTES_PER_SEAL]; SEALS_PER_LOAD], + // Use `Vec` instead of array to avoid thread stack overflow. + pub loaded_chunk: Vec<[u8; BYTES_PER_SEAL]>, pub avalibilities: [bool; SEALS_PER_LOAD], } impl Default for MineLoadChunk { fn default() -> Self { Self { - loaded_chunk: [[0u8; BYTES_PER_SEAL]; SEALS_PER_LOAD], + loaded_chunk: vec![[0u8; BYTES_PER_SEAL]; SEALS_PER_LOAD], avalibilities: [false; SEALS_PER_LOAD], } } @@ -206,14 +206,14 @@ pub trait FlowWrite { /// Append data to the flow. `start_index` is included in `ChunkArray`, so /// it's possible to append arrays in any place. /// Return the list of completed chunks. - fn append_entries(&mut self, data: ChunkArray) -> Result>; + fn append_entries(&self, data: ChunkArray) -> Result>; /// Remove all the entries after `start_index`. /// This is used to remove deprecated data in case of chain reorg. - fn truncate(&mut self, start_index: u64) -> Result<()>; + fn truncate(&self, start_index: u64) -> Result<()>; /// Update the shard config. - fn update_shard_config(&mut self, shard_config: ShardConfig); + fn update_shard_config(&self, shard_config: ShardConfig); } pub struct SealTask { @@ -247,7 +247,7 @@ pub trait FlowSeal { /// Submit sealing result - fn submit_seal_result(&mut self, answers: Vec) -> Result<()>; + fn submit_seal_result(&self, answers: Vec) -> Result<()>; } pub trait Flow: FlowRead + FlowWrite + FlowSeal {} diff --git a/node/storage/src/log_store/tests.rs b/node/storage/src/log_store/tests.rs index 6928540..741b172 100644 --- a/node/storage/src/log_store/tests.rs +++ b/node/storage/src/log_store/tests.rs @@ -12,7 +12,7 @@ use std::cmp; #[test] fn test_put_get() { let config = LogConfig::default(); - let mut store = LogManager::memorydb(config.clone()).unwrap(); + let store = LogManager::memorydb(config.clone()).unwrap(); let chunk_count = config.flow.batch_size + config.flow.batch_size / 2 - 1; // Aligned with size. let start_offset = 1024; diff --git a/node/sync/src/auto_sync/sync_store.rs b/node/sync/src/auto_sync/sync_store.rs index 7fa8bb5..e483d8c 100644 --- a/node/sync/src/auto_sync/sync_store.rs +++ b/node/sync/src/auto_sync/sync_store.rs @@ -1,6 +1,6 @@ use super::tx_store::TxStore; use anyhow::Result; -use std::ops::Deref; + use storage::log_store::config::{ConfigTx, ConfigurableExt}; use storage_async::Store; @@ -29,7 +29,7 @@ impl SyncStore { } pub async fn get_tx_seq_range(&self) -> Result<(Option, Option)> { - let store = self.store.get_store().read().await; + let store = self.store.get_store(); // load next_tx_seq let next_tx_seq = store.get_config_decoded(&KEY_NEXT_TX_SEQ)?; @@ -43,8 +43,6 @@ impl SyncStore { pub async fn set_next_tx_seq(&self, tx_seq: u64) -> Result<()> { self.store .get_store() - .write() - .await .set_config_encoded(&KEY_NEXT_TX_SEQ, &tx_seq) } @@ -52,38 +50,33 @@ impl SyncStore { debug!(%tx_seq, "set_max_tx_seq"); self.store .get_store() - .write() - .await .set_config_encoded(&KEY_MAX_TX_SEQ, &tx_seq) } pub async fn add_pending_tx(&self, tx_seq: u64) -> Result { - let store = self.store.get_store().write().await; + let store = self.store.get_store(); // already in ready queue - if self.ready_txs.has(store.deref(), tx_seq)? { + if self.ready_txs.has(store, tx_seq)? { return Ok(false); } // always add in pending queue - self.pending_txs.add(store.deref(), None, tx_seq) + self.pending_txs.add(store, None, tx_seq) } pub async fn upgrade_tx_to_ready(&self, tx_seq: u64) -> Result { - let store = self.store.get_store().write().await; + let store = self.store.get_store(); let mut tx = ConfigTx::default(); // not in pending queue - if !self - .pending_txs - .remove(store.deref(), Some(&mut tx), tx_seq)? - { + if !self.pending_txs.remove(store, Some(&mut tx), tx_seq)? { return Ok(false); } // move from pending to ready queue - let added = self.ready_txs.add(store.deref(), Some(&mut tx), tx_seq)?; + let added = self.ready_txs.add(store, Some(&mut tx), tx_seq)?; store.exec_configs(tx)?; @@ -91,20 +84,17 @@ impl SyncStore { } pub async fn downgrade_tx_to_pending(&self, tx_seq: u64) -> Result { - let store = self.store.get_store().write().await; + let store = self.store.get_store(); let mut tx = ConfigTx::default(); // not in ready queue - if !self - .ready_txs - .remove(store.deref(), Some(&mut tx), tx_seq)? - { + if !self.ready_txs.remove(store, Some(&mut tx), tx_seq)? { return Ok(false); } // move from ready to pending queue - let added = self.pending_txs.add(store.deref(), Some(&mut tx), tx_seq)?; + let added = self.pending_txs.add(store, Some(&mut tx), tx_seq)?; store.exec_configs(tx)?; @@ -112,27 +102,27 @@ impl SyncStore { } pub async fn random_tx(&self) -> Result> { - let store = self.store.get_store().read().await; + let store = self.store.get_store(); // try to find a tx in ready queue with high priority - if let Some(val) = self.ready_txs.random(store.deref())? { + if let Some(val) = self.ready_txs.random(store)? { return Ok(Some(val)); } // otherwise, find tx in pending queue - self.pending_txs.random(store.deref()) + self.pending_txs.random(store) } pub async fn remove_tx(&self, tx_seq: u64) -> Result { - let store = self.store.get_store().write().await; + let store = self.store.get_store(); // removed in ready queue - if self.ready_txs.remove(store.deref(), None, tx_seq)? { + if self.ready_txs.remove(store, None, tx_seq)? { return Ok(true); } // otherwise, try to remove in pending queue - self.pending_txs.remove(store.deref(), None, tx_seq) + self.pending_txs.remove(store, None, tx_seq) } } diff --git a/node/sync/src/controllers/serial.rs b/node/sync/src/controllers/serial.rs index 474acbc..1fe2d8d 100644 --- a/node/sync/src/controllers/serial.rs +++ b/node/sync/src/controllers/serial.rs @@ -420,8 +420,6 @@ impl SerialSyncController { let validation_result = self .store .get_store() - .write() - .await .validate_and_insert_range_proof(self.tx_seq, &response); match validation_result { @@ -443,13 +441,7 @@ impl SerialSyncController { self.failures = 0; - let shard_config = self - .store - .get_store() - .read() - .await - .flow() - .get_shard_config(); + let shard_config = self.store.get_store().flow().get_shard_config(); let next_chunk = shard_config.next_segment_index( (from_chunk / PORA_CHUNK_SIZE as u64) as usize, (self.tx_start_chunk_in_flow / PORA_CHUNK_SIZE as u64) as usize, @@ -664,7 +656,6 @@ mod tests { use storage::H256; use task_executor::{test_utils::TestRuntime, TaskExecutor}; use tokio::sync::mpsc::{self, UnboundedReceiver}; - use tokio::sync::RwLock; #[test] fn test_status() { @@ -1112,8 +1103,6 @@ mod tests { ); let chunks = peer_store - .read() - .await .get_chunks_with_proof_by_tx_and_index_range(tx_seq, 0, chunk_count) .unwrap() .unwrap(); @@ -1146,8 +1135,6 @@ mod tests { ); let mut chunks = peer_store - .read() - .await .get_chunks_with_proof_by_tx_and_index_range(tx_seq, 0, chunk_count) .unwrap() .unwrap(); @@ -1215,8 +1202,6 @@ mod tests { ); let chunks = peer_store - .read() - .await .get_chunks_with_proof_by_tx_and_index_range(tx_seq, 0, chunk_count) .unwrap() .unwrap(); @@ -1238,7 +1223,6 @@ mod tests { source, msg, } => { - assert_eq!(peer_id, peer_id); match action { PeerAction::Fatal => {} _ => { @@ -1281,8 +1265,6 @@ mod tests { ); let chunks = peer_store - .read() - .await .get_chunks_with_proof_by_tx_and_index_range(tx_seq, 0, chunk_count) .unwrap() .unwrap(); @@ -1355,8 +1337,6 @@ mod tests { ); let chunks = peer_store - .read() - .await .get_chunks_with_proof_by_tx_and_index_range(tx_seq, 0, chunk_count) .unwrap() .unwrap(); @@ -1400,8 +1380,6 @@ mod tests { ); let chunks = peer_store - .read() - .await .get_chunks_with_proof_by_tx_and_index_range(tx_seq, 0, 1024) .unwrap() .unwrap(); @@ -1448,8 +1426,6 @@ mod tests { ); let chunks = peer_store - .read() - .await .get_chunks_with_proof_by_tx_and_index_range(tx_seq, 0, chunk_count) .unwrap() .unwrap(); @@ -1572,7 +1548,7 @@ mod tests { let num_chunks = 123; let config = LogConfig::default(); - let store = Arc::new(RwLock::new(LogManager::memorydb(config).unwrap())); + let store = Arc::new(LogManager::memorydb(config).unwrap()); create_controller(task_executor, peer_id, store, tx_id, num_chunks) } @@ -1580,7 +1556,7 @@ mod tests { fn create_controller( task_executor: TaskExecutor, peer_id: Option, - store: Arc>, + store: Arc, tx_id: TxID, num_chunks: usize, ) -> (SerialSyncController, UnboundedReceiver) { diff --git a/node/sync/src/service.rs b/node/sync/src/service.rs index dfc4b99..7558e85 100644 --- a/node/sync/src/service.rs +++ b/node/sync/src/service.rs @@ -23,7 +23,7 @@ use storage::config::ShardConfig; use storage::error::Result as StorageResult; use storage::log_store::Store as LogStore; use storage_async::Store; -use tokio::sync::{broadcast, mpsc, RwLock}; +use tokio::sync::{broadcast, mpsc}; const HEARTBEAT_INTERVAL_SEC: u64 = 5; @@ -129,7 +129,7 @@ impl SyncService { pub async fn spawn( executor: task_executor::TaskExecutor, network_send: mpsc::UnboundedSender, - store: Arc>, + store: Arc, file_location_cache: Arc, event_recv: broadcast::Receiver, ) -> Result { @@ -148,7 +148,7 @@ impl SyncService { config: Config, executor: task_executor::TaskExecutor, network_send: mpsc::UnboundedSender, - store: Arc>, + store: Arc, file_location_cache: Arc, event_recv: broadcast::Receiver, ) -> Result { @@ -740,8 +740,8 @@ mod tests { runtime: TestRuntime, chunk_count: usize, - store: Arc>, - peer_store: Arc>, + store: Arc, + peer_store: Arc, txs: Vec, init_data: Vec, @@ -930,8 +930,6 @@ mod tests { runtime .peer_store - .read() - .await .validate_range_proof(0, &response) .expect("validate proof"); } @@ -1177,7 +1175,7 @@ mod tests { let config = LogConfig::default(); - let store = Arc::new(RwLock::new(LogManager::memorydb(config.clone()).unwrap())); + let store = Arc::new(LogManager::memorydb(config.clone()).unwrap()); let init_peer_id = identity::Keypair::generate_ed25519().public().to_peer_id(); let file_location_cache: Arc = @@ -1203,10 +1201,7 @@ mod tests { .unwrap(); thread::sleep(Duration::from_millis(1000)); - assert_eq!( - store.read().await.get_tx_by_seq_number(tx_seq).unwrap(), - None - ); + assert_eq!(store.get_tx_by_seq_number(tx_seq).unwrap(), None); assert!(network_recv.try_recv().is_err()); } @@ -1222,18 +1217,13 @@ mod tests { .unwrap(); thread::sleep(Duration::from_millis(1000)); - assert!(runtime - .peer_store - .read() - .await - .check_tx_completed(tx_seq) - .unwrap()); + assert!(runtime.peer_store.check_tx_completed(tx_seq).unwrap()); assert!(runtime.network_recv.try_recv().is_err()); } - async fn wait_for_tx_finalized(store: Arc>, tx_seq: u64) { + async fn wait_for_tx_finalized(store: Arc, tx_seq: u64) { let deadline = Instant::now() + Duration::from_millis(5000); - while !store.read().await.check_tx_completed(tx_seq).unwrap() { + while !store.check_tx_completed(tx_seq).unwrap() { if Instant::now() >= deadline { panic!("Failed to wait tx completed"); } @@ -1255,12 +1245,7 @@ mod tests { receive_dial(&mut runtime, &sync_send).await; - assert!(!runtime - .store - .read() - .await - .check_tx_completed(tx_seq) - .unwrap()); + assert!(!runtime.store.check_tx_completed(tx_seq).unwrap()); assert!(!matches!( sync_send @@ -1330,12 +1315,7 @@ mod tests { receive_dial(&mut runtime, &sync_send).await; - assert!(!runtime - .store - .read() - .await - .check_tx_completed(tx_seq) - .unwrap()); + assert!(!runtime.store.check_tx_completed(tx_seq).unwrap()); receive_chunk_request( &mut runtime.network_recv, @@ -1385,13 +1365,8 @@ mod tests { receive_dial(&mut runtime, &sync_send).await; - assert!(!runtime - .store - .read() - .await - .check_tx_completed(tx_seq) - .unwrap()); - assert!(!runtime.store.read().await.check_tx_completed(0).unwrap()); + assert!(!runtime.store.check_tx_completed(tx_seq).unwrap()); + assert!(!runtime.store.check_tx_completed(0).unwrap()); receive_chunk_request( &mut runtime.network_recv, @@ -1406,7 +1381,7 @@ mod tests { wait_for_tx_finalized(runtime.store.clone(), tx_seq).await; - assert!(!runtime.store.read().await.check_tx_completed(0).unwrap()); + assert!(!runtime.store.check_tx_completed(0).unwrap()); // first file let tx_seq = 0u64; @@ -1461,8 +1436,10 @@ mod tests { #[tokio::test] async fn test_announce_file() { let mut runtime = TestSyncRuntime::new(vec![1023], 0); - let mut config = Config::default(); - config.sync_file_on_announcement_enabled = true; + let config = Config { + sync_file_on_announcement_enabled: true, + ..Default::default() + }; let sync_send = runtime.spawn_sync_service_with_config(false, config).await; let tx_seq = 0u64; @@ -1477,12 +1454,7 @@ mod tests { receive_dial(&mut runtime, &sync_send).await; - assert!(!runtime - .store - .read() - .await - .check_tx_completed(tx_seq) - .unwrap()); + assert!(!runtime.store.check_tx_completed(tx_seq).unwrap()); receive_chunk_request( &mut runtime.network_recv, @@ -1520,12 +1492,7 @@ mod tests { receive_dial(&mut runtime, &sync_send).await; - assert!(!runtime - .store - .read() - .await - .check_tx_completed(tx_seq) - .unwrap()); + assert!(!runtime.store.check_tx_completed(tx_seq).unwrap()); receive_chunk_request( &mut runtime.network_recv, @@ -1606,12 +1573,7 @@ mod tests { receive_dial(&mut runtime, &sync_send).await; - assert!(!runtime - .store - .read() - .await - .check_tx_completed(tx_seq) - .unwrap()); + assert!(!runtime.store.check_tx_completed(tx_seq).unwrap()); assert!(!matches!( sync_send @@ -1637,7 +1599,7 @@ mod tests { async fn receive_chunk_request( network_recv: &mut UnboundedReceiver, sync_send: &SyncSender, - peer_store: Arc>, + peer_store: Arc, init_peer_id: PeerId, tx_seq: u64, index_start: u64, @@ -1671,8 +1633,6 @@ mod tests { }; let chunks = peer_store - .read() - .await .get_chunks_with_proof_by_tx_and_index_range( tx_seq, req.index_start as usize, diff --git a/node/sync/src/test_util.rs b/node/sync/src/test_util.rs index 4fd04f5..d3a0981 100644 --- a/node/sync/src/test_util.rs +++ b/node/sync/src/test_util.rs @@ -8,7 +8,6 @@ use storage::{ }, LogManager, }; -use tokio::sync::RwLock; /// Creates stores for local node and peers with initialized transaction of specified chunk count. /// The first store is for local node, and data not stored. The second store is for peers, and all @@ -17,8 +16,8 @@ use tokio::sync::RwLock; pub fn create_2_store( chunk_count: Vec, ) -> ( - Arc>, - Arc>, + Arc, + Arc, Vec, Vec>, ) { @@ -37,12 +36,7 @@ pub fn create_2_store( offset = ret.2; } - ( - Arc::new(RwLock::new(store)), - Arc::new(RwLock::new(peer_store)), - txs, - data, - ) + (Arc::new(store), Arc::new(peer_store), txs, data) } fn generate_data( @@ -106,7 +100,6 @@ pub mod tests { }; use storage_async::Store; use task_executor::test_utils::TestRuntime; - use tokio::sync::RwLock; pub struct TestStoreRuntime { pub runtime: TestRuntime, @@ -115,7 +108,7 @@ pub mod tests { impl Default for TestStoreRuntime { fn default() -> Self { - let store = Arc::new(RwLock::new(Self::new_store())); + let store = Arc::new(Self::new_store()); Self::new(store) } } @@ -125,7 +118,7 @@ pub mod tests { LogManager::memorydb(LogConfig::default()).unwrap() } - pub fn new(store: Arc>) -> TestStoreRuntime { + pub fn new(store: Arc) -> TestStoreRuntime { let runtime = TestRuntime::default(); let executor = runtime.task_executor.clone(); Self { diff --git a/tests/same_root_test.py b/tests/same_root_test.py index f7c293a..7646559 100755 --- a/tests/same_root_test.py +++ b/tests/same_root_test.py @@ -40,8 +40,8 @@ class SubmissionTest(TestFramework): # Check if all transactions are finalized for tx_offset in range(same_root_tx_count + 1): tx_seq = next_tx_seq - 1 - tx_offset - status = self.nodes[0].zgs_get_file_info_by_tx_seq(tx_seq) - assert status["finalized"] + # old txs are finalized after finalizing the new tx, so we may need to wait here. + wait_until(lambda: self.nodes[0].zgs_get_file_info_by_tx_seq(tx_seq)["finalized"]) # Send tx after uploading data for _ in range(same_root_tx_count): @@ -57,7 +57,7 @@ class SubmissionTest(TestFramework): client = self.nodes[node_idx] wait_until(lambda: client.zgs_get_file_info_by_tx_seq(tx_seq) is not None) - assert_equal(client.zgs_get_file_info_by_tx_seq(tx_seq)["finalized"], data_finalized) + wait_until(lambda: client.zgs_get_file_info_by_tx_seq(tx_seq)["finalized"] == data_finalized) def submit_data(self, chunk_data, node_idx=0): _, data_root = create_submission(chunk_data) diff --git a/version-meld/discv5/src/socket/filter/cache.rs b/version-meld/discv5/src/socket/filter/cache.rs index 6f996f8..3c9270b 100644 --- a/version-meld/discv5/src/socket/filter/cache.rs +++ b/version-meld/discv5/src/socket/filter/cache.rs @@ -23,6 +23,7 @@ use std::{ pub const ENFORCED_SIZE_TIME: u64 = 1; pub struct ReceivedPacket { + #[allow(unused)] /// The source that sent us the packet. pub content: T, /// The time the packet was received.