mirror of
https://github.com/0glabs/0g-storage-node.git
synced 2024-11-20 15:05:19 +00:00
8b225fde43
* Fix stdouterr issue when shutdown storage node * Fix random failure for submission test * add more info for failed rpc * use single file instead of subprocess pipe to launch CLI * Opt test_all for windows * wait for log entry before file sync by admin
158 lines
6.0 KiB
Python
158 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
|
|
|
from test_framework.test_framework import TestFramework
|
|
from utility.submission import create_submission, submit_data
|
|
from utility.utils import wait_until
|
|
|
|
|
|
class CrashTest(TestFramework):
|
|
def setup_params(self):
|
|
self.num_blockchain_nodes = 1
|
|
self.num_nodes = 4
|
|
|
|
def run_test(self):
|
|
# 1. all nodes are running
|
|
chunk_data = b"\x01" * 256
|
|
submissions, data_root = create_submission(chunk_data)
|
|
self.contract.submit(submissions)
|
|
wait_until(lambda: self.contract.num_submissions() == 1)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
|
|
|
|
segment = submit_data(self.nodes[0], chunk_data)
|
|
self.log.info("segment: %s", segment)
|
|
|
|
for i in range(self.num_nodes):
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
|
|
)
|
|
self.nodes[i].admin_start_sync_file(0)
|
|
self.log.info("wait for node: %s", i)
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
|
|
)
|
|
|
|
# 2: first node runnging, other nodes killed
|
|
self.log.info("kill node")
|
|
# kill node to simulate node crash
|
|
for i in range(1, self.num_nodes):
|
|
self.nodes[i].stop(kill=True)
|
|
|
|
chunk_data = b"\x02" * 256 * 1024
|
|
submissions, data_root = create_submission(chunk_data)
|
|
self.contract.submit(submissions)
|
|
wait_until(lambda: self.contract.num_submissions() == 2)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
|
|
|
|
segment = submit_data(self.nodes[0], chunk_data)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
|
|
|
|
self.start_storage_node(1)
|
|
self.nodes[1].wait_for_rpc_connection()
|
|
wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root) is not None)
|
|
self.nodes[1].admin_start_sync_file(1)
|
|
wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root)["finalized"])
|
|
|
|
for i in range(2, self.num_nodes):
|
|
self.start_storage_node(i)
|
|
self.nodes[i].wait_for_rpc_connection()
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
|
|
)
|
|
self.nodes[i].admin_start_sync_file(1)
|
|
|
|
self.nodes[i].stop(kill=True)
|
|
self.start_storage_node(i)
|
|
self.nodes[i].wait_for_rpc_connection()
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
|
|
)
|
|
self.nodes[i].admin_start_sync_file(1)
|
|
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
|
|
)
|
|
|
|
# 4: node[1..] synced contract entries and killed
|
|
self.log.info("kill node 0")
|
|
self.nodes[0].stop(kill=True)
|
|
self.start_storage_node(0)
|
|
self.nodes[0].wait_for_rpc_connection()
|
|
|
|
chunk_data = b"\x03" * 256
|
|
submissions, data_root = create_submission(chunk_data)
|
|
self.contract.submit(submissions)
|
|
wait_until(lambda: self.contract.num_submissions() == 3)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
|
|
|
|
for i in range(1, self.num_nodes):
|
|
self.nodes[i].stop(kill=True)
|
|
|
|
segment = submit_data(self.nodes[0], chunk_data)
|
|
self.log.info("segment: %s", segment)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
|
|
|
|
for i in range(1, self.num_nodes):
|
|
self.log.info("wait for node: %s", i)
|
|
self.start_storage_node(i)
|
|
self.nodes[i].wait_for_rpc_connection()
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
|
|
)
|
|
self.nodes[i].admin_start_sync_file(2)
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
|
|
)
|
|
|
|
# 5: node[1..] synced contract entries and killed, sync disorder
|
|
self.nodes[0].stop(kill=True)
|
|
self.start_storage_node(0)
|
|
self.nodes[0].wait_for_rpc_connection()
|
|
|
|
chunk_data = b"\x04" * 256 * 1561
|
|
submissions, data_root = create_submission(chunk_data)
|
|
self.contract.submit(submissions)
|
|
wait_until(lambda: self.contract.num_submissions() == 4)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root) is not None)
|
|
|
|
for i in range(1, 2):
|
|
self.nodes[i].stop(kill=True)
|
|
|
|
chunk_data1 = b"\x05" * 256
|
|
submissions1, data_root1 = create_submission(chunk_data1)
|
|
self.contract.submit(submissions1)
|
|
wait_until(lambda: self.contract.num_submissions() == 5)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root1) is not None)
|
|
|
|
for i in range(2, self.num_nodes):
|
|
self.nodes[i].stop(kill=True)
|
|
|
|
segment = submit_data(self.nodes[0], chunk_data)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root)["finalized"])
|
|
|
|
segment = submit_data(self.nodes[0], chunk_data1)
|
|
wait_until(lambda: self.nodes[0].zgs_get_file_info(data_root1)["finalized"])
|
|
|
|
for i in range(1, self.num_nodes):
|
|
self.log.info("wait for node: %s", i)
|
|
self.start_storage_node(i)
|
|
self.nodes[i].wait_for_rpc_connection()
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root1) is not None
|
|
)
|
|
self.nodes[i].admin_start_sync_file(4)
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root1)["finalized"]
|
|
)
|
|
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root) is not None
|
|
)
|
|
self.nodes[i].admin_start_sync_file(3)
|
|
wait_until(
|
|
lambda: self.nodes[i].zgs_get_file_info(data_root)["finalized"]
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
CrashTest().main()
|