refactor sync relevant python tests (#301)

tests/config/node_config.py

@@ -3,6 +3,7 @@ import os
 from web3 import Web3
 
 ZGS_CONFIG = {
+    "network_enr_address": "127.0.0.1",
     "log_config_file": "log_config",
     "confirmation_block_count": 1,
     "discv5_disable_ip_limit": True,

tests/sync_auto_historical_test.py

@@ -3,17 +3,15 @@
 from test_framework.test_framework import TestFramework
 from utility.utils import wait_until
 
-class AutoRandomSyncV2Test(TestFramework):
+class AutoSyncHistoricalTest(TestFramework):
     def setup_params(self):
         self.num_nodes = 4
 
-        # Enable random auto sync v2
+        # Enable auto sync
         for i in range(self.num_nodes):
             self.zgs_node_configs[i] = {
                 "sync": {
-                    "auto_sync_enabled": True,
-                    "max_sequential_workers": 0,
-                    "max_random_workers": 3,
+                    "auto_sync_enabled": True
                 }
             }
 
@@ -41,4 +39,4 @@ class AutoRandomSyncV2Test(TestFramework):
         wait_until(lambda: self.nodes[self.num_nodes - 1].zgs_get_file_info(data_root_2)["finalized"])
 
 if __name__ == "__main__":
-    AutoRandomSyncV2Test().main()
+    AutoSyncHistoricalTest().main()

tests/sync_auto_random_test.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python3
-
-from test_framework.test_framework import TestFramework
-from utility.utils import wait_until
-
-class AutoRandomSyncTest(TestFramework):
-    def setup_params(self):
-        self.num_nodes = 2
-
-        # Enable random auto sync only
-        for i in range(self.num_nodes):
-            self.zgs_node_configs[i] = {
-                "sync": {
-                    "auto_sync_enabled": True,
-                    "max_sequential_workers": 0,
-                    "max_random_workers": 3,
-                }
-            }
-
-    def run_test(self):
-        # Submit and upload files on node 0
-        data_root_1 = self.__upload_file__(0, 256 * 1024)
-        data_root_2 = self.__upload_file__(0, 256 * 1024)
-
-        # Files should be available on node 1 via auto sync
-        wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_1) is not None)
-        wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_1)["finalized"])
-        wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_2) is not None)
-        wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_2)["finalized"])
-
-if __name__ == "__main__":
-    AutoRandomSyncTest().main()

tests/sync_auto_sequential_test_ignore.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python3
-
-from test_framework.test_framework import TestFramework
-from utility.utils import wait_until
-
-class AutoSequentialSyncTest(TestFramework):
-    def setup_params(self):
-        self.num_nodes = 2
-
-        # Enable sequential auto sync only
-        for i in range(self.num_nodes):
-            self.zgs_node_configs[i] = {
-                "sync": {
-                    "auto_sync_enabled": True,
-                    "max_sequential_workers": 3,
-                    "max_random_workers": 0,
-                }
-            }
-
-    def run_test(self):
-        # Submit and upload files on node 0
-        data_root_1 = self.__upload_file__(0, 256 * 1024)
-        data_root_2 = self.__upload_file__(0, 256 * 1024)
-
-        # Files should be available on node 1 via auto sync
-        wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_1) is not None)
-        wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_1)["finalized"])
-        wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_2) is not None)
-        wait_until(lambda: self.nodes[1].zgs_get_file_info(data_root_2)["finalized"])
-
-if __name__ == "__main__":
-    AutoSequentialSyncTest().main()

tests/sync_auto_test.py

@@ -11,9 +11,7 @@ class AutoSyncTest(TestFramework):
         for i in range(self.num_nodes):
             self.zgs_node_configs[i] = {
                 "sync": {
-                    "auto_sync_enabled": True,
-                    "max_sequential_workers": 3,
-                    "max_random_workers": 3,
+                    "auto_sync_enabled": True
                 }
             }
 

tests/sync_chunks_test.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
 
 import random
-import time
 
 from test_framework.test_framework import TestFramework
 from utility.submission import data_to_segments
-from utility.utils import (
-    assert_equal,
-    wait_until,
-)
+from utility.utils import assert_equal, wait_until
+
+class SyncChunksTest(TestFramework):
+    """
+    By default, auto_sync_enabled and sync_file_on_announcement_enabled are both false,
+    and chunks sync should be triggered by rpc.
+    """
 
-class SyncTest(TestFramework):
     def setup_params(self):
         self.num_nodes = 2
 
@@ -21,49 +22,6 @@ class SyncTest(TestFramework):
             }
 
     def run_test(self):
-        # By default, auto_sync_enabled and sync_file_on_announcement_enabled are both false,
-        # and file or chunks sync should be triggered by rpc.
-        self.__test_sync_file_by_rpc()
-        self.__test_sync_chunks_by_rpc()
-
-    def __test_sync_file_by_rpc(self):
-        self.log.info("Begin to test file sync by rpc")
-
-        client1 = self.nodes[0]
-        client2 = self.nodes[1]
-
-        # stop client2, preventing it from receiving AnnounceFile
-        client2.shutdown()
-
-        data_root = self.__upload_file__(0, 256 * 1024)
-
-        # restart client2
-        client2.start()
-        client2.wait_for_rpc_connection()
-        
-        # File should not be auto sync on node 2 and there is no cached file locations
-        wait_until(lambda: client2.zgs_get_file_info(data_root) is not None)
-        time.sleep(3)
-        assert_equal(client2.zgs_get_file_info(data_root)["finalized"], False)
-        # file sync use ASK_FILE & ANSWER FILE protocol, and do not cache file announcement anymore.
-        # assert(client2.admin_get_file_location(0) is None)
-
-        # Trigger file sync by rpc
-        assert(client2.admin_start_sync_file(0) is None)
-        wait_until(lambda: client2.sync_status_is_completed_or_unknown(0))
-        wait_until(lambda: client2.zgs_get_file_info(data_root)["finalized"])
-        # file sync use ASK_FILE & ANSWER FILE protocol, and do not cache file announcement anymore.
-        # assert(client2.admin_get_file_location(0) is not None)
-
-        # Validate data
-        assert_equal(
-            client2.zgs_download_segment(data_root, 0, 1024),
-            client1.zgs_download_segment(data_root, 0, 1024),
-        )
-
-    def __test_sync_chunks_by_rpc(self):
-        self.log.info("Begin to test chunks sync by rpc")
-
         client1 = self.nodes[0]
         client2 = self.nodes[1]
 
@@ -98,12 +56,12 @@ class SyncTest(TestFramework):
         self.nodes[0].wait_for_rpc_connection()
 
         # Trigger chunks sync by rpc
-        assert(client2.admin_start_sync_chunks(1, 1024, 2048) is None)
-        wait_until(lambda: client2.sync_status_is_completed_or_unknown(1))
+        assert(client2.admin_start_sync_chunks(0, 1024, 2048) is None)
+        wait_until(lambda: client2.sync_status_is_completed_or_unknown(0))
         wait_until(lambda: client2.zgs_download_segment_decoded(data_root, 1024, 2048) is not None)
 
         # Validate data
         assert_equal(client2.zgs_download_segment_decoded(data_root, 1024, 2048), chunk_data[1024*256:2048*256])
 
 if __name__ == "__main__":
-    SyncTest().main()
+    SyncChunksTest().main()

tests/sync_file_test.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import time
+
+from test_framework.test_framework import TestFramework
+from utility.utils import assert_equal, wait_until
+
+class SyncFileTest(TestFramework):
+    """
+    By default, auto_sync_enabled and sync_file_on_announcement_enabled are both false,
+    and file sync should be triggered by rpc.
+    """
+
+    def setup_params(self):
+        self.num_nodes = 2
+
+    def run_test(self):
+        client1 = self.nodes[0]
+        client2 = self.nodes[1]
+
+        # stop client2, preventing it from receiving AnnounceFile
+        client2.shutdown()
+
+        data_root = self.__upload_file__(0, 256 * 1024)
+
+        # restart client2
+        client2.start()
+        client2.wait_for_rpc_connection()
+        
+        # File should not be auto sync on node 2 and there is no cached file locations
+        wait_until(lambda: client2.zgs_get_file_info(data_root) is not None)
+        time.sleep(3)
+        assert_equal(client2.zgs_get_file_info(data_root)["finalized"], False)
+        # file sync use ASK_FILE & ANSWER FILE protocol, and do not cache file announcement anymore.
+        # assert(client2.admin_get_file_location(0) is None)
+
+        # Trigger file sync by rpc
+        assert(client2.admin_start_sync_file(0) is None)
+        wait_until(lambda: client2.sync_status_is_completed_or_unknown(0))
+        wait_until(lambda: client2.zgs_get_file_info(data_root)["finalized"])
+        # file sync use ASK_FILE & ANSWER FILE protocol, and do not cache file announcement anymore.
+        # assert(client2.admin_get_file_location(0) is not None)
+
+        # Validate data
+        assert_equal(
+            client2.zgs_download_segment(data_root, 0, 1024),
+            client1.zgs_download_segment(data_root, 0, 1024),
+        )
+
+if __name__ == "__main__":
+    SyncFileTest().main()