update sim tooling #440

[R4R] Update sim tooling

Makefile

@@ -109,18 +109,26 @@ link-check:
 # TODO tidy up cli tests to use same -Enable flag as simulations, or the other way round
 # TODO -mod=readonly ?
 # build dependency needed for cli tests
-test-all: build
+test-all: build test-cli test
 	# basic app tests
 	@go test ./app -v
-	# cli tests
-	@go test ./cli_test -tags cli_test -v -p 4
 	# basic simulation (seed "2" happens to not unbond all validators before reaching 100 blocks)
 	@go test ./app -run TestFullAppSimulation        -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h
 	# other sim tests
 	@go test ./app -run TestAppImportExport          -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h
 	@go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h
 	@# AppStateDeterminism does not use Seed flag
-	@go test ./app -run TestAppStateDeterminism      -Enabled -Commit -NumBlocks=100 -BlockSize=200         -v -timeout 24h
+	@go test ./app -run TestAppStateDeterminism      -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h
+
+# run module tests and short simulations
+test-basic: test
+	# basic simulation (seed "2" happens to not unbond all validators before reaching 100 blocks)
+	@go test ./app -run TestFullAppSimulation        -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m
+	# other sim tests
+	@go test ./app -run TestAppImportExport          -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m
+	@go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m
+	@# AppStateDeterminism does not use Seed flag
+	@go test ./app -run TestAppStateDeterminism      -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m
 
 test:
 	@go test ./...
@@ -128,6 +136,9 @@ test:
 test_dredd:
 	rest_test/./run_all_tests_from_make.sh
 
+test-cli:
+	@go test ./cli_test -tags cli_test -v -p 4
+
 # Kick start lots of sims on an AWS cluster.
 # This submits an AWS Batch job to run a lot of sims, each within a docker image. Results are uploaded to S3
 start-remote-sims:
@@ -143,4 +154,4 @@ start-remote-sims:
 		-—job-definition kava-sim-master \
 		-—container-override environment=[{SIM_NAME=master-$(VERSION)}]
 
-.PHONY: all build-linux install clean build test test-all start-remote-sims
+.PHONY: all build-linux install clean build test test-cli test-all test_dredd test-basic start-remote-sims

app/sim_test.go

@@ -662,38 +662,33 @@ func TestAppStateDeterminism(t *testing.T) {
 	config.OnOperation = false
 	config.AllInvariants = false
 
-	numSeeds := 3
+	numTimesToRunPerSeed := 2
-	numTimesToRunPerSeed := 5
 	appHashList := make([]json.RawMessage, numTimesToRunPerSeed)
 
-	for i := 0; i < numSeeds; i++ {
+	for j := 0; j < numTimesToRunPerSeed; j++ {
-		config.Seed = rand.Int63()
+		logger := log.NewNopLogger()
+		db := dbm.NewMemDB()
+		app := NewApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt())
 
-		for j := 0; j < numTimesToRunPerSeed; j++ {
+		fmt.Printf(
-			logger := log.NewNopLogger()
+			"running non-determinism simulation; seed %d: attempt: %d/%d\n",
-			db := dbm.NewMemDB()
+			config.Seed, j+1, numTimesToRunPerSeed,
-			app := NewApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt())
+		)
 
-			fmt.Printf(
+		_, _, err := simulation.SimulateFromSeed(
-				"running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n",
+			t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm),
-				config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed,
+			testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
+		)
+		require.NoError(t, err)
+
+		appHash := app.LastCommitID().Hash
+		appHashList[j] = appHash
+
+		if j != 0 {
+			require.Equal(
+				t, appHashList[0], appHashList[j],
+				"non-determinism in seed %d: attempt: %d/%d\n", config.Seed, j+1, numTimesToRunPerSeed,
 			)
-
-			_, _, err := simulation.SimulateFromSeed(
-				t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm),
-				testAndRunTxs(app, config), app.ModuleAccountAddrs(), config,
-			)
-			require.NoError(t, err)
-
-			appHash := app.LastCommitID().Hash
-			appHashList[j] = appHash
-
-			if j != 0 {
-				require.Equal(
-					t, appHashList[0], appHashList[j],
-					"non-determinism in seed %d: %d/%d, attempt: %d/%d\n", config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed,
-				)
-			}
 		}
 	}
 }