diff --git a/Makefile b/Makefile index 11c1c996..ab532ba0 100644 --- a/Makefile +++ b/Makefile @@ -109,18 +109,26 @@ link-check: # TODO tidy up cli tests to use same -Enable flag as simulations, or the other way round # TODO -mod=readonly ? # build dependency needed for cli tests -test-all: build +test-all: build test-cli test # basic app tests @go test ./app -v - # cli tests - @go test ./cli_test -tags cli_test -v -p 4 # basic simulation (seed "2" happens to not unbond all validators before reaching 100 blocks) @go test ./app -run TestFullAppSimulation -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h # other sim tests @go test ./app -run TestAppImportExport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h @go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h @# AppStateDeterminism does not use Seed flag - @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -v -timeout 24h + @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=100 -BlockSize=200 -Seed 2 -v -timeout 24h + +# run module tests and short simulations +test-basic: test + # basic simulation (seed "2" happens to not unbond all validators before reaching 100 blocks) + @go test ./app -run TestFullAppSimulation -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m + # other sim tests + @go test ./app -run TestAppImportExport -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m + @go test ./app -run TestAppSimulationAfterImport -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m + @# AppStateDeterminism does not use Seed flag + @go test ./app -run TestAppStateDeterminism -Enabled -Commit -NumBlocks=5 -BlockSize=200 -Seed 2 -v -timeout 2m test: @go test ./... @@ -128,6 +136,9 @@ test: test_dredd: rest_test/./run_all_tests_from_make.sh +test-cli: + @go test ./cli_test -tags cli_test -v -p 4 + # Kick start lots of sims on an AWS cluster. # This submits an AWS Batch job to run a lot of sims, each within a docker image. Results are uploaded to S3 start-remote-sims: @@ -143,4 +154,4 @@ start-remote-sims: -—job-definition kava-sim-master \ -—container-override environment=[{SIM_NAME=master-$(VERSION)}] -.PHONY: all build-linux install clean build test test-all start-remote-sims +.PHONY: all build-linux install clean build test test-cli test-all test_dredd test-basic start-remote-sims diff --git a/app/sim_test.go b/app/sim_test.go index 40a0c5e4..c4fb6bcd 100644 --- a/app/sim_test.go +++ b/app/sim_test.go @@ -662,38 +662,33 @@ func TestAppStateDeterminism(t *testing.T) { config.OnOperation = false config.AllInvariants = false - numSeeds := 3 - numTimesToRunPerSeed := 5 + numTimesToRunPerSeed := 2 appHashList := make([]json.RawMessage, numTimesToRunPerSeed) - for i := 0; i < numSeeds; i++ { - config.Seed = rand.Int63() + for j := 0; j < numTimesToRunPerSeed; j++ { + logger := log.NewNopLogger() + db := dbm.NewMemDB() + app := NewApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) - for j := 0; j < numTimesToRunPerSeed; j++ { - logger := log.NewNopLogger() - db := dbm.NewMemDB() - app := NewApp(logger, db, nil, true, simapp.FlagPeriodValue, interBlockCacheOpt()) + fmt.Printf( + "running non-determinism simulation; seed %d: attempt: %d/%d\n", + config.Seed, j+1, numTimesToRunPerSeed, + ) - fmt.Printf( - "running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n", - config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, + _, _, err := simulation.SimulateFromSeed( + t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), + testAndRunTxs(app, config), app.ModuleAccountAddrs(), config, + ) + require.NoError(t, err) + + appHash := app.LastCommitID().Hash + appHashList[j] = appHash + + if j != 0 { + require.Equal( + t, appHashList[0], appHashList[j], + "non-determinism in seed %d: attempt: %d/%d\n", config.Seed, j+1, numTimesToRunPerSeed, ) - - _, _, err := simulation.SimulateFromSeed( - t, os.Stdout, app.BaseApp, simapp.AppStateFn(app.Codec(), app.sm), - testAndRunTxs(app, config), app.ModuleAccountAddrs(), config, - ) - require.NoError(t, err) - - appHash := app.LastCommitID().Hash - appHashList[j] = appHash - - if j != 0 { - require.Equal( - t, appHashList[0], appHashList[j], - "non-determinism in seed %d: %d/%d, attempt: %d/%d\n", config.Seed, i+1, numSeeds, j+1, numTimesToRunPerSeed, - ) - } } } }