test: improve coverage by supporting sims

.coverageignore

@@ -1,6 +1,4 @@
 *.pb.go
 *.pb.gw.go
 *.pulsar.go
-*_simulation.go
 testnet.go
-test_suite.go

Makefile

@@ -149,24 +149,41 @@ test:
 
 .PHONY: test
 
+COV_ROOT="/tmp/manifest-ledger-coverage"
+COV_UNIT_E2E="${COV_ROOT}/unit-e2e"
+COV_SIMULATION="${COV_ROOT}/simulation"
+COV_PKG="github.com/liftedinit/manifest-ledger/..."
+COV_SIM_CMD=${COV_SIMULATION}/simulation.test
+COV_SIM_COMMON=-Enabled=True -NumBlocks=100 -Commit=true -Period=5 -Params=$(shell pwd)/simulation/sim_params.json -Verbose=false -test.v -test.gocoverdir=${COV_SIMULATION}
+
 coverage: ## Run coverage report
 	@echo "--> Creating GOCOVERDIR"
-	@mkdir -p /tmp/manifest-ledger-coverage
+	@mkdir -p ${COV_UNIT_E2E} ${COV_SIMULATION}
 	@echo "--> Cleaning up coverage files, if any"
-	@rm -rf /tmp/manifest-ledger-coverage/*
-	@echo "--> Running coverage"
-	@go test -timeout 30m -race -covermode=atomic -v -cpu=$$(nproc) -cover $$(go list ./...) ./interchaintest/... -coverpkg=github.com/liftedinit/manifest-ledger/... -args -test.gocoverdir="/tmp/manifest-ledger-coverage" > /dev/null 2>&1
+	@rm -rf ${COV_UNIT_E2E}/* ${COV_SIMULATION}/*
+	@echo "--> Building instrumented simulation test binary"
+	@go test -c ./app -mod=readonly -covermode=atomic -coverpkg=${COV_PKG} -cover -o ${COV_SIM_CMD}
+	@echo "  --> Running Full App Simulation"
+	@${COV_SIM_CMD} -test.run TestFullAppSimulation ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "  --> Running App Simulation After Import"
+	@${COV_SIM_CMD} -test.run TestAppSimulationAfterImport ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "  --> Running App State Determinism Simulation"
+	@${COV_SIM_CMD} -test.run TestAppStateDeterminism ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "--> Running unit & e2e tests coverage"
+	@go test -timeout 30m -race -covermode=atomic -v -cpu=$$(nproc) -cover $$(go list ./...) ./interchaintest/... -coverpkg=${COV_PKG} -args -test.gocoverdir="${COV_UNIT_E2E}" > /dev/null 2>&1
+	@echo "--> Merging coverage reports"
+	@go tool covdata merge -i=${COV_UNIT_E2E},${COV_SIMULATION} -o ${COV_ROOT}
 	@echo "--> Converting binary coverage report to text format"
-	@go tool covdata textfmt -i=/tmp/manifest-ledger-coverage -o coverage.out
-	@echo "--> Filtering coverage report"
-	@./scripts/filter-coverage.sh
+	@go tool covdata textfmt -i=${COV_ROOT} -o ${COV_ROOT}/coverage-merged.out
+	@echo "--> Filtering coverage reports"
+	@./scripts/filter-coverage.sh ${COV_ROOT}/coverage-merged.out ${COV_ROOT}/coverage-merged-filtered.out
 	@echo "--> Generating coverage report"
-	@go tool cover -func=coverage-filtered.out
+	@go tool cover -func=${COV_ROOT}/coverage-merged-filtered.out
 	@echo "--> Generating HTML coverage report"
-	@go tool cover -html=coverage-filtered.out -o coverage.html
+	@go tool cover -html=${COV_ROOT}/coverage-merged-filtered.out -o coverage.html
 	@echo "--> Coverage report available at coverage.html"
 	@echo "--> Cleaning up coverage files"
-	@rm coverage.out /tmp/manifest-ledger-coverage/*
+#	@rm -rf ${COV_UNIT_E2E}/* ${COV_SIMULATION}/*
 	@echo "--> Running coverage complete"
 
 .PHONY: coverage

go.mod

@@ -13,6 +13,9 @@ replace (
 	// Fix upstream GHSA-h395-qcrw-5vmq vulnerability.
 	github.com/spf13/viper => github.com/spf13/viper v1.17.0 // v1.18+ breaks app overrides
 
+	// TODO: Remove this once 0.50.3 is released
+	github.com/strangelove-ventures/poa => github.com/strangelove-ventures/poa v0.50.3-0.20240720145936-3595e3d47ba0
+
 	// Downgraded to avoid bugs in following commits which caused simulations to fail.
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 )

go.sum

@@ -960,8 +960,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI=
 github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI=
-github.com/strangelove-ventures/poa v0.50.2 h1:wr0Bif2ARULFtt4cmhsNNoUZPtQ/psB8gEhd17cfboA=
-github.com/strangelove-ventures/poa v0.50.2/go.mod h1:0cOydjyn8T6gZKvLkpXgBYVD636od00GmCddaoOkS2E=
+github.com/strangelove-ventures/poa v0.50.3-0.20240720145936-3595e3d47ba0 h1:9h8TJL5qnTrYOyJmT2xQzUrrLv4cfLmgfvTtTUoBmHY=
+github.com/strangelove-ventures/poa v0.50.3-0.20240720145936-3595e3d47ba0/go.mod h1:0cOydjyn8T6gZKvLkpXgBYVD636od00GmCddaoOkS2E=
 github.com/strangelove-ventures/tokenfactory v0.50.2 h1:Bb6xKHzd+VIJgQpjLc7wuvqnWXmBo1+FllwzF5sIgMI=
 github.com/strangelove-ventures/tokenfactory v0.50.2/go.mod h1:VIByIP8F8jJz8Exk8RrPwJZnyhAnvD6dD7a6tqHOcGY=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=

interchaintest/go.mod

@@ -13,6 +13,8 @@ replace (
 	github.com/cosmos/cosmos-sdk => github.com/liftedinit/cosmos-sdk v0.50.8-liftedinit.1
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	github.com/liftedinit/manifest-ledger => ../
+	// TODO: Remove this once 0.50.3 is released
+	github.com/strangelove-ventures/poa => github.com/strangelove-ventures/poa v0.50.3-0.20240720145936-3595e3d47ba0
 )
 
 require (

interchaintest/go.sum

@@ -1045,8 +1045,8 @@ github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
 github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
 github.com/strangelove-ventures/interchaintest/v8 v8.5.1-0.20240715163423-cbcd20f44321 h1:uemY7a+M5RWC/6kTSgLql0kBEGKy/zY83p1Ne9kTzjY=
 github.com/strangelove-ventures/interchaintest/v8 v8.5.1-0.20240715163423-cbcd20f44321/go.mod h1:UPMvR1joIXO8RMxDozoHIInLZWmd2345PSGLJYWyrwY=
-github.com/strangelove-ventures/poa v0.50.2 h1:wr0Bif2ARULFtt4cmhsNNoUZPtQ/psB8gEhd17cfboA=
-github.com/strangelove-ventures/poa v0.50.2/go.mod h1:0cOydjyn8T6gZKvLkpXgBYVD636od00GmCddaoOkS2E=
+github.com/strangelove-ventures/poa v0.50.3-0.20240720145936-3595e3d47ba0 h1:9h8TJL5qnTrYOyJmT2xQzUrrLv4cfLmgfvTtTUoBmHY=
+github.com/strangelove-ventures/poa v0.50.3-0.20240720145936-3595e3d47ba0/go.mod h1:0cOydjyn8T6gZKvLkpXgBYVD636od00GmCddaoOkS2E=
 github.com/strangelove-ventures/tokenfactory v0.50.2 h1:Bb6xKHzd+VIJgQpjLc7wuvqnWXmBo1+FllwzF5sIgMI=
 github.com/strangelove-ventures/tokenfactory v0.50.2/go.mod h1:VIByIP8F8jJz8Exk8RrPwJZnyhAnvD6dD7a6tqHOcGY=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=

interchaintest/setup.go

@@ -16,7 +16,7 @@ import (
 )
 
 const (
-	ExternalGoCoverDir = "/tmp/manifest-ledger-coverage"
+	ExternalGoCoverDir = "/tmp/manifest-ledger-coverage/unit-e2e"
 )
 
 var (

scripts/filter-coverage.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-coverage_profile="coverage.out"
-filtered_coverage_profile="coverage-filtered.out"
+coverage_profile="$1"
+filtered_coverage_profile="$2"
 exclusion_file=".coverageignore"
 
 cp "$coverage_profile" "$filtered_coverage_profile"