feat!: simulator (#76)

.coverageignore

@@ -1,6 +1,4 @@
 *.pb.go
 *.pb.gw.go
 *.pulsar.go
-*_simulation.go
 testnet.go
-test_suite.go

.github/workflows/simulation.yml

@@ -0,0 +1,39 @@
+name: Simulation Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+    types: [opened, reopened, synchronize]
+
+env:
+    GO_VERSION: 1.22.5
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source
+        uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          check-latest: true
+      - name: Full application simulation (fixed seed)
+        run: make sim-full-app
+      - name: Simulation after state import (fixed seed)
+        run: make sim-after-import
+      #      #  Requires wiring v2
+      #      # https://github.com/liftedinit/manifest-ledger/issues/82
+      #      - name: Simulation export/import (fixed seed)
+      #        run: make sim-export-import
+      - name: Simulate application state determinism (fixed seed)
+        run: make sim-app-determinism

Makefile

@@ -149,24 +149,41 @@ test:
 
 .PHONY: test
 
+COV_ROOT="/tmp/manifest-ledger-coverage"
+COV_UNIT_E2E="${COV_ROOT}/unit-e2e"
+COV_SIMULATION="${COV_ROOT}/simulation"
+COV_PKG="github.com/liftedinit/manifest-ledger/..."
+COV_SIM_CMD=${COV_SIMULATION}/simulation.test
+COV_SIM_COMMON=-Enabled=True -NumBlocks=100 -Commit=true -Period=5 -Params=$(shell pwd)/simulation/sim_params.json -Verbose=false -test.v -test.gocoverdir=${COV_SIMULATION}
+
 coverage: ## Run coverage report
 	@echo "--> Creating GOCOVERDIR"
-	@mkdir -p /tmp/manifest-ledger-coverage
+	@mkdir -p ${COV_UNIT_E2E} ${COV_SIMULATION}
 	@echo "--> Cleaning up coverage files, if any"
-	@rm -rf /tmp/manifest-ledger-coverage/*
-	@echo "--> Running coverage"
-	@go test -timeout 30m -race -covermode=atomic -v -cpu=$$(nproc) -cover $$(go list ./...) ./interchaintest/... -coverpkg=github.com/liftedinit/manifest-ledger/... -args -test.gocoverdir="/tmp/manifest-ledger-coverage" > /dev/null 2>&1
+	@rm -rf ${COV_UNIT_E2E}/* ${COV_SIMULATION}/*
+	@echo "--> Building instrumented simulation test binary"
+	@go test -c ./app -mod=readonly -covermode=atomic -coverpkg=${COV_PKG} -cover -o ${COV_SIM_CMD}
+	@echo "  --> Running Full App Simulation"
+	@${COV_SIM_CMD} -test.run TestFullAppSimulation ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "  --> Running App Simulation After Import"
+	@${COV_SIM_CMD} -test.run TestAppSimulationAfterImport ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "  --> Running App State Determinism Simulation"
+	@${COV_SIM_CMD} -test.run TestAppStateDeterminism ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "--> Running unit & e2e tests coverage"
+	@go test -timeout 30m -race -covermode=atomic -v -cpu=$$(nproc) -cover $$(go list ./...) ./interchaintest/... -coverpkg=${COV_PKG} -args -test.gocoverdir="${COV_UNIT_E2E}" > /dev/null 2>&1
+	@echo "--> Merging coverage reports"
+	@go tool covdata merge -i=${COV_UNIT_E2E},${COV_SIMULATION} -o ${COV_ROOT}
 	@echo "--> Converting binary coverage report to text format"
-	@go tool covdata textfmt -i=/tmp/manifest-ledger-coverage -o coverage.out
-	@echo "--> Filtering coverage report"
-	@./scripts/filter-coverage.sh
+	@go tool covdata textfmt -i=${COV_ROOT} -o ${COV_ROOT}/coverage-merged.out
+	@echo "--> Filtering coverage reports"
+	@./scripts/filter-coverage.sh ${COV_ROOT}/coverage-merged.out ${COV_ROOT}/coverage-merged-filtered.out
 	@echo "--> Generating coverage report"
-	@go tool cover -func=coverage-filtered.out
+	@go tool cover -func=${COV_ROOT}/coverage-merged-filtered.out
 	@echo "--> Generating HTML coverage report"
-	@go tool cover -html=coverage-filtered.out -o coverage.html
+	@go tool cover -html=${COV_ROOT}/coverage-merged-filtered.out -o coverage.html
 	@echo "--> Coverage report available at coverage.html"
 	@echo "--> Cleaning up coverage files"
-	@rm coverage.out /tmp/manifest-ledger-coverage/*
+	@rm -rf ${COV_UNIT_E2E}/* ${COV_SIMULATION}/*
 	@echo "--> Running coverage complete"
 
 .PHONY: coverage
@@ -247,3 +264,47 @@ vet: ## Run go vet
 	@go vet ./...
 
 .PHONY: vet
+
+#### Simulation ####
+
+SIM_PARAMS ?= $(shell pwd)/simulation/sim_params.json
+SIM_NUM_BLOCKS ?= 100
+SIM_PERIOD ?= 5
+SIM_COMMIT ?= true
+SIM_ENABLED ?= true
+SIM_VERBOSE ?= false
+SIM_TIMEOUT ?= 24h
+SIM_SEED ?= 42
+SIM_COMMON_ARGS = -NumBlocks=${SIM_NUM_BLOCKS} -Enabled=${SIM_ENABLED} -Commit=${SIM_COMMIT} -Period=${SIM_PERIOD} -Params=${SIM_PARAMS} -Verbose=${SIM_VERBOSE} -Seed=${SIM_SEED} -v -timeout ${SIM_TIMEOUT}
+
+sim-full-app:
+	@echo "--> Running full app simulation (blocks: ${SIM_NUM_BLOCKS}, commit: ${SIM_COMMIT}, period: ${SIM_PERIOD}, seed: ${SIM_SEED}, params: ${SIM_PARAMS}"
+	@go test ./app -run TestFullAppSimulation ${SIM_COMMON_ARGS}
+
+sim-full-app-random:
+	$(MAKE) sim-full-app SIM_SEED=$$RANDOM
+
+# Note: known to fail when using app wiring v1
+sim-import-export:
+	@echo "--> Running app import/export simulation (blocks: ${SIM_NUM_BLOCKS}, commit: ${SIM_COMMIT}, period: ${SIM_PERIOD}, seed: ${SIM_SEED}, params: ${SIM_PARAMS}"
+	@go test ./app -run TestAppImportExport ${SIM_COMMON_ARGS}
+
+# Note: known to fail when using app wiring v1
+sim-import-export-random:
+	$(MAKE) sim-import-export SIM_SEED=$$RANDOM
+
+sim-after-import:
+	@echo "--> Running app after import simulation (blocks: ${SIM_NUM_BLOCKS}, commit: ${SIM_COMMIT}, period: ${SIM_PERIOD}, seed: ${SIM_SEED}, params: ${SIM_PARAMS}"
+	@go test ./app -run TestAppSimulationAfterImport ${SIM_COMMON_ARGS}
+
+sim-after-import-random:
+	$(MAKE) sim-after-import SIM_SEED=$$RANDOM
+
+sim-app-determinism:
+	@echo "--> Running app determinism simulation (blocks: ${SIM_NUM_BLOCKS}, commit: ${SIM_COMMIT}, period: ${SIM_PERIOD}, seed: ${SIM_SEED}, params: ${SIM_PARAMS}"
+	@go test ./app -run TestAppStateDeterminism ${SIM_COMMON_ARGS}
+
+sim-app-determinism-random:
+	$(MAKE) sim-app-determinism SIM_SEED=$$RANDOM
+
+.PHONY: sim-full-app sim-full-app-random sim-import-export sim-after-import sim-app-determinism sim-import-export-random sim-after-import-random sim-app-determinism-random

README.md

@@ -126,6 +126,28 @@ make ictest-ibc
 make ictest-group-poa
 ```
 
+## Simulation
+
+**To execute the full application simulation run:**
+
+```bash
+make sim-full-app
+````
+
+**To execute the application simulation after state import run:**
+
+```bash
+make sim-after-import
+```
+
+**To test the application determinism run:**
+
+```bash
+make sim-app-determinism
+```
+
+Append `-random` to the end of the commands above to run the simulation with a random seed, e.g., `make sim-full-app-random`.
+
 ## Coverage
 
 To generate a coverage report for the modules run: