From df65bda08bae915d166c15502f0d4c5576d61f3a Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 7 Feb 2024 15:58:43 +0100 Subject: [PATCH 1/2] backport PR 5254 --- .github/workflows/builds.yml | 19 +- .github/workflows/cd.yml | 12 +- .github/workflows/ci.yml | 6 +- .github/workflows/flaky-test-monitor.yml | 4 +- Makefile | 523 ++++++++++++------ README.md | 6 +- cmd/Dockerfile | 24 +- crypto_adx_flag.mk | 10 +- .../ingestion/stop/stop_control_test.go | 10 +- integration/README.md | 20 +- 10 files changed, 435 insertions(+), 199 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index f8b2eeee677..29ada28788b 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -1,6 +1,6 @@ +name: Build Node Docker Images # This workflow is used to build and push one-off images for specific node types. This is useful # when deploying hotfixes or any time a change is not needed for all node roles. -name: Build Node Docker Images on: workflow_dispatch: @@ -38,9 +38,10 @@ on: type: boolean description: 'Observer' required: false - include_without_netgo: + # GHA allows only up to 10 inputs - regroup two entries in one + include_alternative_builds: type: boolean - description: 'Build `without_netgo` images' + description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images' required: false jobs: @@ -111,17 +112,19 @@ jobs: run: | gcloud auth configure-docker - - name: Build/Push ${{ matrix.role }} images + - name: Build/Push ${{ matrix.role }} amd64 images with adx (default) env: IMAGE_TAG: ${{ inputs.docker_tag }} CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} run: | - make docker-build-${{ matrix.role }} docker-push-${{ matrix.role }} + make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx - - name: Build/Push ${{ matrix.role }} without_netgo images - if: ${{ inputs.include_without_netgo }} + - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images + if: ${{ inputs.include_alternative_builds }} env: IMAGE_TAG: ${{ inputs.docker_tag }} CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} run: | - make docker-build-${{ matrix.role }}-without-netgo docker-push-${{ matrix.role }}-without-netgo + make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \ + docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \ + docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 301398077d2..91a8da5682d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -33,9 +33,13 @@ jobs: env: CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} run: | - make docker-build-flow - make docker-build-flow-without-netgo + make docker-build-flow-with-adx + make docker-build-flow-without-adx + make docker-build-flow-without-netgo-without-adx + make docker-cross-build-flow-arm - name: Docker push run: | - make docker-push-flow - make docker-push-flow-without-netgo + make docker-push-flow-with-adx + make docker-push-flow-without-adx + make docker-push-flow-without-netgo-without-adx + make docker-push-flow-arm \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67bd041eb63..de23039a4bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,6 +174,8 @@ jobs: docker-build: name: Docker Build runs-on: buildjet-16vcpu-ubuntu-2204 + env: + CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} steps: - name: Checkout repo uses: actions/checkout@v3 @@ -189,7 +191,7 @@ jobs: - name: Docker build env: CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} - run: make docker-build-flow docker-build-flow-corrupt + run: make docker-native-build-flow docker-native-build-flow-corrupt - name: Save Docker images run: | docker save \ @@ -295,4 +297,4 @@ jobs: with: timeout_minutes: 35 max_attempts: 5 - command: VERBOSE=1 ${{ matrix.make }} + command: VERBOSE=1 ${{ matrix.make }} \ No newline at end of file diff --git a/.github/workflows/flaky-test-monitor.yml b/.github/workflows/flaky-test-monitor.yml index 3a47ef7b829..5073c2b1b01 100644 --- a/.github/workflows/flaky-test-monitor.yml +++ b/.github/workflows/flaky-test-monitor.yml @@ -165,7 +165,7 @@ jobs: go-version: ${{ env.GO_VERSION }} cache: true - name: Docker build - run: make docker-build-flow docker-build-flow-corrupt + run: make docker-native-build-flow docker-native-build-flow-corrupt - name: Run tests run: make -es -C integration ${{ matrix.target }} > test-output timeout-minutes: 100 @@ -175,4 +175,4 @@ jobs: TEST_CATEGORY: ${{ matrix.test_category }} uses: ./.github/workflows/actions/test-monitor-process-results with: - gcp_sa_key: ${{ secrets.GCP_SA_KEY }} + gcp_sa_key: ${{ secrets.GCP_SA_KEY }} \ No newline at end of file diff --git a/Makefile b/Makefile index 37d253f1d6e..de511e58dd8 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,9 @@ ifeq (${IMAGE_TAG},) IMAGE_TAG := ${SHORT_COMMIT} endif -IMAGE_TAG_NO_NETGO := $(IMAGE_TAG)-without-netgo +IMAGE_TAG_NO_ADX := $(IMAGE_TAG)-without-adx +IMAGE_TAG_NO_NETGO_NO_ADX := $(IMAGE_TAG)-without-netgo-without-adx +IMAGE_TAG_ARM := $(IMAGE_TAG)-arm # Name of the cover profile COVER_PROFILE := coverage.txt @@ -39,20 +41,19 @@ K8S_YAMLS_LOCATION_STAGING=./k8s/staging export CONTAINER_REGISTRY := gcr.io/flow-container-registry export DOCKER_BUILDKIT := 1 +# set `CRYPTO_FLAG` when building natively (not cross-compiling) include crypto_adx_flag.mk -CGO_FLAG := CGO_CFLAGS=$(CRYPTO_FLAG) - # needed for CI .PHONY: noop noop: @echo "This is a no-op target" cmd/collection/collection: - $(CGO_FLAG) go build -o cmd/collection/collection cmd/collection/main.go + CGO_CFLAGS=$(CRYPTO_FLAG) go build -o cmd/collection/collection cmd/collection/main.go cmd/util/util: - $(CGO_FLAG) go build -o cmd/util/util cmd/util/main.go + CGO_CFLAGS=$(CRYPTO_FLAG) go build -o cmd/util/util cmd/util/main.go .PHONY: update-core-contracts-version update-core-contracts-version: @@ -71,7 +72,7 @@ update-cadence-version: .PHONY: unittest-main unittest-main: # test all packages - $(CGO_FLAG) go test $(if $(VERBOSE),-v,) -coverprofile=$(COVER_PROFILE) -covermode=atomic $(if $(RACE_DETECTOR),-race,) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) $(GO_TEST_PACKAGES) + CGO_CFLAGS=$(CRYPTO_FLAG) go test $(if $(VERBOSE),-v,) -coverprofile=$(COVER_PROFILE) -covermode=atomic $(if $(RACE_DETECTOR),-race,) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) $(GO_TEST_PACKAGES) .PHONY: install-mock-generators install-mock-generators: @@ -97,7 +98,7 @@ go-math-rand-check: # `exclude` should only specify non production code (test, bench..). # If this check fails, try updating your code by using: # - "crypto/rand" or "flow-go/utils/rand" for non-deterministic randomness - # - "flow-go/crypto/random" for deterministic randomness + # - "onflow/crypto/random" for deterministic randomness grep --include=\*.go \ --exclude=*test* --exclude=*helper* --exclude=*example* --exclude=*fixture* --exclude=*benchmark* --exclude=*profiler* \ --exclude-dir=*test* --exclude-dir=*helper* --exclude-dir=*example* --exclude-dir=*fixture* --exclude-dir=*benchmark* --exclude-dir=*profiler* -rnw '"math/rand"'; \ @@ -111,17 +112,17 @@ code-sanity-check: go-math-rand-check .PHONY: fuzz-fvm fuzz-fvm: # run fuzz tests in the fvm package - cd ./fvm && $(CGO_FLAG) go test -fuzz=Fuzz -run ^$$ + cd ./fvm && CGO_CFLAGS=$(CRYPTO_FLAG) go test -fuzz=Fuzz -run ^$$ .PHONY: test test: verify-mocks unittest-main .PHONY: integration-test -integration-test: docker-build-flow +integration-test: docker-native-build-flow $(MAKE) -C integration integration-test .PHONY: benchmark -benchmark: docker-build-flow +benchmark: docker-native-build-flow $(MAKE) -C integration benchmark .PHONY: coverage @@ -149,14 +150,14 @@ generate-proto: .PHONY: generate-fvm-env-wrappers generate-fvm-env-wrappers: - $(CGO_FLAG) go run ./fvm/environment/generate-wrappers fvm/environment/parse_restricted_checker.go + CGO_CFLAGS=$(CRYPTO_FLAG) go run ./fvm/environment/generate-wrappers fvm/environment/parse_restricted_checker.go .PHONY: generate-mocks generate-mocks: install-mock-generators mockery --name '(Connector|PingInfoProvider)' --dir=network/p2p --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork" - $(CGO_FLAG) mockgen -destination=storage/mocks/storage.go -package=mocks github.com/onflow/flow-go/storage Blocks,Headers,Payloads,Collections,Commits,Events,ServiceEvents,TransactionResults - $(CGO_FLAG) mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Local,Requester - $(CGO_FLAG) mockgen -destination=network/mocknetwork/mock_network.go -package=mocknetwork github.com/onflow/flow-go/network EngineRegistry + CGO_CFLAGS=$(CRYPTO_FLAG) mockgen -destination=storage/mocks/storage.go -package=mocks github.com/onflow/flow-go/storage Blocks,Headers,Payloads,Collections,Commits,Events,ServiceEvents,TransactionResults + CGO_CFLAGS=$(CRYPTO_FLAG) mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Local,Requester + CGO_CFLAGS=$(CRYPTO_FLAG) mockgen -destination=network/mocknetwork/mock_network.go -package=mocknetwork github.com/onflow/flow-go/network EngineRegistry mockery --name='.*' --dir=integration/benchmark/mocksiface --case=underscore --output="integration/benchmark/mock" --outpkg="mock" mockery --name=ExecutionDataStore --dir=module/executiondatasync/execution_data --case=underscore --output="./module/executiondatasync/execution_data/mock" --outpkg="mock" mockery --name=Downloader --dir=module/executiondatasync/execution_data --case=underscore --output="./module/executiondatasync/execution_data/mock" --outpkg="mock" @@ -246,7 +247,7 @@ ci-integration: $(MAKE) -C integration integration-test # Runs benchmark tests -# NOTE: we do not need `docker-build-flow` as this is run as a separate step +# NOTE: we do not need `docker-native-build-flow` as this is run as a separate step # on Teamcity .PHONY: ci-benchmark ci-benchmark: install-tools @@ -274,166 +275,323 @@ docker-ci-integration: -w "/go/flow" "$(CONTAINER_REGISTRY)/golang-cmake:v0.0.7" \ make ci-integration -.PHONY: docker-build-collection -docker-build-collection: - docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ +# only works on Debian +.SILENT: install-cross-build-tools +install-cross-build-tools: + if [ "$(UNAME)" = "Debian" ] ; then \ + apt-get update && apt-get -y install apt-utils gcc-aarch64-linux-gnu ; \ + elif [ "$(UNAME)" = "Linux" ] ; then \ + apt-get update && apt-get -y install apt-utils gcc-aarch64-linux-gnu ; \ + else \ + echo "this target only works on Debian or Linux, host runs on" $(UNAME) ; \ + fi + +.PHONY: docker-native-build-collection +docker-native-build-collection: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/collection:latest" -t "$(CONTAINER_REGISTRY)/collection:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/collection:latest" \ + -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG)" . + +.PHONY: docker-build-collection-with-adx +docker-build-collection-with-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=amd64 --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG)" \ + -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG)" . + +.PHONY: docker-build-collection-without-adx +docker-build-collection-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_ADX) --build-arg GOARCH=amd64 --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_NO_ADX)" . + +.PHONY: docker-build-collection-without-netgo-without-adx +docker-build-collection-without-netgo-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO_NO_ADX) --build-arg GOARCH=amd64 --build-arg TAGS="" --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_NO_NETGO_NO_ADX)" . -.PHONY: docker-build-collection-without-netgo -docker-build-collection-without-netgo: - docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ +.PHONY: docker-cross-build-collection-arm +docker-cross-build-collection-arm: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_ARM) --build-arg CC=aarch64-linux-gnu-gcc --build-arg GOARCH=arm64 --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ - -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_NO_NETGO)" . + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_ARM)" \ + -t "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_ARM)" . -.PHONY: docker-build-collection-debug -docker-build-collection-debug: +.PHONY: docker-native-build-collection-debug +docker-native-build-collection-debug: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/collection --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target debug \ - -t "$(CONTAINER_REGISTRY)/collection-debug:latest" -t "$(CONTAINER_REGISTRY)/collection-debug:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/collection-debug:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/collection-debug:latest" \ + -t "$(CONTAINER_REGISTRY)/collection-debug:$(IMAGE_TAG)" . -.PHONY: docker-build-consensus -docker-build-consensus: +.PHONY: docker-native-build-consensus +docker-native-build-consensus: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/consensus:latest" -t "$(CONTAINER_REGISTRY)/consensus:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG)" . -.PHONY: docker-build-consensus-without-netgo -docker-build-consensus-without-netgo: - docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ + -t "$(CONTAINER_REGISTRY)/consensus:latest" \ + -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG)" . + +.PHONY: docker-build-consensus-with-adx +docker-build-consensus-with-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=amd64 --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG)" \ + -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG)" . + +.PHONY: docker-build-consensus-without-adx +docker-build-consensus-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_ADX) --build-arg GOARCH=amd64 --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_NO_ADX)" . + +.PHONY: docker-build-consensus-without-netgo-without-adx +docker-build-consensus-without-netgo-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO_NO_ADX) --build-arg GOARCH=amd64 --build-arg TAGS="" --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_NO_NETGO_NO_ADX)" . + +.PHONY: docker-cross-build-consensus-arm +docker-cross-build-consensus-arm: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_ARM) --build-arg GOARCH=arm64 --build-arg CC=aarch64-linux-gnu-gcc --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ - -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_NO_NETGO)" . + --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG_ARM}" \ + -t "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_ARM)" . -.PHONY: docker-build-consensus-debug -docker-build-consensus-debug: + +.PHONY: docker-native-build-consensus-debug +docker-build-native-consensus-debug: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/consensus --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target debug \ - -t "$(CONTAINER_REGISTRY)/consensus-debug:latest" -t "$(CONTAINER_REGISTRY)/consensus-debug:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/consensus-debug:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/consensus-debug:latest" \ + -t "$(CONTAINER_REGISTRY)/consensus-debug:$(IMAGE_TAG)" . -.PHONY: docker-build-execution -docker-build-execution: +.PHONY: docker-native-build-execution +docker-native-build-execution: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/execution:latest" -t "$(CONTAINER_REGISTRY)/execution:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/execution:latest" \ + -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG)" . + +.PHONY: docker-build-execution-with-adx +docker-build-execution-with-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=amd64 --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG)" \ + -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG)" . + +.PHONY: docker-build-execution-without-adx +docker-build-execution-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_ADX) --build-arg GOARCH=amd64 --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_NO_ADX)" . + +.PHONY: docker-build-execution-without-netgo-without-adx +docker-build-execution-without-netgo-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO_NO_ADX) --build-arg GOARCH=amd64 --build-arg TAGS="" --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_NO_NETGO_NO_ADX)" . -.PHONY: docker-build-execution-without-netgo -docker-build-execution-without-netgo: - docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ +.PHONY: docker-cross-build-execution-arm +docker-cross-build-execution-arm: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_ARM) --build-arg GOARCH=arm64 --build-arg CC=aarch64-linux-gnu-gcc --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ - -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_NO_NETGO)" . + --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG_ARM}" \ + -t "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_ARM)" . -.PHONY: docker-build-execution-debug -docker-build-execution-debug: +.PHONY: docker-native-build-execution-debug +docker-native-build-execution-debug: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target debug \ - -t "$(CONTAINER_REGISTRY)/execution-debug:latest" -t "$(CONTAINER_REGISTRY)/execution-debug:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/execution-debug:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/execution-debug:latest" \ + -t "$(CONTAINER_REGISTRY)/execution-debug:$(IMAGE_TAG)" . # build corrupt execution node for BFT testing -.PHONY: docker-build-execution-corrupt -docker-build-execution-corrupt: +.PHONY: docker-native-build-execution-corrupt +docker-native-build-execution-corrupt: # temporarily make insecure/ a non-module to allow Docker to use corrupt builders there ./insecure/cmd/mods_override.sh docker build -f cmd/Dockerfile --build-arg TARGET=./insecure/cmd/execution --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - -t "$(CONTAINER_REGISTRY)/execution-corrupted:latest" -t "$(CONTAINER_REGISTRY)/execution-corrupted:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/execution-corrupted:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/execution-corrupted:latest" \ + -t "$(CONTAINER_REGISTRY)/execution-corrupted:$(IMAGE_TAG)" . ./insecure/cmd/mods_restore.sh -.PHONY: docker-build-verification -docker-build-verification: +.PHONY: docker-native-build-verification +docker-native-build-verification: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/verification:latest" -t "$(CONTAINER_REGISTRY)/verification:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/verification:latest" \ + -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG)" . -.PHONY: docker-build-verification-without-netgo -docker-build-verification-without-netgo: - docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ +.PHONY: docker-build-verification-with-adx +docker-build-verification-with-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=amd64 --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ - -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_NO_NETGO)" . + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG)" \ + -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG)" . -.PHONY: docker-build-verification-debug -docker-build-verification-debug: +.PHONY: docker-build-verification-without-adx +docker-build-verification-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_ADX) --build-arg GOARCH=amd64 --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_NO_ADX)" . + +.PHONY: docker-build-verification-without-netgo-without-adx +docker-build-verification-without-netgo-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO_NO_ADX) --build-arg GOARCH=amd64 --build-arg TAGS="" --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_NO_NETGO_NO_ADX)" . + +.PHONY: docker-cross-build-verification-arm +docker-cross-build-verification-arm: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_ARM) --build-arg GOARCH=arm64 --build-arg CC=aarch64-linux-gnu-gcc --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG_ARM}" \ + -t "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_ARM)" . + +.PHONY: docker-native-build-verification-debug +docker-native-build-verification-debug: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target debug \ - -t "$(CONTAINER_REGISTRY)/verification-debug:latest" -t "$(CONTAINER_REGISTRY)/verification-debug:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/verification-debug:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/verification-debug:latest" \ + -t "$(CONTAINER_REGISTRY)/verification-debug:$(IMAGE_TAG)" . # build corrupt verification node for BFT testing -.PHONY: docker-build-verification-corrupt -docker-build-verification-corrupt: +.PHONY: docker-native-build-verification-corrupt +docker-native-build-verification-corrupt: # temporarily make insecure/ a non-module to allow Docker to use corrupt builders there ./insecure/cmd/mods_override.sh docker build -f cmd/Dockerfile --build-arg TARGET=./insecure/cmd/verification --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - -t "$(CONTAINER_REGISTRY)/verification-corrupted:latest" -t "$(CONTAINER_REGISTRY)/verification-corrupted:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/verification-corrupted:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/verification-corrupted:latest" \ + -t "$(CONTAINER_REGISTRY)/verification-corrupted:$(IMAGE_TAG)" . ./insecure/cmd/mods_restore.sh -.PHONY: docker-build-access -docker-build-access: +.PHONY: docker-native-build-access +docker-native-build-access: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - -t "$(CONTAINER_REGISTRY)/access:latest" -t "$(CONTAINER_REGISTRY)/access:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/access:latest" \ + -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG)" . + +.PHONY: docker-build-access-with-adx +docker-build-access-with-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=amd64 --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG)" \ + -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG)" . + +.PHONY: docker-build-access-without-adx +docker-build-access-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_ADX) --build-arg GOARCH=amd64 --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_NO_ADX)" . -.PHONY: docker-build-access-without-netgo -docker-build-access-without-netgo: - docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ +.PHONY: docker-build-access-without-netgo-without-adx +docker-build-access-without-netgo-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO_NO_ADX) --build-arg GOARCH=amd64 --build-arg TAGS="" --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ - -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_NO_NETGO)" . + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_NO_NETGO_NO_ADX)" . -.PHONY: docker-build-access-debug -docker-build-access-debug: +.PHONY: docker-cross-build-access-arm +docker-cross-build-access-arm: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_ARM) --build-arg GOARCH=arm64 --build-arg CC=aarch64-linux-gnu-gcc --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG_ARM}" \ + -t "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_ARM)" . + + +.PHONY: docker-native-build-access-debug +docker-native-build-access-debug: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target debug \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - -t "$(CONTAINER_REGISTRY)/access-debug:latest" -t "$(CONTAINER_REGISTRY)/access-debug:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/access-debug:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/access-debug:latest" \ + -t "$(CONTAINER_REGISTRY)/access-debug:$(IMAGE_TAG)" . # build corrupt access node for BFT testing -.PHONY: docker-build-access-corrupt -docker-build-access-corrupt: +.PHONY: docker-native-build-access-corrupt +docker-native-build-access-corrupt: #temporarily make insecure/ a non-module to allow Docker to use corrupt builders there ./insecure/cmd/mods_override.sh docker build -f cmd/Dockerfile --build-arg TARGET=./insecure/cmd/access --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/access-corrupted:latest" -t "$(CONTAINER_REGISTRY)/access-corrupted:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/access-corrupted:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/access-corrupted:latest" \ + -t "$(CONTAINER_REGISTRY)/access-corrupted:$(IMAGE_TAG)" . ./insecure/cmd/mods_restore.sh -.PHONY: docker-build-observer -docker-build-observer: +.PHONY: docker-native-build-observer +docker-native-build-observer: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/observer:latest" -t "$(CONTAINER_REGISTRY)/observer:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/observer:latest" \ + -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG)" . + +.PHONY: docker-build-observer-with-adx +docker-build-observer-with-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=amd64 --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG)" \ + -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG)" . + +.PHONY: docker-build-observer-without-adx +docker-build-observer-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_ADX) --build-arg GOARCH=amd64 --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_NO_ADX)" . -.PHONY: docker-build-observer-without-netgo -docker-build-observer-without-netgo: - docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ +.PHONY: docker-build-observer-without-netgo-without-adx +docker-build-observer-without-netgo-without-adx: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_NO_NETGO_NO_ADX) --build-arg GOARCH=amd64 --build-arg TAGS="" --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO)" \ - -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_NO_NETGO)" . + --label "git_commit=${COMMIT}" --label "git_tag=$(IMAGE_TAG_NO_NETGO_NO_ADX)" \ + -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_NO_NETGO_NO_ADX)" . +.PHONY: docker-cross-build-observer-arm +docker-cross-build-observer-arm: + docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG_ARM) --build-arg GOARCH=arm64 --build-arg CC=aarch64-linux-gnu-gcc --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ + --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG_ARM}" \ + -t "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_ARM)" . -.PHONY: docker-build-ghost -docker-build-ghost: + +.PHONY: docker-native-build-ghost +docker-native-build-ghost: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/ghost --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ - -t "$(CONTAINER_REGISTRY)/ghost:latest" -t "$(CONTAINER_REGISTRY)/ghost:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/ghost:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/ghost:latest" \ + -t "$(CONTAINER_REGISTRY)/ghost:$(IMAGE_TAG)" . -.PHONY: docker-build-ghost-debug -docker-build-ghost-debug: +.PHONY: docker-native-build-ghost-debug +docker-native-build-ghost-debug: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/ghost --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target debug \ - -t "$(CONTAINER_REGISTRY)/ghost-debug:latest" -t "$(CONTAINER_REGISTRY)/ghost-debug:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/ghost-debug:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/ghost-debug:latest" \ + -t "$(CONTAINER_REGISTRY)/ghost-debug:$(IMAGE_TAG)" . PHONY: docker-build-bootstrap docker-build-bootstrap: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/bootstrap --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/bootstrap:latest" -t "$(CONTAINER_REGISTRY)/bootstrap:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/bootstrap:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/bootstrap:latest" \ + -t "$(CONTAINER_REGISTRY)/bootstrap:$(IMAGE_TAG)" . PHONY: tool-bootstrap tool-bootstrap: docker-build-bootstrap @@ -443,120 +601,171 @@ tool-bootstrap: docker-build-bootstrap docker-build-bootstrap-transit: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/bootstrap/transit --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(VERSION) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --no-cache \ --target production \ - -t "$(CONTAINER_REGISTRY)/bootstrap-transit:latest" -t "$(CONTAINER_REGISTRY)/bootstrap-transit:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/bootstrap-transit:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/bootstrap-transit:latest" \ + -t "$(CONTAINER_REGISTRY)/bootstrap-transit:$(IMAGE_TAG)" . PHONY: tool-transit tool-transit: docker-build-bootstrap-transit docker container create --name transit $(CONTAINER_REGISTRY)/bootstrap-transit:latest;docker container cp transit:/bin/app ./transit;docker container rm transit -.PHONY: docker-build-loader -docker-build-loader: +.PHONY: docker-native-build-loader +docker-native-build-loader: docker build -f ./integration/benchmark/cmd/manual/Dockerfile --build-arg TARGET=./benchmark/cmd/manual --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ --label "git_commit=${COMMIT}" --label "git_tag=${IMAGE_TAG}" \ - -t "$(CONTAINER_REGISTRY)/loader:latest" -t "$(CONTAINER_REGISTRY)/loader:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/loader:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/loader:latest" \ + -t "$(CONTAINER_REGISTRY)/loader:$(IMAGE_TAG)" . + +.PHONY: docker-native-build-flow +docker-native-build-flow: docker-native-build-collection docker-native-build-consensus docker-native-build-execution docker-native-build-verification docker-native-build-access docker-native-build-observer docker-native-build-ghost + +.PHONY: docker-build-flow-with-adx +docker-build-flow-with-adx: docker-build-collection-with-adx docker-build-consensus-with-adx docker-build-execution-with-adx docker-build-verification-with-adx docker-build-access-with-adx docker-build-observer-with-adx -.PHONY: docker-build-flow -docker-build-flow: docker-build-collection docker-build-consensus docker-build-execution docker-build-verification docker-build-access docker-build-observer docker-build-ghost +.PHONY: docker-build-flow-without-adx +docker-build-flow-without-adx: docker-build-collection-without-adx docker-build-consensus-without-adx docker-build-execution-without-adx docker-build-verification-without-adx docker-build-access-without-adx docker-build-observer-without-adx -.PHONY: docker-build-flow-without-netgo -docker-build-flow-without-netgo: docker-build-collection-without-netgo docker-build-consensus-without-netgo docker-build-execution-without-netgo docker-build-verification-without-netgo docker-build-access-without-netgo docker-build-observer-without-netgo +.PHONY: docker-build-flow-without-netgo-without-adx +docker-build-flow-without-netgo-without-adx: docker-build-collection-without-netgo-without-adx docker-build-consensus-without-netgo-without-adx docker-build-execution-without-netgo-without-adx docker-build-verification-without-netgo-without-adx docker-build-access-without-netgo-without-adx docker-build-observer-without-netgo-without-adx -.PHONY: docker-build-flow-corrupt -docker-build-flow-corrupt: docker-build-execution-corrupt docker-build-verification-corrupt docker-build-access-corrupt +# in this target, images are arm64 (aarch64), are build with `netgo` and with `adx`. +# other arm64 images can be built without `netgo` or without `adx` +.PHONY: docker-cross-build-flow-arm +docker-cross-build-flow-arm: docker-cross-build-collection-arm docker-cross-build-consensus-arm docker-cross-build-execution-arm docker-cross-build-verification-arm docker-cross-build-access-arm docker-cross-build-observer-arm -.PHONY: docker-build-benchnet -docker-build-benchnet: docker-build-flow docker-build-loader +.PHONY: docker-native-build-flow-corrupt +docker-native-build-flow-corrupt: docker-native-build-execution-corrupt docker-native-build-verification-corrupt docker-native-build-access-corrupt -.PHONY: docker-push-collection -docker-push-collection: - docker push "$(CONTAINER_REGISTRY)/collection:$(SHORT_COMMIT)" +.PHONY: docker-native-build-benchnet +docker-native-build-benchnet: docker-native-build-flow docker-native-build-loader + +.PHONY: docker-push-collection-with-adx +docker-push-collection-with-adx: docker push "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG)" -.PHONY: docker-push-collection-without-netgo -docker-push-collection-without-netgo: - docker push "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_NO_NETGO)" +.PHONY: docker-push-collection-without-adx +docker-push-collection-without-adx: + docker push "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_NO_ADX)" + +.PHONY: docker-push-collection-without-netgo-without-adx +docker-push-collection-without-netgo-without-adx: + docker push "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_NO_NETGO_NO_ADX)" + +.PHONY: docker-push-collection-arm +docker-push-collection-arm: + docker push "$(CONTAINER_REGISTRY)/collection:$(IMAGE_TAG_ARM)" .PHONY: docker-push-collection-latest docker-push-collection-latest: docker-push-collection docker push "$(CONTAINER_REGISTRY)/collection:latest" -.PHONY: docker-push-consensus -docker-push-consensus: - docker push "$(CONTAINER_REGISTRY)/consensus:$(SHORT_COMMIT)" +.PHONY: docker-push-consensus-with-adx +docker-push-consensus-with-adx: docker push "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG)" -.PHONY: docker-push-consensus-without-netgo -docker-push-consensus-without-netgo: - docker push "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_NO_NETGO)" +.PHONY: docker-push-consensus-without-adx +docker-push-consensus-without-adx: + docker push "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_NO_ADX)" + +.PHONY: docker-push-consensus-without-netgo-without-adx +docker-push-consensus-without-netgo-without-adx: + docker push "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_NO_NETGO_NO_ADX)" + +.PHONY: docker-push-consensus-arm +docker-push-consensus-arm: + docker push "$(CONTAINER_REGISTRY)/consensus:$(IMAGE_TAG_ARM)" .PHONY: docker-push-consensus-latest docker-push-consensus-latest: docker-push-consensus docker push "$(CONTAINER_REGISTRY)/consensus:latest" -.PHONY: docker-push-execution -docker-push-execution: - docker push "$(CONTAINER_REGISTRY)/execution:$(SHORT_COMMIT)" +.PHONY: docker-push-execution-with-adx +docker-push-execution-with-adx: docker push "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG)" .PHONY: docker-push-execution-corrupt docker-push-execution-corrupt: - docker push "$(CONTAINER_REGISTRY)/execution-corrupted:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/execution-corrupted:$(IMAGE_TAG)" +.PHONY: docker-push-execution-without-adx +docker-push-execution-without-adx: + docker push "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_NO_ADX)" + +.PHONY: docker-push-execution-without-netgo-without-adx +docker-push-execution-without-netgo-without-adx: + docker push "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_NO_NETGO_NO_ADX)" -.PHONY: docker-push-execution-without-netgo -docker-push-execution-without-netgo: - docker push "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_NO_NETGO)" +.PHONY: docker-push-execution-arm +docker-push-execution-arm: + docker push "$(CONTAINER_REGISTRY)/execution:$(IMAGE_TAG_ARM)" .PHONY: docker-push-execution-latest docker-push-execution-latest: docker-push-execution docker push "$(CONTAINER_REGISTRY)/execution:latest" -.PHONY: docker-push-verification -docker-push-verification: - docker push "$(CONTAINER_REGISTRY)/verification:$(SHORT_COMMIT)" +.PHONY: docker-push-verification-with-adx +docker-push-verification-with-adx: docker push "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG)" +.PHONY: docker-push-verification-without-adx +docker-push-verification-without-adx: + docker push "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_NO_ADX)" + .PHONY: docker-push-verification-corrupt docker-push-verification-corrupt: - docker push "$(CONTAINER_REGISTRY)/verification-corrupted:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/verification-corrupted:$(IMAGE_TAG)" -.PHONY: docker-push-verification-without-netgo -docker-push-verification-without-netgo: - docker push "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_NO_NETGO)" +.PHONY: docker-push-verification-without-netgo-without-adx +docker-push-verification-without-netgo-without-adx: + docker push "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_NO_NETGO_NO_ADX)" + +.PHONY: docker-push-verification-arm +docker-push-verification-arm: + docker push "$(CONTAINER_REGISTRY)/verification:$(IMAGE_TAG_ARM)" .PHONY: docker-push-verification-latest docker-push-verification-latest: docker-push-verification docker push "$(CONTAINER_REGISTRY)/verification:latest" -.PHONY: docker-push-access -docker-push-access: - docker push "$(CONTAINER_REGISTRY)/access:$(SHORT_COMMIT)" +.PHONY: docker-push-access-with-adx +docker-push-access-with-adx: docker push "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG)" +.PHONY: docker-push-access-without-adx +docker-push-access-without-adx: + docker push "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_NO_ADX)" + .PHONY: docker-push-access-corrupt docker-push-access-corrupt: - docker push "$(CONTAINER_REGISTRY)/access-corrupted:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/access-corrupted:$(IMAGE_TAG)" -.PHONY: docker-push-access-without-netgo -docker-push-access-without-netgo: - docker push "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_NO_NETGO)" +.PHONY: docker-push-access-without-netgo-without-adx +docker-push-access-without-netgo-without-adx: + docker push "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_NO_NETGO_NO_ADX)" + +.PHONY: docker-push-access-arm +docker-push-access-arm: + docker push "$(CONTAINER_REGISTRY)/access:$(IMAGE_TAG_ARM)" .PHONY: docker-push-access-latest docker-push-access-latest: docker-push-access docker push "$(CONTAINER_REGISTRY)/access:latest" -.PHONY: docker-push-observer -docker-push-observer: - docker push "$(CONTAINER_REGISTRY)/observer:$(SHORT_COMMIT)" +.PHONY: docker-push-observer-with-adx +docker-push-observer-with-adx: docker push "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG)" -.PHONY: docker-push-observer-without-netgo -docker-push-observer-without-netgo: - docker push "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_NO_NETGO)" +.PHONY: docker-push-observer-without-adx +docker-push-observer-without-adx: + docker push "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_NO_ADX)" + +.PHONY: docker-push-observer-without-netgo-without-adx +docker-push-observer-without-netgo-without-adx: + docker push "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_NO_NETGO_NO_ADX)" + +.PHONY: docker-push-observer-arm +docker-push-observer-arm: + docker push "$(CONTAINER_REGISTRY)/observer:$(IMAGE_TAG_ARM)" .PHONY: docker-push-observer-latest docker-push-observer-latest: docker-push-observer @@ -564,7 +773,6 @@ docker-push-observer-latest: docker-push-observer .PHONY: docker-push-ghost docker-push-ghost: - docker push "$(CONTAINER_REGISTRY)/ghost:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/ghost:$(IMAGE_TAG)" .PHONY: docker-push-ghost-latest @@ -573,18 +781,23 @@ docker-push-ghost-latest: docker-push-ghost .PHONY: docker-push-loader docker-push-loader: - docker push "$(CONTAINER_REGISTRY)/loader:$(SHORT_COMMIT)" docker push "$(CONTAINER_REGISTRY)/loader:$(IMAGE_TAG)" .PHONY: docker-push-loader-latest docker-push-loader-latest: docker-push-loader docker push "$(CONTAINER_REGISTRY)/loader:latest" -.PHONY: docker-push-flow -docker-push-flow: docker-push-collection docker-push-consensus docker-push-execution docker-push-verification docker-push-access docker-push-observer +.PHONY: docker-push-flow-with-adx +docker-push-flow-with-adx: docker-push-collection-with-adx docker-push-consensus-with-adx docker-push-execution-with-adx docker-push-verification-with-adx docker-push-access-with-adx docker-push-observer-with-adx + +.PHONY: docker-push-flow-without-adx +docker-push-flow-without-adx: docker-push-collection-without-adx docker-push-consensus-without-adx docker-push-execution-without-adx docker-push-verification-without-adx docker-push-access-without-adx docker-push-observer-without-adx + +.PHONY: docker-push-flow-without-netgo-without-adx +docker-push-flow-without-netgo-without-adx: docker-push-collection-without-netgo-without-adx docker-push-consensus-without-netgo-without-adx docker-push-execution-without-netgo-without-adx docker-push-verification-without-netgo-without-adx docker-push-access-without-netgo-without-adx docker-push-observer-without-netgo-without-adx -.PHONY: docker-push-flow-without-netgo -docker-push-flow-without-netgo: docker-push-collection-without-netgo docker-push-consensus-without-netgo docker-push-execution-without-netgo docker-push-verification-without-netgo docker-push-access-without-netgo docker-push-observer-without-netgo +.PHONY: docker-push-flow-arm +docker-push-flow-arm: docker-push-collection-arm docker-push-consensus-arm docker-push-execution-arm docker-push-verification-arm docker-push-access-arm docker-push-observer-arm .PHONY: docker-push-flow-latest docker-push-flow-latest: docker-push-collection-latest docker-push-consensus-latest docker-push-execution-latest docker-push-verification-latest docker-push-access-latest docker-push-observer-latest @@ -632,7 +845,8 @@ docker-all-tools: tool-util tool-remove-execution-fork PHONY: docker-build-util docker-build-util: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ - -t "$(CONTAINER_REGISTRY)/util:latest" -t "$(CONTAINER_REGISTRY)/util:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/util:latest" \ + -t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" . PHONY: tool-util tool-util: docker-build-util @@ -641,7 +855,8 @@ tool-util: docker-build-util PHONY: docker-build-remove-execution-fork docker-build-remove-execution-fork: docker build -f cmd/Dockerfile --ssh default --build-arg TARGET=./cmd/util/cmd/remove-execution-fork --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \ - -t "$(CONTAINER_REGISTRY)/remove-execution-fork:latest" -t "$(CONTAINER_REGISTRY)/remove-execution-fork:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/remove-execution-fork:$(IMAGE_TAG)" . + -t "$(CONTAINER_REGISTRY)/remove-execution-fork:latest" \ + -t "$(CONTAINER_REGISTRY)/remove-execution-fork:$(IMAGE_TAG)" . PHONY: tool-remove-execution-fork tool-remove-execution-fork: docker-build-remove-execution-fork diff --git a/README.md b/README.md index 291e45de347..4bdffcd2eb6 100644 --- a/README.md +++ b/README.md @@ -100,13 +100,13 @@ The recommended way to build and run Flow for local development is using Docker. Build a Docker image for all nodes: ```bash -make docker-build-flow +make docker-native-build-flow ``` Build a Docker image for a particular node role (replace `$ROLE` with `collection`, `consensus`, etc.): ```bash -make docker-build-$ROLE +make docker-native-build-$ROLE ``` ### Local Network @@ -140,4 +140,4 @@ Generate mocks used for unit tests: ```bash make generate-mocks -``` +``` \ No newline at end of file diff --git a/cmd/Dockerfile b/cmd/Dockerfile index aa8690c4c6d..0c7a9d58cf9 100644 --- a/cmd/Dockerfile +++ b/cmd/Dockerfile @@ -6,7 +6,7 @@ FROM golang:1.20-bullseye AS build-setup RUN apt-get update -RUN apt-get -y install zip +RUN apt-get -y install zip apt-utils gcc-aarch64-linux-gnu ## (2) Setup crypto dependencies FROM build-setup AS build-env @@ -27,7 +27,7 @@ COPY . . RUN git config --global url.git@github.com:.insteadOf https://github.com/ RUN mkdir ~/.ssh -# Add GitHub known host +# Add GitHub known host to avoid prompts or failures on key check RUN ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts #################################### @@ -36,19 +36,22 @@ FROM build-env as build-production WORKDIR /app ARG GOARCH=amd64 - # TAGS can be overriden to modify the go build tags (e.g. build without netgo) ARG TAGS="netgo" -# CGO_FLAG can be overwritten -ARG CGO_FLAG +# CC flag can be overwritten to specify a C compiler +ARG CC="" +# CGO_FLAG uses ADX instructions by default, flag can be overwritten to build without ADX +ARG CGO_FLAG="" # Keep Go's build cache between builds. # https://github.com/golang/go/issues/27719#issuecomment-514747274 RUN --mount=type=cache,sharing=locked,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=secret,id=cadence_deploy_key \ + # We evaluate the SSH agent to safely pass in a key for cloning dependencies + # We explicitly use ";" rather than && as we want to safely pass if it is unavailable eval `ssh-agent -s` && printf "%s\n" "$(cat /run/secrets/cadence_deploy_key)" | ssh-add - ; \ - CGO_ENABLED=1 GOOS=linux CGO_FLAGS="${CGO_FLAG}" go build --tags "${TAGS}" -ldflags "-extldflags -static \ + CGO_ENABLED=1 GOOS=linux GOARCH=${GOARCH} CC="${CC}" CGO_FLAGS="${CGO_FLAG}" go build --tags "${TAGS}" -ldflags "-extldflags -static \ -X 'github.com/onflow/flow-go/cmd/build.commit=${COMMIT}' -X 'github.com/onflow/flow-go/cmd/build.semver=${VERSION}'" \ -o ./app ${TARGET} @@ -66,13 +69,16 @@ ENTRYPOINT ["/bin/app"] FROM build-env as build-debug WORKDIR /app ARG GOARCH=amd64 - +ARG CC="" +ARG CGO_FLAG="" RUN --mount=type=ssh \ --mount=type=cache,sharing=locked,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=secret,id=cadence_deploy_key \ + # We evaluate the SSH agent to safely pass in a key for cloning dependencies + # We explicitly use ";" rather than && as we want to safely pass if it is unavailable eval `ssh-agent -s` && printf "%s\n" "$(cat /run/secrets/cadence_deploy_key)" | ssh-add - ; \ - CGO_ENABLED=1 GOOS=linux CGO_FLAGS="${CGO_FLAG}" go build --tags "netgo" -ldflags "-extldflags -static \ + CGO_ENABLED=1 GOOS=linux GOARCH=${GOARCH} CC="${CC}" CGO_FLAGS="${CGO_FLAG}" go build --tags "netgo" -ldflags "-extldflags -static \ -X 'github.com/onflow/flow-go/cmd/build.commit=${COMMIT}' -X 'github.com/onflow/flow-go/cmd/build.semver=${VERSION}'" \ -gcflags="all=-N -l" -o ./app ${TARGET} @@ -88,4 +94,4 @@ COPY --from=build-debug /app/app /bin/app ENTRYPOINT ["dlv", "--listen=:2345", "--headless=true", "--api-version=2", "--accept-multiclient", "exec", "/bin/app", "--"] -FROM build-setup as environment-clean +FROM build-setup as environment-clean \ No newline at end of file diff --git a/crypto_adx_flag.mk b/crypto_adx_flag.mk index 277a4c3fbb4..0d0d5ac7467 100644 --- a/crypto_adx_flag.mk +++ b/crypto_adx_flag.mk @@ -1,5 +1,7 @@ -# This script can be imported by Makefiles in order to set the `CRYPTO_FLAG` automatically. -# The `CRYPTO_FLAG` is a Go command flag that should be used when the machine's CPU executing +# This script can be imported by Makefiles in order to set the `CRYPTO_FLAG` automatically for +# a native build (build and run on the same machine NOT for cross-compilation). +# +# The `CRYPTO_FLAG` is a Go command flag that should be used when the target machine's CPU executing # the command may not support ADX instructions. # For new machines that support ADX instructions, the `CRYPTO_FLAG` flag is not needed (or set # to an empty string). @@ -14,6 +16,8 @@ else ADX_SUPPORT := 1 endif +DISABLE_ADX := "-O2 -D__BLST_PORTABLE__" + # Then, set `CRYPTO_FLAG` # the crypto package uses BLST source files underneath which may use ADX instructions. ifeq ($(ADX_SUPPORT), 1) @@ -21,5 +25,5 @@ ifeq ($(ADX_SUPPORT), 1) CRYPTO_FLAG := "" else # if ADX instructions aren't supported, this CGO flags uses a slower non-ADX implementation - CRYPTO_FLAG := "-O -D__BLST_PORTABLE__" + CRYPTO_FLAG := $(DISABLE_ADX) endif \ No newline at end of file diff --git a/engine/execution/ingestion/stop/stop_control_test.go b/engine/execution/ingestion/stop/stop_control_test.go index 12900d56dad..e44fca75076 100644 --- a/engine/execution/ingestion/stop/stop_control_test.go +++ b/engine/execution/ingestion/stop/stop_control_test.go @@ -865,9 +865,11 @@ func Test_StopControlWorkers(t *testing.T) { func TestPatchedVersion(t *testing.T) { require.True(t, semver.New("0.31.20").LessThan(*semver.New("0.31.21"))) require.True(t, semver.New("0.31.20-patch.1").LessThan(*semver.New("0.31.20"))) // be careful with this one - require.True(t, semver.New("0.31.20-without-netgo").LessThan(*semver.New("0.31.20"))) + require.True(t, semver.New("0.31.20-without-adx").LessThan(*semver.New("0.31.20"))) // a special build created with "+" would not change the version priority for standard and pre-release versions - require.True(t, semver.New("0.31.20+without-netgo").Equal(*semver.New("0.31.20"))) - require.True(t, semver.New("0.31.20-patch.1+without-netgo").Equal(*semver.New("0.31.20-patch.1"))) -} + require.True(t, semver.New("0.31.20+without-adx").Equal(*semver.New("0.31.20"))) + require.True(t, semver.New("0.31.20-patch.1+without-adx").Equal(*semver.New("0.31.20-patch.1"))) + require.True(t, semver.New("0.31.20+without-netgo-without-adx").Equal(*semver.New("0.31.20"))) + require.True(t, semver.New("0.31.20+arm").Equal(*semver.New("0.31.20"))) +} \ No newline at end of file diff --git a/integration/README.md b/integration/README.md index b6b59f4fa82..0ac97ffd831 100644 --- a/integration/README.md +++ b/integration/README.md @@ -14,15 +14,15 @@ Since the test cases run docker instances as a network of nodes, we need to ensu To ensure the latest docker images have been built, you can run: ``` -make docker-build-access -make docker-build-collection -make docker-build-consensus -make docker-build-execution -make docker-build-verification -make docker-build-ghost +make docker-native-build-access +make docker-native-build-collection +make docker-native-build-consensus +make docker-native-build-execution +make docker-native-build-verification +make docker-native-build-ghost ``` -Or simply run `make docker-build-flow` +Or simply run `make docker-native-build-flow` After images have been built, we can run the integration tests: ``` @@ -65,11 +65,11 @@ Because launching a full execution node in the consensus integration tests will ### Rebuild image when debugging During test cases debugging, you might want to update some code. However, if you run `make integration-test` after updating the code, the new change will not be included, because the integration tests still use the old code from the docker image, which was built before adding the changes. -So you need to rebuild all the images by running `make docker-build-flow` again before re-running the integration tests. +So you need to rebuild all the images by running `make docker-native-build-flow` again before re-running the integration tests. Rebuilding all images takes quite some time, here is a shortcut: -If consensus's code was changed, then only consensus's image need to be rebuilt, so simply run `make docker-build-consensus` instead of rebuilding all the images. +If consensus's code was changed, then only consensus's image need to be rebuilt, so simply run `make docker-native-build-consensus` instead of rebuilding all the images. ### Organization @@ -81,4 +81,4 @@ in the Makefile. To send random transactions, for example to load test a network, run `cd integration/localnet; make load`. -In order to build a docker container with the benchmarking binary, run `make docker-build-loader` from the root of this repository. +In order to build a docker container with the benchmarking binary, run `make docker-native-build-loader` from the root of this repository. \ No newline at end of file From c38fd1d0c2b721dab598393eddbde9bc0ae763f7 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 7 Feb 2024 16:21:56 +0100 Subject: [PATCH 2/2] minor cleanup --- .github/workflows/ci.yml | 4 +--- .github/workflows/flaky-test-monitor.yml | 2 +- README.md | 2 +- cmd/Dockerfile | 2 +- crypto_adx_flag.mk | 2 +- engine/execution/ingestion/stop/stop_control_test.go | 2 +- integration/README.md | 2 +- 7 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de23039a4bb..8f783f596df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,8 +174,6 @@ jobs: docker-build: name: Docker Build runs-on: buildjet-16vcpu-ubuntu-2204 - env: - CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} steps: - name: Checkout repo uses: actions/checkout@v3 @@ -297,4 +295,4 @@ jobs: with: timeout_minutes: 35 max_attempts: 5 - command: VERBOSE=1 ${{ matrix.make }} \ No newline at end of file + command: VERBOSE=1 ${{ matrix.make }} diff --git a/.github/workflows/flaky-test-monitor.yml b/.github/workflows/flaky-test-monitor.yml index 5073c2b1b01..146408c67ee 100644 --- a/.github/workflows/flaky-test-monitor.yml +++ b/.github/workflows/flaky-test-monitor.yml @@ -175,4 +175,4 @@ jobs: TEST_CATEGORY: ${{ matrix.test_category }} uses: ./.github/workflows/actions/test-monitor-process-results with: - gcp_sa_key: ${{ secrets.GCP_SA_KEY }} \ No newline at end of file + gcp_sa_key: ${{ secrets.GCP_SA_KEY }} diff --git a/README.md b/README.md index 4bdffcd2eb6..7a30d00ab1c 100644 --- a/README.md +++ b/README.md @@ -140,4 +140,4 @@ Generate mocks used for unit tests: ```bash make generate-mocks -``` \ No newline at end of file +``` diff --git a/cmd/Dockerfile b/cmd/Dockerfile index 0c7a9d58cf9..78b24a4a051 100644 --- a/cmd/Dockerfile +++ b/cmd/Dockerfile @@ -94,4 +94,4 @@ COPY --from=build-debug /app/app /bin/app ENTRYPOINT ["dlv", "--listen=:2345", "--headless=true", "--api-version=2", "--accept-multiclient", "exec", "/bin/app", "--"] -FROM build-setup as environment-clean \ No newline at end of file +FROM build-setup as environment-clean diff --git a/crypto_adx_flag.mk b/crypto_adx_flag.mk index 0d0d5ac7467..5ef51589e34 100644 --- a/crypto_adx_flag.mk +++ b/crypto_adx_flag.mk @@ -26,4 +26,4 @@ ifeq ($(ADX_SUPPORT), 1) else # if ADX instructions aren't supported, this CGO flags uses a slower non-ADX implementation CRYPTO_FLAG := $(DISABLE_ADX) -endif \ No newline at end of file +endif diff --git a/engine/execution/ingestion/stop/stop_control_test.go b/engine/execution/ingestion/stop/stop_control_test.go index e44fca75076..759c3de8e9a 100644 --- a/engine/execution/ingestion/stop/stop_control_test.go +++ b/engine/execution/ingestion/stop/stop_control_test.go @@ -872,4 +872,4 @@ func TestPatchedVersion(t *testing.T) { require.True(t, semver.New("0.31.20-patch.1+without-adx").Equal(*semver.New("0.31.20-patch.1"))) require.True(t, semver.New("0.31.20+without-netgo-without-adx").Equal(*semver.New("0.31.20"))) require.True(t, semver.New("0.31.20+arm").Equal(*semver.New("0.31.20"))) -} \ No newline at end of file +} diff --git a/integration/README.md b/integration/README.md index 0ac97ffd831..8b479f06477 100644 --- a/integration/README.md +++ b/integration/README.md @@ -81,4 +81,4 @@ in the Makefile. To send random transactions, for example to load test a network, run `cd integration/localnet; make load`. -In order to build a docker container with the benchmarking binary, run `make docker-native-build-loader` from the root of this repository. \ No newline at end of file +In order to build a docker container with the benchmarking binary, run `make docker-native-build-loader` from the root of this repository.