From 58d23efa121d16c190e52895064765313da2678e Mon Sep 17 00:00:00 2001 From: Piotr Grabowski Date: Thu, 13 Jul 2023 14:53:31 +0200 Subject: [PATCH 1/2] CI/Makefile: switch from Docker Compose V1 to V2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per Docker: "From July 2023 Docker Compose V1 stopped receiving updates. It’s also no longer available in new releases of Docker Desktop." Therefore, this patch replaces all uses of Docker Compose V1 ("docker-compose" command) with Docker Compose V2 ("docker compose" command). This change will also allow for using new functionality introduced in Docker Compose V2, such as "--wait" flag. --- .github/workflows/cassandra.yml | 8 ++++---- .github/workflows/rust.yml | 8 ++++---- CONTRIBUTING.md | 6 +++--- Makefile | 2 +- test/cluster/wait.sh | 2 +- test/dockerized/run.sh | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cassandra.yml b/.github/workflows/cassandra.yml index df67fc994..58305af1b 100644 --- a/.github/workflows/cassandra.yml +++ b/.github/workflows/cassandra.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v2 - name: Setup 3-node Cassandra cluster run: | - docker-compose -f test/cluster/cassandra/docker-compose.yml up -d + docker compose -f test/cluster/cassandra/docker-compose.yml up -d # A separate step for building to separate measuring time of compilation and testing - name: Build the project run: cargo build --verbose --tests @@ -28,9 +28,9 @@ jobs: CDC='disabled' SCYLLA_URI=172.42.0.2:9042 SCYLLA_URI2=172.42.0.3:9042 SCYLLA_URI3=172.42.0.4:9042 cargo test --verbose -- --skip test_views_in_schema_info - name: Stop the cluster if: ${{ always() }} - run: docker-compose -f test/cluster/cassandra/docker-compose.yml stop + run: docker compose -f test/cluster/cassandra/docker-compose.yml stop - name: Print the cluster logs if: ${{ always() }} - run: docker-compose -f test/cluster/cassandra/docker-compose.yml logs + run: docker compose -f test/cluster/cassandra/docker-compose.yml logs - name: Remove cluster - run: docker-compose -f test/cluster/cassandra/docker-compose.yml down + run: docker compose -f test/cluster/cassandra/docker-compose.yml down diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5fe4c0320..1e96817e5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,7 +21,7 @@ jobs: - name: Setup 3-node Scylla cluster run: | sudo sh -c "echo 2097152 >> /proc/sys/fs/aio-max-nr" - docker-compose -f test/cluster/docker-compose.yml up -d + docker compose -f test/cluster/docker-compose.yml up -d - name: Format check run: cargo fmt --verbose --all -- --check - name: Clippy check @@ -38,12 +38,12 @@ jobs: run: SCYLLA_URI=172.42.0.2:9042 SCYLLA_URI2=172.42.0.3:9042 SCYLLA_URI3=172.42.0.4:9042 cargo test --verbose - name: Stop the cluster if: ${{ always() }} - run: docker-compose -f test/cluster/docker-compose.yml stop + run: docker compose -f test/cluster/docker-compose.yml stop - name: Print the cluster logs if: ${{ always() }} - run: docker-compose -f test/cluster/docker-compose.yml logs + run: docker compose -f test/cluster/docker-compose.yml logs - name: Remove cluster - run: docker-compose -f test/cluster/docker-compose.yml down + run: docker compose -f test/cluster/docker-compose.yml down # Tests that our current minimum supported rust version compiles everything sucessfully min_rust: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b9056483f..fd2debba3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,10 +31,10 @@ Currently, we require new PRs to compile without warnings, pass `cargo fmt` and ## Testing A 3-node ScyllaDB cluster is required to run the tests. -The simplest way to set it up locally is to use a `docker-compose`. -Fortunately there is no need to invoke `docker-compose` manually, everything can be handled by our `Makefile`. +The simplest way to set it up locally is to use a `docker compose`. +Fortunately there is no need to invoke `docker compose` manually, everything can be handled by our `Makefile`. -To run a cargo test suite, use the command below (note that you must have docker and docker-compose installed): +To run a cargo test suite, use the command below (note that you must have Docker and Docker Compose V2 installed): ```bash make test ``` diff --git a/Makefile b/Makefile index bc5447a18..0bd033433 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -COMPOSE := docker-compose -f test/cluster/docker-compose.yml +COMPOSE := docker compose -f test/cluster/docker-compose.yml .PHONY: all all: test diff --git a/test/cluster/wait.sh b/test/cluster/wait.sh index d0c64108c..43bead6e5 100755 --- a/test/cluster/wait.sh +++ b/test/cluster/wait.sh @@ -5,7 +5,7 @@ set -e echo "Waiting for the cluster to become ready." echo " Inspect 'make logs' output if this step takes too long." -DOCKER_COMPOSE='docker-compose -f test/cluster/docker-compose.yml' +DOCKER_COMPOSE='docker compose -f test/cluster/docker-compose.yml' TEST_QUERY='select * from system.local' for NODE in 'scylla1' 'scylla2' 'scylla3'; do diff --git a/test/dockerized/run.sh b/test/dockerized/run.sh index 86b7aa8b5..54c44ea2c 100755 --- a/test/dockerized/run.sh +++ b/test/dockerized/run.sh @@ -2,7 +2,7 @@ cd "$(dirname "$0")"/../../ -if [[ -z $(docker container ls | grep cluster_scylla) ]]; then +if [[ -z $(docker container ls | grep cluster-scylla) ]]; then echo "Scylla container must be running to execute tests (use 'make up')." exit 1 fi From 31beef080fea61b068aff9220eb4eaaf2350d3a0 Mon Sep 17 00:00:00 2001 From: Piotr Grabowski Date: Thu, 13 Jul 2023 16:08:59 +0200 Subject: [PATCH 2/2] CI/Makefile: use "--wait" in "docker compose up" When using "--wait" flag (introduced in Docker Compose V2) in "docker compose up", Docker waits for all healthchecks to be in "healthy" state. Since docker-compose.yml files in this repository use "cqlsh" to determine the health of a container, this means that Docker will wait for all nodes of the cluster to be fully bootstrapped and their CQL port ready. By using this flag, this fixes a potential problem in CI - previously the code would not wait for the last node to be fully started. Additionally, "wait.sh" script can be deleted (this logic now handled by Docker Compose). --- .github/workflows/cassandra.yml | 2 +- .github/workflows/rust.yml | 2 +- Makefile | 10 +++------- test/cluster/wait.sh | 18 ------------------ 4 files changed, 5 insertions(+), 27 deletions(-) delete mode 100755 test/cluster/wait.sh diff --git a/.github/workflows/cassandra.yml b/.github/workflows/cassandra.yml index 58305af1b..8b0f61b91 100644 --- a/.github/workflows/cassandra.yml +++ b/.github/workflows/cassandra.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v2 - name: Setup 3-node Cassandra cluster run: | - docker compose -f test/cluster/cassandra/docker-compose.yml up -d + docker compose -f test/cluster/cassandra/docker-compose.yml up -d --wait # A separate step for building to separate measuring time of compilation and testing - name: Build the project run: cargo build --verbose --tests diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1e96817e5..0b1a5d09a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,7 +21,7 @@ jobs: - name: Setup 3-node Scylla cluster run: | sudo sh -c "echo 2097152 >> /proc/sys/fs/aio-max-nr" - docker compose -f test/cluster/docker-compose.yml up -d + docker compose -f test/cluster/docker-compose.yml up -d --wait - name: Format check run: cargo fmt --verbose --all -- --check - name: Clippy check diff --git a/Makefile b/Makefile index 0bd033433..7cdaeb2d1 100644 --- a/Makefile +++ b/Makefile @@ -30,14 +30,14 @@ clippy: RUSTFLAGS=-Dwarnings cargo clippy --examples --tests -- -Aclippy::uninlined_format_args .PHONY: test -test: up wait-for-cluster +test: up SCYLLA_URI=172.42.0.2:9042 \ SCYLLA_URI2=172.42.0.3:9042 \ SCYLLA_URI3=172.42.0.4:9042 \ cargo test .PHONY: dockerized-test -dockerized-test: up wait-for-cluster +dockerized-test: up test/dockerized/run.sh .PHONY: build @@ -50,7 +50,7 @@ docs: .PHONY: up up: - $(COMPOSE) up -d + $(COMPOSE) up -d --wait @echo @echo "ScyllaDB cluster is running in the background. Use 'make down' to stop it." @echo @@ -75,7 +75,3 @@ shell: clean: down cargo clean rm -rf docs/book - -.PHONY: wait-for-cluster -wait-for-cluster: - @test/cluster/wait.sh diff --git a/test/cluster/wait.sh b/test/cluster/wait.sh deleted file mode 100755 index 43bead6e5..000000000 --- a/test/cluster/wait.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "Waiting for the cluster to become ready." -echo " Inspect 'make logs' output if this step takes too long." - -DOCKER_COMPOSE='docker compose -f test/cluster/docker-compose.yml' -TEST_QUERY='select * from system.local' - -for NODE in 'scylla1' 'scylla2' 'scylla3'; do - until $DOCKER_COMPOSE exec "$NODE" cqlsh -e "$TEST_QUERY" > /dev/null; do - printf '.'; - sleep 1; - done -done - -echo "Done waiting."