refactor(test): redo integration flow

filecoin-project · Sep 21, 2023 · 10c0c50 · 10c0c50
1 parent a6c644d
commit 10c0c50
Show file tree

Hide file tree

Showing 12 changed files with 91 additions and 51 deletions.
diff --git a/.github/workflows/devnet-test.yml b/.github/workflows/devnet-test.yml
@@ -9,7 +9,7 @@ on:
 jobs:
   build:
     name: Boost Devnet
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-4-cores
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -30,34 +30,10 @@ jobs:
             ${{ runner.os }}-build-${{ env.cache-name }}-
             ${{ runner.os }}-build-
             ${{ runner.os }}-
-      - name: Run Boost Devnet from pre-built images
+      - name: Run Motion Integration Tests
         run: |
-          cd integration/test/devnet
-          docker compose up -d
-      - name: Await Lotus full node startup
-        run: |
-          cd integration/test/devnet
-          docker compose exec lotus lotus wait-api --timeout=20m
-      - name: Set up Motion Wallet and API endpoints
-        run: |
-          ./scripts/integration-setup.sh "${GITHUB_ENV}"
-      - name: Run Motionlarity
-        run: |
-          cd integration/test/motionlarity
-          docker compose up -d
-      - name: Wait until motion is running
-        uses: nick-fields/retry@v2
-        with:
-          timeout_seconds: 5
-          max_attempts: 10
-          retry_on: error
-          command: |
-            cd integration/test/motionlarity
-            docker compose ps --services --filter "status=running" | grep motion
-      - name: Run Motion integration tests
-        env:
-          MOTION_INTEGRATION_TEST: 'true'
-        run: go test ./integration/test -v
+          cd integration/test
+          make test
       - name: Start S3 Connector
         run: |
           cd integration/test/s3-connector

diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,7 @@
 /.env.local
 integration/test/devnet/data
 motion
+integration/test/motionlarity/.env.local
+integration/test/motionlarity/.up
+integration/test/devnet/.up
+integration/test/devnet/.boostready
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -23,8 +23,6 @@ services:
     command: admin init
     volumes:
       - motion-singularity-volume:/usr/src/app/storage
-    ports:
-      - 9090:9090
     environment:
       DATABASE_CONNECTION_STRING: postgres://${SINGULARITY_DB_USER:-postgres}:${SINGULARITY_DB_PASSWORD:-postgres}@db:5432/${SINGULARITY_DB_NAME:-singularity}
       LOTUS_TEST: 

diff --git a/integration/singularity/store.go b/integration/singularity/store.go
@@ -443,6 +443,7 @@ func (s *SingularityStore) Describe(ctx context.Context, id blob.ID) (*blob.Desc
 	if err != nil {
 		return nil, err
 	}
+	fmt.Println(getFileDealsRes.Payload)
 	replicas := make([]blob.Replica, 0, len(getFileDealsRes.Payload))
 	for _, deal := range getFileDealsRes.Payload {
 		replicas = append(replicas, blob.Replica{

diff --git a/integration/test/Makefile b/integration/test/Makefile
@@ -0,0 +1,46 @@
+.PHONY: devnet/up devnet/down motionlarity/up motionlarity/down test
+
+./devnet/.up:
+	echo "Run Boost Devnet from pre-built images"
+	rm -rf ./devnet/data && docker compose -f ./devnet/docker-compose.yaml up -d
+	echo "Await Lotus full node startup"
+	docker compose -f ./devnet/docker-compose.yaml exec lotus lotus wait-api --timeout=20m
+	echo "Await Lotus-miner full node startup"
+	docker compose -f ./devnet/docker-compose.yaml exec lotus-miner lotus-miner wait-api --timeout=20m
+	touch ./devnet/.up
+
+devnet/up: ./devnet/.up
+
+./devnet/.boostready: ./devnet/.up
+	./boost-setup.sh
+	touch ./devnet/.boostready
+
+./motionlarity/.env.local: ./devnet/.up
+	echo "Set up Motion Wallet and API endpoints"
+	cat ./motionlarity/.env > ./motionlarity/.env.local
+	echo "" >> ./motionlarity/.env.local
+	./integration-setup.sh ./motionlarity/.env.local
+
+motionlarity/setup: ./motionlarity/.env.local ./devnet/.boostready
+
+devnet/down: motionlarity/down
+	docker compose -f ./devnet/docker-compose.yaml down && sleep 2 && rm -rf ./devnet/data
+	rm ./devnet/.up || true
+
+./motionlarity/.up: ./motionlarity/.env.local ./devnet/.boostready
+	echo "Run Motionlarity"
+	docker compose -f ./motionlarity/docker-compose.yaml --env-file ./motionlarity/.env.local up -d
+	./waitmotion.sh
+	touch ./motionlarity/.up
+
+motionlarity/up: ./motionlarity/.up
+
+motionlarity/down:
+	docker compose -f ./motionlarity/docker-compose.yaml down --rmi=all --volumes
+	rm ./motionlarity/.up || true
+	rm ./motionlarity/.env.local || true
+	rm ./devnet/.boostready || true
+
+test: motionlarity/up
+	echo "Run Motion integration tests"
+	MOTION_INTEGRATION_TEST='true' go test . -v
diff --git a/integration/test/boost-setup.sh b/integration/test/boost-setup.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "setup boost pricing"
+for i in {1..10}
+do
+  curl -X POST -d '{"operationName":"AppStorageAskUpdateMutation","variables":{"update":{"Price":"0", "VerifiedPrice": 0}},"query":"mutation AppStorageAskUpdateMutation($update: StorageAskUpdate!) {\n  storageAskUpdate(update: $update)\n}\n"}' http://localhost:8080/graphql/query && break || sleep 5
+done
diff --git a/integration/test/integration-setup.sh b/integration/test/integration-setup.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+echo "build config file and wallets for motion"
+# Setup Lotus API token
+export `docker compose -f ./devnet/docker-compose.yaml exec lotus lotus auth api-info --perm=admin`
+IFS=: read -r token path <<< "${FULLNODE_API_INFO}"
+# Setup Motion Wallet
+MOTION_WALLET_ADDR=`docker compose -f ./devnet/docker-compose.yaml exec lotus lotus wallet new`
+MOTION_WALLET_KEY=`docker compose -f ./devnet/docker-compose.yaml exec lotus lotus wallet export ${MOTION_WALLET_ADDR}`
+LOTUS_WALLET_DEFAULT_ADDR=`docker compose -f ./devnet/docker-compose.yaml exec lotus lotus wallet default`
+docker compose -f ./devnet/docker-compose.yaml exec lotus lotus send --from=${LOTUS_WALLET_DEFAULT_ADDR} ${MOTION_WALLET_ADDR} 10
+echo "LOTUS_TOKEN=${token}" >> $1
+echo "MOTION_WALLET_ADDR=${MOTION_WALLET_ADDR}" >> $1
+echo "MOTION_WALLET_KEY=${MOTION_WALLET_KEY}" >> $1
+echo "MOTION_STORAGE_PROVIDERS=t01000" >> $1
+echo "MOTION_API_ENDPOINT=http://localhost:40080" >> $1
+echo "SINGULARITY_API_ENDPOINT=http://localhost:9091" >> $1
diff --git a/integration/test/integration_test.go b/integration/test/integration_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -79,6 +80,7 @@ func TestRoundTripPutAndStatus(t *testing.T) {
 			var decoded api.GetStatusResponse
 			err = jsonResp.Decode(&decoded)
 			assert.NoError(c, err)
+			fmt.Println(decoded)
 			assert.Greater(c, len(decoded.Replicas), 0)
 		}, 2*time.Minute, 5*time.Second, "never initiated deal making")
 	}

diff --git a/integration/test/motionlarity/.env b/integration/test/motionlarity/.env
@@ -1,8 +1,9 @@
 SINGULARITY_TAG=main
 LOTUS_TEST='true'
 LOTUS_API=http://lotus:1234/rpc/v1
-MOTION_PRICE_PER_GIB_EPOCH=0.000000000000002
+MOTION_PRICE_PER_GIB_EPOCH=0
 MOTION_SINGULARITY_MAX_CAR_SIZE=7MiB
 MOTION_SINGULARITY_PACK_THRESHOLD=4096
 MOTION_SINGULARITY_SCHEDULE_CRON='* * * * *'
-MOTION_VERIFIED_DEAL=true
+MOTION_VERIFIED_DEAL=false
+MOTION_SINGULARITY_SCHEDULE_DEAL_NUMBER=25
diff --git a/integration/test/motionlarity/docker-compose.yaml b/integration/test/motionlarity/docker-compose.yaml
@@ -31,8 +31,6 @@ services:
     command: admin init
     volumes:
       - motion-singularity-volume:/usr/src/app/storage
-    ports:
-      - 9091:9091
     environment:
       DATABASE_CONNECTION_STRING: postgres://${SINGULARITY_DB_USER:-postgres}:${SINGULARITY_DB_PASSWORD:-postgres}@db:5432/${SINGULARITY_DB_NAME:-singularity}
       LOTUS_TEST: 

diff --git a/integration/test/waitmotion.sh b/integration/test/waitmotion.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "Wait until motion is running"
+for i in {1..10}
+do
+  docker compose -f ./devnet/docker-compose.yaml ps --services --filter "status=running" | grep motion && break || sleep 5
+done
diff --git a/scripts/integration-setup.sh b/scripts/integration-setup.sh