Revert "Revert "Attempt to make CI faster"" (#813)

* Revert "Revert "Attempt to make CI faster" (#809)" This reverts commit a6d1430. * Test Debian packaging without tag * Remove comments * Rename to try to trigger * Comment matrix for now * Empty matrix does not work comment the all thing * Change trigger * Add some log line * More logs * Add random version * Revert comments * Get back to using $VERSION * Fix package * Add build-release to main CI * Make it into 1 workflow * Separate test with postgres * remove env in tests * Experiment * Update readme * Add protoc to clippy2 * Move clippy
helium · May 16, 2024 · c4e3d4a · c4e3d4a
1 parent bdabe6b
commit c4e3d4a
Show file tree

Hide file tree

Showing 4 changed files with 214 additions and 76 deletions.
diff --git a/.github/scripts/make_debian.sh b/.github/scripts/make_debian.sh
@@ -99,14 +99,19 @@ run_fpm()
 sudo apt update
 sudo apt install --yes ruby
 sudo gem install fpm -v 1.14.2 # current as of 2022-11-08
+echo "ruby deps installed" 
 
 for config_path in $( find . -name 'settings-template.toml' )
 do
     oracle=$(echo $config_path | sed -E 's!\./([^/]+)/.+$!\1!' | sed -E 's!_!-!g')
-
+
+    echo "starting  $oracle $config_path $VERSION"
     write_unit_template $oracle
+    echo "write_unit_template  $oracle done"
     write_prepost_template $oracle
+    echo "write_prepost_template  $oracle done"
     run_fpm $oracle $config_path $VERSION
+    echo "run_fpm  $oracle done"
 done
 
 for deb in /tmp/*.deb

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -0,0 +1,201 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: ["main"]
+    tags: ["*"]
+
+env:
+  CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness
+  CARGO_NET_RETRY: 10
+  RUST_BACKTRACE: short
+  RUSTFLAGS: "-D warnings"
+  RUSTUP_MAX_RETRIES: 10
+
+jobs:
+
+  build:
+    runs-on: oracles-20.04
+    concurrency: 
+      group: ${{ github.workflow }}-${{ github.ref }}-build
+      cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Rust install
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install protoc
+        run: sudo apt-get install -y protobuf-compiler
+
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Build
+        run: cargo build --all --tests 
+
+  fmt:
+    runs-on: oracles-20.04
+    concurrency: 
+      group: ${{ github.workflow }}-${{ github.ref }}-fmt
+      cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Rust install
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt
+
+      - name: Check formatting
+        run: cargo fmt  -- --check
+
+  clippy:
+    runs-on: oracles-20.04
+    concurrency: 
+      group: ${{ github.workflow }}-${{ github.ref }}-clippy
+      cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Rust install
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+
+      - name: Install protoc
+        run: sudo apt-get install -y protobuf-compiler
+
+      - name: Clippy
+        run: cargo clippy --all-targets -- -Dclippy::all -D warnings
+
+  tests-postgres:
+    needs: build
+    runs-on: oracles-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        package: [boost-manager,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-verifier]
+    concurrency: 
+      group: ${{ github.workflow }}-${{ github.ref }}-tests-postgres-${{ matrix.package }}
+      cancel-in-progress: true
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+            - 5432:5432
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Rust install
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install protoc
+        run: sudo apt-get install -y protobuf-compiler
+
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Run unit and integration tests
+        env:
+          DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres"
+        run: cargo test -p ${{ matrix.package }}  -- --include-ignored
+
+  tests:
+    needs: build
+    runs-on: oracles-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        package: [file-store,ingest,mobile-packet-verifier,reward-scheduler,task-manager]
+    concurrency: 
+      group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Rust install
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install protoc
+        run: sudo apt-get install -y protobuf-compiler
+
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Run unit and integration tests
+        run: cargo test -p ${{ matrix.package }}  -- --include-ignored
+
+  build-release:
+    needs: [fmt, clippy, tests, tests-postgres]
+    runs-on: oracles-20.04
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-build-release
+      cancel-in-progress: true
+    if: contains(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Install protoc
+        run: sudo apt-get install -y protobuf-compiler
+
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Build Release
+        run: cargo build --all --release
+
+      - name: Debian packaging
+        env:
+          PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }}
+        run: |
+          chmod +x ./.github/scripts/make_debian.sh
+          ./.github/scripts/make_debian.sh
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
diff --git a/README.md b/README.md
@@ -19,7 +19,7 @@ flowchart TD
     MPV("`**Mobile Packet Verifier**
         - Burns DC for data transfer (on solana)
     `")
-    MB("`**Mobile Price**
+    MP("`**Mobile Price**
         - Records Pyth price for MOBILE
     `")
     DB1[(Foundation owned db populated by helius)]
@@ -31,12 +31,15 @@ flowchart TD
         - Writes rewards to foundation db
     `")
     DB2[(Foundation owned db that stores reward totals)]
+    S[(Solana)]
     MI -- S3 --> MV
     MI -- S3 --> MPV
     MPV -- S3 --> MV
-    MB -- S3 --> MV
-    DB1 --> MB
-    MB --> MC
+    MPV -- gRPC --> MC
+    MPV --> S
+    MP <--> S
+    MP -- S3 --> MV
+    DB1 --> MC
     MC -- gRPC --> MV
     MV -- S3 --> MRI
     MRI --> DB2