From c4e3d4ae2c7233d053964e16f11cf8b0def6f786 Mon Sep 17 00:00:00 2001 From: macpie Date: Thu, 16 May 2024 15:11:03 -0700 Subject: [PATCH] Revert "Revert "Attempt to make CI faster"" (#813) * Revert "Revert "Attempt to make CI faster" (#809)" This reverts commit a6d1430067f27ff239ddb54130cc44d688fe21b3. * Test Debian packaging without tag * Remove comments * Rename to try to trigger * Comment matrix for now * Empty matrix does not work comment the all thing * Change trigger * Add some log line * More logs * Add random version * Revert comments * Get back to using $VERSION * Fix package * Add build-release to main CI * Make it into 1 workflow * Separate test with postgres * remove env in tests * Experiment * Update readme * Add protoc to clippy2 * Move clippy --- .github/scripts/make_debian.sh | 7 +- .github/workflows/CI.yml | 201 +++++++++++++++++++++++++++++++++ .github/workflows/rust.yml | 71 ------------ README.md | 11 +- 4 files changed, 214 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/CI.yml delete mode 100644 .github/workflows/rust.yml diff --git a/.github/scripts/make_debian.sh b/.github/scripts/make_debian.sh index 10603f900..1cffe127b 100644 --- a/.github/scripts/make_debian.sh +++ b/.github/scripts/make_debian.sh @@ -99,14 +99,19 @@ run_fpm() sudo apt update sudo apt install --yes ruby sudo gem install fpm -v 1.14.2 # current as of 2022-11-08 +echo "ruby deps installed" for config_path in $( find . -name 'settings-template.toml' ) do oracle=$(echo $config_path | sed -E 's!\./([^/]+)/.+$!\1!' | sed -E 's!_!-!g') - + + echo "starting $oracle $config_path $VERSION" write_unit_template $oracle + echo "write_unit_template $oracle done" write_prepost_template $oracle + echo "write_prepost_template $oracle done" run_fpm $oracle $config_path $VERSION + echo "run_fpm $oracle done" done for deb in /tmp/*.deb diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 000000000..c6258f443 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,201 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: ["main"] + tags: ["*"] + +env: + CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness + CARGO_NET_RETRY: 10 + RUST_BACKTRACE: short + RUSTFLAGS: "-D warnings" + RUSTUP_MAX_RETRIES: 10 + +jobs: + + build: + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Build + run: cargo build --all --tests + + fmt: + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-fmt + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Check formatting + run: cargo fmt -- --check + + clippy: + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-clippy + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Clippy + run: cargo clippy --all-targets -- -Dclippy::all -D warnings + + tests-postgres: + needs: build + runs-on: oracles-20.04 + strategy: + fail-fast: false + matrix: + package: [boost-manager,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-verifier] + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-tests-postgres-${{ matrix.package }} + cancel-in-progress: true + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Run unit and integration tests + env: + DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" + run: cargo test -p ${{ matrix.package }} -- --include-ignored + + tests: + needs: build + runs-on: oracles-20.04 + strategy: + fail-fast: false + matrix: + package: [file-store,ingest,mobile-packet-verifier,reward-scheduler,task-manager] + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Run unit and integration tests + run: cargo test -p ${{ matrix.package }} -- --include-ignored + + build-release: + needs: [fmt, clippy, tests, tests-postgres] + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build-release + cancel-in-progress: true + if: contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Build Release + run: cargo build --all --release + + - name: Debian packaging + env: + PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} + run: | + chmod +x ./.github/scripts/make_debian.sh + ./.github/scripts/make_debian.sh \ No newline at end of file diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index c141ecc9e..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: CI - -on: - pull_request: - branches: [main] - push: - branches: [main] - tags: ["*"] - -env: - CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness - CARGO_NET_RETRY: 10 - RUST_BACKTRACE: short - RUSTFLAGS: "-D warnings" - RUSTUP_MAX_RETRIES: 10 - -jobs: - build: - runs-on: oracles-20.04 - - services: - postgres: - image: postgres - env: - POSTGRES_PASSWORD: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy, rustfmt - - - name: Install protoc - run: sudo apt-get install -y protobuf-compiler - - - name: Setup cache - uses: Swatinem/rust-cache@v2 - - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - - name: Build - run: cargo build --all --release - - - name: Check formatting - run: cargo fmt -- --check - - - name: Clippy - run: cargo clippy --all-targets -- -Dclippy::all -D warnings - - - name: Run unit and integration tests - env: - DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" - run: cargo test -r -- --include-ignored - - - name: Debian packaging - if: contains(github.ref, 'refs/tags/') - env: - PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} - run: | - chmod +x ./.github/scripts/make_debian.sh - ./.github/scripts/make_debian.sh diff --git a/README.md b/README.md index 5d741c250..eae2d542e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ flowchart TD MPV("`**Mobile Packet Verifier** - Burns DC for data transfer (on solana) `") - MB("`**Mobile Price** + MP("`**Mobile Price** - Records Pyth price for MOBILE `") DB1[(Foundation owned db populated by helius)] @@ -31,12 +31,15 @@ flowchart TD - Writes rewards to foundation db `") DB2[(Foundation owned db that stores reward totals)] + S[(Solana)] MI -- S3 --> MV MI -- S3 --> MPV MPV -- S3 --> MV - MB -- S3 --> MV - DB1 --> MB - MB --> MC + MPV -- gRPC --> MC + MPV --> S + MP <--> S + MP -- S3 --> MV + DB1 --> MC MC -- gRPC --> MV MV -- S3 --> MRI MRI --> DB2