chore: README and CI changes

.github/workflows/docker.yml

@@ -1,92 +1,125 @@
-# Builds and uploads docker images whenever a version tag is pushed. When a release is created,
-# the associated image is then tagged with the version number and latest.
-
-name: Docker
+name: "Build multi-arch images"
 
 on:
-  workflow_dispatch:
   push:
     tags:
-      - 'v*'
-  release:
-    types: [published]
-
-env:
-  # Workaround for https://github.com/rust-lang/cargo/issues/8719#issuecomment-1516492970
-  CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      - "v*.*.*"
 
 jobs:
-  # Build a docker image unless this was triggered by a release.
-  build-image:
-    if: github.event_name != 'release'
-    runs-on: pathfinder-large-ubuntu
+  image-info:
+    name: "Extract crate info"
+    runs-on: "ubuntu-latest"
+    outputs:
+      version: ${{ steps.derive.outputs.version }}
+
     steps:
-      - name: Determine Docker image metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: eqlabs/pathfinder
-      - name: Checkout sources
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Generate version
-        id: generate_version
+      - id: "derive"
+        name: "Derive image info from Git tag"
         run: |
-          echo -n "pathfinder_version=" >> $GITHUB_OUTPUT
-          git describe --tags --dirty >> $GITHUB_OUTPUT
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v3
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          buildkitd-config-inline: |
-            [worker.oci]
-              max-parallelism = 4
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
+          FULL_REF="${{ github.ref }}"
+          REGEX="^refs\/tags\/v(.*)$"
+          [[ $FULL_REF =~ $REGEX ]];
+
+          echo "version=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT
+
+  build-amd64:
+    name: "Build for linux/amd64"
+    runs-on: "ubuntu-latest"
+    needs:
+      - "image-info"
+
+    env:
+      DOCKER_REPOSITORY: "starknet/pathfinder-firehose"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v3"
+
+      - name: "Login to Docker Hub"
+        uses: "docker/[email protected]"
         with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-      # Required for git security reasons. See https://github.com/rustyhorde/vergen/pull/126#issuecomment-1201088162
-      - name: Vergen git safe directory
-        run: git config --global --add safe.directory /workspace
-      - name: Build
-        id: docker_build
-        uses: docker/build-push-action@v5
+          username: "${{ secrets.DOCKER_HUB_USERNAME }}"
+          password: "${{ secrets.DOCKER_HUB_PASSWORD }}"
+
+      - name: "Build Docker image"
+        run: |
+          docker buildx build \
+            --build-arg PATHFINDER_FORCE_VERSION="v${{ needs.image-info.outputs.version }}" \
+            --platform "linux/amd64" \
+            -t ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-amd64 -f ./Dockerfile .
+
+      - name: "Push Docker image"
+        run: |
+          docker push ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-amd64
+
+  build-arm64:
+    name: "Build for linux/arm64"
+    runs-on: "ubuntu-latest"
+    needs:
+      - "image-info"
+
+    env:
+      DOCKER_REPOSITORY: "starknet/pathfinder-firehose"
+
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v3"
+
+      - name: "Login to Docker Hub"
+        uses: "docker/[email protected]"
         with:
-          context: .
-          platforms: |
-            linux/amd64
-            linux/arm64
-          file: ./Dockerfile
-          build-args: |
-            PATHFINDER_FORCE_VERSION=${{ steps.generate_version.outputs.pathfinder_version }}
-          builder: ${{ steps.buildx.outputs.name }}
-          push: true
-          labels: ${{ steps.meta.outputs.labels }}
-          tags: eqlabs/pathfinder:snapshot-${{ github.sha }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-  # Add the release labels to the associated docker image.
-  tag-release:
-    if: github.event_name == 'release'
-    runs-on: ubuntu-latest
+          username: "${{ secrets.DOCKER_HUB_USERNAME }}"
+          password: "${{ secrets.DOCKER_HUB_PASSWORD }}"
+
+      - name: "Set up Docker Buildx"
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+          sudo systemctl restart docker
+          docker buildx create --name multi_builder
+          docker buildx use multi_builder
+
+      - name: "Build Docker image"
+        run: |
+          docker buildx build \
+            --build-arg PATHFINDER_FORCE_VERSION="v${{ needs.image-info.outputs.version }}" \
+            --platform "linux/arm64/v8" \
+            --output=type=docker \
+            -t ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-arm64 -f ./Dockerfile .
+
+      - name: "Push Docker image"
+        run: |
+          docker push ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-arm64
+
+  push:
+    name: "Push multi-arch manifest"
+    runs-on: "ubuntu-latest"
+    needs:
+      - "build-amd64"
+      - "build-arm64"
+      - "image-info"
+
+    env:
+      DOCKER_REPOSITORY: "starknet/pathfinder-firehose"
+
     steps:
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
+      - name: "Login to Docker Hub"
+        uses: "docker/login-action@v1.6.0"
         with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-      - name: Tag image with release name
-        run: docker buildx imagetools create -t eqlabs/pathfinder:${{ github.event.release.tag_name }} eqlabs/pathfinder:snapshot-${{ github.sha }}
-        # Only tag the image as 'latest' if its a release i.e. not a prerelease.
-      - name: Tag image with 'latest'
-        if: ${{ !github.event.release.prerelease }}
-        run: docker buildx imagetools create -t eqlabs/pathfinder:latest eqlabs/pathfinder:snapshot-${{ github.sha }}
+          username: "${{ secrets.DOCKER_HUB_USERNAME }}"
+          password: "${{ secrets.DOCKER_HUB_PASSWORD }}"
+
+      - name: "Pull Docker images"
+        run: |
+          docker pull ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-amd64
+          docker pull ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-arm64
+
+      - name: "Push Docker image"
+        run: |
+          docker manifest create ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }} \
+            ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-amd64 \
+            ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-arm64
+          docker manifest create ${DOCKER_REPOSITORY}:latest \
+            ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-amd64 \
+            ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}-arm64
+          docker manifest push ${DOCKER_REPOSITORY}:${{ needs.image-info.outputs.version }}
+          docker manifest push ${DOCKER_REPOSITORY}:latest

.github/workflows/sync.yml

@@ -0,0 +1,81 @@
+on:
+  schedule:
+    # 1:05 past midnight UTC
+    - cron: "5 1 * * *"
+
+name: "Sync with upstream"
+jobs:
+  sync:
+    name: "Sync"
+    runs-on: "ubuntu-latest"
+
+    env:
+      DEFAULT_BRANCH: "main"
+
+    steps:
+      # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+      - name: "Free up disk space"
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: "Checkout source code"
+        uses: "actions/checkout@v3"
+        with:
+          # Subsequent actions are not triggered unless using PAT
+          token: "${{ secrets.GH_PAT }}"
+          fetch-depth: 0
+          submodules: true
+
+      - name: "Setup toolchain"
+        uses: "actions-rs/toolchain@v1"
+        with:
+          toolchain: "stable"
+          profile: "minimal"
+          override: true
+
+      - uses: "Swatinem/rust-cache@v1"
+        with:
+          cache-on-failure: true
+
+      - name: "Install protoc"
+        uses: "Noelware/[email protected]"
+
+      - name: "Config Git"
+        run: |
+          git config user.name "Jonathan LEI"
+          git config user.email "[email protected]"
+
+      - name: "Update branch"
+        run: |
+          git fetch origin
+          git remote add upstream https://github.com/eqlabs/pathfinder
+          git fetch upstream --no-tags
+
+          MERGE_BASE=$(git merge-base origin/$DEFAULT_BRANCH upstream/$DEFAULT_BRANCH)
+
+          # Don't force push unnecessarily unless changes are detected
+          if [[ $(git rev-list $MERGE_BASE..upstream/$DEFAULT_BRANCH | wc -l) -ne 0 ]]; then 
+            # Brings files from `home` to default branch
+            git checkout $DEFAULT_BRANCH
+            git reset --hard upstream/$DEFAULT_BRANCH
+            git checkout origin/home .
+            git add .
+            git commit -m "chore: README and CI changes"
+
+            # Here, we pick commits on the default branch except the first one. We do this instead
+            # of a naive rebase because the `home` branch might have changed, causing merge
+            # conflicts.
+            COMMIT_COUNT=$(git rev-list $MERGE_BASE..origin/$DEFAULT_BRANCH | wc -l)
+            git cherry-pick origin/$DEFAULT_BRANCH~$(($COMMIT_COUNT-1))..origin/$DEFAULT_BRANCH
+
+            # Makes sure the updated local branch builds
+            cargo build --all --all-targets
+
+            # Push updated branch
+            git push --force-with-lease
+          else
+            echo "No changes detected on upstream $DEFAULT_BRANCH"
+          fi