pg17 (#151)

* pg17 + native runners for arm/amd builds --------- Co-authored-by: Nick Hudson <[email protected]>
tembo-io · Oct 15, 2024 · 48d1dd1 · 48d1dd1
1 parent 9e013e3
commit 48d1dd1
Show file tree

Hide file tree

Showing 10 changed files with 243 additions and 104 deletions.
diff --git a/.github/actions/pgx-init/action.yml b/.github/actions/pgx-init/action.yml
@@ -16,6 +16,7 @@ runs:
          mv stoml_linux_amd64 stoml
          chmod +x stoml
          sudo mv stoml /usr/local/bin/
+         sudo apt-get update && sudo apt-get install -y libreadline-dev
       - name: setup pgrx
         shell: bash
         id: pgrx_install

diff --git a/.github/workflows/extension_ci.yml b/.github/workflows/extension_ci.yml
@@ -25,16 +25,13 @@ on:
 jobs:
   dependencies:
     name: Install dependencies
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       # rust needed to install trunk
       - name: Install Rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          cache-directories: /home/runner/.pgrx
+        uses: dtolnay/rust-toolchain@stable
 
       - name: Install stoml and pg-trunk
         shell: bash
@@ -61,13 +58,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Rust minimal nightly with clippy and rustfmt
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly
-          components: rustfmt, clippy
+        uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: "pg-vectorize-extension-lint"
@@ -93,11 +86,9 @@ jobs:
         ports:
           - 3000:3000
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install Rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: "extension-test"
@@ -155,9 +146,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Install Rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: "pg-vectorize-extension-test"
@@ -186,65 +175,3 @@ jobs:
         env:
           TRUNK_API_TOKEN: ${{ secrets.TRUNK_AUTH_TOKEN }}
         run: ~/.cargo/bin/trunk publish
-  build_and_push:
-    name: Build and push images
-    needs:
-      - publish
-    env:
-      TARGET_PLATFORMS: linux/amd64,linux/arm64
-    runs-on:
-      - self-hosted
-      - dind
-      - xlarge-16x16
-    outputs:
-      short_sha: ${{ steps.versions.outputs.SHORT_SHA }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Install stoml and pg-trunk
-        shell: bash
-        run: |
-          set -xe
-          wget https://github.com/freshautomations/stoml/releases/download/v0.7.1/stoml_linux_amd64 &> /dev/null
-          mv stoml_linux_amd64 stoml
-          chmod +x stoml
-          sudo mv stoml /usr/local/bin/
-      - name: Set version strings
-        id: versions
-        run: |
-          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-          echo "TAG_VER=$(/usr/local/bin/stoml Cargo.toml package.version)" >> $GITHUB_OUTPUT
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Quay
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io/tembo
-          username: ${{ secrets.QUAY_USER_TEMBO }}
-          password: ${{ secrets.QUAY_PASSWORD_TEMBO }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          file: ./images/vectorize-pg/Dockerfile
-          context: .
-          platforms: linux/amd64, linux/arm64
-          push: true
-          tags: |
-            quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }}
-            quay.io/tembo/vectorize-pg:latest
diff --git a/.github/workflows/pg-image-build.yml b/.github/workflows/pg-image-build.yml
@@ -0,0 +1,203 @@
+name: Build Postgres with pg_vectorize
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "extension/**"
+      - "images/vectorize-pg/**"
+      - ".github/workflows/pg-image-build.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "extension/**"
+      - "images/vectorize-pg/**"
+      - ".github/workflows/pg-image-build.yml"
+  release:
+    types:
+      - created
+jobs:
+  build_and_push_amd64:
+    name: Build and push AMD64 images
+    runs-on:
+      - self-hosted
+      - dind
+      - large-8x8
+    outputs:
+      short_sha: ${{ steps.versions.outputs.SHORT_SHA }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install stoml and pg-trunk
+        shell: bash
+        run: |
+          set -xe
+          wget https://github.com/freshautomations/stoml/releases/download/v0.7.1/stoml_linux_amd64 &> /dev/null
+          mv stoml_linux_amd64 stoml
+          chmod +x stoml
+          sudo mv stoml /usr/local/bin/
+      - name: Set version strings
+        id: versions
+        run: |
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "TAG_VER=$(/usr/local/bin/stoml Cargo.toml package.version)" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Quay
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io/tembo
+          username: ${{ secrets.QUAY_USER_TEMBO }}
+          password: ${{ secrets.QUAY_PASSWORD_TEMBO }}
+      - name: Build and push -- Commit
+        # push commit build when not a release
+        if: github.event_name != 'release'
+        run: |
+            docker build \
+                -f ./images/vectorize-pg/Dockerfile \
+                -t quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}-amd64 .
+            docker push quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}-amd64
+
+      - name: Build and push -- Release
+        if: github.event_name == 'release'
+        run: |
+            docker build \
+                -f ./images/vectorize-pg/Dockerfile \
+                -t quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }}-amd64 \
+                -t quay.io/tembo/vectorize-pg:latest-amd64 .
+            docker push quay.io/tembo/vectorize-pg
+
+  build_and_push_arm64:
+    name: Build and push ARM64 images
+    runs-on:
+      - arm64-4x16
+    outputs:
+      short_sha: ${{ steps.versions.outputs.SHORT_SHA }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install stoml and pg-trunk
+        shell: bash
+        run: |
+          set -xe
+          wget https://github.com/freshautomations/stoml/releases/download/v0.7.1/stoml_linux_amd64 &> /dev/null
+          mv stoml_linux_amd64 stoml
+          chmod +x stoml
+          sudo mv stoml /usr/local/bin/
+      - name: Set version strings
+        id: versions
+        run: |
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "TAG_VER=$(/usr/local/bin/stoml Cargo.toml package.version)" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Quay
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io/tembo
+          username: ${{ secrets.QUAY_USER_TEMBO }}
+          password: ${{ secrets.QUAY_PASSWORD_TEMBO }}
+
+      - name: Build and push -- Commit
+        # push commit build when not a release
+        if: github.event_name != 'release'
+        run: |
+            docker build \
+                -f ./images/vectorize-pg/Dockerfile \
+                --platform linux/arm64 \
+                -t quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}-arm64 .
+            docker push quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}-arm64
+
+      - name: Build and push -- Release
+        if: github.event_name == 'release'
+        run: |
+            docker build \
+                -f ./images/vectorize-pg/Dockerfile \
+                --platform linux/arm64 \
+                -t quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }}-arm64 \
+                -t quay.io/tembo/vectorize-pg:latest-arm64 .
+            docker push quay.io/tembo/vectorize-pg
+
+  create_manifest:
+    name: Create and Push Manifest
+
+    runs-on: ubuntu-latest
+    needs: [build_and_push_arm64, build_and_push_amd64]
+    outputs:
+      short_sha: ${{ steps.versions.outputs.SHORT_SHA }}
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Quay
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io/tembo
+          username: ${{ secrets.QUAY_USER_TEMBO }}
+          password: ${{ secrets.QUAY_PASSWORD_TEMBO }}
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install stoml and pg-trunk
+        shell: bash
+        run: |
+          set -xe
+          wget https://github.com/freshautomations/stoml/releases/download/v0.7.1/stoml_linux_amd64 &> /dev/null
+          mv stoml_linux_amd64 stoml
+          chmod +x stoml
+          sudo mv stoml /usr/local/bin/
+      - name: Set version strings
+        id: versions
+        run: |
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "TAG_VER=$(/usr/local/bin/stoml Cargo.toml package.version)" >> $GITHUB_OUTPUT
+
+      - name: Create and push Docker manifest -- Commit
+        if: github.event_name != 'release'
+        run: |
+          docker manifest create quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }} \
+            quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}-arm64 \
+            quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}-amd64
+        
+          docker manifest annotate quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }} quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}-arm64 --arch arm64 --os linux
+          docker manifest annotate quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }} quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}-amd64 --arch amd64 --os linux
+          docker manifest push quay.io/tembo/vectorize-pg:${{ steps.versions.outputs.SHORT_SHA }}
+
+      - name: Create and push Docker manifest -- Release
+        if: github.event_name == 'release'
+        run: |
+          docker manifest create quay.io/tembo/vectorize-pg:latest \
+            quay.io/tembo/vectorize-pg:latest-arm64 \
+            quay.io/tembo/vectorize-pg:latest-amd64
+        
+          docker manifest annotate quay.io/tembo/vectorize-pg:latest quay.io/tembo/vectorize-pg:latest-arm64 --arch arm64 --os linux
+          docker manifest annotate quay.io/tembo/vectorize-pg:latest quay.io/tembo/vectorize-pg:latest-amd64 --arch amd64 --os linux
+          docker manifest push quay.io/tembo/vectorize-pg:latest
+
+          docker manifest create quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }} \
+            quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }}-arm64 \
+            quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }}-amd64
+        
+          docker manifest annotate quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }} quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }}-arm64 --arch arm64 --os linux
+          docker manifest annotate quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }} quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }}-amd64 --arch amd64 --os linux
+          docker manifest push quay.io/tembo/vectorize-pg:v${{ steps.versions.outputs.TAG_VER }}
diff --git a/core/Cargo.toml b/core/Cargo.toml
@@ -25,7 +25,7 @@ regex = "1.9.2"
 reqwest = {version = "0.11.18", features = ["json"] }
 serde = { version = "1.0.173", features = ["derive"] }
 serde_json = "1.0.103"
-sqlx = { version = "=0.7.3", features = [
+sqlx = { version = "=0.8", features = [
     "runtime-tokio-native-tls",
     "postgres",
     "chrono",

diff --git a/extension/Cargo.toml b/extension/Cargo.toml
@@ -1,17 +1,22 @@
 [package]
 name = "vectorize"
-version = "0.18.3"
+version = "0.19.0"
 edition = "2021"
 publish = false
 
 [lib]
-crate-type = ["cdylib"]
+crate-type = ["cdylib", "lib"]
+
+[[bin]]
+name = "pgrx_embed_vectorize"
+path = "./src/bin/pgrx_embed.rs"
 
 [features]
-default = ["pg16"]
+default = ["pg17"]
 pg14 = ["pgrx/pg14", "pgrx-tests/pg14"]
 pg15 = ["pgrx/pg15", "pgrx-tests/pg15"]
 pg16 = ["pgrx/pg16", "pgrx-tests/pg16"]
+pg17 = ["pgrx/pg17", "pgrx-tests/pg17"]
 pg_test = []
 
 [dependencies]
@@ -21,13 +26,13 @@ handlebars = "5.1.0"
 lazy_static = "1.4.0"
 log = "0.4.21"
 pgmq = "0.29"
-pgrx = "0.11.4"
+pgrx = "=0.12.5"
 postgres-types = "0.2.5"
 regex = "1.9.2"
 reqwest = {version = "0.11.18", features = ["json"] }
 serde = { version = "1.0.173", features = ["derive"] }
 serde_json = "1.0.103"
-sqlx = { version = "=0.7.3", features = [
+sqlx = { version = "0.8", features = [
     "runtime-tokio-native-tls",
     "postgres",
     "chrono",
@@ -40,7 +45,7 @@ url = "2.4.0"
 vectorize_core = { path = "../core", package = "vectorize-core" }
 
 [dev-dependencies]
-pgrx-tests = "0.11.4"
+pgrx-tests = "=0.12.5"
 rand = "0.8.5"
 whoami = "1.4.1"