diff --git a/.github/workflows/cargo_publish.yml b/.github/workflows/cargo_publish.yml new file mode 100644 index 00000000..c7e0b2a3 --- /dev/null +++ b/.github/workflows/cargo_publish.yml @@ -0,0 +1,83 @@ +name: Cargo publish or validate + +on: + pull_request: + branches: + - main + - "release/[0-9]+.[0-9]+" + push: + branches: + - 'main' + - 'release/[0-9]+.[0-9]+' + +jobs: + find_directories: + name: Find crates that changed + runs-on: ubuntu-20.04 + outputs: + changed_crates: ${{ steps.find_directories.outputs.build_matrix }} + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - name: Find directories including Cargo.toml that changed + id: find_directories + uses: ./.github/actions/find-changed-directories + with: + contains_the_file: Cargo.toml + # If the branch does not exist, then it will not + # filter any directories containing the file. + # This allows for filtering out unchanged directories + # in a pull request, and using all directories on the release + # or main branches. + changed_relative_to_ref: origin/${{ github.base_ref || 'not-a-branch' }} + + + cargo_publish: + # On a pull request, this is validating that the version + # is not already published to crates.io, and on a push to + # main or release branches, it is publishing if the version + # does not exist, ignoring if the version is already + # published. + name: Cargo publish or validate + runs-on: ubuntu-20.04 + needs: + - find_directories + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.find_directories.outputs.changed_crates) }} + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - name: Determine which flags to use on cargo publish + id: cargo_flags + run: | + set -x + BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) + if [ "${BRANCH_NAME}" == "main" ]; then + echo "dry_run=false" >> $GITHUB_OUTPUT + echo "fail_if_version_published=false" >> $GITHUB_OUTPUT + elif [[ "${BRANCH_NAME}" == release/* ]]; then + echo "dry_run=false" >> $GITHUB_OUTPUT + echo "fail_if_version_published=false" >> $GITHUB_OUTPUT + else + echo "dry_run=true" >> $GITHUB_OUTPUT + echo "fail_if_version_published=true" >> $GITHUB_OUTPUT + fi + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: ${{ matrix.name }} + workspaces: | + ${{ matrix.path }} + # Additional directories to cache + cache-directories: | + /home/runner/.pgrx + - uses: ./.github/actions/pgx-init + with: + working-directory: ${{ matrix.path }} + - name: Publish or validate + uses: ./.github/actions/publish-crate + with: + working-directory: ${{ matrix.path }} + dry-run: ${{ steps.cargo_flags.outputs.dry_run }} + fail-if-version-published: ${{ steps.cargo_flags.outputs.fail_if_version_published }} + cargo-registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/.github/workflows/crate_ci.yml b/.github/workflows/crate_ci.yml index e9ffd7ae..aea1136b 100644 --- a/.github/workflows/crate_ci.yml +++ b/.github/workflows/crate_ci.yml @@ -10,7 +10,7 @@ on: branches: - main paths: - - '.github/workflows/pgmq.yml' + - '.github/workflows/crate_ci.yml' - 'core/**' push: branches: @@ -56,21 +56,3 @@ jobs: run: cargo sqlx prepare --check - name: teardown run: make test.cleanup - - publish: - # only run when tag release - if: startsWith(github.ref, 'refs/tags/') - name: Publish Crate & Extension - runs-on: ubuntu-22.04 - needs: [lint, tests] - steps: - - uses: actions/checkout@v2 - - name: Install Rust stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - uses: Swatinem/rust-cache@v2 - - name: publish - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - run: cargo publish diff --git a/.github/workflows/extension_ci.yml b/.github/workflows/extension_ci.yml index c705e658..6c6aea58 100644 --- a/.github/workflows/extension_ci.yml +++ b/.github/workflows/extension_ci.yml @@ -9,10 +9,12 @@ on: pull_request: branches: - main - push: branches: - main + release: + types: + - created jobs: lint: @@ -76,8 +78,9 @@ jobs: cargo pgrx test ${pg_version} publish: - # only publish off main branch - if: github.ref == 'refs/heads/main' + # only publish on tagged release events + # if: github.ref == 'refs/heads/main' + if: startsWith(github.ref, 'refs/tags/') name: trunk publish runs-on: ubuntu-22.04 steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c482ea5f..32aeb794 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Postgres Message Queue (PGMQ) +# Contributing to Postgres Message Queue (PGMQ) ## Installation @@ -8,44 +8,68 @@ The fastest way to get started is by running the Tembo docker image, where PGMQ docker run -d --name postgres -e POSTGRES_PASSWORD=postgres -p 5432:5432 quay.io/tembo/pgmq-pg:latest ``` -# Development +## Building from source -Setup `pgrx`. +PGMQ is written as a Rust extension and requires [pgrx](https://github.com/pgcentralfoundation/pgrx). + +To build pgmq from source, you need +* A toolchain capable of building Postgres +* Rust toolchain +* [pg_partman](https://github.com/pgpartman/pg_partman). + +Once you have those pre-requisites, you need to setup `pgrx`. ```bash -cargo install --locked cargo-pgrx -cargo pgrx init +cargo install --locked cargo-pgrx --version 0.9.8 ``` -Then, clone this repo and change into this directory. +Clone the repo and change into the directory. ```bash -git clone git@https://github.com/tembo-io/pgmq.git +git clone https://github.com/tembo-io/pgmq.git cd pgmq ``` -### Setup dependencies +After this point, the steps differ slightly based on if you'd like to build +and install against an existing Postgres setup or develop against pgrx managed +development environment (which installs and allows you to test against multiple +Postgres versions). -Install: -- [pg_partman](https://github.com/pgpartman/pg_partman), which is required for partitioned tables. +### Install to a pre-existing Postgres +Initialize `cargo-pgrx`, and tell it the path to the your `pg_config`. For example, +if `pg_config` is on your `$PATH` and you have Postgres 15, you can run: -Update postgresql.conf in the development environment. +```bash +cargo pgrx init --pg15=`which pg_config` ``` -# ~/.pgrx/data-14/postgresql.conf -shared_preload_libraries = 'pg_partman_bgw' +Then, to install the release build, you can simply run: +``` +cargo pgrx install --release ``` +### Install against pgrx managed Postgres (Recommended for Development) -Run the dev environment +Initialize `cargo-pgrx` development environment: + +```bash +cargo pgrx init +``` + +**Note**: Make sure you build and install `pg_partman` against the postgres installation +you want to build against (`PG_CONFIG` in `~/.pgrx/PG_VERSION/pgrx-install/bin/pg_config` +and `PGDATA` in `/Users/samaysharma/.pgrx/data-PG_MAJOR_VERSION`) + +Then, you can use the run command, which will build and install the extension +and drop you into psql: ```bash cargo pgrx run pg15 ``` -Create the extension +Finally, you can create the extension and get started with the example in the [README.md](README.md). -```pql +```psql CREATE EXTENSION pgmq cascade; ``` @@ -56,3 +80,7 @@ Run this script to package into a `.deb` file, which can be installed on Ubuntu. ``` /bin/bash build-extension.sh ``` + +# Releases + +PGMQ Postgres Extension releases are automated through a [Github workflow](https://github.com/tembo-io/pgmq/blob/main/.github/workflows/extension_ci.yml). The compiled binaries are publish to and hosted at [pgt.dev](https://pgt.dev). To create a release, create a new tag follow a valid [semver](https://semver.org/), then create a release with the same name. Auto-generate the release notes and/or add more relevant details as needed. See subdirectories for the [Rust](https://github.com/tembo-io/pgmq/tree/main/core) and [Python](https://github.com/tembo-io/pgmq/tree/main/tembo-pgmq-python) SDK release processes. diff --git a/Cargo.lock b/Cargo.lock index 75678b71..3238b88b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -236,9 +236,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +dependencies = [ + "libc", +] [[package]] name = "cexpr" @@ -524,18 +527,18 @@ dependencies = [ [[package]] name = "enum-map" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "017b207acb4cc917f4c31758ed95c0bc63ddb0f358b22eb38f80a2b2a43f6b1f" +checksum = "9705d8de4776df900a4a0b2384f8b0ab42f775e93b083b42f8ce71bdc32a47e3" dependencies = [ "enum-map-derive", ] [[package]] name = "enum-map-derive" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8560b409800a72d2d7860f8e5f4e0b0bd22bea6a352ea2a9ce30ccdef7f16d2f" +checksum = "ccb14d927583dd5c2eac0f2cf264fc4762aefe1ae14c47a8a20fc1939d3a5fc0" dependencies = [ "proc-macro2", "quote", @@ -1088,9 +1091,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -1120,9 +1123,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" dependencies = [ "cc", "libc", @@ -1243,7 +1246,7 @@ name = "pgmq" version = "0.11.0" dependencies = [ "chrono", - "pgmq 0.14.0", + "pgmq 0.14.1", "pgrx", "pgrx-tests", "rand", @@ -1257,7 +1260,7 @@ dependencies = [ [[package]] name = "pgmq" -version = "0.14.0" +version = "0.14.1" dependencies = [ "chrono", "log", @@ -1404,9 +1407,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "2c516611246607d0c04186886dbb3a754368ef82c79e9827a802c6d836dd111c" [[package]] name = "pin-utils" @@ -1576,9 +1579,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", @@ -1588,9 +1591,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" dependencies = [ "aho-corasick", "memchr", @@ -1635,9 +1638,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.4" +version = "0.38.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399" dependencies = [ "bitflags 2.3.3", "errno", @@ -1728,9 +1731,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.180" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] @@ -1747,9 +1750,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.180" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", @@ -2033,9 +2036,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.7.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ "cfg-if", "fastrand", @@ -2500,9 +2503,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd122eb777186e60c3fdf765a58ac76e41c582f1f535fbf3314434c6b58f3f7" +checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64" dependencies = [ "memchr", ] diff --git a/README.md b/README.md index 404e80ee..3e9007ac 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ The fastest way to get started is by running the Tembo docker image, where PGMQ docker run -d --name postgres -e POSTGRES_PASSWORD=postgres -p 5432:5432 quay.io/tembo/pgmq-pg:latest ``` +If you'd like to build from source, you can follow the instructions in [CONTRIBUTING.md](CONTRIBUTING.md) + ## Client Libraries @@ -223,7 +225,7 @@ Add the following to `postgresql.conf`. Note, changing `shared_preload_libraries `pg_partman_bgw.interval` sets the interval at which `pg_partman` conducts maintenance. This creates new partitions and dropping of partitions falling out of the `retention_interval`. By default, `pg_partman` will keep 4 partitions "ahead" of the currently active partition. ``` -shared_preload_libraries = 'pg_partman_bgw' # requires restart of Postgrs +shared_preload_libraries = 'pg_partman_bgw' # requires restart of Postgres pg_partman_bgw.interval = 60 pg_partman_bgw.role = 'postgres' pg_partman_bgw.dbname = 'postgres' diff --git a/core/Cargo.toml b/core/Cargo.toml index f33f8fbd..e5072bea 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgmq" -version = "0.14.0" +version = "0.14.1" edition = "2021" authors = ["Tembo.io"] description = "A distributed message queue for Rust applications, on Postgres." diff --git a/core/Makefile b/core/Makefile index e28cf4aa..1dd84aa7 100644 --- a/core/Makefile +++ b/core/Makefile @@ -15,7 +15,7 @@ run.postgres: docker run --rm -d --name pgmq-pg -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -p 5432:5432 quay.io/tembo/pgmq-pg:latest test: run.postgres - sleep 2; + sleep 4; echo "Running all tests..." sqlx migrate run DATABASE_URL=postgres://postgres:postgres@0.0.0.0:5432 cargo test diff --git a/core/tests/integration_test.rs b/core/tests/integration_test.rs index 63366af3..25f9e93b 100644 --- a/core/tests/integration_test.rs +++ b/core/tests/integration_test.rs @@ -16,6 +16,9 @@ async fn init_queue(qname: &str) -> pgmq::PGMQueue { // make sure queue doesn't exist before the test let _ = queue.destroy(qname).await.unwrap(); // CREATE QUEUE + // jiggle to mitigate race condition in concurrent `create if not exists` statements + let random_sleep_ms = rand::thread_rng().gen_range(0..1000); + tokio::time::sleep(std::time::Duration::from_millis(random_sleep_ms)).await; let q_success = queue.create(qname).await; println!("q_success: {:?}", q_success); assert!(q_success.is_ok()); diff --git a/src/lib.rs b/src/lib.rs index 03f4c293..ec7f318b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,7 +10,9 @@ pub mod metrics; pub mod partition; use pgmq_crate::errors::PgmqError; -use pgmq_crate::query::{archive, check_input, delete, init_queue, pop, read, TABLE_PREFIX}; +use pgmq_crate::query::{ + archive, check_input, delete, init_queue, pop, read, PGMQ_SCHEMA, TABLE_PREFIX, +}; use thiserror::Error; #[derive(Error, Debug)] @@ -95,7 +97,7 @@ fn enqueue_str(name: &str) -> Result { check_input(name)?; Ok(format!( " - INSERT INTO {TABLE_PREFIX}_{name} (vt, message) + INSERT INTO {PGMQ_SCHEMA}.{TABLE_PREFIX}_{name} (vt, message) VALUES (now() at time zone 'utc', $1) RETURNING msg_id; "