From 29dc39c9b91a9e0495fd2fbf45000124abf55419 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Fri, 15 Mar 2024 17:35:00 +0800 Subject: [PATCH] update pg-ext (#16) * update pg-ext * update jdk naming * fix gpdb py3 bug * debug pgext --- .github/workflows/build-docker.yml | 6 ++-- docker_bigdata/Dockerfile | 2 +- docker_greenplum/Dockerfile | 2 +- docker_kafka_confluent/Dockerfile | 2 +- .../kafka-standalone-confluent/README.md | 2 +- docker_postgres/README.md | 23 +++++++++++++ docker_postgres/postgres-ext.Dockerfile | 32 ++++++++++++------- docker_postgres/work/install_list_pgext.apt | 11 ------- .../work/install_list_pgext.tpl.apt | 12 +++++++ 9 files changed, 63 insertions(+), 29 deletions(-) create mode 100644 docker_postgres/README.md delete mode 100644 docker_postgres/work/install_list_pgext.apt create mode 100644 docker_postgres/work/install_list_pgext.tpl.apt diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 9937f9a..815e67e 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -63,13 +63,13 @@ jobs: source ./tool.sh build_image greenplum latest docker_greenplum/Dockerfile && push_image qpod_postgres: - name: "postgres" + name: "postgres-ext" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - run: | source ./tool.sh - build_image postgres latest docker_postgres/postgres-ext.Dockerfile && push_image + build_image postgres-ext latest docker_postgres/postgres-ext.Dockerfile && push_image ## Sync all images in this build (listed by "names") to mirror registry. sync_images: @@ -84,4 +84,4 @@ jobs: --env-file /tmp/docker.env \ -v $(pwd):/tmp \ -w /tmp \ - qpod/docker-kit python /opt/utils/image-syncer/run_jobs.py + qpod/docker-kit /opt/conda/bin/python /opt/utils/image-syncer/run_jobs.py diff --git a/docker_bigdata/Dockerfile b/docker_bigdata/Dockerfile index 8971d43..9c2aa70 100644 --- a/docker_bigdata/Dockerfile +++ b/docker_bigdata/Dockerfile @@ -1,7 +1,7 @@ # Distributed under the terms of the Modified BSD License. ARG BASE_NAMESPACE -ARG BASE_IMG="jdk11" +ARG BASE_IMG="jdk-11" FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} LABEL maintainer="haobibo@gmail.com" diff --git a/docker_greenplum/Dockerfile b/docker_greenplum/Dockerfile index 2584678..ffc642f 100644 --- a/docker_greenplum/Dockerfile +++ b/docker_greenplum/Dockerfile @@ -37,7 +37,7 @@ RUN set -x && source /opt/utils/script-utils.sh \ && chown -R ${GPUSER}:${GPUSER} ${GPHOME} ${GPDATA} /home/${GPUSER} \ && install_apt /opt/utils/install_list_greenplum.apt \ && export PATH="${PATH}:${GPHOME}/bin" && cd ${GPHOME}/bin && ls -alh \ - && PYTHON_SITE=$(python3 -c 'import sys;print(list(filter(lambda s: "site" in s, sys.path))[0])') \ + && PYTHON_SITE=$(/opt/conda/bin/python3 -c 'import sys;print(list(filter(lambda s: "site" in s, sys.path))[0])') \ && sudo ln -s ${GPHOME}/lib/python/* ${PYTHON_SITE}/ \ && pip install -U psutil pygresql pyyaml \ && cp ${GPHOME}/docs/cli_help/gpconfigs/gpinitsystem_config ${GPHOME}/conf/gpinitsystem_config \ diff --git a/docker_kafka_confluent/Dockerfile b/docker_kafka_confluent/Dockerfile index e11fc9e..191f734 100644 --- a/docker_kafka_confluent/Dockerfile +++ b/docker_kafka_confluent/Dockerfile @@ -1,6 +1,6 @@ # Distributed under the terms of the Modified BSD License. ARG BASE_NAMESPACE -ARG BASE_IMG="jdk11" +ARG BASE_IMG="jdk-11" FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} ARG KAFKA_VERSION="7.3.3" diff --git a/docker_kafka_confluent/example/kafka-standalone-confluent/README.md b/docker_kafka_confluent/example/kafka-standalone-confluent/README.md index 6847e24..98ff5ff 100644 --- a/docker_kafka_confluent/example/kafka-standalone-confluent/README.md +++ b/docker_kafka_confluent/example/kafka-standalone-confluent/README.md @@ -41,7 +41,7 @@ docker run -it \ -p=9092:9092 \ -v /data/kafka-confluent/broker/:/var/lib/kafka-broker/ \ -v $(pwd):/root/dev \ - qpod/jdk11 bash + qpod/jdk-11 bash export KAFKA_BROKER_ID=1 export KAFKA_LISTENER_SECURITY_PROTOCOL_MAP='CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' diff --git a/docker_postgres/README.md b/docker_postgres/README.md new file mode 100644 index 0000000..d5c7991 --- /dev/null +++ b/docker_postgres/README.md @@ -0,0 +1,23 @@ +# PostgreSQL with extensions + +## Debug + +```shell +BUILDKIT_PROGRESS=plain docker build -t postgres-ext -f ./postgres-ext.Dockerfile --build-arg BASE_NAMESPACE=qpod0dev . + +IMG="qpod/postgres-ext" + +docker run -d \ + --name db-postgres \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=pg-password \ + $IMG + +ls -alh /usr/share/postgresql/15/extension/*.control +``` + +```sql +SELECT * FROM pg_extension; +SELECT * FROM pg_available_extensions ORDER BY name; +CREATE EXTENSION "vector"; +``` diff --git a/docker_postgres/postgres-ext.Dockerfile b/docker_postgres/postgres-ext.Dockerfile index 7cb89e5..7daa2d1 100644 --- a/docker_postgres/postgres-ext.Dockerfile +++ b/docker_postgres/postgres-ext.Dockerfile @@ -1,22 +1,32 @@ # Distributed under the terms of the Modified BSD License. ARG BASE_NAMESPACE -ARG BASE_IMG="base" -FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} as builder - -ARG PG_MAJOR=15 -FROM library/postgres:${PG_MAJOR:-latest} +ARG BASE_IMG="postgres" +FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} LABEL maintainer="haobibo@gmail.com" COPY work /opt/utils/ -COPY --from=builder /opt /opt -RUN set -x \ +RUN set -x && . /opt/utils/script-utils.sh \ && apt-get update && apt-get install -y gettext \ - apt-utils apt-transport-https ca-certificates gnupg2 dirmngr locales sudo lsb-release curl \ - && envsubst < /opt/utils/install_list_pgext.apt > /opt/utils/install_list_pgext.apt \ - && . /opt/utils/script-utils.sh \ - && install_apt /opt/utils/install_list_base.apt \ + && DISTRO_NAME=$(awk '{ print tolower($0) }' <<< $(lsb_release -is)) \ + && DISTRO_CODE_NAME=$(lsb_release -cs) \ + # apt source for: https://packagecloud.io/citusdata/community + && curl -fsSL "https://packagecloud.io/citusdata/community/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \ + && echo "deb https://packagecloud.io/citusdata/community/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/citusdata_community.list \ + # apt source for: https://packagecloud.io/timescale/timescaledb + && curl -fsSL "https://packagecloud.io/timescale/timescaledb/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/timescale_timescaledb.gpg \ + && echo "deb https://packagecloud.io/timescale/timescaledb/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/timescale_timescaledb.list \ + # apt source for: https://packagecloud.io/pigsty/pgsql + && curl -fsSL "https://packagecloud.io/pigsty/pgsql/gpgkey" | gpg --dearmor > /etc/apt/trusted.gpg.d/pigsty_pgsql.gpg \ + && echo "deb https://packagecloud.io/pigsty/pgsql/${DISTRO_NAME}/ ${DISTRO_CODE_NAME} main" | sudo tee /etc/apt/sources.list.d/pigsty_pgsql.list \ + && envsubst < /opt/utils/install_list_pgext.tpl.apt > /opt/utils/install_list_pgext.apt \ + && rm -rf /opt/utils/install_list_pgext.tpl.apt \ + && echo "To install PG extensions: $(cat /opt/utils/install_list_pgext.apt)" \ && install_apt /opt/utils/install_list_pgext.apt \ + && ls -alh /usr/share/postgresql/*/extension/*.control | sort \ && echo "Clean up" && list_installed_packages && install__clean + +USER postgres +WORKDIR /var/lib/postgresql diff --git a/docker_postgres/work/install_list_pgext.apt b/docker_postgres/work/install_list_pgext.apt deleted file mode 100644 index 675a5a3..0000000 --- a/docker_postgres/work/install_list_pgext.apt +++ /dev/null @@ -1,11 +0,0 @@ -postgresql-contrib -postgresql-${PG_MAJOR}-postgis* -postgresql-${PG_MAJOR}-pgvector -postgresql-${PG_MAJOR}-cron -postgresql-${PG_MAJOR}-wal2json - -% https://packagecloud.io/citusdata/community/${distro_name}/ ${distro_codename} main -% https://packagecloud.io/timescale/timescaledb/${distro_name}/ ${distro_codename} main -% https://apt.postgresml.org ${distro_codename} maintainer -% timescaledb-2-postgresql-${PG_MAJOR} -% postgresql-${PG_MAJOR}-citus-12.1 diff --git a/docker_postgres/work/install_list_pgext.tpl.apt b/docker_postgres/work/install_list_pgext.tpl.apt new file mode 100644 index 0000000..e88fa0a --- /dev/null +++ b/docker_postgres/work/install_list_pgext.tpl.apt @@ -0,0 +1,12 @@ +postgresql-contrib +postgresql-${PG_MAJOR}-postgis* +postgresql-${PG_MAJOR}-pgvector +postgresql-${PG_MAJOR}-cron +postgresql-${PG_MAJOR}-wal2json +timescaledb-2-postgresql-${PG_MAJOR} +postgresql-${PG_MAJOR}-citus-12.1 +pg-graphql % https://github.com/supabase/pg_graphql +pg-net % https://github.com/supabase/pg_net +% postgresql-*-pgdg-pgroonga % TODO: postgresql-15-pgdg-pgroonga : Depends: libgroonga0 (>= 13.0.8) but 13.0.0+dfsg-3~deb12u1 is to be installed +% postgresql-pgml-* % TODO: many python packages +% echo "deb https://apt.postgresml.org ${DISTRO_CODE_NAME} maintainer" | sudo tee /etc/apt/sources.list.d/postgresml.list