fix parallel segfault by releasing global GIL and aqcquire GIL in each

partition thread
sfu-db · May 9, 2024 · 33176e8 · 33176e8
1 parent 795c1da
commit 33176e8
Show file tree

Hide file tree

Showing 46 changed files with 1,786 additions and 1,346 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,131 +10,109 @@ on:
       - main
 
 jobs:
-  # rust:
-  #   runs-on: ubuntu-latest
-  #   container: ubuntu:20.04
-  #   services:
-  #     # Label used to access the service container
-  #     postgres:
-  #       # Docker Hub image
-  #       image: postgres
-  #       env:
-  #         POSTGRES_PASSWORD: postgres
-  #       # Set health checks to wait until postgres has started
-  #       options: >-
-  #         --health-cmd pg_isready
-  #         --health-interval 10s
-  #         --health-timeout 5s
-  #         --health-retries 5
-  #     # mysql
-  #     mysql:
-  #       image: ghcr.io/wangxiaoying/mysql:latest
-  #       env:
-  #         MYSQL_DATABASE: mysql
-  #         MYSQL_ROOT_PASSWORD: mysql
-  #         LANG: C.UTF-8
-  #       ports:
-  #         - 3306:3306
-  #       options: >-
-  #         --health-cmd "mysqladmin ping"
-  #         --health-interval 10s
-  #         --health-timeout 10s
-  #         --health-retries 5
-  #     mssql:
-  #       image: mcr.microsoft.com/mssql/server:2019-latest
-  #       env:
-  #         ACCEPT_EULA: y
-  #         SA_PASSWORD: mssql!Password
-  #       ports:
-  #         - 1433:1433
-  #       options: >-
-  #         --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' || exit 1"
-  #         --health-interval 10s
-  #         --health-timeout 5s
-  #         --health-retries 20
-  #   steps:
-  #     - uses: actions/checkout@v4
+  rust:
+    runs-on: ubuntu-latest
+    container: ubuntu:22.04
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      mysql:
+        image: ghcr.io/wangxiaoying/mysql:latest
+        env:
+          MYSQL_DATABASE: mysql
+          MYSQL_ROOT_PASSWORD: mysql
+          LANG: C.UTF-8
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 10s
+          --health-retries 5
 
-  #     - name: Install tools
-  #       run: |
-  #         apt-get update
-  #         apt-get install -y curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client python3 python3-pip libicu66 libkrb5-dev libclang-dev
-  #         pip3 install mssql-cli
-  #         pip3 install cli-helpers==2.2.0
-  #         ln -s /usr/bin/python3 /usr/bin/python
-  #         echo "Cache Version ${{ secrets.CACHE_VERSION }}"
-  #       env:
-  #         DEBIAN_FRONTEND: noninteractive
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install tools
+        run: |
+          apt-get update
+          apt-get install -y curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client python3 python3-pip libkrb5-dev libclang-dev
+          echo "Cache Version ${{ secrets.CACHE_VERSION }}"
+        env:
+          DEBIAN_FRONTEND: noninteractive
 
-  #     - name: Install Rust
-  #       uses: actions-rs/toolchain@v1
-  #       with:
-  #         toolchain: stable
-  #         components: rustfmt
-  #         default: true
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          components: rustfmt
+          default: true
 
-  #     - uses: actions/cache@v4
-  #       with:
-  #         path: |
-  #           ~/.cargo/bin/
-  #           ~/.cargo/registry/index/
-  #           ~/.cargo/registry/cache/
-  #           ~/.cargo/git/db/
-  #           target/
-  #         key: ${{ runner.os }}-cargo-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
 
-  #     - name: Install other dependent tools
-  #       run: |
-  #         if [ ! -f "$HOME/.cargo/bin/just" ]; then curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.cargo/bin; fi
+      - uses: extractions/setup-just@v2
 
-  #     - name: Seed the database
-  #       run: just seed-db
-  #       env:
-  #         POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
-  #         SQLITE_URL: "sqlite:///tmp/test.db"
-  #         MYSQL_HOST: mysql
-  #         MYSQL_PORT: 3306
-  #         MYSQL_DB: mysql
-  #         MYSQL_USER: root
-  #         MYSQL_PASSWORD: mysql
-  #         MSSQL_HOST: mssql
-  #         MSSQL_PORT: 1433
-  #         MSSQL_DB: tempdb
-  #         MSSQL_USER: sa
-  #         MSSQL_PASSWORD: mssql!Password
+      - name: Seed the database
+        run: just seed-db
+        env:
+          POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
+          SQLITE_URL: "sqlite:///tmp/test.db"
+          MYSQL_HOST: mysql
+          MYSQL_PORT: 3306
+          MYSQL_DB: mysql
+          MYSQL_USER: root
+          MYSQL_PASSWORD: mysql
 
-  #     - name: Is the code formatted?
-  #       uses: actions-rs/cargo@v1
-  #       with:
-  #         command: fmt
-  #         args: --all -- --check -q
+      - name: Is the code formatted?
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check -q
 
-  #     - name: Clippy linting
-  #       uses: actions-rs/cargo@v1
-  #       with:
-  #         command: clippy
-  #         args: --features all
+      - name: Clippy linting
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --features all
 
-  #     - name: Check each feature gate
-  #       run: just test-feature-gate
+      - name: Check each feature gate
+        run: just test-feature-gate
 
-  #     - name: Run tests
-  #       run: cargo clean && just test
-  #       env:
-  #         POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
-  #         SQLITE_URL: "sqlite:///tmp/test.db"
-  #         MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"
-  #         MSSQL_URL: "mssql://sa:mssql!Password@mssql:1433/tempdb"
+      - name: Run tests
+        run: cargo clean && just test
+        env:
+          POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
+          SQLITE_URL: "sqlite:///tmp/test.db"
+          MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"
 
-  #     - name: Test build docs
-  #       uses: actions-rs/cargo@v1
-  #       with:
-  #         command: doc
-  #         args: --no-deps --features all
+      - name: Test build docs
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --no-deps --features all
 
   python:
     runs-on: ubuntu-latest
-    container: ubuntu:20.04
+    container: ubuntu:22.04
+    strategy:
+      matrix:
+        python-version: ["3.12"]
     services:
       # Label used to access the service container
       postgres:
@@ -162,32 +140,33 @@ jobs:
           --health-interval 10s
           --health-timeout 10s
           --health-retries 5
-      mssql:
-        image: mcr.microsoft.com/mssql/server:2019-latest
-        env:
-          ACCEPT_EULA: y
-          SA_PASSWORD: mssql!Password
-        ports:
-          - 1433:1433
-        options: >-
-          --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' || exit 1"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 20
 
     steps:
       - uses: actions/checkout@v4
 
+      - uses: actions/setup-python@v5
+        with:
+            python-version: ${{ matrix.python-version }}
+      - name: Setup python env
+        run: | 
+          which python
+          which pip
+          # ln -s `which python` /usr/bin/python
+          # ln -s `which pip` /usr/bin/pip
+          echo "$Python3_ROOT_DIR/bin" >> $GITHUB_PATH
+          echo $GITHUB_PATH
+
       - name: Install tools
         run: |
           apt-get update
-          apt-get install -y curl postgresql-client build-essential python3-dev python3-pip pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev python3 python3-pip libicu66 libkrb5-dev libclang-dev
-          pip3 install mssql-cli
-          pip3 install cli-helpers==2.2.0
-          ln -s /usr/bin/python3 /usr/bin/python
+          apt-get install -y wget curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev libkrb5-dev libclang-dev
         env:
           DEBIAN_FRONTEND: noninteractive
 
+      - uses: Gr1N/setup-poetry@v9
+
+      - uses: extractions/setup-just@v2
+
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
@@ -208,12 +187,7 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-python-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Install other dependent tools
-        run: |
-          pip3 install poetry
-          if [ ! -f "$HOME/.cargo/bin/just" ]; then curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.cargo/bin; fi
+          key: ${{ runner.os }}-cargo-python-${{ matrix.python-version }}-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Seed the database
         run: just seed-db
@@ -225,11 +199,6 @@ jobs:
           MYSQL_DB: mysql
           MYSQL_USER: root
           MYSQL_PASSWORD: mysql
-          MSSQL_HOST: mssql
-          MSSQL_PORT: 1433
-          MSSQL_DB: tempdb
-          MSSQL_USER: sa
-          MSSQL_PASSWORD: mssql!Password
 
       - name: Clippy linting
         uses: actions-rs/cargo@v1
@@ -241,18 +210,16 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-venv-${{ secrets.CACHE_VERSION }}-${{ hashFiles('connectorx-python/poetry.lock') }}
+          key: ${{ runner.os }}-${{ matrix.python-version }}-venv-${{ secrets.CACHE_VERSION }}-${{ hashFiles('connectorx-python/poetry.lock') }}
 
-      - name: Install python dependencies
-        run: just bootstrap-python
+      - uses: extractions/setup-just@v2
 
       - name: Test python
         run: just test-python
         env:
           POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
           SQLITE_URL: "sqlite:///tmp/test.db"
           MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"
-          MSSQL_URL: "mssql://sa:mssql!Password@mssql:1433/tempdb"
           DB1: "postgresql://postgres:postgres@postgres:5432/postgres"
           DB2: "postgresql://postgres:postgres@postgres:5432/postgres"
           FED_CONFIG_PATH: ${{ github.workspace }}/.github/config

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -12,7 +12,7 @@ jobs:
     container: quay.io/pypa/manylinux_2_28_x86_64
     strategy:
       matrix:
-        python-version: [[38, "3.8"], [39, "3.9"], [310, "3.10"], [311, "3.11"], [312, "3.12"]]
+        python-version: [[39, "3.9"], [310, "3.10"], [311, "3.11"], [312, "3.12"]]
     steps:
       - uses: actions/checkout@v4
 
@@ -162,7 +162,7 @@ jobs:
     strategy:
       matrix:
         os: ["windows-latest", "macos-latest"]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         include:
           - os: "macos-latest"
             features: "--features integrated-auth-gssapi"
@@ -226,7 +226,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
 
@@ -290,7 +290,7 @@ jobs:
     needs: [win-and-mac, linux, apple-arm]
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         os: [macos-latest, ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v4

diff --git a/Justfile b/Justfile
@@ -54,10 +54,10 @@ seed-db:
     psql $POSTGRES_URL -f scripts/postgres.sql
     sqlite3 ${SQLITE_URL#sqlite://} < scripts/sqlite.sql
     mysql --protocol tcp -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB < scripts/mysql.sql
-    mssql-cli -S$MSSQL_HOST -U$MSSQL_USER -P$MSSQL_PASSWORD -d$MSSQL_DB -i scripts/mssql.sql
 
 # dbs not included in ci
 seed-db-more:
+    mssql-cli -S$MSSQL_HOST -U$MSSQL_USER -P$MSSQL_PASSWORD -d$MSSQL_DB -i scripts/mssql.sql
     mysql --protocol tcp -h$CLICKHOUSE_HOST -P$CLICKHOUSE_PORT -u$CLICKHOUSE_USER -p$CLICKHOUSE_PASSWORD $CLICKHOUSE_DB < scripts/clickhouse.sql
     psql $REDSHIFT_URL -f scripts/redshift.sql
     ORACLE_URL_SCRIPT=`echo ${ORACLE_URL#oracle://} | sed "s/:/\//"`