Merge branch 'master' into master

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.11
+current_version = 0.50.13
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/actions/run-dagger-pipeline/action.yml

@@ -47,6 +47,9 @@ inputs:
     description: "Bucket name for metadata service"
     required: false
     default: "prod-airbyte-cloud-connector-metadata-service"
+  sentry_dsn:
+    description: "Sentry DSN"
+    required: false
   spec_cache_bucket_name:
     description: "Bucket name for GCS spec cache"
     required: false
@@ -89,12 +92,13 @@ runs:
       shell: bash
       run: |
         pip install pipx
+        pipx ensurepath
         pipx install airbyte-ci/connectors/pipelines/
     - name: Run airbyte-ci
       shell: bash
       run: |
         export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
-        airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+        airbyte-ci-internal --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
         _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
         CI_CONTEXT: "${{ inputs.context }}"
@@ -110,6 +114,7 @@ runs:
         METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }}
         PRODUCTION: ${{ inputs.production }}
         PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+        SENTRY_DSN: ${{ inputs.sentry_dsn }}
         SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
         SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
         SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}

.github/workflows/airbyte-ci-tests.yml

@@ -0,0 +1,33 @@
+name: Airbyte CI pipeline tests
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+    paths:
+      - airbyte-ci/**
+jobs:
+  run-airbyte-ci-tests:
+    name: Run Airbyte CI tests
+    runs-on: "conn-prod-xlarge-runner"
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Run pipelines tests
+        id: run-pipelines-tests
+        uses: ./.github/actions/run-dagger-pipeline
+        with:
+          context: "pull_request"
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          subcommand: "tests connectors/pipelines"

.github/workflows/connector-performance-command.yml

@@ -113,6 +113,7 @@ jobs:
       - name: Install CI scripts
         run: |
           pip install pipx
+          pipx ensurepath
           pipx install airbyte-ci/connectors/ci_credentials
           pipx install airbyte-ci/connectors/connector_ops
       - name: Source or Destination harness

.github/workflows/connector_metadata_checks.yml

@@ -20,6 +20,7 @@ jobs:
       - name: Install ci-connector-ops package
         run: |
           pip install pipx
+          pipx ensurepath
           pipx install airbyte-ci/connectors/connector_ops/
       - name: Check test strictness level
         run: check-test-strictness-level

.github/workflows/connector_teams_review_requirements.yml

@@ -25,6 +25,7 @@ jobs:
       - name: Install ci-connector-ops package
         run: |
           pip install pipx
+          pipx ensurepath
           pipx install airbyte-ci/connectors/connector_ops
       - name: Write review requirements file
         id: write-review-requirements-file

.github/workflows/legacy-publish-command.yml

@@ -249,13 +249,20 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+      - name: Install Pyenv
+        run: |
+          python3 -m pip install --quiet virtualenv --user
+          rm -r venv || echo "no pre-existing venv"
+          python3 -m virtualenv venv
+          source venv/bin/activate
       - name: Install CI scripts
         run: |
-          pip install pipx
-          pipx install airbyte-ci/connectors/ci_credentials
-          pipx install airbyte-ci/connectors/connector_ops
+          source venv/bin/activate
+          pip install --quiet -e ./airbyte-ci/connectors/ci_credentials
+          pip install --quiet -e ./airbyte-ci/connectors/connector_ops
       - name: Write Integration Test Credentials for ${{ matrix.connector }}
         run: |
+          source venv/bin/activate
           ci_credentials ${{ matrix.connector }} write-to-storage
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ matrix.connector }}" ] || [ 'base-normalization' = "${{ matrix.connector }}" ]; then
@@ -283,6 +290,7 @@ jobs:
         id: qa_checks
         if: always()
         run: |
+          source venv/bin/activate
           run-qa-checks ${{ matrix.connector }}
       - name: Publish ${{ matrix.connector }}
         id: publish
@@ -301,6 +309,7 @@ jobs:
       - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}
         if: always()
         run: |
+          source venv/bin/activate
           ci_credentials ${{ matrix.connector }} update-secrets
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ matrix.connector }}" ] || [ 'base-normalization' = "${{ matrix.connector }}" ]; then

.github/workflows/legacy-test-command.yml

@@ -101,10 +101,10 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install CI scripts
+        # all CI python packages have the prefix "ci_"
         run: |
-          pip install pipx
-          pipx install airbyte-ci/connectors/ci_credentials
-          pipx install airbyte-ci/connectors/connector_ops
+          pip install --quiet -e ./airbyte-ci/connectors/ci_credentials
+          pip install --quiet -e ./airbyte-ci/connectors/connector_ops
       - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
         run: |
           ci_credentials ${{ github.event.inputs.connector }} write-to-storage

.github/workflows/publish_connectors.yml

@@ -37,9 +37,10 @@ jobs:
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
           spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
-          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --modified publish --main-release"
+          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
 
       - name: Publish connectors [manual]
         id: publish-connectors
@@ -53,6 +54,7 @@ jobs:
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
           spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
           subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"

.github/workflows/publish_pypi.yml

@@ -0,0 +1,16 @@
+name: Publish connectors to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      runs-on:
+        type: string
+        default: conn-prod-xlarge-runner
+        required: true
+
+jobs:
+  no-op:
+    name: No-op
+    runs-on: ${{ inputs.runs-on || 'conn-prod-xlarge-runner' }}
+    steps:
+      - run: echo 'hi!'

.github/workflows/test-performance-command.yml

@@ -92,6 +92,7 @@ jobs:
       - name: Install CI scripts
         run: |
           pip install pipx
+          pipx ensurepath
           pipx install airbyte-ci/connectors/ci_credentials
       - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
         run: |

.gitignore

@@ -98,4 +98,7 @@ dd-java-agent.jar
 gha-creds-*.json
 
 # Legacy pipeline reports path
-tools/ci_connector_ops/pipeline_reports/
+tools/ci_connector_ops/pipeline_reports/
+
+# ignore local build scan uri output
+scan-journal.log