Merge branch 'master' into pedro/sso-domain-docs

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.58.0
+current_version = 0.64.7
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.devcontainer/python-connectors-generic/devcontainer.json

@@ -2,7 +2,7 @@
 {
   "name": "Python Development DevContainer (Generic)",
 
-  "image": "mcr.microsoft.com/devcontainers/python:0-3.10",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.10",
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker": {},
     "ghcr.io/devcontainers/features/python:1": {

.github/CODEOWNERS

@@ -4,23 +4,31 @@
 /airbyte-integrations/connectors/destination-milvus @airbytehq/ai-language-models
 /airbyte-integrations/connectors/destination-qdrant @airbytehq/ai-language-models
 /airbyte-integrations/connectors/destination-chroma @airbytehq/ai-language-models
+/airbyte-integrations/connectors/destination-snowflake-cortex @airbytehq/ai-language-models
 /airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based @airbytehq/ai-language-models
 
 # CI/CD
-/.github/ @airbytehq/connector-extensibility
-/airbyte-ci/ @airbytehq/connector-extensibility
+/.github/ @airbytehq/dev-tooling
+/airbyte-ci/ @airbytehq/dev-tooling
 
 # Python CDK and Connector Acceptance Tests
-/airbyte-cdk/python @airbytehq/connector-extensibility
-/airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
-/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-extensibility @lazebnyi @oustynova
+/airbyte-cdk/python @airbytehq/python-team
+/airbyte-integrations/connector-templates/ @airbytehq/dev-marketplace-contributions
+/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/dev-tooling
 
 # Build customization file change
-/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-extensibility
+/airbyte-integrations/connectors/**/build_customization.py @airbytehq/dev-tooling
 
 # Protocol related items
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
+# Bulk CDK
+/airbyte-cdk/bulk @airbytehq/dbsources @airbytehq/destinations
+/airbyte-cdk/bulk/core/extract/ @airbytehq/dbsources
+/airbyte-cdk/bulk/core/load/ @airbytehq/destinations
+/airbyte-cdk/bulk/toolkits/extract-*/ @airbytehq/dbsources
+/airbyte-cdk/bulk/toolkits/load-*/ @airbytehq/destinations
+
 # Java CDK
 /airbyte-cdk/java/airbyte-cdk @airbytehq/dbsources @airbytehq/destinations
 /airbyte-cdk/java/airbyte-cdk/*-sources/ @airbytehq/dbsources
@@ -51,3 +59,10 @@
 /airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-redshift/ @airbytehq/destinations
+
+# Python critical connectors
+/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-hubspot/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-salesforce/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-shopify/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-stripe/ @airbytehq/python-team

.github/ISSUE_TEMPLATE/2-issue-ai-assist.yaml

@@ -0,0 +1,51 @@
+name: 🤖 [AI Assist] Report an issue using the AI Assist
+description: Use this template when you have a problem creating a connector using the AI Connector Assist.
+labels: [type/bug, area/connector-builder, needs-triage, ai-assist]
+body:
+  - type: markdown
+    attributes:
+      value: >
+        <p align="center">
+          <a target="_blank" href="https://airbyte.com">
+            <image>
+              <source srcset="https://raw.githubusercontent.com/airbytehq/airbyte/master/.github/octavia-issue-template.svg">
+              <img alt="octavia-welcome" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/.github/octavia-issue-template.svg" width="auto" height="120">
+            </image>
+          </a>
+        </p>
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report...
+        Make sure to update this issue with a concise title and provide all information you have to
+        help us debug the problem together.
+  - type: input
+    id: connector-url-link
+    attributes:
+      label: Connector Documentation URL
+      description: Share the documentation URL you used to create the connector.
+    validations:
+      required: true
+  - type: textarea
+    id: description
+    attributes:
+      label: What error are you seeing?
+      description: Please give any information you have and steps to reproduce the problem.
+  - type: textarea
+    id: replicate
+    attributes:
+      label: How to replicate
+      description: Please provide what steps you made so far
+  - type: textarea
+    id: expect
+    attributes:
+      label: What do you expect to happen?
+      description: What output were you expecting?
+  - type: checkboxes
+    id: alpha-release
+    attributes:
+      label: Is this in the Alpha Release?
+      description: Are you noticing this issue in the alpha release of AI Assist?
+      options:
+        - label: Yes, the issue is in the AI Assist Alpha Release.
+          required: false

.github/actions/install-airbyte-ci/action.yml

@@ -36,8 +36,10 @@ runs:
         if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
           echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
           echo "install-mode=source" >> $GITHUB_OUTPUT
+          echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV
         else
           echo "install-mode=binary" >> $GITHUB_OUTPUT
+          echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV
         fi
 
     - name: Install Airbyte CI from binary

.github/actions/run-airbyte-ci/action.yml

@@ -9,16 +9,16 @@ inputs:
     required: true
   github_token:
     description: "GitHub token"
-    required: true
+    required: false
   dagger_cloud_token:
     description: "Dagger Cloud token"
-    required: true
+    required: false
   docker_hub_username:
     description: "Dockerhub username"
-    required: true
+    required: false
   docker_hub_password:
     description: "Dockerhub password"
-    required: true
+    required: false
   options:
     description: "Options for the subcommand"
     required: false
@@ -34,6 +34,10 @@ inputs:
     description: "GCP credentials for GCP Secret Manager"
     required: false
     default: ""
+  gcp_integration_tester_credentials:
+    description: "GCP credentials for integration tests"
+    required: false
+    default: ""
   git_repo_url:
     description: "Git repository URL"
     default: https://github.com/airbytehq/airbyte.git
@@ -87,21 +91,29 @@ inputs:
     description: "Whether the PR is from a fork"
     required: false
     default: "false"
+  max_attempts:
+    description: "Number of attempts at running the airbyte-ci command"
+    required: false
+    default: 1
+  retry_wait_seconds:
+    description: "Number of seconds to wait between retry attempts"
+    required: false
+    default: 60
+
 runs:
   using: "composite"
   steps:
     - name: Get start timestamp
       id: get-start-timestamp
       shell: bash
-      run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
-
+      run: echo "start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
     - name: Docker login
       id: docker-login
       uses: docker/login-action@v3
+      if: ${{ inputs.docker_hub_username != '' && inputs.docker_hub_password != '' }}
       with:
         username: ${{ inputs.docker_hub_username }}
         password: ${{ inputs.docker_hub_password }}
-
     - name: Install Airbyte CI
       id: install-airbyte-ci
       uses: ./.github/actions/install-airbyte-ci
@@ -110,20 +122,20 @@ runs:
         is_fork: ${{ inputs.is_fork }}
     - name: Run airbyte-ci
       id: run-airbyte-ci
-      shell: bash
-      run: |
-        airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+      uses: nick-fields/retry@v3
       env:
+        CI: "True"
+        CI_GIT_USER: ${{ github.repository_owner }}
+        CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
+        PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+        # Next environment variables are workflow inputs based and can be set with empty values if the inputs are not required and passed
         CI_CONTEXT: "${{ inputs.context }}"
-        CI_GIT_REPO_URL: ${{ inputs.git_repo_url }}
         CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }}
+        CI_GIT_REPO_URL: ${{ inputs.git_repo_url }}
         CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }}
-        CI_GIT_USER: ${{ github.repository_owner }}
         CI_GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }}
         CI_JOB_KEY: ${{ inputs.ci_job_key }}
-        CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
         CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }}
-        CI: "True"
         DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}"
         DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }}
         DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }}
@@ -133,47 +145,52 @@ runs:
         METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }}
         METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }}
         PRODUCTION: ${{ inputs.production }}
-        PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
         PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}
         PYTHON_REGISTRY_URL: ${{ inputs.python_registry_url }}
-        PYTHON_REGISTRY_CHECK_URL: ${{ inputs.python_registry_check_url }}
         S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }}
         S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }}
         SENTRY_DSN: ${{ inputs.sentry_dsn }}
-        SENTRY_ENVIRONMENT: ${{ steps.determine-install-mode.outputs.install-mode }}
         SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
         SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
         SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}
-    # give the Dagger Engine more time to push cache data to Dagger Cloud
+      with:
+        shell: bash
+        max_attempts: ${{ inputs.max_attempts }}
+        retry_wait_seconds: ${{ inputs.retry_wait_seconds }}
+        # 360mn > 6 hours: it's the GitHub runner max job duration
+        timeout_minutes: 360
+        command: |
+          airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
     - name: Stop Engine
       id: stop-engine
       if: always()
       shell: bash
       run: |
         mapfile -t containers < <(docker ps --filter name="dagger-engine-*" -q)
         if [[ "${#containers[@]}" -gt 0 ]]; then
+          # give 5mn to the Dagger Engine to push cache data to Dagger Cloud
           docker stop -t 300 "${containers[@]}";
         fi
 
-    - name: Collect docker logs on failure
-      id: collect-docker-logs
+    - name: Collect dagger engine logs
+      id: collect-dagger-engine-logs
       if: always()
       uses: jwalton/gh-docker-logs@v2
       with:
-        dest: "./docker_logs"
+        dest: "./dagger_engine_logs"
         images: "registry.dagger.io/engine"
 
     - name: Tar logs
       id: tar-logs
       if: always()
       shell: bash
-      run: tar cvzf ./docker_logs.tgz ./docker_logs
+      run: tar cvzf ./dagger_engine_logs.tgz ./dagger_engine_logs
 
     - name: Upload logs to GitHub
-      id: upload-docker-logs
+      id: upload-dagger-engine-logs
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: docker_logs.tgz
-        path: ./docker_logs.tgz
+        name: ${{ github.job }}_dagger_engine_logs.tgz
+        path: ./dagger_engine_logs.tgz
         retention-days: 7

.github/actions/send-pager-duty-event/action.yml

@@ -0,0 +1,36 @@
+name: "Send PagerDuty event"
+description: "Use Event API V2 to send a PagerDuty event."
+inputs:
+  integration_key:
+    description: "The integration key for the PagerDuty service."
+    required: true
+  summary:
+    description: "A brief text summary of the event."
+    required: true
+  severity:
+    description: "The severity of the event. (info, warning, error, critical)"
+    required: true
+  source:
+    description: "Specific human-readable unique identifier, such as a hostname, for the system having the problem."
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Send PagerDuty event
+      id: send-pager-duty-event
+      shell: bash
+      run: |
+        curl --request 'POST' \
+          --fail \
+          --url "https://events.pagerduty.com/v2/enqueue" \
+          --header 'Content-Type: application/json' \
+          --data '{
+            "payload": {
+              "summary": "${{ inputs.summary }}",
+              "severity": "${{ inputs.severity }}",
+              "source": "${{ inputs.source }}"
+            },
+            "event_action": "trigger",
+            "routing_key": "${{ inputs.integration_key }}"
+          }'

.github/workflows/airbyte-ci-release.yml

@@ -54,7 +54,7 @@ jobs:
         working-directory: airbyte-ci/connectors/pipelines/
         run: poetry run poe build-release-binary ${{ env.BINARY_FILE_NAME }}
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: airbyte-ci-${{ matrix.os }}-${{ steps.get_short_sha.outputs.sha }}
           path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}