better CDK dev workflow

better CDK dev workflow

.github/workflows/bump_connector_versions.yml

@@ -0,0 +1,93 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+name: Connector Ops CI - Bump Connector Versions
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "airbyte-integrations/connectors/**/.changelog_entries/*"
+
+jobs:
+  bump_connector_versions:
+    name: Publish connectors
+    runs-on: connector-publish-large
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Publish modified connectors [On merge to master]
+        id: publish-modified-connectors
+        if: github.event_name == 'push'
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: "master"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
+          spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
+          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          subcommand: "connectors bump_version"
+          python_registry_token: ${{ secrets.PYPI_TOKEN }}
+
+  set-instatus-incident-on-failure:
+    name: Create Instatus Incident on Failure
+    runs-on: ubuntu-latest
+    needs:
+      - bump_connector_versions
+    if: ${{ failure() && github.ref == 'refs/heads/master' }}
+    steps:
+      - name: Call Instatus Webhook
+        uses: joelwmale/webhook-action@master
+        with:
+          url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }}
+          body: '{ "trigger": "down", "status": "HASISSUES" }'
+
+  set-instatus-incident-on-success:
+    name: Create Instatus Incident on Success
+    runs-on: ubuntu-latest
+    needs:
+      - bump_connector_versions
+    if: ${{ success() && github.ref == 'refs/heads/master' }}
+    steps:
+      - name: Call Instatus Webhook
+        uses: joelwmale/webhook-action@master
+        with:
+          url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }}
+          body: '{ "trigger": "up" }'
+
+  notify-failure-slack-channel:
+    name: "Notify Slack Channel on Build Failures"
+    runs-on: ubuntu-latest
+    needs:
+      - bump_connector_versions
+    if: ${{ failure() && github.ref == 'refs/heads/master' }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Match GitHub User to Slack User
+        id: match-github-to-slack-user
+        uses: ./.github/actions/match-github-to-slack-user
+        env:
+          AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
+          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Publish to OSS Build Failure Slack Channel
+        uses: abinoda/slack-action@master
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+        with:
+          args: >-
+            {\"channel\":\"C056HGD1QSW\", \"blocks\":[
+            {\"type\":\"divider\"},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Connector Version Bump Failed! :bangbang: \n\n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
+            {\"type\":\"divider\"}]}

.github/workflows/create_connector_changelog_entry_files.yml

@@ -0,0 +1,24 @@
+name: Create connector changelog entry files
+run-name: Create connector changelog entry files
+on:
+  pull_request:
+    types:[opened, edited, synchronize]
+
+env:
+  GITREF: ${{ github.event.inputs.gitref || github.ref }}
+
+
+jobs:
+  create-connector-changelog-entry-files:
+    # IMPORTANT: This name must match the require check name on the branch protection settings
+    name: "Delete existing files for the current PR"
+    runs-on: tooling-test-small
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.GITREF }}
+
+      - name: Write PR body to file
+
+      - name: Create Changelog Entry Files

.github/workflows/publish-java-cdk-command.yml

@@ -34,6 +34,11 @@ on:
         required: true
         type: boolean
         default: false
+      remove-useLocalCdk:
+        description: "Once the CDK is published, set useLocalCdk to false for all connectors where it is true"
+        required: true
+        type: boolean
+        default: false
       gitref:
         description: "The git ref to check out from the specified repository."
         required: true
@@ -50,9 +55,11 @@ env:
   GITREF: ${{ github.event.inputs.gitref || github.ref }}
   FORCE: "${{ github.event_name == 'push' || github.event.inputs.force == null && 'false' ||  github.event.inputs.force }}"
   DRY_RUN: "${{ github.event.inputs.dry-run == null && 'true' || github.event.inputs.dry-run }}"
+  REMOVE_USELOCALCDK: "${{ github.event_name == 'push' || github.event.inputs.remove-useLocalCdk == null && 'false' || github.event.inputs.remove-useLocalCdk }}"
   CDK_VERSION_FILE_PATH: "./airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties"
   S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
   S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+  AIRBYTE_CI_CONTEXT: "${{ github.event_name == 'push' && 'master' || 'manual' }}
 
 jobs:
   publish-cdk:
@@ -135,6 +142,19 @@ jobs:
           gradle-distribution-sha-256-sum-warning: false
           arguments: --scan :airbyte-cdk:java:airbyte-cdk:cdkPublish
 
+      - name: remove usages of useLocalCdk
+        if: ${{ env.DRY_RUN == 'false' && env.REMOVE_USELOCALCDK != 'false' }}
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: ${{ env.AIRBYTE_CI_CONTEXT }}
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          subcommand: "connectors --with-use-local-cdk upgrade_cdk"
+
       - name: Add Success Comment
         if: github.event.inputs.comment-id && success()
         uses: peter-evans/create-or-update-comment@v1

.github/workflows/publish_connectors.yml

@@ -42,7 +42,7 @@ jobs:
           spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
+          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only --without-use-local-cdk publish --main-release"
           python_registry_token: ${{ secrets.PYPI_TOKEN }}
 
       - name: Publish connectors [manual]

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/cdk_java/commands.py

@@ -0,0 +1,187 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import os
+from pathlib import Path
+from typing import List, Optional, Set, Tuple
+
+import asyncclick as click
+from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo  # type: ignore
+from pipelines import main_logger
+from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.cli.lazy_group import LazyGroup
+from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors
+from pipelines.helpers.git import get_modified_files
+from pipelines.helpers.utils import transform_strs_to_paths
+
+ALL_CONNECTORS = get_all_connectors_in_repo()
+
+
+def log_selected_connectors(selected_connectors_with_modified_files: List[ConnectorWithModifiedFiles]) -> None:
+    if selected_connectors_with_modified_files:
+        selected_connectors_names = [c.technical_name for c in selected_connectors_with_modified_files]
+        main_logger.info(f"Will run on the following {len(selected_connectors_names)} connectors: {', '.join(selected_connectors_names)}.")
+    else:
+        main_logger.info("No connectors to run.")
+
+def get_selected_connectors_with_modified_files(
+    selected_names: Tuple[str],
+    selected_support_levels: Tuple[str],
+    selected_languages: Tuple[str],
+    modified: bool,
+    metadata_changes_only: bool,
+    with_changelog_entry_files: bool,
+    metadata_query: str,
+    modified_files: Set[Path],
+    enable_dependency_scanning: bool = False,
+) -> List[ConnectorWithModifiedFiles]:
+    """Get the connectors that match the selected criteria.
+
+    Args:
+        selected_names (Tuple[str]): Selected connector names.
+        selected_support_levels (Tuple[str]): Selected connector support levels.
+        selected_languages (Tuple[str]): Selected connector languages.
+        modified (bool): Whether to select the modified connectors.
+        metadata_changes_only (bool): Whether to select only the connectors with metadata changes.
+        with_changelog_entry_files (bool): Whether to select the connectors with files in .changelog_entries
+        modified_files (Set[Path]): The modified files.
+        enable_dependency_scanning (bool): Whether to enable the dependency scanning.
+    Returns:
+        List[ConnectorWithModifiedFiles]: The connectors that match the selected criteria.
+    """
+
+    if metadata_changes_only and not modified:
+        main_logger.info("--metadata-changes-only overrides --modified")
+        modified = True
+
+    selected_modified_connectors = (
+        get_modified_connectors(modified_files, ALL_CONNECTORS, enable_dependency_scanning) if modified else set()
+    )
+    selected_connectors_by_name = {c for c in ALL_CONNECTORS if c.technical_name in selected_names}
+    selected_connectors_by_support_level = {connector for connector in ALL_CONNECTORS if connector.support_level in selected_support_levels}
+    selected_connectors_by_language = {connector for connector in ALL_CONNECTORS if connector.language in selected_languages}
+    selected_connectors_by_query = (
+        {connector for connector in ALL_CONNECTORS if connector.metadata_query_match(metadata_query)} if metadata_query else set()
+    )
+    selected_connectors_by_changelog_entry_files = {c for c in ALL_CONNECTORS if c.changelog_entry_files} if with_changelog_entry_files else set()
+
+    non_empty_connector_sets = [
+        connector_set
+        for connector_set in [
+            selected_connectors_by_name,
+            selected_connectors_by_support_level,
+            selected_connectors_by_language,
+            selected_connectors_by_query,
+            selected_modified_connectors,
+            selected_connectors_by_changelog_entry_files
+        ]
+        if connector_set
+    ]
+    # The selected connectors are the intersection of the selected connectors by name, support_level, language, simpleeval query and modified.
+    selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set()
+
+    selected_connectors_with_modified_files = []
+    for connector in selected_connectors:
+        connector_with_modified_files = ConnectorWithModifiedFiles(
+            relative_connector_path=connector.relative_connector_path,
+            modified_files=get_connector_modified_files(connector, modified_files),
+        )
+        if not metadata_changes_only:
+            selected_connectors_with_modified_files.append(connector_with_modified_files)
+        else:
+            if connector_with_modified_files.has_metadata_change:
+                selected_connectors_with_modified_files.append(connector_with_modified_files)
+    return selected_connectors_with_modified_files
+
+
+def validate_environment(is_local: bool) -> None:
+    """Check if the required environment variables exist."""
+    if is_local:
+        if not Path(".git").is_dir():
+            raise click.UsageError("You need to run this command from the repository root.")
+    else:
+        required_env_vars_for_ci = [
+            "GCP_GSM_CREDENTIALS",
+            "CI_REPORT_BUCKET_NAME",
+            "CI_GITHUB_ACCESS_TOKEN",
+            "DOCKER_HUB_USERNAME",
+            "DOCKER_HUB_PASSWORD",
+        ]
+        for required_env_var in required_env_vars_for_ci:
+            if os.getenv(required_env_var) is None:
+                raise click.UsageError(f"When running in a CI context a {required_env_var} environment variable must be set.")
+
+
+def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool:
+    """Check if the connector secrets should be loaded from Airbyte GSM or from the local secrets directory.
+
+    Args:
+        use_remote_secrets (Optional[bool]): Whether to use remote connector secrets or local connector secrets according to user inputs.
+
+    Raises:
+        click.UsageError: If the --use-remote-secrets flag was provided but no GCP_GSM_CREDENTIALS environment variable was found.
+
+    Returns:
+        bool: Whether to use remote connector secrets (True) or local connector secrets (False).
+    """
+    gcp_gsm_credentials_is_set = bool(os.getenv("GCP_GSM_CREDENTIALS"))
+    if use_remote_secrets is None:
+        if gcp_gsm_credentials_is_set:
+            main_logger.info("GCP_GSM_CREDENTIALS environment variable found, using remote connector secrets.")
+            return True
+        else:
+            main_logger.info("No GCP_GSM_CREDENTIALS environment variable found, using local connector secrets.")
+            return False
+    if use_remote_secrets:
+        if gcp_gsm_credentials_is_set:
+            main_logger.info("GCP_GSM_CREDENTIALS environment variable found, using remote connector secrets.")
+            return True
+        else:
+            raise click.UsageError("The --use-remote-secrets flag was provided but no GCP_GSM_CREDENTIALS environment variable was found.")
+    else:
+        main_logger.info("Using local connector secrets as the --use-local-secrets flag was provided")
+        return False
+
+
+@click.group(
+    cls=LazyGroup,
+    help="Commands related to connectors and connector acceptance tests.",
+    lazy_subcommands={
+        "bump_version": "pipelines.airbyte_ci.cdk_java.bump_version.commands.bump_version",
+    },
+)
+@click_merge_args_into_context_obj
+@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"]))
+@click.pass_context
+@click_ignore_unused_kwargs
+async def connectors(
+    ctx: click.Context,
+) -> None:
+    """Group all the connectors-ci command."""
+    validate_environment(ctx.obj["is_local"])
+
+    modified_files = []
+    if ctx.obj["modified"] or ctx.obj["metadata_changes_only"]:
+        modified_files = transform_strs_to_paths(
+            await get_modified_files(
+                ctx.obj["git_branch"],
+                ctx.obj["git_revision"],
+                ctx.obj["diffed_branch"],
+                ctx.obj["is_local"],
+                ctx.obj["ci_context"],
+            )
+        )
+
+    ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files(
+        ctx.obj["names"],
+        ctx.obj["support_levels"],
+        ctx.obj["languages"],
+        ctx.obj["modified"],
+        ctx.obj["metadata_changes_only"],
+        ctx.obj["with_changelog_entry_files"],
+        ctx.obj["metadata_query"],
+        set(modified_files),
+        ctx.obj["enable_dependency_scanning"],
+    )
+    log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"])

airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py

@@ -137,6 +137,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool:
         "metadata": "pipelines.airbyte_ci.metadata.commands.metadata",
         "test": "pipelines.airbyte_ci.test.commands.test",
         "update": "pipelines.airbyte_ci.update.commands.update",
+        "cdk-java": "pipelines.airbyte_ci.cdk_java.commands.cdk_java"
     },
 )
 @click.version_option(__installed_version__)