airbyte-ci: always couple selection of strict-encrypt variants (e vic…

…e versa) (#42903)

airbyte-ci/connectors/pipelines/README.md

@@ -773,6 +773,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.28.0  | [#42849](https://github.com/airbytehq/airbyte/pull/42849)  | Couple selection of strict-encrypt variants (e vice versa)                                                                   |
 | 4.27.0  | [#42574](https://github.com/airbytehq/airbyte/pull/42574)  | Live tests: run from connectors test pipeline for connectors with sandbox connections                                        |
 | 4.26.1  | [#42905](https://github.com/airbytehq/airbyte/pull/42905)  | Rename the docker cache volume to avoid using the corrupted previous volume.                                                 |
 | 4.26.0  | [#42849](https://github.com/airbytehq/airbyte/pull/42849)  | Send publish failures messages to `#connector-publish-failures`                                                              |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py

@@ -7,7 +7,7 @@
 from typing import List, Set, Tuple
 
 import asyncclick as click
-from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo  # type: ignore
+from connector_ops.utils import Connector, ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo  # type: ignore
 from pipelines import main_logger
 from pipelines.cli.airbyte_ci import wrap_in_secret
 from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj
@@ -17,6 +17,11 @@
 from pipelines.helpers.utils import transform_strs_to_paths
 
 ALL_CONNECTORS = get_all_connectors_in_repo()
+CONNECTORS_WITH_STRICT_ENCRYPT_VARIANTS = {
+    Connector(c.relative_connector_path.replace("-strict-encrypt", ""))
+    for c in ALL_CONNECTORS
+    if c.technical_name.endswith("-strict-encrypt")
+}
 
 
 def log_selected_connectors(selected_connectors_with_modified_files: List[ConnectorWithModifiedFiles]) -> None:
@@ -27,6 +32,23 @@ def log_selected_connectors(selected_connectors_with_modified_files: List[Connec
         main_logger.info("No connectors to run.")
 
 
+def get_base_connector_and_variants(connector: Connector) -> Set[Connector]:
+    base_connector_path = connector.relative_connector_path.replace("-strict-encrypt", "")
+    base_connector = Connector(base_connector_path)
+    strict_encrypt_connector_path = f"{base_connector.relative_connector_path}-strict-encrypt"
+    if base_connector in CONNECTORS_WITH_STRICT_ENCRYPT_VARIANTS:
+        return {base_connector, Connector(strict_encrypt_connector_path)}
+    else:
+        return {base_connector}
+
+
+def update_selected_connectors_with_variants(selected_connectors: Set[Connector]) -> Set[Connector]:
+    updated_selected_connectors = set()
+    for selected_connector in selected_connectors:
+        updated_selected_connectors.update(get_base_connector_and_variants(selected_connector))
+    return updated_selected_connectors
+
+
 def get_selected_connectors_with_modified_files(
     selected_names: Tuple[str],
     selected_support_levels: Tuple[str],
@@ -79,6 +101,9 @@ def get_selected_connectors_with_modified_files(
     # The selected connectors are the intersection of the selected connectors by name, support_level, language, simpleeval query and modified.
     selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set()
 
+    # We always want to pair selection of a base connector with its variant, e vice versa
+    selected_connectors = update_selected_connectors_with_variants(selected_connectors)
+
     selected_connectors_with_modified_files = []
     for connector in selected_connectors:
         connector_with_modified_files = ConnectorWithModifiedFiles(
@@ -90,6 +115,7 @@ def get_selected_connectors_with_modified_files(
         else:
             if connector_with_modified_files.has_metadata_change:
                 selected_connectors_with_modified_files.append(connector_with_modified_files)
+
     return selected_connectors_with_modified_files
 
 

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.27.0"
+version = "4.28.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py

@@ -14,7 +14,7 @@
 from pipelines.airbyte_ci.connectors.test import commands as connectors_test_command
 from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles
 from pipelines.models.secrets import InMemorySecretStore
-from tests.utils import pick_a_random_connector
+from tests.utils import pick_a_random_connector, pick_a_strict_encrypt_variant_pair
 
 
 @pytest.fixture(scope="session")
@@ -227,6 +227,40 @@ def test_get_selected_connectors_with_metadata_query():
     assert not selected_connectors[0].modified_files
 
 
+def test_strict_encrypt_variant_is_selected():
+    main_connector, strict_encrypt_variant = pick_a_strict_encrypt_variant_pair()
+    selected_connectors = connectors_commands.get_selected_connectors_with_modified_files(
+        selected_names=(main_connector.technical_name,),
+        selected_support_levels=(),
+        selected_languages=(),
+        modified=False,
+        metadata_changes_only=False,
+        metadata_query=None,
+        modified_files={},
+    )
+    assert len(selected_connectors) == 2
+    selected_names = [c.technical_name for c in selected_connectors]
+    assert main_connector.technical_name in selected_names
+    assert strict_encrypt_variant.technical_name in selected_names
+
+
+def test_main_connector_selected_when_variant_is_selected():
+    main_connector, strict_encrypt_variant = pick_a_strict_encrypt_variant_pair()
+    selected_connectors = connectors_commands.get_selected_connectors_with_modified_files(
+        selected_names=(strict_encrypt_variant.technical_name,),
+        selected_support_levels=(),
+        selected_languages=(),
+        modified=False,
+        metadata_changes_only=False,
+        metadata_query=None,
+        modified_files={},
+    )
+    assert len(selected_connectors) == 2
+    selected_names = [c.technical_name for c in selected_connectors]
+    assert main_connector.technical_name in selected_names
+    assert strict_encrypt_variant.technical_name in selected_names
+
+
 @pytest.fixture()
 def in_memory_secret_store():
     store = InMemorySecretStore()

airbyte-ci/connectors/pipelines/tests/utils.py

@@ -21,10 +21,16 @@ async def with_exec(self, *args, **kwargs):
 
 
 def pick_a_random_connector(
-    language: ConnectorLanguage = None, support_level: str = None, other_picked_connectors: list = None
+    language: ConnectorLanguage = None,
+    support_level: str = None,
+    other_picked_connectors: list = None,
+    ignore_strict_encrypt_variants: bool = True,
 ) -> Connector:
     """Pick a random connector from the list of all connectors."""
-    all_connectors = [c for c in list(ALL_CONNECTORS)]
+    if not ignore_strict_encrypt_variants:
+        all_connectors = [c for c in list(ALL_CONNECTORS)]
+    else:
+        all_connectors = [c for c in list(ALL_CONNECTORS) if "-strict-encrypt" not in c.technical_name]
     if language:
         all_connectors = [c for c in all_connectors if c.language is language]
     if support_level:
@@ -36,6 +42,13 @@ def pick_a_random_connector(
     return picked_connector
 
 
+def pick_a_strict_encrypt_variant_pair():
+    for c in ALL_CONNECTORS:
+        if c.technical_name.endswith("-strict-encrypt"):
+            main_connector = Connector(c.relative_connector_path.replace("-strict-encrypt", ""))
+            return main_connector, c
+
+
 def mock_container():
     container_mock = AsyncMock(MockContainerClass)
     container_mock.with_label.return_value = container_mock