Merge branch 'master' into baz/source-shopify-bump-to-2023-07

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.16
+current_version = 0.50.18
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

airbyte-ci/connectors/connector_ops/connector_ops/allowed_hosts_checks.py

@@ -8,36 +8,13 @@
 
 from connector_ops import utils
 
-ALLOWED_HOST_THRESHOLD = {
-    "sl": 200,
-    "ql": 300,
-}
-
-
-def _requires_allowed_hosts(connector: utils.Connector) -> bool:
-    """Check if a connector requires allowed hosts.
-
-    Args:
-        connector (utils.Connector): The connector to check.
-
-    Returns:
-        bool: True if the connector requires allowed hosts, False otherwise.
-    """
-    if connector.ab_internal_sl >= ALLOWED_HOST_THRESHOLD["sl"]:
-        return True
-
-    if connector.ab_internal_ql >= ALLOWED_HOST_THRESHOLD["ql"]:
-        return True
-
-    return False
-
 
 def get_connectors_missing_allowed_hosts() -> List[utils.Connector]:
     connectors_missing_allowed_hosts: List[utils.Connector] = []
     changed_connectors = utils.get_changed_connectors(destination=False, third_party=False)
 
     for connector in changed_connectors:
-        if _requires_allowed_hosts(connector):
+        if connector.requires_allowed_hosts_check:
             missing = not connector_has_allowed_hosts(connector)
             if missing:
                 connectors_missing_allowed_hosts.append(connector)

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -51,6 +51,10 @@ def download_catalog(catalog_url):
     "ql": 400,
 }
 
+ALLOWED_HOST_THRESHOLD = {
+    "ql": 300,
+}
+
 
 class ConnectorInvalidNameError(Exception):
     pass
@@ -346,6 +350,15 @@ def requires_high_test_strictness_level(self) -> bool:
         """
         return self.ab_internal_ql >= IMPORTANT_CONNECTOR_THRESHOLDS["ql"]
 
+    @property
+    def requires_allowed_hosts_check(self) -> bool:
+        """Check if a connector requires allowed hosts.
+
+        Returns:
+            bool: True if the connector requires allowed hosts, False otherwise.
+        """
+        return self.ab_internal_ql >= ALLOWED_HOST_THRESHOLD["ql"]
+
     @property
     def allowed_hosts(self) -> Optional[List[str]]:
         return self.metadata.get("allowedHosts") if self.metadata else None

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py

@@ -8,6 +8,7 @@
     simple_link_html,
     icon_image_html,
     test_badge_html,
+    internal_level_html,
     ColumnInfo,
 )
 from orchestrator.config import CONNECTOR_REPO_NAME, CONNECTOR_TEST_SUMMARY_FOLDER, REPORT_FOLDER, get_public_metadata_service_url
@@ -83,6 +84,22 @@ def test_summary_url(row: pd.DataFrame) -> str:
     return get_public_metadata_service_url(path)
 
 
+def ab_internal_sl(row: pd.DataFrame) -> str:
+    ab_internal = row.get("ab_internal_oss")
+    if not isinstance(ab_internal, dict) or "sl" not in ab_internal:
+        return None
+
+    return ab_internal["sl"]
+
+
+def ab_internal_ql(row: pd.DataFrame) -> str:
+    ab_internal = row.get("ab_internal_oss")
+    if not isinstance(ab_internal, dict) or "ql" not in ab_internal:
+        return None
+
+    return ab_internal["ql"]
+
+
 # 📊 Dataframe Augmentation
 
 
@@ -118,6 +135,10 @@ def augment_and_normalize_connector_dataframes(
     total_registry["issue_url"] = total_registry.apply(issue_url, axis=1)
     total_registry["test_summary_url"] = total_registry.apply(test_summary_url, axis=1)
 
+    # Show Internal Fields
+    total_registry["ab_internal_ql"] = total_registry.apply(ab_internal_ql, axis=1)
+    total_registry["ab_internal_sl"] = total_registry.apply(ab_internal_sl, axis=1)
+
     # Merge docker repo and version into separate columns
     total_registry["docker_image_oss"] = total_registry.apply(lambda x: merge_docker_repo_and_version(x, OSS_SUFFIX), axis=1)
     total_registry["docker_image_cloud"] = total_registry.apply(lambda x: merge_docker_repo_and_version(x, CLOUD_SUFFIX), axis=1)
@@ -225,10 +246,24 @@ def connector_registry_report(context, all_destinations_dataframe, all_sources_d
             "column": "connector_type",
             "title": "Connector Type",
         },
+        {
+            "column": "releaseStage_oss",
+            "title": "Release Stage",
+        },
         {
             "column": "supportLevel_oss",
             "title": "Support Level",
         },
+        {
+            "column": "ab_internal_sl",
+            "title": "Internal SL",
+            "formatter": internal_level_html,
+        },
+        {
+            "column": "ab_internal_ql",
+            "title": "Internal QL",
+            "formatter": internal_level_html,
+        },
         {
             "column": "test_summary_url",
             "title": "Build Status",

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py

@@ -39,6 +39,15 @@ def test_badge_html(test_summary_url: str) -> str:
     return f'<a href="{report_url}" target="_blank">{icon_image}</a>'
 
 
+def internal_level_html(level_value: float) -> str:
+    level = level_value / 100
+
+    # remove trailing zeros
+    level = f"{level:.2f}".rstrip("0").rstrip(".")
+
+    return f"Level <b>{level}</b>"
+
+
 # Dataframe to HTML
 
 

airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcConstants.java

@@ -27,5 +27,6 @@ public final class JdbcConstants {
   public static final String INTERNAL_COLUMN_SIZE = "columnSize";
   public static final String INTERNAL_IS_NULLABLE = "isNullable";
   public static final String INTERNAL_DECIMAL_DIGITS = "decimalDigits";
+  public static final String KEY_SEQ = "KEY_SEQ";
 
 }

airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 200
-    ql: 300
+    ql: 200
   supportLevel: certified
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml

@@ -25,6 +25,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-bigquery/metadata.yaml

@@ -30,6 +30,6 @@ data:
     - language:java
   ab_internal:
     sl: 200
-    ql: 300
+    ql: 400
   supportLevel: certified
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-cassandra/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-clickhouse/metadata.yaml

@@ -25,6 +25,6 @@ data:
     - language:java
   ab_internal:
     sl: 200
-    ql: 300
+    ql: 200
   supportLevel: certified
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-convex/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-csv/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-cumulio/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-databend/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-databricks/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-dev-null/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-doris/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-duckdb/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-dynamodb/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 200
-    ql: 300
+    ql: 200
   supportLevel: certified
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-e2e-test/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-elasticsearch/metadata.yaml

@@ -20,6 +20,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-exasol/metadata.yaml

@@ -18,6 +18,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-firebolt/metadata.yaml

@@ -20,6 +20,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-firestore/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-google-sheets/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 300
-    ql: 300
+    ql: 200
   supportLevel: certified
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-iceberg/metadata.yaml

@@ -18,6 +18,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-kafka/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-keen/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-kinesis/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-langchain/metadata.yaml

@@ -20,6 +20,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-local-json/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-mariadb-columnstore/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:java
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-meilisearch/metadata.yaml

@@ -19,6 +19,6 @@ data:
     - language:python
   ab_internal:
     sl: 100
-    ql: 200
+    ql: 100
   supportLevel: community
 metadataSpecVersion: "1.0"