Source Hubspot: Web Analytics API (#33042)

Co-authored-by: roman-yermilov-gl <[email protected]>

airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml

@@ -45,6 +45,32 @@ acceptance_tests:
             bypass_reason: Unable to populate
           - name: owners_archived
             bypass_reason: unable to populate
+          - name: tickets_web_analytics
+            bypass_reason: Unable to populate
+          - name: deals_web_analytics
+            bypass_reason: Unable to populate
+          - name: companies_web_analytics
+            bypass_reason: Unable to populate
+          - name: engagements_calls_web_analytics
+            bypass_reason: Unable to populate
+          - name: engagements_emails_web_analytics
+            bypass_reason: Unable to populate
+          - name: engagements_meetings_web_analytics
+            bypass_reason: Unable to populate
+          - name: engagements_notes_web_analytics
+            bypass_reason: Unable to populate
+          - name: engagements_tasks_web_analytics
+            bypass_reason: Unable to populate
+          - name: goals_web_analytics
+            bypass_reason: Unable to populate
+          - name: line_items_web_analytics
+            bypass_reason: Unable to populate
+          - name: products_web_analytics
+            bypass_reason: Unable to populate
+          - name: pets_web_analytics
+            bypass_reason: Unable to populate
+          - name: cars_web_analytics
+            bypass_reason: Unable to populate
         ignored_fields:
           contact_lists:
             - name: ilsFilterBranch
@@ -215,3 +241,4 @@ acceptance_tests:
         configured_catalog_path: sample_files/incremental_catalog.json
         future_state:
           future_state_path: integration_tests/abnormal_state.json
+        timeout_seconds: 7200

airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json

@@ -206,5 +206,21 @@
       "stream_descriptor": { "name": "property_history" },
       "stream_state": { "updatedAt": 7945393076000 }
     }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_descriptor": { "name": "contacts_web_analytics" },
+      "stream_state": {
+        "151": { "occurredAt": "2050-01-01T00:00:00Z" },
+        "251": { "occurredAt": "2050-01-01T00:00:00Z" },
+        "401": { "occurredAt": "2050-01-01T00:00:00Z" },
+        "601": { "occurredAt": "2050-01-01T00:00:00Z" },
+        "651": { "occurredAt": "2050-01-01T00:00:00Z" },
+        "2501": { "occurredAt": "2050-01-01T00:00:00Z" },
+        "2551": { "occurredAt": "2050-01-01T00:00:00Z" },
+        "2601": { "occurredAt": "2050-01-01T00:00:00Z" }
+      }
+    }
   }
 ]

airbyte-integrations/connectors/source-hubspot/metadata.yaml

@@ -6,11 +6,11 @@ data:
     hosts:
       - api.hubapi.com
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c
+    baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9
   connectorSubtype: api
   connectorType: source
   definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
-  dockerImageTag: 1.8.0
+  dockerImageTag: 1.9.0
   dockerRepository: airbyte/source-hubspot
   documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot
   githubIssueLabel: source-hubspot

airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json

@@ -322,6 +322,15 @@
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "contacts_web_analytics",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
     }
   ]
 }

airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json

@@ -287,6 +287,18 @@
       "sync_mode": "incremental",
       "cursor_field": ["timestamp"],
       "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "contacts_web_analytics",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["occurredAt"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["occurredAt"],
+      "destination_sync_mode": "append"
     }
   ]
 }

airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json

@@ -89,5 +89,37 @@
       "stream_state": { "timestamp": 1700681340514 },
       "stream_descriptor": { "name": "property_history" }
     }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_descriptor": { "name": "contacts_web_analytics" },
+      "stream_state": {
+        "601": {
+          "occurredAt": "2023-12-01T22:09:29.626Z"
+        },
+        "2601": {
+          "occurredAt": "2023-12-01T22:11:15.870Z"
+        },
+        "251": {
+          "occurredAt": "2023-12-01T22:08:24.346Z"
+        },
+        "2501": {
+          "occurredAt": "2023-12-01T22:10:02.613Z"
+        },
+        "401": {
+          "occurredAt": "2023-12-01T22:08:57.083Z"
+        },
+        "151": {
+          "occurredAt": "2023-12-01T22:07:17.836Z"
+        },
+        "2551": {
+          "occurredAt": "2023-12-01T22:10:43.512Z"
+        },
+        "651": {
+          "occurredAt": "2023-12-01T22:07:51.996Z"
+        }
+      }
+    }
   }
 ]

airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/shared/default_event_properties.json

@@ -0,0 +1,118 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": ["null", "object"],
+  "additionalProperties": true,
+  "properties": {
+    "properties_hs_asset_description": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_asset_type": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_browser": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_campaign_id": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_city": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_country": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_device_name": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_device_type": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_element_class": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_element_id": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_element_text": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_language": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_link_href": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_operating_system": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_operating_version": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_page_content_type": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_page_id": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_page_title": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_page_url": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_parent_module_id": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_referrer": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_region": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_screen_height": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_screen_width": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_touchpoint_source": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_tracking_name": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_user_agent": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_utm_campaign": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_utm_content": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_utm_medium": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_utm_source": {
+      "type": ["null", "string"]
+    },
+    "properties_hs_utm_term": {
+      "type": ["null", "string"]
+    },
+    "id": {
+      "type": ["null", "string"]
+    },
+    "objectId": {
+      "type": ["null", "string"]
+    },
+    "objectType": {
+      "type": ["null", "string"]
+    },
+    "eventType": {
+      "type": ["null", "string"]
+    },
+    "occurredAt": {
+      "type": ["null", "string"]
+    }
+  }
+}

airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py

@@ -17,34 +17,47 @@
     API,
     Campaigns,
     Companies,
+    CompaniesWebAnalytics,
     ContactLists,
     Contacts,
     ContactsListMemberships,
     ContactsMergedAudit,
+    ContactsWebAnalytics,
     CustomObject,
     DealPipelines,
     Deals,
     DealsArchived,
+    DealsWebAnalytics,
     EmailEvents,
     EmailSubscriptions,
     Engagements,
     EngagementsCalls,
+    EngagementsCallsWebAnalytics,
     EngagementsEmails,
+    EngagementsEmailsWebAnalytics,
     EngagementsMeetings,
+    EngagementsMeetingsWebAnalytics,
     EngagementsNotes,
+    EngagementsNotesWebAnalytics,
     EngagementsTasks,
+    EngagementsTasksWebAnalytics,
     Forms,
     FormSubmissions,
     Goals,
+    GoalsWebAnalytics,
     LineItems,
+    LineItemsWebAnalytics,
     MarketingEmails,
     Owners,
     OwnersArchived,
     Products,
+    ProductsWebAnalytics,
     PropertyHistory,
     SubscriptionChanges,
     TicketPipelines,
     Tickets,
+    TicketsWebAnalytics,
+    WebAnalyticsStream,
     Workflows,
 )
 
@@ -130,6 +143,26 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Workflows(**common_params),
         ]
 
+        enable_experimental_streams = "enable_experimental_streams" in config and config["enable_experimental_streams"]
+
+        if enable_experimental_streams:
+            streams.extend(
+                [
+                    ContactsWebAnalytics(**common_params),
+                    CompaniesWebAnalytics(**common_params),
+                    DealsWebAnalytics(**common_params),
+                    TicketsWebAnalytics(**common_params),
+                    EngagementsCallsWebAnalytics(**common_params),
+                    EngagementsEmailsWebAnalytics(**common_params),
+                    EngagementsMeetingsWebAnalytics(**common_params),
+                    EngagementsNotesWebAnalytics(**common_params),
+                    EngagementsTasksWebAnalytics(**common_params),
+                    GoalsWebAnalytics(**common_params),
+                    LineItemsWebAnalytics(**common_params),
+                    ProductsWebAnalytics(**common_params),
+                ]
+            )
+
         api = API(credentials=credentials)
         if api.is_oauth2():
             authenticator = api.get_authenticator()
@@ -151,6 +184,13 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
 
         available_streams.extend(self.get_custom_object_streams(api=api, common_params=common_params))
 
+        if enable_experimental_streams:
+            custom_objects_web_analytics_streams = self.get_web_analytics_custom_objects_stream(
+                custom_object_stream_instances=self.get_custom_object_streams(api=api, common_params=common_params),
+                common_params=common_params,
+            )
+            available_streams.extend(custom_objects_web_analytics_streams)
+
         return available_streams
 
     def get_custom_object_streams(self, api: API, common_params: Mapping[str, Any]):
@@ -162,3 +202,24 @@ def get_custom_object_streams(self, api: API, common_params: Mapping[str, Any]):
                 custom_properties=custom_properties,
                 **common_params,
             )
+
+    def get_web_analytics_custom_objects_stream(
+        self, custom_object_stream_instances: List[CustomObject], common_params: Any
+    ) -> WebAnalyticsStream:
+        for custom_object_stream_instance in custom_object_stream_instances:
+
+            def __init__(self, **kwargs: Any):
+                parent = custom_object_stream_instance.__class__(
+                    entity=custom_object_stream_instance.entity,
+                    schema=custom_object_stream_instance.schema,
+                    fully_qualified_name=custom_object_stream_instance.fully_qualified_name,
+                    custom_properties=custom_object_stream_instance.custom_properties,
+                    **common_params,
+                )
+                super(self.__class__, self).__init__(parent=parent, **kwargs)
+
+            custom_web_analytics_stream_class = type(
+                f"{custom_object_stream_instance.name.capitalize()}WebAnalytics", (WebAnalyticsStream,), {"__init__": __init__}
+            )
+
+            yield custom_web_analytics_stream_class(**common_params)

airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml

@@ -86,6 +86,11 @@ connectionSpecification:
                 if you need help finding this token.
               type: string
               airbyte_secret: true
+    enable_experimental_streams:
+      title: Enable experimental streams
+      description: If enabled then experimental streams become available for sync.
+      type: boolean
+      default: false
 advanced_auth:
   auth_flow_type: oauth2.0
   predicate_key: