Implement periscope connector 1.3

.github/workflows/docker-openmetadata-db.yml

@@ -31,7 +31,7 @@ jobs:
     steps:
       - name: Check trigger type
         if: ${{ env.input == '' }}
-        run: echo "input=1.3.0-SNAPSHOT" >> $GITHUB_ENV
+        run: echo "input=1.3.0" >> $GITHUB_ENV
 
       - name: Check out the Repo
         uses: actions/checkout@v3

.github/workflows/docker-openmetadata-ingestion-base-slim.yml

@@ -31,7 +31,7 @@ jobs:
     steps:
       - name: Check trigger type
         if: ${{ env.input == '' }}
-        run: echo "input=1.3.0-SNAPSHOT" >> $GITHUB_ENV
+        run: echo "input=1.3.0" >> $GITHUB_ENV
 
       - name: Check out the Repo
         uses: actions/checkout@v3

.github/workflows/docker-openmetadata-ingestion-base.yml

@@ -31,7 +31,7 @@ jobs:
     steps:
       - name: Check trigger type
         if: ${{ env.input == '' }}
-        run: echo "input=1.3.0-SNAPSHOT" >> $GITHUB_ENV
+        run: echo "input=1.3.0" >> $GITHUB_ENV
 
       - name: Check out the Repo
         uses: actions/checkout@v3

.github/workflows/docker-openmetadata-ingestion.yml

@@ -31,7 +31,7 @@ jobs:
     steps:
       - name: Check trigger type
         if: ${{ env.input == '' }}
-        run: echo "input=1.3.0-SNAPSHOT" >> $GITHUB_ENV
+        run: echo "input=1.3.0" >> $GITHUB_ENV
 
       - name: Check out the Repo
         uses: actions/checkout@v3

.github/workflows/docker-openmetadata-postgres.yml

@@ -31,7 +31,7 @@ jobs:
     steps:
       - name: Check trigger type
         if: ${{ env.input == '' }}
-        run: echo "input=1.3.0-SNAPSHOT" >> $GITHUB_ENV
+        run: echo "input=1.3.0" >> $GITHUB_ENV
 
       - name: Check out the Repo
         uses: actions/checkout@v3

.github/workflows/docker-openmetadata-server.yml

@@ -64,7 +64,7 @@ jobs:
     steps:
       - name: Check trigger type
         id: check_trigger
-        run: echo "DOCKER_RELEASE_TAG=1.3.0-SNAPSHOT" >> $GITHUB_OUTPUT
+        run: echo "DOCKER_RELEASE_TAG=1.3.0" >> $GITHUB_OUTPUT
 
       - name: Download application from Artifiact
         uses: actions/download-artifact@v2
@@ -129,7 +129,7 @@ jobs:
       - name: Check trigger type
         id: check_trigger
         if: ${{ env.DOCKER_RELEASE_TAG == '' }} 
-        run: echo "DOCKER_RELEASE_TAG=1.3.0-SNAPSHOT" >> $GITHUB_ENV
+        run: echo "DOCKER_RELEASE_TAG=1.3.0" >> $GITHUB_ENV
 
       - name: Check out the Repo
         uses: actions/checkout@v3

common/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>platform</artifactId>
     <groupId>org.open-metadata</groupId>
-    <version>1.3.0-SNAPSHOT</version>
+    <version>1.3.0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <properties>

docker/development/docker-compose-postgres.yml

@@ -70,7 +70,7 @@ services:
       context: ../../.
       dockerfile: docker/development/Dockerfile
     container_name: execute_migrate_all
-    command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
+    command: "./bootstrap/bootstrap_storage.sh migrate-all debug"
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}

docker/development/docker-compose.yml

@@ -70,7 +70,7 @@ services:
       context: ../../.
       dockerfile: docker/development/Dockerfile
     container_name: execute_migrate_all
-    command: "./bootstrap/openmetadata-ops.sh -d migrate --force"
+    command: "./bootstrap/bootstrap_storage.sh migrate-all debug"
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}

docker/docker-compose-ingestion/docker-compose-ingestion.yml

@@ -18,7 +18,7 @@ volumes:
 services:  
   ingestion:
     container_name: openmetadata_ingestion
-    image: docker.getcollate.io/openmetadata/ingestion:1.3.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/ingestion:1.3.0
     environment:
       AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
       AIRFLOW__CORE__EXECUTOR: LocalExecutor

docker/docker-compose-openmetadata/docker-compose-openmetadata.yml

@@ -13,8 +13,8 @@ version: "3.9"
 services:  
   execute-migrate-all:
     container_name: execute_migrate_all
-    command: "./bootstrap/openmetadata-ops.sh migrate"
-    image: docker.getcollate.io/openmetadata/server:1.3.0-SNAPSHOT
+    command: "./bootstrap/bootstrap_storage.sh migrate-all"
+    image: docker.getcollate.io/openmetadata/server:1.3.0
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}
@@ -202,7 +202,7 @@ services:
   openmetadata-server:
     container_name: openmetadata_server
     restart: always
-    image: docker.getcollate.io/openmetadata/server:1.3.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.3.0
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}

docker/docker-compose-quickstart/Dockerfile

@@ -11,7 +11,7 @@
 
 # Build stage
 FROM alpine:3.19 AS build
-ARG RI_VERSION="1.3.0-SNAPSHOT"
+ARG RI_VERSION="1.3.0"
 ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/${RI_VERSION}-release/openmetadata-${RI_VERSION}.tar.gz"
 
 RUN mkdir -p /opt/openmetadata && \
@@ -21,7 +21,7 @@ RUN mkdir -p /opt/openmetadata && \
 
 # Final stage
 FROM alpine:3.19
-ARG RI_VERSION="1.3.0-SNAPSHOT"
+ARG RI_VERSION="1.3.0"
 ARG BUILD_DATE
 ARG COMMIT_ID
 LABEL maintainer="OpenMetadata"

docker/docker-compose-quickstart/docker-compose-postgres.yml

@@ -18,7 +18,7 @@ volumes:
 services:
   postgresql:
     container_name: openmetadata_postgresql
-    image: docker.getcollate.io/openmetadata/postgresql:1.3.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/postgresql:1.3.0
     restart: always
     command: "--work_mem=10MB"
     environment:
@@ -61,8 +61,8 @@ services:
 
   execute-migrate-all:
     container_name: execute_migrate_all
-    image: docker.getcollate.io/openmetadata/server:1.3.0-SNAPSHOT
-    command: "./bootstrap/openmetadata-ops.sh migrate"
+    image: docker.getcollate.io/openmetadata/server:1.3.0
+    command: "./bootstrap/bootstrap_storage.sh migrate-all"
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}
@@ -250,7 +250,7 @@ services:
   openmetadata-server:
     container_name: openmetadata_server
     restart: always
-    image: docker.getcollate.io/openmetadata/server:1.3.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.3.0
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}
@@ -439,7 +439,7 @@ services:
 
   ingestion:
     container_name: openmetadata_ingestion
-    image: docker.getcollate.io/openmetadata/ingestion:1.3.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/ingestion:1.3.0
     depends_on:
       elasticsearch:
         condition: service_started

docker/docker-compose-quickstart/docker-compose.yml

@@ -18,7 +18,7 @@ volumes:
 services:
   mysql:
     container_name: openmetadata_mysql
-    image: docker.getcollate.io/openmetadata/db:1.3.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/db:1.3.0
     command: "--sort_buffer_size=10M"
     restart: always
     environment:
@@ -59,8 +59,8 @@ services:
 
   execute-migrate-all:
     container_name: execute_migrate_all
-    image: docker.getcollate.io/openmetadata/server:1.3.0-SNAPSHOT
-    command: "./bootstrap/openmetadata-ops.sh migrate"
+    image: docker.getcollate.io/openmetadata/server:1.3.0
+    command: "./bootstrap/bootstrap_storage.sh migrate-all"
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}
@@ -248,7 +248,7 @@ services:
   openmetadata-server:
     container_name: openmetadata_server
     restart: always
-    image: docker.getcollate.io/openmetadata/server:1.3.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.3.0
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}
@@ -437,7 +437,7 @@ services:
 
   ingestion:
     container_name: openmetadata_ingestion
-    image: docker.getcollate.io/openmetadata/ingestion:1.3.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/ingestion:1.3.0
     depends_on:
       elasticsearch:
         condition: service_started

ingestion/Dockerfile

@@ -81,7 +81,7 @@ ARG INGESTION_DEPENDENCY="all"
 ENV PIP_NO_CACHE_DIR=1
 # Make pip silent
 ENV PIP_QUIET=1
-ARG RI_VERSION="1.3.0.0.dev0"
+ARG RI_VERSION="1.3.0.0"
 RUN pip install --upgrade pip
 RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
 RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}"

ingestion/operators/docker/Dockerfile

@@ -87,7 +87,7 @@ ENV PIP_QUIET=1
 RUN pip install --upgrade pip
 
 ARG INGESTION_DEPENDENCY="all"
-ARG RI_VERSION="1.3.0.0.dev0"
+ARG RI_VERSION="1.3.0.0"
 RUN pip install --upgrade pip
 RUN pip install "openmetadata-ingestion[airflow]~=${RI_VERSION}"
 RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}"

ingestion/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 # since it helps us organize and isolate version management
 [project]
 name = "openmetadata-ingestion"
-version = "1.3.0.0.dev0"
+version = "1.3.0.0"
 dynamic = ["readme", "dependencies", "optional-dependencies"]
 authors = [
     {name = "OpenMetadata Committers"}

ingestion/setup.py

@@ -41,7 +41,7 @@
     "azure-storage-blob": "azure-storage-blob~=12.14",
     "azure-identity": "azure-identity~=1.12",
     "sqlalchemy-databricks": "sqlalchemy-databricks~=0.1",
-    "databricks-sdk": "databricks-sdk~=0.1",
+    "databricks-sdk": "databricks-sdk>=0.18.0,<0.20.0",
     "google": "google>=3.0.0",
     "trino": "trino[sqlalchemy]",
     "spacy": "spacy==3.5.0",

ingestion/src/metadata/ingestion/source/dashboard/periscope/client.py

@@ -0,0 +1,133 @@
+"""
+REST Auth & Client for Periscope
+"""
+import json
+import traceback
+from typing import List, Optional, Dict
+from copy import deepcopy
+
+import requests
+
+from metadata.ingestion.source.dashboard.periscope.ometa_client import REST, ClientConfig
+from metadata.ingestion.source.dashboard.periscope.models import (
+    PeriscopeDashboard,
+    PeriscopeDashboardList,
+    PeriscopeDashboardDetails,
+    PeriscopeViewList,
+    PeriscopeView
+)
+
+from metadata.ingestion.connections.test_connections import SourceConnectionException
+from metadata.utils.constants import AUTHORIZATION_HEADER, NO_ACCESS_TOKEN
+from metadata.utils.logger import ingestion_logger
+
+
+logger = ingestion_logger()
+
+SESSION_HEADERS = {"Content-Type": "application/json", "Accept": "*/*"}
+DEFAULT_TIMEOUT = 30
+
+class PeriscopeClient:
+    """
+    Client Handling API communication with Periscope
+    """
+
+    def _check_connection(self) -> dict:
+        try:
+            headers = deepcopy(SESSION_HEADERS)
+            headers["cookie"] = self.config.cookies
+            self.resp = requests.get(
+                f"https://app.periscopedata.com/login_state/dashboards",
+                headers=headers,
+                timeout=DEFAULT_TIMEOUT,
+                params={"client_site_id": self.config.client_site_id},
+            )
+
+            if not self.resp.ok:
+                msg = "Failed to fetch Periscope, please validate credentials"
+                raise SourceConnectionException(msg)
+
+        except Exception as exc:
+            msg = f"Unknown error in connection: {exc}."
+            raise SourceConnectionException(msg) from exc
+
+        logger.debug(f"Periscope connection status: {self.resp.status_code}")
+        return headers
+
+    def __init__(
+        self,
+        config,
+    ):
+        self.config = config
+        headers = self._check_connection()
+
+        client_config: ClientConfig = ClientConfig(
+            base_url="https://app.periscopedata.com",
+            api_version="",
+            auth_header="no-auth",
+            auth_token=lambda: (NO_ACCESS_TOKEN, 0),
+            extra_headers=headers
+        )
+        self.client = REST(client_config)
+        self.cached_dashboard_dict = self.get_cached_dashboard_per_id()
+
+    def get_dashboards_list(self) -> List[PeriscopeDashboard]:
+        """
+        Get List of all dashboards
+        """
+        try:
+            resp_dashboards = self.client.get(f"login_state/dashboards",
+                data={
+                    "client_site_id": self.config.client_site_id
+                }
+            )
+            if resp_dashboards:
+                dashboard_list = PeriscopeDashboardList.parse_obj(resp_dashboards)
+                return dashboard_list.Dashboard
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the dashboard list")
+        return []
+
+    def get_cached_dashboard_per_id(self) -> Dict[str, PeriscopeDashboard]:
+        dashboards = self.get_dashboards_list()
+        return {dashboard.id: dashboard for dashboard in dashboards}
+
+    def get_views_list(self) -> List[PeriscopeView]:
+        """
+        Get List of all views
+        """
+        try:
+            resp_collections = self.client.get("login_state/sql_views", data={"client_site_id": self.config.client_site_id})
+            if resp_collections:
+                collection_list = PeriscopeViewList.parse_obj(resp_collections)
+                return collection_list.SqlView
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the collections list")
+        return []
+
+    def get_dashboard_details(
+        self, dashboard_id: str
+    ) -> Optional[PeriscopeDashboardDetails]:
+        """
+        Get Dashboard Details
+        """
+        if not dashboard_id:
+            return None  # don't call api if dashboard_id is None
+        try:
+            resp_dashboard = self.client.get(f"welcome/remaining_widgets",
+                data={
+                    "current_dashboard": dashboard_id,
+                    "client_site_id": self.config.client_site_id
+                }
+            )
+            if resp_dashboard:
+                return PeriscopeDashboardDetails(
+                    charts=resp_dashboard.get("Widget"),
+                    dashboard=self.cached_dashboard_dict[dashboard_id]
+                )
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")
+        return None