feat: add feature for multiple data_source secrets

Signed-off-by: ImMin5 <[email protected]>

src/cloudforet/cost_analysis/main.py

@@ -22,7 +22,6 @@ def data_source_init(params: dict) -> dict:
         }
     """
     data_source_mgr = DataSourceManager()
-    print(params)
     return data_source_mgr.init_response(**params)
 
 
@@ -67,9 +66,27 @@ def job_get_tasks(params: dict) -> dict:
         }
 
     """
+    tasks = {
+        "tasks": [],
+        "changed": [],
+        "synced_accounts": [],
+    }
+
     job_mgr = JobManager()
     params["schema"] = params.pop("schema_name", None)
-    return job_mgr.get_tasks(**params)
+    secret_data = params.pop("secret_data")
+
+    secrets = secret_data.get("secrets", [secret_data])
+    for _secret_data in secrets:
+        params["secret_data"] = _secret_data
+        job_tasks = job_mgr.get_tasks(**params)
+
+        tasks["tasks"].extend(job_tasks.get("tasks", []))
+        tasks["changed"].extend(job_tasks.get("changed", []))
+        tasks["synced_accounts"].extend(job_tasks.get("synced_accounts", []))
+
+    tasks["changed"] = __remove_duplicate_list_of_dict(tasks.get("changed", []))
+    return tasks
 
 
 @app.route("Cost.get_linked_accounts")
@@ -89,9 +106,17 @@ def cost_get_linked_accounts(params: dict) -> dict:
             'results': 'list'
     }
     """
+    result = []
     cost_mgr = CostManager()
     params["schema"] = params.pop("schema_name", None)
-    return cost_mgr.get_linked_accounts(**params)
+    secret_data = params.pop("secret_data")
+
+    secrets = secret_data.get("secrets", [secret_data])
+    for _secret_data in secrets:
+        params["secret_data"] = _secret_data
+        result.extend(cost_mgr.get_linked_accounts(**params))
+
+    return {"results": result}
 
 
 @app.route("Cost.get_data")
@@ -127,7 +152,12 @@ def cost_get_data(params: dict) -> Generator[dict, None, None]:
     cost_mgr = CostManager()
     options = params.get("options", {})
     task_options = params.get("task_options", {})
+    secret_data = params.pop("secret_data")
+
     params["schema"] = params.pop("schema_name", None)
+    params["secret_data"] = __get_secret_data_with_tenant_id(
+        secret_data, task_options.get("tenant_id")
+    )
 
     if options.get("cost_metric") == "AmortizedCost" and task_options.get(
         "is_benefit_job", False
@@ -137,3 +167,27 @@ def cost_get_data(params: dict) -> Generator[dict, None, None]:
     else:
         for cost_response in cost_mgr.get_data(**params):
             yield {"results": cost_response}
+
+
+def __remove_duplicate_list_of_dict(changed: list) -> list:
+    seen = set()
+    unique_list = []
+    for changed_info in changed:
+        # Convert dictionary to frozenset of tuples
+        frozenset_changed_info = frozenset(changed_info.items())
+        if frozenset_changed_info not in seen:
+            seen.add(frozenset_changed_info)
+            unique_list.append(changed_info)
+    return unique_list
+
+
+def __get_secret_data_with_tenant_id(secret_data: dict, tenant_id: str = None) -> dict:
+    secrets = secret_data.get("secrets", [secret_data])
+    if len(secrets) == 1:
+        return secrets[0]
+
+    for _secret_data in secrets:
+        if _secret_data["tenant_id"] == tenant_id:
+            secret_data = _secret_data
+
+    return secret_data

src/cloudforet/cost_analysis/manager/cost_manager.py

@@ -26,12 +26,12 @@ def __init__(self, *args, **kwargs):
         self.retail_price_map = {}
 
     def get_linked_accounts(
-            self,
-            options: dict,
-            secret_data: dict,
-            schema: str,
-            domain_id: str,
-    ) -> dict:
+        self,
+        options: dict,
+        secret_data: dict,
+        schema: str,
+        domain_id: str,
+    ) -> list:
         self.azure_cm_connector.create_session(options, secret_data, schema)
         billing_account_info = self.azure_cm_connector.get_billing_account()
         agreement_type = billing_account_info.get("agreement_type")
@@ -54,15 +54,15 @@ def get_linked_accounts(
         _LOGGER.debug(
             f"[get_linked_accounts] total accounts count: {len(accounts_info)}, domain_id: {domain_id}"
         )
-        return {"results": accounts_info}
+        return accounts_info
 
     def get_benefit_data(
-            self,
-            options: dict,
-            secret_data: dict,
-            schema: str,
-            task_options: dict,
-            domain_id: str,
+        self,
+        options: dict,
+        secret_data: dict,
+        schema: str,
+        task_options: dict,
+        domain_id: str,
     ):
         self.azure_cm_connector.create_session(options, secret_data, schema)
         start: datetime = self._get_first_date_of_month(task_options["start"])
@@ -85,12 +85,12 @@ def get_benefit_data(
                 )
 
     def _make_benefit_cost_data(
-            self,
-            results: dict,
-            end: datetime,
-            options: dict,
-            tenant_id: str = None,
-            agreement_type: str = None,
+        self,
+        results: dict,
+        end: datetime,
+        options: dict,
+        tenant_id: str = None,
+        agreement_type: str = None,
     ) -> list:
         benefit_costs_data = []
         try:
@@ -143,12 +143,12 @@ def _make_benefit_cost_info(self, result: dict, billed_at: str) -> dict:
         return data
 
     def get_data(
-            self,
-            options: dict,
-            secret_data: dict,
-            schema: str,
-            task_options: dict,
-            domain_id: str,
+        self,
+        options: dict,
+        secret_data: dict,
+        schema: str,
+        task_options: dict,
+        domain_id: str,
     ) -> list:
         self.azure_cm_connector.create_session(options, secret_data, schema)
         self._check_task_options(task_options)
@@ -202,12 +202,12 @@ def get_data(
         yield []
 
     def _make_cost_data(
-            self,
-            results: list,
-            end: datetime,
-            options: dict,
-            tenant_id: str = None,
-            agreement_type: str = None,
+        self,
+        results: list,
+        end: datetime,
+        options: dict,
+        tenant_id: str = None,
+        agreement_type: str = None,
     ) -> list:
         """Source Data Model"""
 
@@ -248,14 +248,14 @@ def _make_transaction_cost_data(self, tenant_id: str, end: datetime) -> list:
 
         try:
             for (
-                    reservation_transaction
+                reservation_transaction
             ) in self.azure_cm_connector.list_reservation_transactions_by_billing_profile_id(
                 query_filter
             ):
 
                 if (
-                        reservation_transaction.invoice_section_id.split("/")[-1]
-                        == invoice_section_id
+                    reservation_transaction.invoice_section_id.split("/")[-1]
+                    == invoice_section_id
                 ):
                     reservation_transaction_info = (
                         self.azure_cm_connector.convert_nested_dictionary(
@@ -312,12 +312,12 @@ def _make_transaction_cost_data(self, tenant_id: str, end: datetime) -> list:
         return transaction_cost_data
 
     def _make_data_info(
-            self,
-            result: dict,
-            billed_date: str,
-            options: dict,
-            tenant_id: str = None,
-            agreement_type: str = None,
+        self,
+        result: dict,
+        billed_date: str,
+        options: dict,
+        tenant_id: str = None,
+        agreement_type: str = None,
     ):
         additional_info: dict = self._get_additional_info(result, options, tenant_id)
         cost: float = self._get_cost_from_result_with_options(result, options)
@@ -391,8 +391,8 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
             additional_info["Benefit Name"] = benefit_name
 
             if (
-                    result.get("pricingmodel") == "Reservation"
-                    and result["metercategory"] == ""
+                result.get("pricingmodel") == "Reservation"
+                and result["metercategory"] == ""
             ):
                 result["metercategory"] = self._set_product_from_benefit_name(
                     benefit_name
@@ -404,14 +404,14 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
         if result.get("metersubcategory") != "" and result.get("metersubcategory"):
             additional_info["Meter SubCategory"] = result.get("metersubcategory")
             if (
-                    result.get("pricingmodel") == "OnDemand"
-                    and result.get("metercategory") == ""
+                result.get("pricingmodel") == "OnDemand"
+                and result.get("metercategory") == ""
             ):
                 result["metercategory"] = result.get("metercategory")
 
         if result.get("customername") is None:
             if result.get("invoicesectionname") != "" and result.get(
-                    "invoicesectionname"
+                "invoicesectionname"
             ):
                 additional_info["Department Name"] = result.get("invoicesectionname")
             elif result.get("departmentname") != "" and result.get("departmentname"):
@@ -420,7 +420,7 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
         if result.get("accountname") != "" and result.get("accountname"):
             additional_info["Enrollment Account Name"] = result["accountname"]
         elif result.get("enrollmentaccountname") != "" and result.get(
-                "enrollmentaccountname"
+            "enrollmentaccountname"
         ):
             additional_info["Enrollment Account Name"] = result["enrollmentaccountname"]
 
@@ -429,9 +429,9 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
 
         collect_resource_id = options.get("collect_resource_id", False)
         if (
-                collect_resource_id
-                and result.get("resourceid") != ""
-                and result.get("resourceid")
+            collect_resource_id
+            and result.get("resourceid") != ""
+            and result.get("resourceid")
         ):
             additional_info["Resource Id"] = result["resourceid"]
             additional_info["Resource Name"] = result["resourceid"].split("/")[-1]
@@ -569,10 +569,10 @@ def _get_tenant_ids(task_options: dict, collect_scope: str) -> list:
 
     @staticmethod
     def _make_scope(
-            secret_data: dict,
-            task_options: dict,
-            collect_scope: str,
-            customer_tenant_id: str = None,
+        secret_data: dict,
+        task_options: dict,
+        collect_scope: str,
+        customer_tenant_id: str = None,
     ):
         if collect_scope == "subscription_id":
             subscription_id = task_options["subscription_id"]
@@ -673,7 +673,7 @@ def _convert_date_format_to_utc(date_format: str) -> datetime:
         return datetime.strptime(date_format, "%Y-%m-%d").replace(tzinfo=timezone.utc)
 
     def _make_monthly_time_period(
-            self, start_date: datetime, end_date: datetime
+        self, start_date: datetime, end_date: datetime
     ) -> list:
         monthly_time_period = []
         current_date = end_date
@@ -703,7 +703,7 @@ def _make_monthly_time_period(
 
     @staticmethod
     def _get_linked_customer_tenants(
-            secret_data: dict, billing_accounts_info: list
+        secret_data: dict, billing_accounts_info: list
     ) -> list:
         customer_tenants = secret_data.get("customer_tenants", [])
         if not customer_tenants:
@@ -716,7 +716,7 @@ def _get_linked_customer_tenants(
 
     @staticmethod
     def _make_accounts_info_from_customer_tenants(
-            billing_accounts_info: list, customer_tenants: list
+        billing_accounts_info: list, customer_tenants: list
     ) -> list:
         accounts_info = []
         for billing_account_info in billing_accounts_info:
@@ -754,7 +754,7 @@ def _exclude_cost_data_with_options(result: dict, options: dict) -> bool:
 
     @staticmethod
     def _set_network_traffic_cost(
-            additional_info: dict, product: str, usage_type: str
+        additional_info: dict, product: str, usage_type: str
     ) -> dict:
         if product in ["Bandwidth", "Content Delivery Network"]:
             additional_info["Usage Type Details"] = usage_type

src/cloudforet/cost_analysis/manager/job_manager.py

@@ -64,6 +64,7 @@ def get_tasks(
                                 "account_agreement_type": billing_account_agreement_type,
                                 "collect_scope": "customer_tenant_id",
                                 "customer_tenants": divided_customer_tenant_info,
+                                "tenant_id": secret_data["tenant_id"],
                             }
                         }
                     )
@@ -81,6 +82,7 @@ def get_tasks(
                                 "account_agreement_type": billing_account_agreement_type,
                                 "collect_scope": "customer_tenant_id",
                                 "customer_tenants": first_sync_tenants,
+                                "tenant_id": secret_data["tenant_id"],
                                 "is_sync": False,
                             }
                         }
@@ -104,6 +106,7 @@ def get_tasks(
                                 "start": start_month,
                                 "account_agreement_type": billing_account_agreement_type,
                                 "collect_scope": "billing_account_id",
+                                "tenant_id": secret_data["tenant_id"],
                                 "is_benefit_job": True,
                             }
                         }
@@ -142,9 +145,6 @@ def get_tasks(
             raise ERROR_INVALID_SECRET_TYPE(secret_type=options.get("secret_type"))
 
         tasks = {"tasks": tasks, "changed": changed, "synced_accounts": synced_accounts}
-
-        print(tasks)
-
         return tasks
 
     def _get_tenants_from_billing_account(self):