diff --git a/README.md b/README.md index f81f1e19..0dfeda94 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,27 @@ This tool support the following policies: * [tag_baremetal](cloud_governance/policy/ibm/tag_baremetal.py): Tag IBM baremetal machines * [tag_vm](cloud_governance/policy/ibm/tag_vm.py): Tga IBM Virtual Machines machines +* [tag_resources](./cloud_governance/policy/ibm/tag_resources.py): Tag IBM resources + list of supported IBM Resources + +- virtual_servers +- volumes +- floating_ips +- vpcs +- virtual_network_interfaces +- security_groups +- public_gateways +- vpc_endpoint_gateways +- load_balancers +- schematics_workspaces + +Environment Variables required: + +| KeyName | Value | Description | +|----------------------------|--------|----------------------------------------------------------------------------| +| IBM_CUSTOM_TAGS_LIST | string | pass string with separated with comma. i.e: "cost-center: test, env: test" | +| RESOURCE_TO_TAG (optional) | string | pass the resource name to tag. ex: virtual_servers | +| IBM_CLOUD_API_KEY | string | IBM Cloud API Key | ** You can write your own policy using [Cloud-Custodian](https://cloudcustodian.io/docs/quickstart/index.html) and run it (see 'custom cloud custodian policy' in [Policy workflows](#policy-workloads)). diff --git a/cloud_governance/common/clouds/ibm/account/ibm_authenticator.py b/cloud_governance/common/clouds/ibm/account/ibm_authenticator.py new file mode 100644 index 00000000..46d72e0b --- /dev/null +++ b/cloud_governance/common/clouds/ibm/account/ibm_authenticator.py @@ -0,0 +1,18 @@ +import logging + +from ibm_cloud_sdk_core.authenticators import IAMAuthenticator + +from cloud_governance.main.environment_variables import environment_variables + + +class IBMAuthenticator: + """ + Refer: https://cloud.ibm.com/apidocs/vpc/latest + Refer: https://github.com/IBM/ibm-cloud-sdk-common?tab=readme-ov-file + """ + + def __init__(self): + logging.disable(logging.DEBUG) + self.env_config = environment_variables + self.__api_key = self.env_config.IBM_CLOUD_API_KEY + self.iam_authenticator = IAMAuthenticator(self.__api_key) diff --git a/cloud_governance/common/clouds/ibm/developer_tools/__init__.py b/cloud_governance/common/clouds/ibm/developer_tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/clouds/ibm/developer_tools/schematic_operations.py b/cloud_governance/common/clouds/ibm/developer_tools/schematic_operations.py new file mode 100644 index 00000000..e3d0e41f --- /dev/null +++ b/cloud_governance/common/clouds/ibm/developer_tools/schematic_operations.py @@ -0,0 +1,55 @@ +from ibm_schematics.schematics_v1 import SchematicsV1 + +from cloud_governance.common.clouds.ibm.account.ibm_authenticator import IBMAuthenticator + + +class SchematicOperations(IBMAuthenticator): + """ + This class performs schematic operations. + """ + REGION_SCHEMATICS_URL = "https://%s.schematics.cloud.ibm.com/" + + def __init__(self): + super().__init__() + self.__client = SchematicsV1(self.iam_authenticator) + self.__client.set_service_url('https://us.schematics.cloud.ibm.com') + + def set_service_url(self, region: str): + """ + This method sets the service URL. + :param region: + :return: + """ + service_url = self.REGION_SCHEMATICS_URL % region + self.__client.set_service_url(service_url) + + def get_workspaces(self): + """ + This method lists all available schematics workspaces + :return: + """ + response = self.__client.list_workspaces().get_result() + return response['workspaces'] + + def get_supported_locations(self): + """ + This method lists supported locations + :return: + """ + response = self.__client.list_locations().get_result() + return response['locations'] + + def get_all_workspaces(self): + """ + This method lists all available schematics workspaces + :return: + """ + locations = self.get_supported_locations() + resources_list = {} + for location in locations: + region = location['region'] + geography_code = location['geography_code'] + self.set_service_url(region) + if geography_code not in resources_list: + resources_list[geography_code] = self.get_workspaces() + return resources_list diff --git a/cloud_governance/common/clouds/ibm/tagging/__init__.py b/cloud_governance/common/clouds/ibm/tagging/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/clouds/ibm/tagging/global_tagging_operations.py b/cloud_governance/common/clouds/ibm/tagging/global_tagging_operations.py new file mode 100644 index 00000000..8bccc2b6 --- /dev/null +++ b/cloud_governance/common/clouds/ibm/tagging/global_tagging_operations.py @@ -0,0 +1,40 @@ +from ibm_platform_services.global_tagging_v1 import GlobalTaggingV1, Resource + +from cloud_governance.common.clouds.ibm.account.ibm_authenticator import IBMAuthenticator +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp + + +class GlobalTaggingOperations(IBMAuthenticator): + """ + This class performs tagging operations on cloud resources. + """ + BATCH_SIZE = 100 + + def __init__(self): + super().__init__() + self.__tag_service = GlobalTaggingV1(authenticator=self.iam_authenticator) + + @logger_time_stamp + def update_tags(self, resources_crn: list, tags: list): + """ + This method updates the tags associated with an instance. + :param resources_crn: + :param tags: + :return: + """ + resources_list = [Resource(resource_crn) for resource_crn in resources_crn] + resources_batch_list = [resources_list[i:i + self.BATCH_SIZE] + for i in range(0, len(resources_list), self.BATCH_SIZE)] + success = 0 + errors = [] + for resource_batch in resources_batch_list: + responses = self.__tag_service.attach_tag(resources=resource_batch, tag_names=tags) \ + .get_result()['results'] + for resource in responses: + if resource['is_error']: + errors.append(resource.get('resource_id')) + logger.error(f'Unable to attach resource tags to: {resource["resource_id"]}') + else: + success += 1 + return success == len(resources_crn), errors diff --git a/cloud_governance/common/clouds/ibm/vpc/__init__.py b/cloud_governance/common/clouds/ibm/vpc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/common/clouds/ibm/vpc/vpc_infra_operations.py b/cloud_governance/common/clouds/ibm/vpc/vpc_infra_operations.py new file mode 100644 index 00000000..a1411581 --- /dev/null +++ b/cloud_governance/common/clouds/ibm/vpc/vpc_infra_operations.py @@ -0,0 +1,222 @@ +from functools import wraps + +import ibm_vpc +from typing import Callable + +from cloud_governance.common.clouds.ibm.account.ibm_authenticator import IBMAuthenticator + + +def region_wrapper(func): + @wraps(func) + def wrapper(*args, **kwargs): + vpc_obj = VpcInfraOperations() + regions = vpc_obj.get_regions() + resources_list = {} + for region in regions: + region_name = region.get('name') + if region['status'] == 'available': + vpc_obj.set_service_url(region_name) + resources_list[region_name] = func(*args, **kwargs) + return resources_list + + return wrapper + + +class VpcInfraOperations(IBMAuthenticator): + """ + This class contains methods to perform operations on VPC Infra Operations. + """ + + REGION_SERVICE_URL = "https://%s.iaas.cloud.ibm.com/v1" + + def __init__(self): + super().__init__() + self.__client = ibm_vpc.vpc_v1.VpcV1(authenticator=self.iam_authenticator) + + def get_regions(self): + """ + This method lists all available regions. + :return: + """ + regions = self.__client.list_regions().get_result()['regions'] + return regions + + def set_service_url(self, region_name: str): + """ + This method sets the service URL. + :param region_name: + :return: + """ + service_url = self.REGION_SERVICE_URL % region_name + self.__client.set_service_url(service_url) + + def iter_next_resources(self, exec_func: Callable, resource_name: str, region_name: str = None): + """ + This method . + :param region_name: + :param exec_func: + :param resource_name: + :return: + """ + if region_name: + self.set_service_url(region_name) + response = exec_func().get_result() + resources = response[resource_name] + while response.get('next'): + href = response['next']['href'] + start = href.split('&')[-1].split('=')[-1] + response = exec_func(start=start).get_result() + resources.extend(response[resource_name]) + return resources + + def get_instances(self, region_name: str = None): + """ + This method lists available instances in one region, default 'us-south' + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_instances, + resource_name='instances', region_name=region_name) + + def get_volumes(self, region_name: str = None): + """ + This method lists available volumes. + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_volumes, + resource_name='volumes', region_name=region_name) + + def get_floating_ips(self, region_name: str = None): + """ + This method lists available floating ips. + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_floating_ips, + resource_name='floating_ips', region_name=region_name) + + def get_vpcs(self, region_name: str = None): + """ + This method lists available vpcs. + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_vpcs, + resource_name='vpcs', region_name=region_name) + + def get_virtual_network_interfaces(self, region_name: str = None): + """ + This method lists available virtual network interfaces. + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_virtual_network_interfaces, + resource_name='virtual_network_interfaces', region_name=region_name) + + def get_security_groups(self, region_name: str = None): + """ + This method lists available security_groups + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_security_groups, + resource_name='security_groups', region_name=region_name) + + def get_public_gateways(self, region_name: str = None): + """ + This method lists available public_gateways + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_public_gateways, + resource_name='public_gateways', region_name=region_name) + + def get_vpc_endpoint_gateways(self, region_name: str = None): + """ + This method lists available vpc endpoint gateways + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_endpoint_gateways, + resource_name='endpoint_gateways', region_name=region_name) + + def get_load_balancers(self, region_name: str = None): + """ + This method lists available load balancers + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_load_balancers, + resource_name='load_balancers', region_name=region_name) + + @region_wrapper + def get_all_instances(self): + """ + This method lists all available instances. + :return: + """ + return self.get_instances() + + @region_wrapper + def get_all_volumes(self): + """ + This method lists all available volumes. + :return: + """ + return self.get_volumes() + + @region_wrapper + def get_all_vpcs(self): + """ + This method lists all available vpc's. + :return: + """ + return self.get_vpcs() + + @region_wrapper + def get_all_floating_ips(self): + """ + This method lists all floating ips. + :return: + """ + return self.get_floating_ips() + + @region_wrapper + def get_all_virtual_network_interfaces(self): + """ + This method lists all available virtual network interfaces. + :return: + """ + return self.get_virtual_network_interfaces() + + @region_wrapper + def get_all_security_groups(self): + """ + This method lists all available security_groups + :return: + """ + return self.get_security_groups() + + @region_wrapper + def get_all_public_gateways(self): + """ + This method lists all available public_gateways + :return: + """ + return self.get_public_gateways() + + @region_wrapper + def get_all_vpc_endpoint_gateways(self): + """ + This method lists all available vpc endpoint gateways + :return: + """ + return self.get_vpc_endpoint_gateways() + + @region_wrapper + def get_all_load_balancers(self): + """ + This method lists all available load balancers. + :return: + """ + return self.get_load_balancers() diff --git a/cloud_governance/common/logger/logger_time_stamp.py b/cloud_governance/common/logger/logger_time_stamp.py index 259b5983..7178638b 100644 --- a/cloud_governance/common/logger/logger_time_stamp.py +++ b/cloud_governance/common/logger/logger_time_stamp.py @@ -13,6 +13,7 @@ def logger_time_stamp(method): @param method: @return: method wrapper """ + @wraps(method) # solve method help doc def method_wrapper(*args, **kwargs): """ @@ -31,13 +32,15 @@ def method_wrapper(*args, **kwargs): date_time_end = datetime.datetime.now().strftime(datetime_format) total_time = time_end - time_start total_time_str = f'Total time: {round(total_time, 2)} sec' - logger.warn(f'Method name: {method.__name__} , End time: {date_time_end} , {total_time_str}') + logger.info(f'Method name: {method.__name__} , End time: {date_time_end} , {total_time_str}') except Exception as err: time_end = time.time() total_time = time_end - time_start date_time_end = datetime.datetime.now().strftime(datetime_format) - logger.error(f'Method name: {method.__name__} , End time with errors: {date_time_end} , Total time: {round(total_time, 2)} sec') + logger.error( + f'Method name: {method.__name__} , End time with errors: {date_time_end} , Total time: {round(total_time, 2)} sec') raise err # Exception(method.__name__, err) return result + return method_wrapper diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index c410b7c3..e1188dbf 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -49,6 +49,7 @@ def load_from_yaml(self, ): yaml_data = yaml.safe_load(yaml_file) if isinstance(yaml_data, dict): for key, value in yaml_data.items(): + setattr(self, key, value) if key not in os.environ: # Prefer existing env variables os.environ[key] = str(value) except FileNotFoundError: @@ -164,7 +165,9 @@ def __init__(self): self._environment_variables_dict['IBM_API_KEY'] = EnvironmentVariables.get_env('IBM_API_KEY', '') self._environment_variables_dict['USAGE_REPORTS_APIKEY'] = EnvironmentVariables.get_env('USAGE_REPORTS_APIKEY', '') - if self._environment_variables_dict['USAGE_REPORTS_APIKEY']: + self._environment_variables_dict['IBM_CLOUD_API_KEY'] = EnvironmentVariables.get_env('IBM_CLOUD_API_KEY', '') + + if self._environment_variables_dict['USAGE_REPORTS_APIKEY'] or hasattr(self, "IBM_CLOUD_API_KEY"): self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'IBM' self._environment_variables_dict['month'] = EnvironmentVariables.get_env('month', '') self._environment_variables_dict['year'] = EnvironmentVariables.get_env('year', '') diff --git a/cloud_governance/main/main_oerations/main_operations.py b/cloud_governance/main/main_oerations/main_operations.py index 3c5193d1..edfb2293 100644 --- a/cloud_governance/main/main_oerations/main_operations.py +++ b/cloud_governance/main/main_oerations/main_operations.py @@ -2,6 +2,7 @@ from cloud_governance.main.environment_variables import environment_variables from cloud_governance.policy.policy_runners.azure.policy_runner import PolicyRunner as AzurePolicyRunner from cloud_governance.policy.policy_runners.aws.policy_runner import PolicyRunner as AWSPolicyRunner +from cloud_governance.policy.policy_runners.ibm.policy_runner import PolicyRunner as IBMPolicyRunner class MainOperations: @@ -21,9 +22,11 @@ def get_policy_runner(self): policy_runner = None if Utils.equal_ignore_case(self._public_cloud_name, 'AWS'): policy_runner = AWSPolicyRunner() + elif Utils.equal_ignore_case(self._public_cloud_name, 'Azure'): + policy_runner = AzurePolicyRunner() else: - if Utils.equal_ignore_case(self._public_cloud_name, 'AZURE'): - policy_runner = AzurePolicyRunner() + if Utils.equal_ignore_case(self._public_cloud_name, 'IBM'): + policy_runner = IBMPolicyRunner() return policy_runner @@ -40,7 +43,7 @@ def run(self): if self._policy in policies and self._policy in ["instance_run", "unattached_volume", "cluster_run", "ip_unattached", "unused_nat_gateway", "instance_idle", "zombie_snapshots", "database_idle", "s3_inactive", - "empty_roles"]: + "empty_roles", "tag_resources"]: source = policy_type if Utils.equal_ignore_case(policy_type, self._public_cloud_name): source = '' diff --git a/cloud_governance/policy/ibm/tag_resources.py b/cloud_governance/policy/ibm/tag_resources.py new file mode 100644 index 00000000..1a474537 --- /dev/null +++ b/cloud_governance/policy/ibm/tag_resources.py @@ -0,0 +1,176 @@ +from functools import wraps + +from cloud_governance.common.clouds.ibm.developer_tools.schematic_operations import SchematicOperations +from cloud_governance.common.clouds.ibm.tagging.global_tagging_operations import GlobalTaggingOperations +from cloud_governance.common.clouds.ibm.vpc.vpc_infra_operations import VpcInfraOperations +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp + + +def get_resources_wrapper(func): + @wraps(func) + def wrapper(*args, **kwargs): + resources_crn = [] + resource_list = func(*args, **kwargs) + for region, resources in resource_list.items(): + resources_crn.extend([resource.get('crn') for resource in resources]) + return resources_crn + + return wrapper + + +class TagResources: + """ + This class tags the Virtual PrivateCloud Resources + Virtual Servers + VPC Resources + """ + + def __init__(self): + self.vpc_infra_operations = VpcInfraOperations() + self.tag_operations = GlobalTaggingOperations() + self.schematic_operations = SchematicOperations() + self.__env_config = self.vpc_infra_operations.env_config + self.__ibm_custom_tags_list = self.__env_config.IBM_CUSTOM_TAGS_LIST \ + if hasattr(self.__env_config, 'IBM_CUSTOM_TAGS_LIST') else None + self.resource_to_tag = self.__env_config.RESOURCE_TO_TAG \ + if hasattr(self.__env_config, 'RESOURCE_TO_TAG') else None + + @get_resources_wrapper + @logger_time_stamp + def get_virtual_servers_crn(self): + """ + This method returns all virtual server crn's + :return: + """ + return self.vpc_infra_operations.get_all_instances() + + @get_resources_wrapper + @logger_time_stamp + def get_volumes_crn(self): + """ + This method returns all volumes crn's + :return: + """ + return self.vpc_infra_operations.get_all_volumes() + + @get_resources_wrapper + @logger_time_stamp + def get_floating_ips_crn(self): + """ + This method returns all floating ips crn's' + :return: + """ + return self.vpc_infra_operations.get_all_floating_ips() + + @get_resources_wrapper + @logger_time_stamp + def get_vpcs_crn(self): + """ + This method returns all vpcs crn's' + :return: + """ + return self.vpc_infra_operations.get_all_vpcs() + + @get_resources_wrapper + @logger_time_stamp + def get_virtual_network_interfaces_crn(self): + """ + This method returns all virtual network interfaces crn's' + :return: + """ + return self.vpc_infra_operations.get_all_virtual_network_interfaces() + + @get_resources_wrapper + @logger_time_stamp + def get_security_groups_crn(self): + """ + This method returns all virtual security_groups crn's' + :return: + """ + return self.vpc_infra_operations.get_all_security_groups() + + @get_resources_wrapper + @logger_time_stamp + def get_public_gateways_crn(self): + """ + This method returns all virtual public_gateways crn's' + :return: + """ + return self.vpc_infra_operations.get_all_public_gateways() + + @get_resources_wrapper + @logger_time_stamp + def get_vpc_endpoint_gateways_crn(self): + """ + This method returns all vpc endpoint gateways crn's' + :return: + """ + return self.vpc_infra_operations.get_all_vpc_endpoint_gateways() + + @get_resources_wrapper + @logger_time_stamp + def get_schematics_workspaces_crn(self): + """ + This method returns all schematics workspaces crn's' + :return: + """ + return self.schematic_operations.get_all_workspaces() + + @get_resources_wrapper + @logger_time_stamp + def get_load_balancers_crn(self): + """ + This method returns all load balancers crn's' + :return: + """ + return self.vpc_infra_operations.get_all_load_balancers() + + @logger_time_stamp + def tag_all_vpc_resources(self): + """ + This method tags all Virtual PrivateCloud Resources + :return: + """ + if not self.__ibm_custom_tags_list: + return {'ok': False, 'errors': {}, + 'message': 'No tags to add resources, please export IBM_CUSTOM_TAGS_LIST in ' + 'str format. i.e "key:value, env:test"'} + tags_list = self.__ibm_custom_tags_list.split(',') + vpc_resources = [ + "virtual_servers", + "volumes", + "floating_ips", + "vpcs", + "virtual_network_interfaces", + "security_groups", + "public_gateways", + "vpc_endpoint_gateways", + "load_balancers", + "schematics_workspaces" + ] + if self.resource_to_tag and self.resource_to_tag in vpc_resources: + vpc_resources = [self.resource_to_tag] + logger.info(f"Running tag operation on total of {len(vpc_resources)} resources") + errors = [] + messages = {} + for vpc_resource in vpc_resources: + message = 'tagged are added to all resources' + resources_crn = getattr(self, f'get_{vpc_resource}_crn')() + logger.info(f"Started the tagging operation for {vpc_resource}") + ok, errors = self.tag_operations.update_tags(resources_crn, tags=tags_list) + if not ok: + message = 'Unable to tag all resources' + logger.info(f'{message}, please find the servers that are not tagged: {errors}') + errors.update({vpc_resource: message, 'crns': errors}) + else: + messages.update({vpc_resource: message}) + + return {'errors': errors, 'messages': messages} + + def run(self): + """ + This method runs the tag operations + :return: + """ + return self.tag_all_vpc_resources() diff --git a/cloud_governance/policy/policy_runners/ibm/__init__.py b/cloud_governance/policy/policy_runners/ibm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloud_governance/policy/policy_runners/ibm/policy_runner.py b/cloud_governance/policy/policy_runners/ibm/policy_runner.py new file mode 100644 index 00000000..3a29a942 --- /dev/null +++ b/cloud_governance/policy/policy_runners/ibm/policy_runner.py @@ -0,0 +1,32 @@ +from typing import Callable + +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.policy.policy_runners.common.abstract_policy_runner import AbstractPolicyRunner + + +class PolicyRunner(AbstractPolicyRunner): + + def execute_policy(self, policy_class_name: str, run_policy: Callable, upload: bool = False): + """ + This method executes the policy + :param policy_class_name: + :type policy_class_name: + :param run_policy: + :type run_policy: + :param upload: + :type upload: + :return: + :rtype: + """ + policy_result = [] + response = run_policy().run() + if isinstance(response, str): + logger.info(response) + else: + logger.info(response) + policy_result.extend(response) + self._upload_elastic_search.upload(data=policy_result) + return policy_result + + def __init__(self): + super().__init__() diff --git a/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile b/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile index 066210bf..2126fd05 100644 --- a/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile +++ b/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile @@ -16,6 +16,9 @@ pipeline { AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id') GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials') LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') + IBM_CUSTOM_TAGS_LIST = credentials('IBM_CUSTOM_TAGS_LIST') + IBM_CLOUD_API_KEY = credentials('IBM_CLOUD_API_KEY') + account = "IBM-PERF" contact1 = "ebattat@redhat.com" contact2 = "athiruma@redhat.com" } diff --git a/jenkins/clouds/ibm/hourly/tagging/tagging.py b/jenkins/clouds/ibm/hourly/tagging/tagging.py index 5b261d85..c36cb138 100644 --- a/jenkins/clouds/ibm/hourly/tagging/tagging.py +++ b/jenkins/clouds/ibm/hourly/tagging/tagging.py @@ -5,13 +5,63 @@ IBM_API_KEY = os.environ['IBM_API_KEY'] IBM_API_USERNAME = os.environ['IBM_API_USERNAME'] SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID'] +IBM_CUSTOM_TAGS_LIST = os.environ['IBM_CUSTOM_TAGS_LIST'] +IBM_CLOUD_API_KEY = os.environ['IBM_CLOUD_API_KEY'] LOGS = os.environ.get('LOGS', 'logs') +account = os.environ['account'] QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', 'quay.io/cloud-governance/cloud-governance:latest') -print('Run IBM tagging on baremetal, vm') -os.system( - f"""podman run --rm --name cloud-governance -e account="IBM-PERF" -e policy="tag_baremetal" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") -os.system( - f"""podman run --rm --name cloud-governance -e account="IBM-PERF" -e policy="tag_vm" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS} -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e IBM_API_USERNAME="{IBM_API_USERNAME}" -e IBM_API_KEY="{IBM_API_KEY}" -e tag_operation="update" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" {QUAY_CLOUD_GOVERNANCE_REPOSITORY}""") +def run_cmd(cmd): + os.system(cmd) + + +def generate_env_vars(**kwargs): + items = [] + for k, v in kwargs.items(): + items.append(f'-e {k}="{v}"') + return ' '.join(items) + + +def generate_volume_mounts(volume_mounts): + items = [] + for mount in volume_mounts: + items.append(f'-v {mount}="{mount}"') + return ' '.join(items) + + +def get_podman_run_cmd(volume_mounts: list = None, **kwargs): + container_name = "--name cloud-governance" + if not volume_mounts: + volume_mounts = [] + if "/etc/localtime" not in volume_mounts: + volume_mounts.append("/etc/localtime") + cmd = f"podman run --rm {container_name} {generate_env_vars(**kwargs)} {generate_volume_mounts(volume_mounts)} {QUAY_CLOUD_GOVERNANCE_REPOSITORY}" + return cmd + + +run_cmd('Run IBM tagging on baremetal, vm') + +run_cmd("Run IBM tag baremetal") +volume_mounts_targets = [GOOGLE_APPLICATION_CREDENTIALS] + +input_env_keys = {'volume_mounts': volume_mounts_targets, 'account': account, 'LDAP_HOST_NAME': LDAP_HOST_NAME, + 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS, 'SPREADSHEET_ID': SPREADSHEET_ID, + 'IBM_API_USERNAME': IBM_API_USERNAME, 'IBM_API_KEY': IBM_API_KEY, 'tag_operation': "update", + 'log_level': "INFO", 'policy': 'tag_baremetal'} + +baremetal_cmd = get_podman_run_cmd(volume_mounts=volume_mounts_targets, **input_env_keys) +run_cmd(baremetal_cmd) + +run_cmd("Run IBM tag Virtual Machines") +input_env_keys['policy'] = 'tag_vm' +virtual_machine_cmd = get_podman_run_cmd(volume_mounts=volume_mounts_targets, **input_env_keys) +run_cmd(virtual_machine_cmd) + +# Run tag resources +run_cmd("Run tag resources command") +podman_run_cmd = get_podman_run_cmd(policy="tag_resources", account=account, + IBM_CLOUD_API_KEY=IBM_CLOUD_API_KEY, + IBM_CUSTOM_TAGS_LIST=IBM_CUSTOM_TAGS_LIST) +run_cmd(podman_run_cmd) diff --git a/requirements.txt b/requirements.txt index b506123d..ab325533 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,10 @@ google-auth-httplib2==0.1.0 google-auth-oauthlib==0.5.2 google-cloud-bigquery==3.5.0 google-cloud-billing==1.9.1 -ibm_platform_services==0.27.0 +ibm-cloud-sdk-core==3.18.0 +ibm-platform-services==0.27.0 +ibm-schematics==1.1.0 +ibm-vpc==0.21.0 myst-parser==1.0.0 numpy<=1.26.4 # opensearch 1.2.4 for elasticsearch oauthlib~=3.1.1 diff --git a/setup.py b/setup.py index 8e0559a5..e02fd52b 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,10 @@ 'google-auth-oauthlib==0.5.2', # google drive 'google-cloud-bigquery==3.5.0', # google cloud cost 'google-cloud-billing==1.9.1', # google cloud cost - 'ibm_platform_services==0.27.0', # IBM Usage reports + 'ibm-cloud-sdk-core==3.18.0', + 'ibm-platform-services==0.27.0', # IBM Usage reports + 'ibm-schematics==1.1.0', + 'ibm-vpc==0.21.0', 'myst-parser==1.0.0', # readthedocs 'numpy<=1.26.4', # opensearch 1.2.4 for elasticsearch 'oauthlib~=3.1.1', # required by jira diff --git a/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/__init__.py b/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/test_ibm_global_tagging.py b/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/test_ibm_global_tagging.py new file mode 100644 index 00000000..53520d44 --- /dev/null +++ b/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/test_ibm_global_tagging.py @@ -0,0 +1,18 @@ +from cloud_governance.common.clouds.ibm.tagging.global_tagging_operations import GlobalTaggingOperations +from cloud_governance.main.environment_variables import environment_variables +from tests.unittest.mocks.ibm.mock_ibm_global_tagging import mock_ibm_global_tagging + +environment_variables.IBM_CLOUD_API_KEY = 'mock_ibm_api_key' + + +@mock_ibm_global_tagging +def test_update_tags(): + """ + This method tests the update_tags function of GlobalTagging Operations of IBM Cloud + :return: + """ + global_tagging_operations = GlobalTaggingOperations() + crns = ['id123'] + tags = ["cost-center: test"] + response = global_tagging_operations.update_tags(resources_crn=crns, tags=tags) + assert response[0] diff --git a/tests/unittest/cloud_governance/common/clouds/ibm/vpc/__init__.py b/tests/unittest/cloud_governance/common/clouds/ibm/vpc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/cloud_governance/common/clouds/ibm/vpc/test_vpc_infra_operations.py b/tests/unittest/cloud_governance/common/clouds/ibm/vpc/test_vpc_infra_operations.py new file mode 100644 index 00000000..ba67bacf --- /dev/null +++ b/tests/unittest/cloud_governance/common/clouds/ibm/vpc/test_vpc_infra_operations.py @@ -0,0 +1,29 @@ +from cloud_governance.common.clouds.ibm.vpc.vpc_infra_operations import VpcInfraOperations +from cloud_governance.main.environment_variables import environment_variables +from tests.unittest.mocks.ibm.mock_ibm_vpc import mock_ibm_vpc + +environment_variables.IBM_CLOUD_API_KEY = 'mock_ibm_api_key' + + +@mock_ibm_vpc +def test_get_regions(): + """ + This test checks that the get_regions function works. + :return: + """ + vpc_infra_operations = VpcInfraOperations() + response = vpc_infra_operations.get_regions() + assert response is not None + assert len(response) == 1 + + +@mock_ibm_vpc +def test_get_instances(): + """ + This test checks that the get_instances function works. + :return: + """ + vpc_infra_operations = VpcInfraOperations() + response = vpc_infra_operations.get_instances() + assert response is not None + assert len(response) == 1 diff --git a/tests/unittest/cloud_governance/policy/ibm/__init__.py b/tests/unittest/cloud_governance/policy/ibm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/cloud_governance/policy/ibm/test_tag_resources.py b/tests/unittest/cloud_governance/policy/ibm/test_tag_resources.py new file mode 100644 index 00000000..6fd47b40 --- /dev/null +++ b/tests/unittest/cloud_governance/policy/ibm/test_tag_resources.py @@ -0,0 +1,15 @@ +from cloud_governance.main.environment_variables import environment_variables +from cloud_governance.policy.ibm.tag_resources import TagResources +from tests.unittest.mocks.ibm.mock_ibm_global_tagging import mock_ibm_global_tagging +from tests.unittest.mocks.ibm.mock_ibm_vpc import mock_ibm_vpc + + +@mock_ibm_global_tagging +@mock_ibm_vpc +def test_tag_all_vpc_resources(): + environment_variables.IBM_CLOUD_API_KEY = 'mock_ibm_api_key' + environment_variables.RESOURCE_TO_TAG = 'virtual_servers' + environment_variables.IBM_CUSTOM_TAGS_LIST = "cost-center: test" + tag_resources = TagResources() + res = tag_resources.tag_all_vpc_resources() + assert res.get('messages').get('virtual_servers') diff --git a/tests/unittest/mocks/ibm/__init__.py b/tests/unittest/mocks/ibm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unittest/mocks/ibm/mock_ibm_global_tagging.py b/tests/unittest/mocks/ibm/mock_ibm_global_tagging.py new file mode 100644 index 00000000..8c3411c1 --- /dev/null +++ b/tests/unittest/mocks/ibm/mock_ibm_global_tagging.py @@ -0,0 +1,47 @@ +from typing import List +from unittest.mock import patch + +from ibm_cloud_sdk_core import DetailedResponse +from ibm_platform_services.global_tagging_v1 import Resource, GlobalTaggingV1 + + +class MockGlobalTaggingV1(GlobalTaggingV1): + + def __init__(self, *args, **kwargs): + self.resources = {} + + def attach_tag(self, + resources: List[Resource], + tag_name: str = None, + tag_names: List[str] = None, + *args, + **kwargs + ) -> DetailedResponse: + results = [] + for resource in resources: + if tag_names: + self.resources[resource.resource_id] = tag_names + else: + if tag_name: + self.resources[resource.resource_id] = [tag_name] + results.append({ + 'is_error': False, + 'resource_id': resource.resource_id, + }) + return DetailedResponse(response={'results': results}) + + +def mock_ibm_global_tagging(method): + def method_wrapper(*args, **kwargs): + """ + This is the wrapper method to wraps the method inside the function + @param args: + @param kwargs: + @return: + """ + with patch.object(GlobalTaggingV1, 'attach_tag', + MockGlobalTaggingV1().attach_tag): + result = method(*args, **kwargs) + return result + + return method_wrapper diff --git a/tests/unittest/mocks/ibm/mock_ibm_vpc.py b/tests/unittest/mocks/ibm/mock_ibm_vpc.py new file mode 100644 index 00000000..40a54a52 --- /dev/null +++ b/tests/unittest/mocks/ibm/mock_ibm_vpc.py @@ -0,0 +1,60 @@ +from unittest.mock import patch +import ibm_vpc +from ibm_cloud_sdk_core import DetailedResponse + + +class MockDetailedResponse(DetailedResponse): + + def __init__(self, response): + super().__init__() + self.result = response + + +class MockVpcV1(ibm_vpc.vpc_v1.VpcV1): + + def __init__(self, *args, **kwargs): + self.regions = [{ + 'endpoint': 'https://au-syd.iaas.cloud.ibm.com', + 'href': 'https://us-south.iaas.cloud.ibm.com/v1/regions/au-syd', + 'name': 'au-syd', + 'status': 'available' + }] + self.instances = [{ + 'crn': 'id123', + 'name': 'test-mock-vm' + }] + + def set_region(self, region: dict): + """ + This method set region + :param region: + :return: + """ + self.regions.append(region) + + def list_regions(self, *args, **kwargs) -> MockDetailedResponse: + response = { + 'regions': self.regions + } + return MockDetailedResponse(response) + + def list_instances(self, *args, **kwargs) -> MockDetailedResponse: + response = { + 'instances': self.instances + } + return MockDetailedResponse(response) + + +def mock_ibm_vpc(method): + def method_wrapper(*args, **kwargs): + """ + This is the wrapper method to wraps the method inside the function + @param args: + @param kwargs: + @return: + """ + with patch.object(ibm_vpc.vpc_v1, 'VpcV1', MockVpcV1): + result = method(*args, **kwargs) + return result + + return method_wrapper diff --git a/tests_requirements.txt b/tests_requirements.txt index 50d80e7a..b1d156c5 100644 --- a/tests_requirements.txt +++ b/tests_requirements.txt @@ -8,6 +8,10 @@ boto3==1.26.4 elasticsearch==7.13.4 elasticsearch-dsl==7.4.0 freezegun==1.5.1 +ibm-cloud-sdk-core==3.18.0 +ibm-platform-services==0.27.0 +ibm-schematics==1.1.0 +ibm-vpc==0.21.0 moto==4.0.1 numpy<=1.26.4 oauthlib~=3.1.1