diff --git a/simple_vm_client/openstack_connector/openstack_connector.py b/simple_vm_client/openstack_connector/openstack_connector.py index e1890fa..6e50472 100644 --- a/simple_vm_client/openstack_connector/openstack_connector.py +++ b/simple_vm_client/openstack_connector/openstack_connector.py @@ -138,7 +138,7 @@ def load_env_config(self) -> None: sys.exit(1) self.USE_APPLICATION_CREDENTIALS = ( - os.environ.get("USE_APPLICATION_CREDENTIALS", "False").lower() == "true" + os.environ.get("USE_APPLICATION_CREDENTIALS", "False").lower() == "true" ) if self.USE_APPLICATION_CREDENTIALS: @@ -180,15 +180,15 @@ def load_env_config(self) -> None: self.PROJECT_DOMAIN_ID = os.environ["OS_PROJECT_DOMAIN_ID"] def create_server( - self, - name: str, - image_id: str, - flavor_id: str, - network_id: str, - userdata: str, - key_name: str, - metadata: dict[str, str], - security_groups: list[str], + self, + name: str, + image_id: str, + flavor_id: str, + network_id: str, + userdata: str, + key_name: str, + metadata: dict[str, str], + security_groups: list[str], ) -> Server: logger.info( f"Create Server:\n\tname: {name}\n\timage_id:{image_id}\n\tflavor_id:{flavor_id}\n\tmetadata:{metadata}" @@ -229,7 +229,7 @@ def delete_volume(self, volume_id: str) -> None: raise DefaultException(message=e.message) def create_volume_snapshot( - self, volume_id: str, name: str, description: str + self, volume_id: str, name: str, description: str ) -> str: try: logger.info(f"Create Snapshot for Volume {volume_id}") @@ -269,7 +269,7 @@ def delete_volume_snapshot(self, snapshot_id: str) -> None: raise DefaultException(message=e.message) def create_volume_by_source_volume( - self, volume_name: str, metadata: dict[str, str], source_volume_id: str + self, volume_name: str, metadata: dict[str, str], source_volume_id: str ) -> Volume: logger.info(f"Creating volume from source volume with id {source_volume_id}") try: @@ -285,7 +285,7 @@ def create_volume_by_source_volume( raise ResourceNotAvailableException(message=e.message) def create_volume_by_volume_snap( - self, volume_name: str, metadata: dict[str, str], volume_snap_id: str + self, volume_name: str, metadata: dict[str, str], volume_snap_id: str ) -> Volume: logger.info(f"Creating volume from volume snapshot with id {volume_snap_id}") try: @@ -319,7 +319,7 @@ def get_servers_by_ids(self, ids: list[str]) -> list[Server]: return servers def attach_volume_to_server( - self, openstack_id: str, volume_id: str + self, openstack_id: str, volume_id: str ) -> dict[str, str]: server = self.get_server(openstack_id=openstack_id) volume = self.get_volume(name_or_id=volume_id) @@ -360,7 +360,7 @@ def resize_volume(self, volume_id: str, size: int) -> None: raise DefaultException(message=str(e)) def create_volume( - self, volume_name: str, volume_storage: int, metadata: dict[str, str] + self, volume_name: str, volume_storage: int, metadata: dict[str, str] ) -> Volume: logger.info(f"Creating volume with {volume_storage} GB storage") try: @@ -477,9 +477,9 @@ def get_active_image_by_os_version(self, os_version: str, os_distro: str) -> Ima image_os_distro = metadata.get("os_distro", None) base_image_ref = metadata.get("base_image_ref", None) if ( - os_version == image_os_version - and image.status == "active" - and base_image_ref is None + os_version == image_os_version + and image.status == "active" + and base_image_ref is None ): if os_distro and os_distro == image_os_distro: return image @@ -491,11 +491,11 @@ def get_active_image_by_os_version(self, os_version: str, os_distro: str) -> Ima ) def get_image( - self, - name_or_id: str, - replace_inactive: bool = False, - ignore_not_active: bool = False, - ignore_not_found: bool = False, + self, + name_or_id: str, + replace_inactive: bool = False, + ignore_not_active: bool = False, + ignore_not_found: bool = False, ) -> Image: logger.info(f"Get Image {name_or_id}") @@ -519,12 +519,12 @@ def get_image( return image def create_snapshot( - self, - openstack_id: str, - name: str, - username: str, - base_tags: list[str], - description: str, + self, + openstack_id: str, + name: str, + username: str, + base_tags: list[str], + description: str, ) -> str: logger.info( f"Create Snapshot from Instance {openstack_id} with name {name} for {username}" @@ -636,9 +636,9 @@ def get_gateway_ip(self) -> dict[str, str]: return {"gateway_ip": self.GATEWAY_IP} def create_mount_init_script( - self, - new_volumes: list[dict[str, str]] = None, # type: ignore - attach_volumes: list[dict[str, str]] = None, # type: ignore + self, + new_volumes: list[dict[str, str]] = None, # type: ignore + attach_volumes: list[dict[str, str]] = None, # type: ignore ) -> str: logger.info(f"Create init script for volume ids:{new_volumes}") if not new_volumes and not attach_volumes: @@ -719,7 +719,7 @@ def delete_security_group_rule(self, openstack_id): ) def open_port_range_for_vm_in_project( - self, range_start, range_stop, openstack_id, ethertype="IPV4", protocol="TCP" + self, range_start, range_stop, openstack_id, ethertype="IPV4", protocol="TCP" ): server: Server = self.openstack_connection.get_server_by_id(id=openstack_id) if server is None: @@ -768,13 +768,13 @@ def open_port_range_for_vm_in_project( raise OpenStackConflictException(message=e.message) def create_security_group( - self, - name: str, - udp_port: int = None, # type: ignore - ssh: bool = True, - udp: bool = False, - description: str = "", - research_environment_metadata: ResearchEnvironmentMetadata = None, + self, + name: str, + udp_port: int = None, # type: ignore + ssh: bool = True, + udp: bool = False, + description: str = "", + research_environment_metadata: ResearchEnvironmentMetadata = None, ) -> SecurityGroup: logger.info(f"Create new security group {name}") sec: SecurityGroup = self.openstack_connection.get_security_group( @@ -881,7 +881,7 @@ def is_security_group_in_use(self, security_group_id): return False def get_or_create_research_environment_security_group( - self, resenv_metadata: ResearchEnvironmentMetadata + self, resenv_metadata: ResearchEnvironmentMetadata ): if not resenv_metadata.needs_forc_support: return None @@ -1115,9 +1115,9 @@ def delete_server(self, openstack_id: str) -> None: server=server, security_group=sec ) if ( - sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and ("bibigrid" not in sec.name or "master" not in server.name) - and not self.is_security_group_in_use(security_group_id=sec.id) + sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME + and ("bibigrid" not in sec.name or "master" not in server.name) + and not self.is_security_group_in_use(security_group_id=sec.id) ): self.openstack_connection.delete_security_group(sg) self.openstack_connection.compute.delete_server(server.id, force=True) @@ -1154,47 +1154,51 @@ def get_vm_ports(self, openstack_id: str) -> dict[str, str]: return {"port": str(ssh_port), "udp": str(udp_port)} def create_userdata( - self, - volume_ids_path_new: list[dict[str, str]], - volume_ids_path_attach: list[dict[str, str]], - additional_keys: list[str], + self, + volume_ids_path_new: list[dict[str, str]], + volume_ids_path_attach: list[dict[str, str]], + additional_keys: list[str], ) -> str: - init_script = self.create_mount_init_script( - new_volumes=volume_ids_path_new, - attach_volumes=volume_ids_path_attach, + + unlock_ubuntu_user_script = "\npasswd -u ubuntu\n" + unlock_ubuntu_user_script_encoded = encodeutils.safe_encode( + unlock_ubuntu_user_script.encode("utf-8") ) + init_script = unlock_ubuntu_user_script_encoded if additional_keys: - if init_script: - add_key_script = self.create_add_keys_script(keys=additional_keys) - init_script = ( + add_key_script = self.create_add_keys_script(keys=additional_keys) + init_script = ( add_key_script + encodeutils.safe_encode("\n".encode("utf-8")) - + init_script - ) + ) + if volume_ids_path_new or volume_ids_path_attach: + mount_script = self.create_mount_init_script( + new_volumes=volume_ids_path_new, + attach_volumes=volume_ids_path_attach, + ) + init_script = ( + init_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + mount_script + + ) - else: - init_script = self.create_add_keys_script(keys=additional_keys) - unlock_ubuntu_user_script = "\npasswd -u ubuntu\n" - unlock_ubuntu_user_script = encodeutils.safe_encode( - unlock_ubuntu_user_script.encode("utf-8") - ) - init_script = init_script + unlock_ubuntu_user_script return init_script def start_server( - self, - flavor_name: str, - image_name: str, - servername: str, - metadata: dict[str, str], - public_key: str, - research_environment_metadata: Union[ResearchEnvironmentMetadata, None] = None, - volume_ids_path_new: Union[list[dict[str, str]], None] = None, - volume_ids_path_attach: Union[list[dict[str, str]], None] = None, - additional_keys: Union[list[str], None] = None, - additional_security_group_ids: Union[list[str], None] = None, + self, + flavor_name: str, + image_name: str, + servername: str, + metadata: dict[str, str], + public_key: str, + research_environment_metadata: Union[ResearchEnvironmentMetadata, None] = None, + volume_ids_path_new: Union[list[dict[str, str]], None] = None, + volume_ids_path_attach: Union[list[dict[str, str]], None] = None, + additional_keys: Union[list[str], None] = None, + additional_security_group_ids: Union[list[str], None] = None, ) -> str: logger.info(f"Start Server {servername}") @@ -1276,16 +1280,16 @@ def start_server( raise DefaultException(message=str(e)) def start_server_with_playbook( - self, - flavor_name: str, - image_name: str, - servername: str, - metadata: dict[str, str], - research_environment_metadata: ResearchEnvironmentMetadata, - volume_ids_path_new: list[dict[str, str]] = None, # type: ignore - volume_ids_path_attach: list[dict[str, str]] = None, # type: ignore - additional_keys: list[str] = None, # type: ignore - additional_security_group_ids=None, # type: ignore + self, + flavor_name: str, + image_name: str, + servername: str, + metadata: dict[str, str], + research_environment_metadata: ResearchEnvironmentMetadata, + volume_ids_path_new: list[dict[str, str]] = None, # type: ignore + volume_ids_path_attach: list[dict[str, str]] = None, # type: ignore + additional_keys: list[str] = None, # type: ignore + additional_security_group_ids=None, # type: ignore ) -> tuple[str, str]: logger.info(f"Start Server {servername}") @@ -1421,16 +1425,16 @@ def add_udp_security_group(self, server_id): return def add_cluster_machine( - self, - cluster_id: str, - cluster_user: str, - cluster_group_id: list[str], - image_name: str, - flavor_name: str, - name: str, - key_name: str, - batch_idx: int, - worker_idx: int, + self, + cluster_id: str, + cluster_user: str, + cluster_group_id: list[str], + image_name: str, + flavor_name: str, + name: str, + key_name: str, + batch_idx: int, + worker_idx: int, ) -> str: logger.info(f"Add machine to {cluster_id}") image: Image = self.get_image(name_or_id=image_name, replace_inactive=True) diff --git a/simple_vm_client/openstack_connector/scripts/bash/mount.sh b/simple_vm_client/openstack_connector/scripts/bash/mount.sh index 685c53c..dc23ac0 100644 --- a/simple_vm_client/openstack_connector/scripts/bash/mount.sh +++ b/simple_vm_client/openstack_connector/scripts/bash/mount.sh @@ -1,5 +1,4 @@ #!/bin/bash -sudo touch test declare -a volumes_new=VOLUME_IDS_NEW declare -a paths_new=VOLUME_PATHS_NEW declare -a volumes_attach=VOLUME_IDS_ATTACH