diff --git a/common/scylla_cloud_io_setup b/common/scylla_cloud_io_setup index 6c5f8217..5cd51a66 100755 --- a/common/scylla_cloud_io_setup +++ b/common/scylla_cloud_io_setup @@ -81,6 +81,172 @@ class aws_io_setup(cloud_io_setup): self.disk_properties["read_bandwidth"] = 507338935 * nr_disks self.disk_properties["write_iops"] = 57100 * nr_disks self.disk_properties["write_bandwidth"] = 483141731 * nr_disks + elif self.idata.instance_class() in ("m5d", "r5d"): + if self.idata.instance_size() == "large": + self.disk_properties["read_iops"] = 33271 + self.disk_properties["read_bandwidth"] = 158538149 + self.disk_properties["write_iops"] = 16820 + self.disk_properties["write_bandwidth"] = 70219810 + elif self.idata.instance_size() == "xlarge": + self.disk_properties["read_iops"] = 65979 + self.disk_properties["read_bandwidth"] = 260654293 + self.disk_properties["write_iops"] = 32534 + self.disk_properties["write_bandwidth"] = 135897424 + elif self.idata.instance_size() == "2xlarge": + self.disk_properties["read_iops"] = 130095 + self.disk_properties["read_bandwidth"] = 621758272 + self.disk_properties["write_iops"] = 63644 + self.disk_properties["write_bandwidth"] = 267667525 + elif self.idata.instance_size() == "4xlarge": + self.disk_properties["read_iops"] = 129822 * nr_disks + self.disk_properties["read_bandwidth"] = 620878826 * nr_disks + self.disk_properties["write_iops"] = 63212 * nr_disks + self.disk_properties["write_bandwidth"] = 267703397 * nr_disks + elif self.idata.instance_size() == "8xlarge": + self.disk_properties["read_iops"] = 257069 * nr_disks + self.disk_properties["read_bandwidth"] = 1250134869 * nr_disks + self.disk_properties["write_iops"] = 115433 * nr_disks + self.disk_properties["write_bandwidth"] = 532868032 * nr_disks + elif self.idata.instance_size() == "12xlarge": + self.disk_properties["read_iops"] = 381626 * nr_disks + self.disk_properties["read_bandwidth"] = 1865794816 * nr_disks + self.disk_properties["write_iops"] = 115333 * nr_disks + self.disk_properties["write_bandwidth"] = 795884800 * nr_disks + elif self.idata.instance_size() == "16xlarge": + self.disk_properties["read_iops"] = 257054 * nr_disks + self.disk_properties["read_bandwidth"] = 1254133802 * nr_disks + self.disk_properties["write_iops"] = 108163 * nr_disks + self.disk_properties["write_bandwidth"] = 532996277 * nr_disks + elif self.idata.instance_size() == "24xlarge": + self.disk_properties["read_iops"] = 374737 * nr_disks + self.disk_properties["read_bandwidth"] = 1855833386 * nr_disks + self.disk_properties["write_iops"] = 125214 * nr_disks + self.disk_properties["write_bandwidth"] = 796082133 * nr_disks + elif self.idata.instance_size() == "metal": + self.disk_properties["read_iops"] = 381441 * nr_disks + self.disk_properties["read_bandwidth"] = 1874585429 * nr_disks + self.disk_properties["write_iops"] = 108789 * nr_disks + self.disk_properties["write_bandwidth"] = 796443221 * nr_disks + elif self.idata.instancetype == "m5ad.large": + self.disk_properties["read_iops"] = 33306 + self.disk_properties["read_bandwidth"] = 158338864 + self.disk_properties["write_iops"] = 16817 + self.disk_properties["write_bandwidth"] = 70194288 + elif self.idata.instancetype == "m5ad.xlarge": + self.disk_properties["read_iops"] = 66127 + self.disk_properties["read_bandwidth"] = 260377466 + self.disk_properties["write_iops"] = 32893 + self.disk_properties["write_bandwidth"] = 135897696 + elif self.idata.instancetype == "m5ad.2xlarge": + self.disk_properties["read_iops"] = 129977 + self.disk_properties["read_bandwidth"] = 621997248 + self.disk_properties["write_iops"] = 63442 + self.disk_properties["write_bandwidth"] = 267648736 + elif self.idata.instancetype == "m5ad.4xlarge": + self.disk_properties["read_iops"] = 129937 * nr_disks + self.disk_properties["read_bandwidth"] = 620231082 * nr_disks + self.disk_properties["write_iops"] = 62666 * nr_disks + self.disk_properties["write_bandwidth"] = 267639125 * nr_disks + elif self.idata.instancetype == "m5ad.8xlarge": + self.disk_properties["read_iops"] = 257095 * nr_disks + self.disk_properties["read_bandwidth"] = 1249927637 * nr_disks + self.disk_properties["write_iops"] = 114446 * nr_disks + self.disk_properties["write_bandwidth"] = 532821760 * nr_disks + elif self.idata.instancetype == "m5ad.12xlarge": + self.disk_properties["read_iops"] = 376431 * nr_disks + self.disk_properties["read_bandwidth"] = 1865866709 * nr_disks + self.disk_properties["write_iops"] = 115985 * nr_disks + self.disk_properties["write_bandwidth"] = 796003477 * nr_disks + elif self.idata.instancetype == "m5ad.16xlarge": + self.disk_properties["read_iops"] = 256358 * nr_disks + self.disk_properties["read_bandwidth"] = 1250889770 * nr_disks + self.disk_properties["write_iops"] = 114707 * nr_disks + self.disk_properties["write_bandwidth"] = 532998506 * nr_disks + elif self.idata.instancetype == "m5ad.24xlarge": + self.disk_properties["read_iops"] = 258951 * nr_disks + self.disk_properties["read_bandwidth"] = 1865871317 * nr_disks + self.disk_properties["write_iops"] = 116030 * nr_disks + self.disk_properties["write_bandwidth"] = 796217706 * nr_disks + elif self.idata.instancetype == "c5d.large": + self.disk_properties["read_iops"] = 22095 + self.disk_properties["read_bandwidth"] = 104797834 + self.disk_properties["write_iops"] = 10125 + self.disk_properties["write_bandwidth"] = 41982906 + elif self.idata.instancetype == "c5d.xlarge": + self.disk_properties["read_iops"] = 44355 + self.disk_properties["read_bandwidth"] = 212593018 + self.disk_properties["write_iops"] = 20025 + self.disk_properties["write_bandwidth"] = 84213472 + elif self.idata.instancetype == "c5d.2xlarge": + self.disk_properties["read_iops"] = 89036 + self.disk_properties["read_bandwidth"] = 426821429 + self.disk_properties["write_iops"] = 41697 + self.disk_properties["write_bandwidth"] = 173730709 + elif self.idata.instancetype == "c5d.4xlarge": + self.disk_properties["read_iops"] = 193970 + self.disk_properties["read_bandwidth"] = 928278314 + self.disk_properties["write_iops"] = 83058 + self.disk_properties["write_bandwidth"] = 351839733 + elif self.idata.instancetype == "c5d.9xlarge": + self.disk_properties["read_iops"] = 381800 + self.disk_properties["read_bandwidth"] = 1865831893 + self.disk_properties["write_iops"] = 112264 + self.disk_properties["write_bandwidth"] = 795731264 + elif self.idata.instancetype == "c5d.12xlarge": + self.disk_properties["read_iops"] = 381775 * nr_disks + self.disk_properties["read_bandwidth"] = 1866481792 * nr_disks + self.disk_properties["write_iops"] = 114302 * nr_disks + self.disk_properties["write_bandwidth"] = 795607616 * nr_disks + elif self.idata.instancetype == "c5d.18xlarge": + self.disk_properties["read_iops"] = 381270 * nr_disks + self.disk_properties["read_bandwidth"] = 1856972330 * nr_disks + self.disk_properties["write_iops"] = 125638 * nr_disks + self.disk_properties["write_bandwidth"] = 795813866 * nr_disks + elif self.idata.instancetype == "c5d.24xlarge": + self.disk_properties["read_iops"] = 381355 * nr_disks + self.disk_properties["read_bandwidth"] = 1876056704 * nr_disks + self.disk_properties["write_iops"] = 104946 * nr_disks + self.disk_properties["write_bandwidth"] = 795901013 * nr_disks + elif self.idata.instancetype == "c5d.metal": + self.disk_properties["read_iops"] = 381330 * nr_disks + self.disk_properties["read_bandwidth"] = 1865216426 * nr_disks + self.disk_properties["write_iops"] = 115484 * nr_disks + self.disk_properties["write_bandwidth"] = 796109546 * nr_disks + elif self.idata.instancetype == "z1d.large": + self.disk_properties["read_iops"] = 33286 + self.disk_properties["read_bandwidth"] = 158206858 + self.disk_properties["write_iops"] = 16956 + self.disk_properties["write_bandwidth"] = 70226280 + elif self.idata.instancetype == "z1d.xlarge": + self.disk_properties["read_iops"] = 66076 + self.disk_properties["read_bandwidth"] = 260565488 + self.disk_properties["write_iops"] = 32769 + self.disk_properties["write_bandwidth"] = 135891989 + elif self.idata.instancetype == "z1d.2xlarge": + self.disk_properties["read_iops"] = 130235 + self.disk_properties["read_bandwidth"] = 622297194 + self.disk_properties["write_iops"] = 63891 + self.disk_properties["write_bandwidth"] = 267679509 + elif self.idata.instancetype == "z1d.3xlarge": + self.disk_properties["read_iops"] = 193840 + self.disk_properties["read_bandwidth"] = 927493696 + self.disk_properties["write_iops"] = 82864 + self.disk_properties["write_bandwidth"] = 351608480 + elif self.idata.instancetype == "z1d.6xlarge": + self.disk_properties["read_iops"] = 381902 + self.disk_properties["read_bandwidth"] = 1865543381 + self.disk_properties["write_iops"] = 117874 + self.disk_properties["write_bandwidth"] = 795786901 + elif self.idata.instancetype == "z1d.12xlarge": + self.disk_properties["read_iops"] = 381648 * nr_disks + self.disk_properties["read_bandwidth"] = 1865706538 * nr_disks + self.disk_properties["write_iops"] = 115834 * nr_disks + self.disk_properties["write_bandwidth"] = 795876778 * nr_disks + elif self.idata.instancetype == "z1d.metal": + self.disk_properties["read_iops"] = 381378 * nr_disks + self.disk_properties["read_bandwidth"] = 1857873109 * nr_disks + self.disk_properties["write_iops"] = 122453 * nr_disks + self.disk_properties["write_bandwidth"] = 795593024 * nr_disks elif self.idata.instance_class() in ("c6gd", "m6gd", "r6gd", "x2gd"): if self.idata.instance_size() == "medium": self.disk_properties["read_iops"] = 14808 @@ -127,6 +293,36 @@ class aws_io_setup(cloud_io_setup): self.disk_properties["read_bandwidth"] = 2527296683 * nr_disks self.disk_properties["write_iops"] = 156326 * nr_disks self.disk_properties["write_bandwidth"] = 1063657088 * nr_disks + elif self.idata.instancetype == "i4g.large": + self.disk_properties["read_iops"] = 34035 + self.disk_properties["read_bandwidth"] = 288471904 + self.disk_properties["write_iops"] = 27943 + self.disk_properties["write_bandwidth"] = 126763269 + elif self.idata.instancetype == "i4g.xlarge": + self.disk_properties["read_iops"] = 68111 + self.disk_properties["read_bandwidth"] = 571766890 + self.disk_properties["write_iops"] = 47622 + self.disk_properties["write_bandwidth"] = 254230192 + elif self.idata.instancetype == "i4g.2xlarge": + self.disk_properties["read_iops"] = 136352 + self.disk_properties["read_bandwidth"] = 1148509696 + self.disk_properties["write_iops"] = 82746 + self.disk_properties["write_bandwidth"] = 508828810 + elif self.idata.instancetype == "i4g.4xlarge": + self.disk_properties["read_iops"] = 272704 + self.disk_properties["read_bandwidth"] = 2297019392 + self.disk_properties["write_iops"] = 165492 + self.disk_properties["write_bandwidth"] = 1017657620 + elif self.idata.instancetype == "i4g.8xlarge": + self.disk_properties["read_iops"] = 271495 * nr_disks + self.disk_properties["read_bandwidth"] = 2293024938 * nr_disks + self.disk_properties["write_iops"] = 93653 * nr_disks + self.disk_properties["write_bandwidth"] = 1031956586 * nr_disks + elif self.idata.instancetype == "i4g.16xlarge": + self.disk_properties["read_iops"] = 250489 * nr_disks + self.disk_properties["read_bandwidth"] = 2286635861 * nr_disks + self.disk_properties["write_iops"] = 93737 * nr_disks + self.disk_properties["write_bandwidth"] = 1034256042 * nr_disks elif self.idata.instancetype == "im4gn.large": self.disk_properties["read_iops"] = 33943 self.disk_properties["read_bandwidth"] = 288433525 diff --git a/lib/scylla_cloud.py b/lib/scylla_cloud.py index b3b5b433..073242bd 100644 --- a/lib/scylla_cloud.py +++ b/lib/scylla_cloud.py @@ -745,7 +745,7 @@ def instance_class(self): return self._type.split(".")[0] def is_supported_instance_class(self): - if self.instance_class() in ['i2', 'i3', 'i3en', 'c5d', 'm5d', 'm5ad', 'r5d', 'z1d', 'c6gd', 'm6gd', 'r6gd', 'x2gd', 'im4gn', 'is4gen', 'i4i']: + if self.instance_class() in ['i2', 'i3', 'i3en', 'c5d', 'm5d', 'm5ad', 'r5d', 'z1d', 'c6gd', 'm6gd', 'r6gd', 'x2gd', 'im4gn', 'is4gen', 'i4i', 'i4g']: return True return False @@ -754,7 +754,7 @@ def get_en_interface_type(self): instance_size = self.instance_size() if instance_class in ['c3', 'c4', 'd2', 'i2', 'r3']: return 'ixgbevf' - if instance_class in ['a1', 'c5', 'c5a', 'c5d', 'c5n', 'c6g', 'c6gd', 'f1', 'g3', 'g4', 'h1', 'i3', 'i3en', 'inf1', 'm5', 'm5a', 'm5ad', 'm5d', 'm5dn', 'm5n', 'm6g', 'm6gd', 'p2', 'p3', 'r4', 'r5', 'r5a', 'r5ad', 'r5b', 'r5d', 'r5dn', 'r5n', 't3', 't3a', 'u-6tb1', 'u-9tb1', 'u-12tb1', 'u-18tn1', 'u-24tb1', 'x1', 'x1e', 'z1d', 'c6g', 'c6gd', 'm6g', 'm6gd', 't4g', 'r6g', 'r6gd', 'x2gd', 'im4gn', 'is4gen', 'i4i']: + if instance_class in ['a1', 'c5', 'c5a', 'c5d', 'c5n', 'c6g', 'c6gd', 'f1', 'g3', 'g4', 'h1', 'i3', 'i3en', 'inf1', 'm5', 'm5a', 'm5ad', 'm5d', 'm5dn', 'm5n', 'm6g', 'm6gd', 'p2', 'p3', 'r4', 'r5', 'r5a', 'r5ad', 'r5b', 'r5d', 'r5dn', 'r5n', 't3', 't3a', 'u-6tb1', 'u-9tb1', 'u-12tb1', 'u-18tn1', 'u-24tb1', 'x1', 'x1e', 'z1d', 'c6g', 'c6gd', 'm6g', 'm6gd', 't4g', 'r6g', 'r6gd', 'x2gd', 'im4gn', 'is4gen', 'i4i', 'i4g']: return 'ena' if instance_class == 'm4': if instance_size == '16xlarge': diff --git a/packer/ami_variables.json b/packer/ami_variables.json index 69111a39..0d23822d 100644 --- a/packer/ami_variables.json +++ b/packer/ami_variables.json @@ -1,5 +1,5 @@ { - "security_group_id": "sg-c5e1f7a0", + "security_group_id": "sg-088128b2712c264d1", "region": "us-east-1", "associate_public_ip_address": "true", "instance_type": "c4.xlarge" diff --git a/packer/build_image.sh b/packer/build_image.sh index ffb1d62f..c497f863 100755 --- a/packer/build_image.sh +++ b/packer/build_image.sh @@ -29,6 +29,7 @@ print_usage() { echo " [--scylla-build-sha-id] Scylla build SHA id form metadata file" echo " [--branch] Set the release branch for GCE label. Default: master" echo " [--ami-regions] Set regions to copy the AMI when done building it (including permissions and tags)" + echo " [--ami-users] A list of account IDs that have access to launch the AMI" echo " [--build-tag] Jenkins Build tag" echo " --download-no-server Download all deb needed excluding scylla from repo-for-install" echo " [--build-mode] Choose which build mode to use for Scylla installation. Default: release. Valid options: release|debug" @@ -98,6 +99,11 @@ while [ $# -gt 0 ]; do echo "--ami-regions prameter: AMI_REGIONS |$AMI_REGIONS|" shift 2 ;; + "--ami-users"): + AMI_USERS=$2 + echo "--ami-users parameter: AMI_USERS |$AMI_USERS|" + shift 2 + ;; "--log-file") PACKER_LOG_PATH=$2 echo "--log-file parameter: PACKER_LOG_PATH |$PACKER_LOG_PATH|" @@ -345,6 +351,7 @@ set -x -var operating_system="$OPERATING_SYSTEM" \ -var branch="$BRANCH" \ -var ami_regions="$AMI_REGIONS" \ + -var ami_users="$AMI_USERS" \ -var arch="$ARCH" \ -var product="$PRODUCT" \ -var build_mode="$BUILD_MODE" \ diff --git a/packer/scylla.json b/packer/scylla.json index fcda121a..f06cc7c8 100644 --- a/packer/scylla.json +++ b/packer/scylla.json @@ -89,6 +89,7 @@ "build_mode": "{{user `build_mode`| clean_resource_name}}" }, "ami_regions": "{{user `ami_regions`}}", + "ami_users": "{{user `ami_users`}}", "aws_polling": { "delay_seconds": "30", "max_attempts": "100" @@ -176,14 +177,14 @@ { "type": "shell", "inline": [ - "if [ {{build_name}} = aws -o {{build_name}} = azure ]; then sudo apt-get update; sudo DEBIAN_FRONTEND=noninteractive apt purge -y linux-{{build_name}}* linux-headers-{{build_name}}* linux-image*{{build_name}}* linux-modules*-{{build_name}}*; sudo DEBIAN_FRONTEND=noninteractive apt-get install -y linux-{{build_name}}-lts-22.04; sudo reboot ; fi" + "if [ {{build_name}} = aws -o {{build_name}} = azure ]; then sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt purge -y linux-{{build_name}}* linux-headers-{{build_name}}* linux-image*{{build_name}}* linux-modules*-{{build_name}}* && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y linux-{{build_name}}-lts-22.04 && sudo reboot; fi" ], "expect_disconnect": true }, { "type": "shell", "inline": [ - "if [ {{build_name}} = gce ]; then sudo apt-get update; sudo DEBIAN_FRONTEND=noninteractive apt purge -y linux-gcp* linux-headers-gcp* linux-image*gcp* linux-modules*-gcp*; sudo DEBIAN_FRONTEND=noninteractive apt-get install -y linux-gcp-lts-22.04; sudo reboot ; fi" + "if [ {{build_name}} = gce ]; then sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt purge -y linux-gcp* linux-headers-gcp* linux-image*gcp* linux-modules*-gcp* && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y linux-gcp-lts-22.04 && sudo reboot; fi" ], "expect_disconnect": true }, diff --git a/packer/scylla_install_image b/packer/scylla_install_image index 07801f85..a93d39e8 100755 --- a/packer/scylla_install_image +++ b/packer/scylla_install_image @@ -125,6 +125,8 @@ if __name__ == '__main__': kernel_opt = '' grub_variable = 'GRUB_CMDLINE_LINUX_DEFAULT' elif args.target_cloud == 'gce': + # align with other clouds image + run('apt-get purge -y rsyslog', shell=True, check=True) sysconfig_opt = '--disable-writeback-cache' kernel_opt = '' grub_variable = 'GRUB_CMDLINE_LINUX_DEFAULT' @@ -170,6 +172,7 @@ WantedBy=multi-user.target os.remove('{}/.ssh/authorized_keys'.format(homedir)) os.remove('/var/lib/scylla-housekeeping/housekeeping.uuid') + os.remove('/var/cache/debconf/config.dat') with open('/etc/default/grub.d/50-cloudimg-settings.cfg') as f: grub = f.read()