Merge branch 'next' into exponential_backoff

scylladb · Sep 18, 2023 · 5e75298 · 5e75298
2 parents cbbbdd0 + a6f7f2e
commit 5e75298
Show file tree

Hide file tree

Showing 6 changed files with 212 additions and 5 deletions.
diff --git a/common/scylla_cloud_io_setup b/common/scylla_cloud_io_setup
@@ -81,6 +81,172 @@ class aws_io_setup(cloud_io_setup):
             self.disk_properties["read_bandwidth"] = 507338935 * nr_disks
             self.disk_properties["write_iops"] = 57100 * nr_disks
             self.disk_properties["write_bandwidth"] = 483141731 * nr_disks
+        elif self.idata.instance_class() in ("m5d", "r5d"):
+            if self.idata.instance_size() == "large":
+                self.disk_properties["read_iops"] = 33271
+                self.disk_properties["read_bandwidth"] = 158538149
+                self.disk_properties["write_iops"] = 16820
+                self.disk_properties["write_bandwidth"] = 70219810
+            elif self.idata.instance_size() == "xlarge":
+                self.disk_properties["read_iops"] = 65979
+                self.disk_properties["read_bandwidth"] = 260654293
+                self.disk_properties["write_iops"] = 32534
+                self.disk_properties["write_bandwidth"] = 135897424
+            elif self.idata.instance_size() == "2xlarge":
+                self.disk_properties["read_iops"] = 130095
+                self.disk_properties["read_bandwidth"] = 621758272
+                self.disk_properties["write_iops"] = 63644
+                self.disk_properties["write_bandwidth"] = 267667525
+            elif self.idata.instance_size() == "4xlarge":
+                self.disk_properties["read_iops"] = 129822 * nr_disks
+                self.disk_properties["read_bandwidth"] = 620878826 * nr_disks
+                self.disk_properties["write_iops"] = 63212 * nr_disks
+                self.disk_properties["write_bandwidth"] = 267703397 * nr_disks
+            elif self.idata.instance_size() == "8xlarge":
+                self.disk_properties["read_iops"] = 257069 * nr_disks
+                self.disk_properties["read_bandwidth"] = 1250134869 * nr_disks
+                self.disk_properties["write_iops"] = 115433 * nr_disks
+                self.disk_properties["write_bandwidth"] = 532868032 * nr_disks
+            elif self.idata.instance_size() == "12xlarge":
+                self.disk_properties["read_iops"] = 381626 * nr_disks
+                self.disk_properties["read_bandwidth"] = 1865794816 * nr_disks
+                self.disk_properties["write_iops"] = 115333 * nr_disks
+                self.disk_properties["write_bandwidth"] = 795884800 * nr_disks
+            elif self.idata.instance_size() == "16xlarge":
+                self.disk_properties["read_iops"] = 257054 * nr_disks
+                self.disk_properties["read_bandwidth"] = 1254133802 * nr_disks
+                self.disk_properties["write_iops"] = 108163 * nr_disks
+                self.disk_properties["write_bandwidth"] = 532996277 * nr_disks
+            elif self.idata.instance_size() == "24xlarge":
+                self.disk_properties["read_iops"] = 374737 * nr_disks
+                self.disk_properties["read_bandwidth"] = 1855833386 * nr_disks
+                self.disk_properties["write_iops"] = 125214 * nr_disks
+                self.disk_properties["write_bandwidth"] = 796082133 * nr_disks
+            elif self.idata.instance_size() == "metal":
+                self.disk_properties["read_iops"] = 381441 * nr_disks
+                self.disk_properties["read_bandwidth"] = 1874585429 * nr_disks
+                self.disk_properties["write_iops"] = 108789 * nr_disks
+                self.disk_properties["write_bandwidth"] = 796443221 * nr_disks
+        elif self.idata.instancetype == "m5ad.large":
+            self.disk_properties["read_iops"] = 33306
+            self.disk_properties["read_bandwidth"] = 158338864
+            self.disk_properties["write_iops"] = 16817
+            self.disk_properties["write_bandwidth"] = 70194288
+        elif self.idata.instancetype == "m5ad.xlarge":
+            self.disk_properties["read_iops"] = 66127
+            self.disk_properties["read_bandwidth"] = 260377466
+            self.disk_properties["write_iops"] = 32893
+            self.disk_properties["write_bandwidth"] = 135897696
+        elif self.idata.instancetype == "m5ad.2xlarge":
+            self.disk_properties["read_iops"] = 129977
+            self.disk_properties["read_bandwidth"] = 621997248
+            self.disk_properties["write_iops"] = 63442
+            self.disk_properties["write_bandwidth"] = 267648736
+        elif self.idata.instancetype == "m5ad.4xlarge":
+            self.disk_properties["read_iops"] = 129937 * nr_disks
+            self.disk_properties["read_bandwidth"] = 620231082 * nr_disks
+            self.disk_properties["write_iops"] = 62666 * nr_disks
+            self.disk_properties["write_bandwidth"] = 267639125 * nr_disks
+        elif self.idata.instancetype == "m5ad.8xlarge":
+            self.disk_properties["read_iops"] = 257095 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1249927637 * nr_disks
+            self.disk_properties["write_iops"] = 114446 * nr_disks
+            self.disk_properties["write_bandwidth"] = 532821760 * nr_disks
+        elif self.idata.instancetype == "m5ad.12xlarge":
+            self.disk_properties["read_iops"] = 376431 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1865866709 * nr_disks
+            self.disk_properties["write_iops"] = 115985 * nr_disks
+            self.disk_properties["write_bandwidth"] = 796003477 * nr_disks
+        elif self.idata.instancetype == "m5ad.16xlarge":
+            self.disk_properties["read_iops"] = 256358 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1250889770 * nr_disks
+            self.disk_properties["write_iops"] = 114707 * nr_disks
+            self.disk_properties["write_bandwidth"] = 532998506 * nr_disks
+        elif self.idata.instancetype == "m5ad.24xlarge":
+            self.disk_properties["read_iops"] = 258951 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1865871317 * nr_disks
+            self.disk_properties["write_iops"] = 116030 * nr_disks
+            self.disk_properties["write_bandwidth"] = 796217706 * nr_disks
+        elif self.idata.instancetype == "c5d.large":
+            self.disk_properties["read_iops"] = 22095
+            self.disk_properties["read_bandwidth"] = 104797834
+            self.disk_properties["write_iops"] = 10125
+            self.disk_properties["write_bandwidth"] = 41982906
+        elif self.idata.instancetype == "c5d.xlarge":
+            self.disk_properties["read_iops"] = 44355
+            self.disk_properties["read_bandwidth"] = 212593018
+            self.disk_properties["write_iops"] = 20025
+            self.disk_properties["write_bandwidth"] = 84213472
+        elif self.idata.instancetype == "c5d.2xlarge":
+            self.disk_properties["read_iops"] = 89036
+            self.disk_properties["read_bandwidth"] = 426821429
+            self.disk_properties["write_iops"] = 41697
+            self.disk_properties["write_bandwidth"] = 173730709
+        elif self.idata.instancetype == "c5d.4xlarge":
+            self.disk_properties["read_iops"] = 193970
+            self.disk_properties["read_bandwidth"] = 928278314
+            self.disk_properties["write_iops"] = 83058
+            self.disk_properties["write_bandwidth"] = 351839733
+        elif self.idata.instancetype == "c5d.9xlarge":
+            self.disk_properties["read_iops"] = 381800
+            self.disk_properties["read_bandwidth"] = 1865831893
+            self.disk_properties["write_iops"] = 112264
+            self.disk_properties["write_bandwidth"] = 795731264
+        elif self.idata.instancetype == "c5d.12xlarge":
+            self.disk_properties["read_iops"] = 381775 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1866481792 * nr_disks
+            self.disk_properties["write_iops"] = 114302 * nr_disks
+            self.disk_properties["write_bandwidth"] = 795607616 * nr_disks
+        elif self.idata.instancetype == "c5d.18xlarge":
+            self.disk_properties["read_iops"] = 381270 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1856972330 * nr_disks
+            self.disk_properties["write_iops"] = 125638 * nr_disks
+            self.disk_properties["write_bandwidth"] = 795813866 * nr_disks
+        elif self.idata.instancetype == "c5d.24xlarge":
+            self.disk_properties["read_iops"] = 381355 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1876056704 * nr_disks
+            self.disk_properties["write_iops"] = 104946 * nr_disks
+            self.disk_properties["write_bandwidth"] = 795901013 * nr_disks
+        elif self.idata.instancetype == "c5d.metal":
+            self.disk_properties["read_iops"] = 381330 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1865216426 * nr_disks
+            self.disk_properties["write_iops"] = 115484 * nr_disks
+            self.disk_properties["write_bandwidth"] = 796109546 * nr_disks
+        elif self.idata.instancetype == "z1d.large":
+            self.disk_properties["read_iops"] = 33286
+            self.disk_properties["read_bandwidth"] = 158206858
+            self.disk_properties["write_iops"] = 16956
+            self.disk_properties["write_bandwidth"] = 70226280
+        elif self.idata.instancetype == "z1d.xlarge":
+            self.disk_properties["read_iops"] = 66076
+            self.disk_properties["read_bandwidth"] = 260565488
+            self.disk_properties["write_iops"] = 32769
+            self.disk_properties["write_bandwidth"] = 135891989
+        elif self.idata.instancetype == "z1d.2xlarge":
+            self.disk_properties["read_iops"] = 130235
+            self.disk_properties["read_bandwidth"] = 622297194
+            self.disk_properties["write_iops"] = 63891
+            self.disk_properties["write_bandwidth"] = 267679509
+        elif self.idata.instancetype == "z1d.3xlarge":
+            self.disk_properties["read_iops"] = 193840
+            self.disk_properties["read_bandwidth"] = 927493696
+            self.disk_properties["write_iops"] = 82864
+            self.disk_properties["write_bandwidth"] = 351608480
+        elif self.idata.instancetype == "z1d.6xlarge":
+            self.disk_properties["read_iops"] = 381902
+            self.disk_properties["read_bandwidth"] = 1865543381
+            self.disk_properties["write_iops"] = 117874
+            self.disk_properties["write_bandwidth"] = 795786901
+        elif self.idata.instancetype == "z1d.12xlarge":
+            self.disk_properties["read_iops"] = 381648 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1865706538 * nr_disks
+            self.disk_properties["write_iops"] = 115834 * nr_disks
+            self.disk_properties["write_bandwidth"] = 795876778 * nr_disks
+        elif self.idata.instancetype == "z1d.metal":
+            self.disk_properties["read_iops"] = 381378 * nr_disks
+            self.disk_properties["read_bandwidth"] = 1857873109 * nr_disks
+            self.disk_properties["write_iops"] = 122453 * nr_disks
+            self.disk_properties["write_bandwidth"] = 795593024 * nr_disks
         elif self.idata.instance_class() in ("c6gd", "m6gd", "r6gd", "x2gd"):
             if self.idata.instance_size() == "medium":
                 self.disk_properties["read_iops"] = 14808
@@ -127,6 +293,36 @@ class aws_io_setup(cloud_io_setup):
                 self.disk_properties["read_bandwidth"] = 2527296683 * nr_disks
                 self.disk_properties["write_iops"] = 156326 * nr_disks
                 self.disk_properties["write_bandwidth"] = 1063657088 * nr_disks
+        elif self.idata.instancetype == "i4g.large":
+            self.disk_properties["read_iops"] = 34035
+            self.disk_properties["read_bandwidth"] = 288471904
+            self.disk_properties["write_iops"] = 27943
+            self.disk_properties["write_bandwidth"] = 126763269
+        elif self.idata.instancetype == "i4g.xlarge":
+            self.disk_properties["read_iops"] = 68111
+            self.disk_properties["read_bandwidth"] = 571766890
+            self.disk_properties["write_iops"] = 47622
+            self.disk_properties["write_bandwidth"] = 254230192
+        elif self.idata.instancetype == "i4g.2xlarge":
+            self.disk_properties["read_iops"] = 136352
+            self.disk_properties["read_bandwidth"] = 1148509696
+            self.disk_properties["write_iops"] = 82746
+            self.disk_properties["write_bandwidth"] = 508828810
+        elif self.idata.instancetype == "i4g.4xlarge":
+            self.disk_properties["read_iops"] = 272704
+            self.disk_properties["read_bandwidth"] = 2297019392
+            self.disk_properties["write_iops"] = 165492
+            self.disk_properties["write_bandwidth"] = 1017657620
+        elif self.idata.instancetype == "i4g.8xlarge":
+            self.disk_properties["read_iops"] = 271495 * nr_disks
+            self.disk_properties["read_bandwidth"] = 2293024938 * nr_disks
+            self.disk_properties["write_iops"] = 93653 * nr_disks
+            self.disk_properties["write_bandwidth"] = 1031956586 * nr_disks
+        elif self.idata.instancetype == "i4g.16xlarge":
+            self.disk_properties["read_iops"] = 250489 * nr_disks
+            self.disk_properties["read_bandwidth"] = 2286635861 * nr_disks
+            self.disk_properties["write_iops"] = 93737 * nr_disks
+            self.disk_properties["write_bandwidth"] = 1034256042 * nr_disks
         elif self.idata.instancetype == "im4gn.large":
             self.disk_properties["read_iops"] = 33943
             self.disk_properties["read_bandwidth"] = 288433525

diff --git a/lib/scylla_cloud.py b/lib/scylla_cloud.py
@@ -745,7 +745,7 @@ def instance_class(self):
         return self._type.split(".")[0]
 
     def is_supported_instance_class(self):
-        if self.instance_class() in ['i2', 'i3', 'i3en', 'c5d', 'm5d', 'm5ad', 'r5d', 'z1d', 'c6gd', 'm6gd', 'r6gd', 'x2gd', 'im4gn', 'is4gen', 'i4i']:
+        if self.instance_class() in ['i2', 'i3', 'i3en', 'c5d', 'm5d', 'm5ad', 'r5d', 'z1d', 'c6gd', 'm6gd', 'r6gd', 'x2gd', 'im4gn', 'is4gen', 'i4i', 'i4g']:
             return True
         return False
 
@@ -754,7 +754,7 @@ def get_en_interface_type(self):
         instance_size = self.instance_size()
         if instance_class in ['c3', 'c4', 'd2', 'i2', 'r3']:
             return 'ixgbevf'
-        if instance_class in ['a1', 'c5', 'c5a', 'c5d', 'c5n', 'c6g', 'c6gd', 'f1', 'g3', 'g4', 'h1', 'i3', 'i3en', 'inf1', 'm5', 'm5a', 'm5ad', 'm5d', 'm5dn', 'm5n', 'm6g', 'm6gd', 'p2', 'p3', 'r4', 'r5', 'r5a', 'r5ad', 'r5b', 'r5d', 'r5dn', 'r5n', 't3', 't3a', 'u-6tb1', 'u-9tb1', 'u-12tb1', 'u-18tn1', 'u-24tb1', 'x1', 'x1e', 'z1d', 'c6g', 'c6gd', 'm6g', 'm6gd', 't4g', 'r6g', 'r6gd', 'x2gd', 'im4gn', 'is4gen', 'i4i']:
+        if instance_class in ['a1', 'c5', 'c5a', 'c5d', 'c5n', 'c6g', 'c6gd', 'f1', 'g3', 'g4', 'h1', 'i3', 'i3en', 'inf1', 'm5', 'm5a', 'm5ad', 'm5d', 'm5dn', 'm5n', 'm6g', 'm6gd', 'p2', 'p3', 'r4', 'r5', 'r5a', 'r5ad', 'r5b', 'r5d', 'r5dn', 'r5n', 't3', 't3a', 'u-6tb1', 'u-9tb1', 'u-12tb1', 'u-18tn1', 'u-24tb1', 'x1', 'x1e', 'z1d', 'c6g', 'c6gd', 'm6g', 'm6gd', 't4g', 'r6g', 'r6gd', 'x2gd', 'im4gn', 'is4gen', 'i4i', 'i4g']:
             return 'ena'
         if instance_class == 'm4':
             if instance_size == '16xlarge':

diff --git a/packer/ami_variables.json b/packer/ami_variables.json
@@ -1,5 +1,5 @@
 {
-	"security_group_id": "sg-c5e1f7a0",
+	"security_group_id": "sg-088128b2712c264d1",
 	"region": "us-east-1",
 	"associate_public_ip_address": "true",
 	"instance_type": "c4.xlarge"

diff --git a/packer/build_image.sh b/packer/build_image.sh
@@ -29,6 +29,7 @@ print_usage() {
     echo "  [--scylla-build-sha-id] Scylla build SHA id form metadata file"
     echo "  [--branch]              Set the release branch for GCE label. Default: master"
     echo "  [--ami-regions]         Set regions to copy the AMI when done building it (including permissions and tags)"
+    echo "  [--ami-users]           A list of account IDs that have access to launch the AMI"
     echo "  [--build-tag]           Jenkins Build tag"
     echo "  --download-no-server    Download all deb needed excluding scylla from repo-for-install"
     echo "  [--build-mode]          Choose which build mode to use for Scylla installation. Default: release. Valid options: release|debug"
@@ -98,6 +99,11 @@ while [ $# -gt 0 ]; do
             echo "--ami-regions prameter: AMI_REGIONS |$AMI_REGIONS|"
             shift 2
             ;;
+        "--ami-users"):
+            AMI_USERS=$2
+            echo "--ami-users parameter: AMI_USERS |$AMI_USERS|"
+            shift 2
+            ;;
         "--log-file")
             PACKER_LOG_PATH=$2
             echo "--log-file parameter: PACKER_LOG_PATH |$PACKER_LOG_PATH|"
@@ -345,6 +351,7 @@ set -x
   -var operating_system="$OPERATING_SYSTEM" \
   -var branch="$BRANCH" \
   -var ami_regions="$AMI_REGIONS" \
+  -var ami_users="$AMI_USERS" \
   -var arch="$ARCH" \
   -var product="$PRODUCT" \
   -var build_mode="$BUILD_MODE" \

diff --git a/packer/scylla.json b/packer/scylla.json
@@ -89,6 +89,7 @@
           "build_mode": "{{user `build_mode`| clean_resource_name}}"
       },
       "ami_regions": "{{user `ami_regions`}}",
+      "ami_users": "{{user `ami_users`}}",
       "aws_polling": {
         "delay_seconds": "30",
         "max_attempts": "100"
@@ -176,14 +177,14 @@
     {
       "type": "shell",
       "inline": [
-        "if [ {{build_name}} = aws -o {{build_name}} = azure ]; then sudo apt-get update; sudo DEBIAN_FRONTEND=noninteractive apt purge -y linux-{{build_name}}* linux-headers-{{build_name}}* linux-image*{{build_name}}* linux-modules*-{{build_name}}*; sudo DEBIAN_FRONTEND=noninteractive apt-get install -y linux-{{build_name}}-lts-22.04; sudo reboot ; fi"
+        "if [ {{build_name}} = aws -o {{build_name}} = azure ]; then sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt purge -y linux-{{build_name}}* linux-headers-{{build_name}}* linux-image*{{build_name}}* linux-modules*-{{build_name}}* && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y linux-{{build_name}}-lts-22.04 && sudo reboot; fi"
       ],
       "expect_disconnect": true
     },
     {
       "type": "shell",
       "inline": [
-        "if [ {{build_name}} = gce ]; then sudo apt-get update; sudo DEBIAN_FRONTEND=noninteractive apt purge -y linux-gcp* linux-headers-gcp* linux-image*gcp* linux-modules*-gcp*; sudo DEBIAN_FRONTEND=noninteractive apt-get install -y linux-gcp-lts-22.04; sudo reboot ; fi"
+        "if [ {{build_name}} = gce ]; then sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt purge -y linux-gcp* linux-headers-gcp* linux-image*gcp* linux-modules*-gcp* && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y linux-gcp-lts-22.04 && sudo reboot; fi"
       ],
       "expect_disconnect": true
     },

diff --git a/packer/scylla_install_image b/packer/scylla_install_image
@@ -125,6 +125,8 @@ if __name__ == '__main__':
         kernel_opt = ''
         grub_variable = 'GRUB_CMDLINE_LINUX_DEFAULT'
     elif args.target_cloud == 'gce':
+        # align with other clouds image
+        run('apt-get purge -y rsyslog', shell=True, check=True)
         sysconfig_opt = '--disable-writeback-cache'
         kernel_opt = ''
         grub_variable = 'GRUB_CMDLINE_LINUX_DEFAULT'
@@ -170,6 +172,7 @@ WantedBy=multi-user.target
 
     os.remove('{}/.ssh/authorized_keys'.format(homedir))
     os.remove('/var/lib/scylla-housekeeping/housekeeping.uuid')
+    os.remove('/var/cache/debconf/config.dat')
 
     with open('/etc/default/grub.d/50-cloudimg-settings.cfg') as f:
         grub = f.read()