NBS-5217: Allow images from different folders in tests (#2413)

* NBS-5217: Allow images from different folders in tests

cloud/blockstore/pylibs/clusters/test_config/test_config.py

@@ -15,6 +15,7 @@ class FolderDesc:
     subnet_name: str
     subnet_id: str
     image_name: str
+    image_folder_id: str
     filesystem_id: str
     platform_id: str = None
     service_account_id: str = None
@@ -60,7 +61,15 @@ def get_cluster_test_config(name: str, zone_id: str, config_path: str = None) ->
     res.__dict__.update(cfg)
     folders = dict()
     for name in cfg['folder_desc']:
-        folder = FolderDesc(None, None, None, None, None, None)
+        folder = FolderDesc(
+            folder_id=None,
+            zone_id=None,
+            subnet_name=None,
+            subnet_id=None,
+            image_name=None,
+            image_folder_id=None,
+            filesystem_id=None,
+        )
         folder.__dict__.update(cfg['folder_desc'][name])
         folders[name] = folder
     res.__dict__.update({

cloud/blockstore/pylibs/ycp/ycp.py

@@ -143,6 +143,7 @@ class CreateInstanceConfig:
         memory: int
         folder_id: str
         image_name: str
+        image_folder_id: str
         zone_id: str
         subnet_name: str
         subnet_id: str
@@ -167,6 +168,7 @@ class CreateDiskConfig:
         placement_group_partition_count: int
         zone_id: str
         image_name: str
+        image_folder_id: str
         snapshot_name: str
         description: str
         kek_id: str
@@ -315,9 +317,9 @@ def _resolve_entity_name(self, cmd, name, stderr, fail_if_not_found=True):
 
         return None
 
-    def _resolve_image_name(self, folder_id, image_name, stderr):
+    def _resolve_image_name(self, image_folder_id, image_name, stderr):
         return self._resolve_entity_name(
-            self._ycp.compute.image.list(folder_id=folder_id),
+            self._ycp.compute.image.list(folder_id=image_folder_id),
             image_name,
             stderr)
 
@@ -377,7 +379,7 @@ def create_instance(self, config: CreateInstanceConfig) -> Instance:
                     placement_group_id = entity['id']
 
             image_id = self._resolve_image_name(
-                config.folder_id,
+                config.image_folder_id,
                 config.image_name,
                 stderr)
 
@@ -440,7 +442,7 @@ def create_disk(self, config: CreateDiskConfig) -> Disk:
             snapshot_id = None
             if config.image_name is not None:
                 image_id = self._resolve_image_name(
-                    config.folder_id,
+                    config.image_folder_id,
                     config.image_name,
                     stderr)
             elif config.snapshot_name is not None:

cloud/blockstore/pylibs/ycp/ycp_wrapper.py

@@ -57,6 +57,7 @@ def create_instance(self,
                         placement_group_name: str = None,
                         host_group: str = None,
                         image_name: str = None,
+                        image_folder_id: str = None,
                         name: str = None,
                         platform_id: str = None,
                         auto_delete: bool = True,
@@ -69,6 +70,7 @@ def create_instance(self,
             cores=cores,
             memory=memory * 1024 ** 3,
             image_name=image_name or self._folder_desc.image_name,
+            image_folder_id=image_folder_id or self._folder_desc.image_folder_id or self._folder_desc.folder_id,
             zone_id=self._folder_desc.zone_id,
             subnet_name=self._folder_desc.subnet_name,
             subnet_id=self._folder_desc.subnet_id,
@@ -113,6 +115,7 @@ def create_disk(self,
                     name: str = None,
                     kek_id: str = None,
                     image_name: str = None,
+                    image_folder_id: str = None,
                     snapshot_name: str = None,
                     auto_delete: bool = True,
                     description: str = None) -> Ycp.Disk:
@@ -129,6 +132,7 @@ def create_disk(self,
             folder_id=self._folder_desc.folder_id,
             kek_id=kek_id,
             image_name=image_name,
+            image_folder_id=image_folder_id or self._folder_desc.image_folder_id or self._folder_desc.folder_id,
             snapshot_name=snapshot_name,
             description=description,
         )

cloud/blockstore/tools/ci/check_emptiness_test/tests/canondata/test.test_check_emptiness_test_cluster1_/testing_out_stuff_results.txt

@@ -5,7 +5,7 @@
 ##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute instance list --folder-id fake-folder' status='NORMAL' ]
 ##teamcity[message text='|[ #0/zone1 |] Creating instance' status='NORMAL' ]
 ##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder' status='NORMAL' ]
-##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder' status='NORMAL' ]
+##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1' status='NORMAL' ]
 ##teamcity[message text='|[ #0/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute instance create --request -' status='NORMAL' ]
 ##teamcity[message text='|[ #0/zone1 |] Created instance <id=fake-instance-id, host=fake, ip=1:1:1:1:1:1:1:1>' status='NORMAL' ]
 ##teamcity[message text='|[ #0/zone1 |] Creating disk' status='NORMAL' ]
@@ -30,7 +30,7 @@
 ##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute instance list --folder-id fake-folder' status='NORMAL' ]
 ##teamcity[message text='|[ #1/zone1 |] Creating instance' status='NORMAL' ]
 ##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder' status='NORMAL' ]
-##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder' status='NORMAL' ]
+##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1' status='NORMAL' ]
 ##teamcity[message text='|[ #1/zone1 |] sent ycp request: ycp --format json --profile fake-cluster1 compute instance create --request -' status='NORMAL' ]
 ##teamcity[message text='|[ #1/zone1 |] Created instance <id=fake-instance-id, host=fake, ip=1:1:1:1:1:1:1:1>' status='NORMAL' ]
 ##teamcity[message text='|[ #1/zone1 |] Creating disk' status='NORMAL' ]
@@ -59,7 +59,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=
@@ -82,7 +82,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/check_emptiness_test/tests/test-configs/cluster1.json

@@ -8,6 +8,7 @@
             "folder_id": "fake-folder",
             "hot_attach_fs": true,
             "image_name": "fake-name1",
+            "image_folder_id": "fake-images-folder1",
             "platform_id": null,
             "service_account_id": null,
             "subnet_id": null,
@@ -20,6 +21,7 @@
             "folder_id": "fake-folder",
             "hot_attach_fs": true,
             "image_name": "fake-name2",
+            "image_folder_id": "fake-images-folder2",
             "platform_id": null,
             "service_account_id": null,
             "subnet_id": null,
@@ -32,6 +34,7 @@
             "folder_id": "nbs.tests.folder",
             "hot_attach_fs": true,
             "image_name": "fake-name3",
+            "image_folder_id": "fake-images-folder3",
             "platform_id": null,
             "service_account_id": null,
             "subnet_id": null,

cloud/blockstore/tools/ci/corruption_test_suite/lib/main.py

@@ -191,6 +191,7 @@ def run_corruption_test(module_factories: common.ModuleFactories, args, logger):
             memory=_TEST_INSTANCE_MEMORY,
             compute_node=args.compute_node,
             image_name=image,
+            image_folder_id=folder.image_folder_id,
             description="Corruption test") as instance:
 
         server_version = module_factories.fetch_server_version(

cloud/blockstore/tools/ci/corruption_test_suite/tests/canondata/test.test_corruption_test_suite_nbs-cluster1-512bytes-bs_/512bytes-bs_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/corruption_test_suite/tests/canondata/test.test_corruption_test_suite_nbs-cluster1-64MB-bs_/64MB-bs_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/corruption_test_suite/tests/canondata/test.test_corruption_test_suite_nbs-cluster1-ranges-intersection_/ranges-intersection_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/corruption_test_suite/tests/canondata/test.test_corruption_test_suite_nfs-cluster1-512bytes-bs_/512bytes-bs_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/corruption_test_suite/tests/canondata/test.test_corruption_test_suite_nfs-cluster1-64MB-bs_/64MB-bs_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/corruption_test_suite/tests/canondata/test.test_corruption_test_suite_nfs-cluster1-ranges-intersection_/ranges-intersection_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/corruption_test_suite/tests/test-configs/cluster1.json

@@ -8,6 +8,7 @@
             "folder_id": "fake-folder",
             "hot_attach_fs": true,
             "image_name": "fake-name1",
+            "image_folder_id": "fake-images-folder1",
             "platform_id": null,
             "service_account_id": null,
             "subnet_id": null,
@@ -20,6 +21,7 @@
             "folder_id": "fake-folder",
             "hot_attach_fs": true,
             "image_name": "fake-name2",
+            "image_folder_id": "fake-images-folder2",
             "platform_id": null,
             "service_account_id": null,
             "subnet_id": null,
@@ -32,6 +34,7 @@
             "folder_id": "nbs.tests.folder",
             "hot_attach_fs": true,
             "image_name": "fake-name3",
+            "image_folder_id": "fake-images-folder3",
             "platform_id": null,
             "service_account_id": null,
             "subnet_id": null,

cloud/blockstore/tools/ci/fio_performance_test_suite/lib/main.py

@@ -99,6 +99,11 @@ def parse_args() -> argparse.Namespace:
         type=str,
         default=None,
         help='use image with the specified name to create disks')
+    test_arguments_group.add_argument(
+        '--image-folder-id',
+        type=str,
+        default=None,
+        help='use image from the specified folder to create disks')
     test_arguments_group.add_argument(
         '--instance-cores',
         type=int,
@@ -142,8 +147,8 @@ def __init__(
         self._logger = logger
         self._module_factories = module_factories
 
-        self._version_method_name = None
-        self._instance_image_name = None
+    def _version_method_name(self) -> str:
+        raise NotImplementedError()
 
     def _run_tests(
         self,
@@ -193,11 +198,12 @@ def run_test_suite(self) -> None:
                     platform_id=self._args.platform_id,
                     compute_node=self._args.compute_node,
                     placement_group_name=self._args.placement_group_name,
-                    image_name=self._instance_image_name,
+                    image_name=self._args.image_name,
+                    image_folder_id=self._args.image_folder_id,
                     description='fio performance test') as instance:
                 version = self._module_factories.fetch_server_version(
                     self._args.dry_run,
-                    self._version_method_name,
+                    self._version_method_name(),
                     instance.compute_node,
                     cluster,
                     self._logger)
@@ -335,7 +341,9 @@ def __init__(
         logger: logging.Logger,
     ) -> None:
         super().__init__(module_factories, args, profiler, logger)
-        self._version_method_name = 'get_current_nbs_version'
+
+    def _version_method_name(self) -> str:
+        return 'get_current_nbs_version'
 
     def _teardown_parallel_run(
         self,
@@ -425,6 +433,7 @@ def _run_sequential_tests(
                     type_id=translate_disk_type(self._args.cluster, test_case.type),
                     bs=test_case.device_bs,
                     image_name=self._args.image_name,
+                    image_folder_id=self._args.image_folder_id,
                     description=f'fio performance test: {test_case.name}') as disk:
                 try:
                     with ycp.attach_disk(
@@ -473,6 +482,7 @@ def _run_parallel_tests(
                     type_id=translate_disk_type(self._args.cluster, test_case.type),
                     bs=test_case.device_bs,
                     image_name=self._args.image_name,
+                    image_folder_id=self._args.image_folder_id,
                     auto_delete=False,
                     description=f'fio performance test: {test_case.name}') as disk:
                 with ycp.attach_disk(
@@ -557,8 +567,9 @@ def __init__(
         logger: logging.Logger,
     ) -> None:
         super().__init__(module_factories, args, profiler, logger)
-        self._version_method_name = 'get_current_nfs_version'
-        self._instance_image_name = 'ubuntu-2004-eternal'
+
+    def _version_method_name(self) -> str:
+        return 'get_current_nfs_version'
 
     def _mount_fs(
         self,

cloud/blockstore/tools/ci/fio_performance_test_suite/tests/canondata/test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-default_/default_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/fio_performance_test_suite/tests/canondata/test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-large_block_size_/large_block_size_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/fio_performance_test_suite/tests/canondata/test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-max_bandwidth_/max_bandwidth_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/fio_performance_test_suite/tests/canondata/test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-max_iops_/max_iops_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/fio_performance_test_suite/tests/canondata/test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-mirrored_iops_/mirrored_iops_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/fio_performance_test_suite/tests/canondata/test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-mirrored_throughput_/mirrored_throughput_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/fio_performance_test_suite/tests/canondata/test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-nrd_/nrd_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=

cloud/blockstore/tools/ci/fio_performance_test_suite/tests/canondata/test.test_fio_performance_test_suite_False-nbs-cluster_and_platform0-nrd_iops_/nrd_iops_results.txt

@@ -2,7 +2,7 @@ Command=ycp --format json --profile fake-cluster1 compute instance list --folder
 Input=
 Command=ycp --format json --profile fake-cluster1 vpc subnet list --folder-id fake-folder
 Input=
-Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-folder
+Command=ycp --format json --profile fake-cluster1 compute image list --folder-id fake-images-folder1
 Input=
 Command=ycp --format json --profile fake-cluster1 compute instance create --request -
 Input=