diff --git a/README.md b/README.md index e0af807..f8f024d 100644 --- a/README.md +++ b/README.md @@ -209,3 +209,9 @@ To install `cf-remote` so that it reflects any changes in this source directory ``` $ pip install --editable . ``` + +## cloud_data.py tips + +In order to find AWS images for a particular owner to work on cloud_data.py name_pattern list the names for an owner with the following `aws` command: + +aws ec2 describe-images --region us-east-2 --owners 801119661308 --query 'Images[*].[Name]' --output text diff --git a/cf_remote/cloud_data.py b/cf_remote/cloud_data.py index ce32185..9ce8b62 100644 --- a/cf_remote/cloud_data.py +++ b/cf_remote/cloud_data.py @@ -1,179 +1,73 @@ -aws_platforms = { - "ubuntu-22-04-arm64": { - "ami": "ami-00c50882a52d323a6", - "user": "ubuntu", - "size": "t4g.micro", - "xlsize": "t4g.xlarge", - }, - "ubuntu-22-04-x64": { - "ami": "ami-01dd271720c1ba44f", - "user": "ubuntu", - "size": "t2.small", - "xlsize": "t3.xlarge", - }, - "ubuntu-20-04-x64": { - "ami": "ami-0aef57767f5404a3c", - "user": "ubuntu", - "size": "t2.small", - "xlsize": "t3.xlarge", - }, - "ubuntu-18-04-x64": { - "ami": "ami-0ee3436f275c4f2e8", - "user": "ubuntu", - "size": "m1.small", - "xlsize": "m3.xlarge", - }, - "ubuntu-14-04-x32": { - "ami": "ami-07a1e6256cb43b99c", - "user": "ubuntu", - "size": "m1.small", - }, - "debian-8-x64": { - "ami": "ami-402f1a33", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-7-x64": { - "ami": "ami-61e56916", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-9-x64": { - "ami": "ami-035c67e6a9ef8f024", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-10-x64": { - "ami": "ami-0a9d04ba7d4df6c3b", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-11-arm64": { - "ami": "ami-0353cb95279bf4f20", - "user": "admin", - "size": "t4g.micro", - "xlsize": "t4g.xlarge", - }, - "debian-11-x64": { - "ami": "ami-0293236c9a0c23a77", - "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "debian-12-arm64": { - "ami": "ami-03820227fb3e4ffad", +aws_defaults = { + "architecture": "x86_64", + "sizes": { + "x86_64": { + "size": "t2.micro", + "xlsize": "t2.xlarge", + }, + "arm64": { + "size": "t4g.micro", + "xlsize": "t4g.xlarge", + }, + }, + "user": "ec2-user", +} +aws_image_criteria = { + "debian-9": { + "owner_id": "379101102735", + "name_pattern": "debian-stretch-hvm-x86_64*", "user": "admin", - "size": "t4g.micro", - "xlsize": "t4g.xlarge", }, - "debian-12-x64": { - "ami": "ami-07024fbdfd1aab8a0", + "debian": { + "owner_id": "136693071363", + "name_pattern": "debian-{version}*", "user": "admin", - "size": "t1.micro", - "xlsize": "m3.xlarge", - }, - "centos-6-x64": { - "ami": "ami-05bd23226cb7c2896", - "user": "centos", - "size": "t2.micro", - "xlsize": "m3.xlarge", - }, - "centos-7-x64": { - "ami": "ami-0f4775c518fa29365", - "user": "centos", - "size": "t2.micro", - "xlsize": "m3.xlarge", - }, - "rhel-5-x64": { - "ami": "ami-ea94369d", - "size": "t1.micro", - "user": "root", - "xlsize": "t1.micro", - }, - "rhel-6-x64": { - "ami": "ami-c1bb06b2", - "size": "t2.micro", - "user": "ec2-user", - "xlsize": "t2.large", }, - "rhel-7-x64": { - "ami": "ami-065ec1e661d619058", - "size": "t2.micro", - "user": "ec2-user", - "xlsize": "t2.large", - }, - "rhel-8-x64": { - "ami": "ami-08f4717d06813bf00", - "size": "t3a.micro", - "user": "ec2-user", - "xlsize": "m3.xlarge", - }, - "rhel-9-x64": { - "ami": "ami-049b0abf844cab8d7", - "size": "t3a.micro", - "user": "ec2-user", - "xlsize": "m3.xlarge" - }, - "centos-5-x32": {"ami": "ami-fe11398a", "user": "root", "size": "m1.small"}, - "debian-6-x64": {"ami": "ami-879e4ff0", "user": "admin", "size": "t1.micro"}, - "debian-5-x32": {"ami": "ami-8398b3f7", "user": "root", "size": "m1.small"}, - "debian-7-x32": {"ami": "ami-1be06c6c", "user": "admin", "size": "t1.micro"}, - "debian-4-x32": {"ami": "ami-8198b3f5", "user": "root", "size": "m1.small"}, - "ubuntu-12-04-x64": { - "ami": "ami-d1767bb7", + "ubuntu-16": { + "owner_id": "099720109477", + "name_pattern": "ubuntu-pro-server/images/hvm-ssd/ubuntu-xenial-16.04-amd64-pro-server*", "user": "ubuntu", - "size": "m1.small", - "xlsize": "m3.xlarge", }, - "debian-6-x32": {"ami": "ami-8d9e4ffa", "user": "admin", "size": "t1.micro"}, - "ubuntu-16-04-x64": { - "ami": "ami-0d47c52ffe8fef155", + "ubuntu": { + "owner_id": "099720109477", + "name_pattern": "ubuntu/images/hvm-ssd/ubuntu-*-{version}*", "user": "ubuntu", - "size": "m1.small", - "xlsize": "m3.xlarge", }, - "debian-5-x64": {"ami": "ami-8f98b3fb", "user": "root", "size": "m1.small"}, - "debian-4-x64": {"ami": "ami-8d98b3f9", "user": "root", "size": "m1.small"}, - "ubuntu-14-04-x64": { - "ami": "ami-0c68b4b8bbbdc39de", - "user": "ubuntu", - "size": "m1.small", - "xlsize": "m3.xlarge", + "centos": { + "note": "This owner is our nt-dev account in AWS so these are private custom images.", + "owner_id": "304194462000", + "name_pattern": "centos-{version}-x64", + "region": "eu-west-1", + }, + "rhel": { + "owner_id": "309956199498", + "name_pattern": "RHEL-{version}*", }, - "ubuntu-12-04-x32": {"ami": "ami-5c78753a", "user": "ubuntu", "size": "m1.small"}, - "centos-5-x64": {"ami": "ami-f2113986", "user": "root", "size": "m1.small"}, - "windows-2012-x64": { - "ami": "ami-045768fc2ae3fa829", + "windows-2008": { + "ami": "ami-09046e654c804633f", "user": "Administrator", - "size": "m1.small", - "xlsize": "m3.xlarge", + "region": "eu-west-1", }, - "windows-2016-x64": { - "ami": "ami-08f68fefe026532ea", + "windows-2012": { + "ami": "ami-0444b0c023c7f3671", "user": "Administrator", - "size": "m1.small", - "xlsize": "m3.xlarge", + "region": "eu-west-1", }, - "windows-2019-x64": { - "ami": "ami-0311c2819c6a29312", + "windows-2016": { + "ami": "ami-00a7e5468b339302c", "user": "Administrator", - "size": "t2.small", - "xlsize": "t2.xlarge", + "region": "eu-west-1", }, - "suse-12-x64": { - "ami": "ami-0d5622d69a166848b", - "user": "ec2-user", - "size": "t2.small", - "xlsize": "t2.xlarge", + "windows-2019": { + "ami": "ami-0311c2819c6a29312", + "user": "Administrator", + "region": "eu-west-1", }, - "suse-15-x64": { - "ami": "ami-0e5e442298b8e7f5a", - "user": "ec2-user", - "size": "t2.small", - "xlsize": "t2.xlarge", + "windows": { + "note": "Note that typically we rely on custom pre-configured windows imimages with ssh installed and pre-populated public keys so an image spawned from this criteria will not come with ssh built-in and ready to go.", + "owner_id": "801119661308", + "name_pattern": "Windows_Server-{version}-English-Core-Base*", + "user": "Administrator", }, + "suse": {"owner_id": "013907871322", "name_pattern": "suse-sles-{version}*"}, } diff --git a/cf_remote/commands.py b/cf_remote/commands.py index 4245f2f..42e542d 100644 --- a/cf_remote/commands.py +++ b/cf_remote/commands.py @@ -556,8 +556,14 @@ def destroy(group_name=None): def list_platforms(): + print() + print("Platform images are queried based on the platform name, version and architecture.") + print("The form of platform specified is: [-][-]. e.g. debian, debian-12 or debian-12-x64") + print("Ubuntu version can be just major (20) or major+minor (20-04)") + print("Architecture can either be x64 or arm64") + print() print("Available platforms:") - for key in sorted(cloud_data.aws_platforms.keys()): + for key in sorted(cloud_data.aws_image_criteria.keys()): print(key) return 0 diff --git a/cf_remote/spawn.py b/cf_remote/spawn.py index f3279ab..e24e223 100644 --- a/cf_remote/spawn.py +++ b/cf_remote/spawn.py @@ -9,7 +9,7 @@ from libcloud.compute.providers import get_driver from libcloud.compute.base import NodeSize, NodeImage -from cf_remote.cloud_data import aws_platforms +from cf_remote.cloud_data import aws_image_criteria, aws_defaults from cf_remote.utils import whoami from cf_remote import log from cf_remote import cloud_data @@ -298,6 +298,69 @@ def get_cloud_driver(provider, creds, region): return driver +# the string platform_name can be platform, platform-version(partial even), or platform-version-architecture +# The data in cloud_data.py aws_image_criteria can have general information for just +# `platform` or include all the components if necessary. +# +# Generally up-to-date versions should use a generic criteria which pulls the most up to date +# image for that platform and version. +def _get_image_criteria(platform_name): + log.debug("Looking for AWS AMI for platform_name '%s'" % (platform_name)) + platform = platform_name.split("-")[0] + if platform == "ubuntu": + if platform_name.count("-") > 0: + platform_version = ".".join(platform_name.split("-")[1:-1]) + else: + platform_version = "" + else: + platform_version = ( + platform_name.count("-") > 0 and platform_name.split("-")[1] or "*" + ) + log.debug( + "Parsed platform_version '%s' from platform_name '%s'" + % (platform_version, platform_name) + ) + platform_with_major_version = "-".join(platform_name.split("-")[0:2]) + architecture = platform_name.split("-")[-1] + # architecture should be either x64 or arm64 + if not (architecture == "x64" or architecture == "arm64"): + # default to x64 + architecture = "x64" + # translate cf-remote x64 to amazon x86_64 + if architecture == "x64": + architecture = "x86_64" + log.debug("Determined architecture to be '%s'" % (architecture)) + + # Assign a value to criteria variable based on the given conditions + if platform_with_major_version in aws_image_criteria: + criteria = aws_image_criteria[platform_with_major_version] + else: + criteria = aws_image_criteria[platform] + + criteria["architecture"] = architecture + criteria["version"] = platform_version + log.debug("Determined image criteria: %s" % (criteria)) + return criteria + + +def _get_ami(criteria, driver): + candidates = driver.list_images( + ex_owner=criteria["owner_id"], + ex_filters={ + "name": criteria["name_pattern"].format(version=criteria["version"]), + "architecture": criteria["architecture"], + "virtualization-type": "hvm", + }, + ) + if len(candidates) == 0: + raise ValueError("No images found for criteria: %s" % (criteria)) + selected = sorted(candidates, key=lambda x: x.extra["creation_date"], reverse=True)[ + 0 + ] + log.debug("Selected image %s" % (selected)) + return selected.id + + def spawn_vm_in_aws( platform, aws_creds, @@ -308,15 +371,19 @@ def spawn_vm_in_aws( size=None, role=None, ): - if platform not in aws_platforms: - raise ValueError("Platform '%s' does not exist. (Available platforms: %s)" % (platform, - ", ".join(cloud_data.aws_platforms.keys()))) + platform_name = platform.split("-")[0] + if platform_name not in aws_image_criteria: + raise ValueError( + "Platform '%s' is not in our set of image criteria. (Available platforms: %s)" + % (platform, ", ".join(cloud_data.aws_image_criteria.keys())) + ) try: driver = get_cloud_driver(Providers.AWS, aws_creds, region) existing_vms = driver.list_nodes() except InvalidCredsError as error: raise ValueError( - "Invalid credentials, check cloud_config.json (%s.)" % str(error)[1:-1]) + "Invalid credentials, check cloud_config.json (%s.)" % str(error)[1:-1] + ) if name is None: name = _get_unused_name( [vm.name for vm in existing_vms], platform, _NAME_RANDOM_PART_LENGTH @@ -324,31 +391,50 @@ def spawn_vm_in_aws( else: if any(vm.state in (0, "running") and vm.name == name for vm in existing_vms): raise ValueError("VM with the name '%s' already exists" % name) - aws_platform = aws_platforms[platform] - size = size or aws_platform.get("xlsize") or aws_platform["size"] - user = aws_platform.get("user") - ami = aws_platform["ami"] - - log.info("Spawning new '%s' VM in AWS (AMI: %s, size=%s)" % (platform, ami, size)) - node = driver.create_node( - name=name, - image=NodeImage(id=ami, name=None, driver=driver), - size=NodeSize( - id=size, - name=None, - ram=None, - disk=None, - bandwidth=None, - price=None, - driver=driver, - ), - ex_keyname=key_pair, - ex_security_groups=security_groups, - ex_metadata={ - "created-by": "cf-remote", - "owner": whoami(), - }, + criteria = _get_image_criteria(platform) + architecture = criteria["architecture"] or aws_defaults["architecture"] + sizes = criteria.get("sizes") or aws_defaults["sizes"] + small = sizes[architecture]["size"] + large = sizes[architecture]["xlsize"] + if size == None: + size = (large or small) if (role == "hub") else (small or large) + user = criteria.get("user") or aws_defaults["user"] + ami = criteria.get("ami") or _get_ami(criteria, driver) + if "region" in criteria and region != criteria["region"]: + raise ValueError( + "AMI for platform '%s'(%s) is only available in region '%s' and not in your configured region of '%s'." + % (platform, ami, criteria["region"], region) + ) + + print( + "Spawning new platform '%s' VM in AWS (AMI: %s, size=%s) %s" + % (platform, ami, size, criteria.get("note", "")) ) + try: + node = driver.create_node( + name=name, + image=NodeImage(id=ami, name=None, driver=driver), + size=NodeSize( + id=size, + name=None, + ram=None, + disk=None, + bandwidth=None, + price=None, + driver=driver, + ), + ex_keyname=key_pair, + ex_security_groups=security_groups, + ex_metadata={ + "created-by": "cf-remote", + "owner": whoami(), + }, + ) + except Exception as e: + raise ValueError( + "Problem spawning '%s' VM in AWS (AMI: %s, size=%s). Error: %s" + % (platform, ami, size, e) + ) return VM( name, diff --git a/tests/aws-spawn-test.sh b/tests/aws-spawn-test.sh new file mode 100644 index 0000000..a007b4f --- /dev/null +++ b/tests/aws-spawn-test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -ex + +function cleanup() { + cf-remote destroy --all +} + +trap cleanup ERR +trap cleanup EXIT + +# this is a fairly exhaustive test and will take some time +# spawn all reasonable "platform" specifications +function test() { + platform=$1 + version=$2 + + for role in client hub; do + cf-remote spawn --count 1 --platform "$platform-$version" --role "$role" --name "$platform-$version-$role" + cleanup + done +} + +function fail() { + echo "FAIL: $@" + exit 1 +} + +# start with cleanup +cleanup + +# test some negative cases +set +e +cf-remote spawn --count 1 --platform ubuntu --role client --name test && fail "ubuntu platform requires a version" +cleanup + +set -e + +# test some basic day to day cases +# for testing, include ubuntu and centos which require versions +for platform in debian-12-x64 debian-12-arm64; do + cf-remote spawn --count 1 --platform $platform --role client --name $platform + cleanup +done +for platform in debian rhel windows debian-9 ubuntu-22 centos-7 rhel-9 windows-2019; do + cf-remote spawn --count 1 --platform $platform --role client --name $platform + cleanup +done +for version in 9 10 11 12; do + test debian $version +done +for version in 7 8; do + test centos $version +done +for version in 7 8 9; do + test rhel $version +done +for version in 2008 2012 2016 2019 2022; do + test windows $version +done +for version in 16-04 18-04 20-04 22-04; do + test ubuntu "$version" +done diff --git a/tests/test_spawn.py b/tests/test_spawn.py new file mode 100644 index 0000000..c8a08f4 --- /dev/null +++ b/tests/test_spawn.py @@ -0,0 +1,71 @@ +from cf_remote.spawn import _get_image_criteria + +def test_get_image_criteria(): + criteria = _get_image_criteria("ubuntu-22-04-x86") + assert criteria["version"] == "22.04" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("ubuntu-22-04") + """ It says version is "22", not "22.04" """ + # assert criteria["version"] == "22.04" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("ubuntu") + assert criteria["version"] == "" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("ubuntu-22-04-arm64") + assert criteria["version"] == "22.04" + assert criteria["architecture"] == "arm64" + + criteria = _get_image_criteria("rhel-9-x64") + assert criteria["version"] == "9" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("rhel-9") + assert criteria["version"] == "9" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("rhel") + assert criteria["version"] == "*" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("debian-12-x64") + assert criteria["version"] == "12" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("debian-12") + assert criteria["version"] == "12" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("debian") + assert criteria["version"] == "*" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("debian-11-arm64") + assert criteria["version"] == "11" + assert criteria["architecture"] == "arm64" + + criteria = _get_image_criteria("centos-7-x64") + assert criteria["version"] == "7" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("centos-7") + assert criteria["version"] == "7" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("centos") + assert criteria["version"] == "*" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("windows-2019-x64") + assert criteria["version"] == "2019" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("windows-2019") + assert criteria["version"] == "2019" + assert criteria["architecture"] == "x86_64" + + criteria = _get_image_criteria("windows") + assert criteria["version"] == "*" + assert criteria["architecture"] == "x86_64"