From bdcad27a7cab6e24867f6bdbeb5467c863f0fb73 Mon Sep 17 00:00:00 2001 From: Yaron Kaikov Date: Tue, 30 Jan 2024 20:09:39 +0200 Subject: [PATCH] images: clean old and un-used code since they not in use anymore removing them --- aws/ami/.dockerignore | 1 - aws/ami/README.md | 84 -------------- aws/ami/build_deb_ami.sh | 1 - aws/ami/build_deb_with_docker.sh | 1 - aws/ami/files/.gitkeep | 0 aws/ami/user_data.txt | 2 - aws/ami/variables.json.example | 9 -- aws/cloudformation/Jenkinsfile | 77 ------------- aws/cloudformation/README.md | 36 ------ aws/cloudformation/conftest.py | 5 - aws/cloudformation/pytest.ini | 3 - aws/cloudformation/requirements.txt | 5 - aws/cloudformation/test_scylla_cfn.py | 145 ------------------------- azure/README | 1 - azure/image/README.md | 65 ----------- azure/image/build_azure_image.sh | 1 - azure/image/build_azure_with_docker.sh | 1 - azure/image/files/.gitkeep | 0 azure/image/variables.json | 9 -- gce/README | 29 ----- gce/image/build_deb_image.sh | 1 - gce/image/files/.gitkeep | 0 gce/image/variables.json.example | 7 -- 23 files changed, 483 deletions(-) delete mode 100644 aws/ami/.dockerignore delete mode 100644 aws/ami/README.md delete mode 120000 aws/ami/build_deb_ami.sh delete mode 120000 aws/ami/build_deb_with_docker.sh delete mode 100644 aws/ami/files/.gitkeep delete mode 100644 aws/ami/user_data.txt delete mode 100644 aws/ami/variables.json.example delete mode 100644 aws/cloudformation/Jenkinsfile delete mode 100644 aws/cloudformation/README.md delete mode 100644 aws/cloudformation/conftest.py delete mode 100644 aws/cloudformation/pytest.ini delete mode 100644 aws/cloudformation/requirements.txt delete mode 100644 aws/cloudformation/test_scylla_cfn.py delete mode 100644 azure/README delete mode 100644 azure/image/README.md delete mode 120000 azure/image/build_azure_image.sh delete mode 120000 azure/image/build_azure_with_docker.sh delete mode 100644 azure/image/files/.gitkeep delete mode 100644 azure/image/variables.json delete mode 100644 gce/README delete mode 120000 gce/image/build_deb_image.sh delete mode 100644 gce/image/files/.gitkeep delete mode 100644 gce/image/variables.json.example diff --git a/aws/ami/.dockerignore b/aws/ami/.dockerignore deleted file mode 100644 index 678ea2b8..00000000 --- a/aws/ami/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -**/*.rpm diff --git a/aws/ami/README.md b/aws/ami/README.md deleted file mode 100644 index 7d2ab57e..00000000 --- a/aws/ami/README.md +++ /dev/null @@ -1,84 +0,0 @@ -Edit variables.json and run aws/ami/build_ami.sh, it will build & upload AMI. - -When deploying the AMI, be sure to add two more disks(sdb, sdc) to construct RAID. - -### Prerequire -AMI build script only works on distrubution with docker installed. - -### AMI variables -Create dist/ami/variables.json with following format, specify AMI variables: -``` -{ - "access_key": "xxx", - "secret_key": "xxx", - "subnet_id": "subnet-xxx", - "security_group_id": "sg-xxx", - "region": "us-east-1", - "associate_public_ip_address": "true", - "instance_type": "c4.xlarge", - "ami_prefix": "YOUR_NAME-" -} -``` - -You can use dist/ami/variables.json.example as template. - -### Build AMI from locally built rpm -If you want to make AMI with modified scylla source code, this is what you want. -and move all files into `aws/ami/files`, to find building instruction for all needed packages see: -https://github.com/scylladb/scylla/blob/master/docs/building-packages.md#scylla-server - -To build AMI from locally built rpm, run -``` -# build scyll-machine-image (it's not yet part of the repo, since not merge to master yet) -./dist/redhat/build_rpm.sh -t centos -c aws -cp build/RPMS/noarch/scylla-machine-image-*.rpm ./aws/ami/files/ - -SCYLLA_DIR=~/Projects/scylla - -cd $SCYLLA_DIR -./SCYLLA-VERSION-GEN -PRODUCT=$(cat build/SCYLLA-PRODUCT-FILE) -REPO=`./scripts/scylla_current_repo --target centos` - -cd ./aws/ami - -# optionaly: download all other rpms except scylla-server from the repo -./build_with_docker.sh --product $PRODUCT --repo $REPO --download-no-server - -# copy the built scylla RPMs -cp $SCYLLA_DIR/build/redhat/RPMS/x86_64/*.rpm ./files/ - -./build_with_docker.sh --localrpm --product $PRODUCT --repo-for-update $REPO - -``` - -### Build AMI locally from rpms: - -To build ami locally, you will need all rpms. you could find them for example by this link: -http://downloads.scylladb.com/relocatable/unstable/master/2020-12-20T00%3A11%3A59Z/rpm/ - -1. Download all rpms to aws/ami/files folder - -2. Build scylla-machine-image rpm file. If you use the redhat os, the you could -use instructions above. If you use debian like os, then you could use docker -_commit and push your changes_ - -``` -cd scylla-machine-image -docker run -v `pwd`:/scylla-machine-image scylladb/packer-builder /bin/bash -c "cd /scylla-machine-image; ./dist/redhat/build_rpm.sh -t centos" -cp build/RPMS/noarch/scylla-machine-image-*.rpm ./aws/ami/files/ -``` - -3. create file aws/ami/variables.json with valid settings. See example variables.json.example -4. then you can build your ami: -``` -./build_with_docker.sh --localrpm --product scylla -``` - -5. At the end you will get the ami id in us-east-1 region. - -### Build AMI from yum repository -To build AMI from unstable yum repository, run -``` -./build_with_docker.sh --repo http://downloads.scylladb.com.s3.amazonaws.com/rpm/unstable/centos/master/latest/scylla.repo -``` diff --git a/aws/ami/build_deb_ami.sh b/aws/ami/build_deb_ami.sh deleted file mode 120000 index 7796f008..00000000 --- a/aws/ami/build_deb_ami.sh +++ /dev/null @@ -1 +0,0 @@ -../../packer/build_deb_image.sh \ No newline at end of file diff --git a/aws/ami/build_deb_with_docker.sh b/aws/ami/build_deb_with_docker.sh deleted file mode 120000 index d2f5b35e..00000000 --- a/aws/ami/build_deb_with_docker.sh +++ /dev/null @@ -1 +0,0 @@ -../../packer/build_deb_with_docker.sh \ No newline at end of file diff --git a/aws/ami/files/.gitkeep b/aws/ami/files/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/aws/ami/user_data.txt b/aws/ami/user_data.txt deleted file mode 100644 index f5bf0d5e..00000000 --- a/aws/ami/user_data.txt +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -sed -i 's/Defaults requiretty/#Defaults requiretty/g' /etc/sudoers diff --git a/aws/ami/variables.json.example b/aws/ami/variables.json.example deleted file mode 100644 index 31e551e7..00000000 --- a/aws/ami/variables.json.example +++ /dev/null @@ -1,9 +0,0 @@ -{ - "access_key": "", - "secret_key": "", - "subnet_id": "", - "security_group_id": "", - "region": "us-east-1", - "associate_public_ip_address": "true", - "instance_type": "" -} diff --git a/aws/cloudformation/Jenkinsfile b/aws/cloudformation/Jenkinsfile deleted file mode 100644 index 2811275f..00000000 --- a/aws/cloudformation/Jenkinsfile +++ /dev/null @@ -1,77 +0,0 @@ -#!groovy -pipeline { - agent { - label { - label "aws-sct-builders-eu-west-1" - } - } - parameters { - string(defaultValue: "us-east-1", - description: 'aws region to use, ex. us-east-1 or eu-west-1', - name: 'aws_region') - - string(defaultValue: '', description: '', name: 'scylla_ami_id') - } - options { - timestamps() - disableConcurrentBuilds() - timeout([time: 30, unit: 'MINUTES']) - buildDiscarder(logRotator(numToKeepStr: '20')) - } - stages { - stage('Checkout') { - steps { - checkout scm - } - } - stage('create virtualenv') { - steps { - script { - wrap([$class: 'BuildUser']) { - dir('aws/cloudformation') { - sh """ - #!/bin/bash - set -xe - - python3 -m venv .venv3 - source .venv3/bin/activate - - pip install --upgrade pip - pip install -r requirements.txt - """ - } - } - } - } - } - stage('test') { - environment { - AWS_ACCESS_KEY_ID = credentials('jenkins2-aws-secret-key-id') - AWS_SECRET_ACCESS_KEY = credentials('jenkins2-aws-secret-access-key') - } - steps { - script { - wrap([$class: 'BuildUser']) { - dir('aws/cloudformation') { - sh """ - #!/bin/bash - set -xe - - source .venv3/bin/activate - - # regenerate the teample - jinja2 scylla.yaml.j2 > scylla.yaml - - # lint the template - cfn-lint --template scylla.yaml --region us-east-1 - - # running the test - pytest --log-cli-level info --ami ${params.scylla_ami_id} --region ${params.aws_region} - """ - } - } - } - } - } - } -} diff --git a/aws/cloudformation/README.md b/aws/cloudformation/README.md deleted file mode 100644 index bc1ab865..00000000 --- a/aws/cloudformation/README.md +++ /dev/null @@ -1,36 +0,0 @@ -## Installation -``` -python3 -m venv .venv3 -source .venv3/bin/activate -install -r requirements.txt -``` - -## Testing -``` bash -# regenerate the teample -jinja2 scylla.yaml.j2 > scylla.yaml - -# lint the template -cfn-lint --template scylla.yaml --region us-east-1 - -# running the test -pytest --log-cli-level info -``` - -# Using the template -``` -# run the template from command line -aws cloudformation create-stack --region eu-west-1 --stack-name fruch-test-05 --template-body file://scylla.yaml \ - --parameters ParameterKey=KeyName,ParameterValue=scylla-qa-ec2 \ - ParameterKey=InstanceType,ParameterValue=i3.large \ - ParameterKey=AvailabilityZone,ParameterValue=eu-west-1a \ - ParameterKey=ClusterName,ParameterValue=fruch \ - ParameterKey=InstanceCount,ParameterValue=3 \ - ParameterKey=ScyllaAmi,ParameterValue=ami-0ececa5cacea302a8 - -``` -Example of link to start the cloudforamtion: - -https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/create/review?templateURL=https://s3-eu-west-1.amazonaws.com/cf-templates-1jy8um4tbzwit-eu-west-1/2019241R3e-scylla.templateenk889k0zz&stackName=fruch-test¶m_ScyllaAmi=ami-0ececa5cacea302a8 - - diff --git a/aws/cloudformation/conftest.py b/aws/cloudformation/conftest.py deleted file mode 100644 index 212b0757..00000000 --- a/aws/cloudformation/conftest.py +++ /dev/null @@ -1,5 +0,0 @@ -def pytest_addoption(parser): - parser.addoption("--keep-cfn", action="store_true", default=False) - parser.addoption("--stack-name", action="store", default=None) - parser.addoption("--region", action="store", default="us-east-1") - parser.addoption("--ami", action="store", default="ami-0791dd37ca6da66c5") diff --git a/aws/cloudformation/pytest.ini b/aws/cloudformation/pytest.ini deleted file mode 100644 index 96735eb5..00000000 --- a/aws/cloudformation/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -log_format = %(asctime)s %(levelname)s %(message)s -log_date_format = %Y-%m-%d %H:%M:%S diff --git a/aws/cloudformation/requirements.txt b/aws/cloudformation/requirements.txt deleted file mode 100644 index fe8d24c8..00000000 --- a/aws/cloudformation/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -cfn-lint==0.23.4 -boto3==1.9.216 -pytest==5.1.1 -Jinja2==2.11.3 -jinja2-cli==0.7.0 \ No newline at end of file diff --git a/aws/cloudformation/test_scylla_cfn.py b/aws/cloudformation/test_scylla_cfn.py deleted file mode 100644 index bddd7f58..00000000 --- a/aws/cloudformation/test_scylla_cfn.py +++ /dev/null @@ -1,145 +0,0 @@ -# -# Copyright 2020 ScyllaDB -# -# SPDX-License-Identifier: Apache-2.0 - -import pprint -import logging -import subprocess -import uuid - -import pytest -import boto3 -from botocore.errorfactory import ClientError - - -def run_on_node(node_ip, cmd): - output = subprocess.run( - [ - "/bin/bash", - "-c", - f'ssh -i ~/.ssh/scylla-qa-ec2 -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" centos@{node_ip} {cmd}', - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - logging.info(output.stderr.decode("utf-8").strip()) - return output.stdout.decode("utf-8").strip() - - -@pytest.fixture(scope="session") -def cfn_scylla_cluster(request): - region = request.config.getoption("--region") - client = boto3.client("cloudformation", region_name=region) - ec2 = boto3.client('ec2', region_name=region) - availability_zone = ec2.describe_availability_zones()['AvailabilityZones'][0]['ZoneName'] - - if request.config.getoption("--stack-name"): - name = request.config.getoption("--stack-name") - else: - rand_suffix = uuid.uuid4().hex[:6] - name = f"scylla-cfn-test-{rand_suffix}" - - try: - response = client.create_stack( - StackName=name, - TemplateBody=open("scylla.yaml").read(), - Parameters=[ - {"ParameterKey": "KeyName", "ParameterValue": "scylla-qa-ec2"}, - {"ParameterKey": "InstanceType", "ParameterValue": "i3.large"}, - {"ParameterKey": "AvailabilityZone", "ParameterValue": availability_zone}, - {"ParameterKey": "ClusterName", "ParameterValue": name}, - {"ParameterKey": "InstanceCount", "ParameterValue": "3"}, - { - "ParameterKey": "ScyllaAmi", - "ParameterValue": request.config.getoption("--ami"), - }, - ], - ) - logging.info(pprint.pformat(response)) - - except ClientError as ex: - logging.info(ex) - - logging.info(f"waiting for cloudformation [{name}] to complete") - - waiter = client.get_waiter("stack_create_complete") - response = waiter.wait(StackName=name) - - resources = client.list_stack_resources(StackName=name) - - outputs = client.describe_stacks(StackName=name) - logging.info(pprint.pformat(outputs)) - outputs = { - item["OutputKey"]: item["OutputValue"] - for item in outputs["Stacks"][0]["Outputs"] - } - - yield resources, outputs - - if not request.session.testsfailed and not request.config.getoption("--keep-cfn"): - response = client.delete_stack(StackName=name) - logging.info(response) - - waiter = client.get_waiter("stack_delete_complete") - response = waiter.wait(StackName=name) - logging.info(response) - - -def wait_nodes_ready(resources, region): - ec2 = boto3.client("ec2", region_name=region) - ok_waiter = ec2.get_waiter("instance_status_ok") - - instances = [ - r["PhysicalResourceId"] - for r in resources["StackResourceSummaries"] - if r["ResourceType"] == "AWS::EC2::Instance" - ] - - logging.debug(pprint.pformat(instances)) - - response = ok_waiter.wait(InstanceIds=instances) - logging.info(pprint.pformat(response)) - - -def test_cluster_up(request, cfn_scylla_cluster): - resources, outputs = cfn_scylla_cluster - region = request.config.getoption("--region") - wait_nodes_ready(resources, region) - - -def test_connect(cfn_scylla_cluster): - resources, outputs = cfn_scylla_cluster - nodes_ip_addresses = [v for k, v in outputs.items() if "PublicIp" in k] - - for node_ip in nodes_ip_addresses: - logging.info(f"connecting to node {node_ip}") - output = run_on_node(node_ip, "scylla --version") - logging.info(output) - - -def test_nodetool_status(cfn_scylla_cluster): - resources, outputs = cfn_scylla_cluster - nodes_ip_addresses = [v for k, v in outputs.items() if "PublicIp" in k] - node_private_ips = [v for k, v in outputs.items() if "PrivateIp" in k] - - for node_ip in nodes_ip_addresses: - logging.info(f"running nodetool on node {node_ip}") - output = run_on_node(node_ip, "nodetool status") - logging.info(output) - for private_ip in node_private_ips: - assert f"UN {private_ip}" in output - - -def test_cassandra_stress(cfn_scylla_cluster): - resources, outputs = cfn_scylla_cluster - nodes_ip_addresses = [v for k, v in outputs.items() if "PublicIp" in k] - - for node_ip in nodes_ip_addresses: - logging.info(f"running c-s to node {node_ip}") - output = run_on_node( - node_ip, "cassandra-stress write n=40000 -rate threads=40 -node 172.31.0.11" - ) - logging.info(output) - assert "Total errors : 0 [WRITE: 0]" in output diff --git a/azure/README b/azure/README deleted file mode 100644 index dde1bb4a..00000000 --- a/azure/README +++ /dev/null @@ -1 +0,0 @@ -Here will be located Microsoft Azure Cloud Image setup files \ No newline at end of file diff --git a/azure/image/README.md b/azure/image/README.md deleted file mode 100644 index 93dbbda5..00000000 --- a/azure/image/README.md +++ /dev/null @@ -1,65 +0,0 @@ -Edit variables.json and run azure/image/build_azure_image.sh, it will build & upload Azure image. - -### Azure variables -Create azure/image/variables.json with following format, specify Azure variables: -``` -{ - "client_id": "", - "client_secret": "", - "tenant_id": "", - "subscription_id": "", - "security_group_id": "", - "region": "", - "vm_size": "" -} -``` - -You can use azure/image/variables.json.example as template. - -### Build Azure image from locally built rpm -If you want to make Azure with modified scylla source code, this is what you want. -and move all files into `azure/image/files`, to find building instruction for all needed packages see: -https://github.com/scylladb/scylla/blob/master/docs/building-packages.md#scylla-server - -To build Azure from locally built rpm, run -``` -# build scyll-machine-image (it's not yet part of the repo, since not merge to master yet) -./dist/debian/build_deb.sh -cp build/debian/scylla-machine-image-*.deb ./azure/image/files - -SCYLLA_DIR=~/Projects/scylla - -cd $SCYLLA_DIR -./SCYLLA-VERSION-GEN -PRODUCT=$(cat build/SCYLLA-PRODUCT-FILE) - -cd azure/image - - -# copy the built scylla RPMs -cp $SCYLLA_DIR/build/dist//debian/*.deb ./files/ - -./build_with_docker.sh --localrpm --product $PRODUCT --repo-for-update $REPO - -``` - -### Build Azure image locally from deb: - -To build Azure locally, you will need all deb. you could find them for example by this link: -http://downloads.scylladb.com/relocatable/unstable/master/2020-12-20T00%3A11%3A59Z/deb/ - -1. Download all debs to azure/image/files folder - -2. Build scylla-machine-image deb file. - -3. create file azure/image/variables.json with valid settings. See example variables.json.example -4. then you can build your Azure image: -``` -./build_azure_with_docker.sh --localrpm --product scylla -``` - -### Build Azure from deb repository -To build Azure image from unstable deb repository, run -``` -./build_azure_with_docker.sh --repo http://downloads.scylladb.com/unstable/scylla/master/deb/unified/latest/scylladb-master/scylla.list -``` diff --git a/azure/image/build_azure_image.sh b/azure/image/build_azure_image.sh deleted file mode 120000 index 7796f008..00000000 --- a/azure/image/build_azure_image.sh +++ /dev/null @@ -1 +0,0 @@ -../../packer/build_deb_image.sh \ No newline at end of file diff --git a/azure/image/build_azure_with_docker.sh b/azure/image/build_azure_with_docker.sh deleted file mode 120000 index d2f5b35e..00000000 --- a/azure/image/build_azure_with_docker.sh +++ /dev/null @@ -1 +0,0 @@ -../../packer/build_deb_with_docker.sh \ No newline at end of file diff --git a/azure/image/files/.gitkeep b/azure/image/files/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/azure/image/variables.json b/azure/image/variables.json deleted file mode 100644 index 1a845c52..00000000 --- a/azure/image/variables.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "client_id": "", - "client_secret": "", - "tenant_id": "", - "subscription_id": "", - "security_group_id": "", - "region": "EAST US", - "vm_size": "Standard_D2_v4" -} diff --git a/gce/README b/gce/README deleted file mode 100644 index 8c8123c4..00000000 --- a/gce/README +++ /dev/null @@ -1,29 +0,0 @@ -Instructions to build the GCE Image - - -Create gce/image/variables.json using gce/image/variables.json.example. -Example: -``` -{ - "project_id": "skilled-adapter-452", - "zone": "us-east-1-a", - "instance_type": "n1-standard-1" -} -``` - -If running locally make sure you are authenticated - -``` -https://www.packer.io/docs/builders/googlecompute.html#precedence-of-authentication-methods -``` - -or set `GOOGLE_APPLICATION_CREDENTIALS` env variable with path to the credentials file. - -Example of the command: -``` -./build_image.sh --localrpm --repo https://s3.amazonaws.com/downloads.scylladb.com/rpm/unstable/centos/branch-4.1/2020-06-18T14:49:56Z/scylla.repo --product scylla -``` - -User data - -To store user data create key named "user-data" in the instance meta data. \ No newline at end of file diff --git a/gce/image/build_deb_image.sh b/gce/image/build_deb_image.sh deleted file mode 120000 index 7796f008..00000000 --- a/gce/image/build_deb_image.sh +++ /dev/null @@ -1 +0,0 @@ -../../packer/build_deb_image.sh \ No newline at end of file diff --git a/gce/image/files/.gitkeep b/gce/image/files/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/gce/image/variables.json.example b/gce/image/variables.json.example deleted file mode 100644 index 4c6c518b..00000000 --- a/gce/image/variables.json.example +++ /dev/null @@ -1,7 +0,0 @@ -{ - "project_id": "skilled-adapter-452", - "region": "europe-west1", - "zone": "europe-west1-b", - "image_storage_location": "europe-west1", - "instance_type": "n1-standard-1" -}