Enabling Haim policies

jenkins/poc/haim/appeng/PolicyJenkinsfileDaily

@@ -0,0 +1,63 @@
+pipeline {
+    options {
+        disableConcurrentBuilds()
+    }
+    agent {
+        docker {
+            label 'cloud-governance-worker'
+            image 'quay.io/athiru/centos-stream8-podman:latest'
+            args  '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged'
+        }
+    }
+    environment {
+        access_key = credentials('appeng-aws-access-key-id')
+        secret_key = credentials('appeng-aws-secret-key-id')
+        s3_bucket = credentials('appeng-s3-bucket')
+        account_name = "appeng"
+        AWS_IAM_USER_SPREADSHEET_ID = credentials('cloud-governance-aws-iam-user-spreadsheet-id')
+        GOOGLE_APPLICATION_CREDENTIALS = credentials('cloud-governance-google-application-credentials')
+        LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name')
+        ES_HOST = credentials('haim-cloud-governance-elasticsearch-url')
+        ES_PORT = credentials('haim-cloud-governance-elasticsearch-port')
+        contact1 = "[email protected]"
+        contact2 = "[email protected]"
+    }
+    stages {
+        stage('Checkout') { // Checkout (git clone ...) the projects repository
+           steps {
+                 checkout scm
+           }
+        }
+        stage('Initial Cleanup') {
+            steps {
+                 sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
+            }
+        }
+        stage('Run Policies the Cost Policies') {
+            steps {
+                 sh 'python3 jenkins/poc/haim/common/run_cost_policies.py'
+            }
+        }
+        stage('Run Policies the Daily polices') {
+            steps {
+                 sh 'python3 jenkins/poc/haim/common/run_policies.py'
+            }
+        }
+        stage('Finalize Cleanup') {
+            steps {
+                 sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
+                 deleteDir()
+            }
+        }
+    }
+    post {
+        failure {
+              script {
+                msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})"
+                emailext body: """\
+            Jenkins job: ${env.BUILD_URL}\nSee the console output for more details:  ${env.BUILD_URL}consoleFull\n\n
+            """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}"
+            }
+          }
+    }
+}

jenkins/poc/haim/appeng/TaggingJenkinsfileHourly

@@ -0,0 +1,53 @@
+pipeline {
+    options {
+        disableConcurrentBuilds()
+    }
+    agent {
+        docker {
+            label 'cloud-governance-worker'
+            image 'quay.io/athiru/centos-stream8-podman:latest'
+            args  '-u root -v /etc/postfix/main.cf:/etc/postfix/main.cf --privileged'
+        }
+    }
+    environment {
+        access_key = credentials('appeng-aws-access-key-id')
+        secret_key = credentials('appeng-aws-secret-key-id')
+        LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name')
+        account_name = "appeng"
+        contact1 = "[email protected]"
+        contact2 = "[email protected]"
+    }
+    stages {
+        stage('Checkout') { // Checkout (git clone ...) the projects repository
+           steps {
+                 checkout scm
+           }
+        }
+        stage('Initial Cleanup') {
+            steps {
+                 sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
+            }
+        }
+        stage('Run Tagging Cluster & Non-Cluster') {
+            steps {
+                 sh 'python3 jenkins/poc/haim/common/run_tagging.py'
+            }
+        }
+        stage('Finalize Cleanup') {
+            steps {
+                 sh '''if [[ "$(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null)" != "" ]]; then podman rmi -f $(podman images -q quay.io/ebattat/cloud-governance 2> /dev/null); fi'''
+                 deleteDir()
+            }
+        }
+    }
+    post {
+        failure {
+              script {
+                msg = "Build error for ${env.JOB_NAME} ${env.BUILD_NUMBER} (${env.BUILD_URL})"
+                emailext body: """\
+            Jenkins job: ${env.BUILD_URL}\nSee the console output for more details:  ${env.BUILD_URL}consoleFull\n\n
+            """,subject: msg, to: "${contact1}, ${contact2}, ${contact3}"
+            }
+          }
+    }
+}

jenkins/poc/haim/common/run_cost_policies.py

@@ -0,0 +1,17 @@
+import os
+
+access_key = os.environ['access_key']
+secret_key = os.environ['secret_key']
+s3_bucket = os.environ['s3_bucket']
+account_name = os.environ['account_name']
+ES_HOST = os.environ['ES_HOST']
+ES_PORT = os.environ['ES_PORT']
+
+
+cost_tags = ['PurchaseType', 'ChargeType', 'User', 'Budget', 'Project', 'Manager', 'Owner',
+             'LaunchTime', 'Name', 'Email', 'Environment', 'User:Spot']
+cost_metric = 'UnblendedCost'  # UnblendedCost/BlendedCost
+granularity = 'DAILY'  # DAILY/MONTHLY/HOURLY
+cost_explorer_index = 'cloud-governance-haim-cost-explorer-global-index'
+os.system(f"""echo "Running the CloudGovernance CostExplorer Policies" """)
+os.system(f"""podman run --rm --name cloud-governance --net="host" -e AWS_DEFAULT_REGION="us-east-1" -e account="{account_name}" -e policy="cost_explorer" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_index="{cost_explorer_index}" -e cost_explorer_tags="{cost_tags}" -e granularity="{granularity}" -e cost_metric="{cost_metric}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")

jenkins/poc/haim/common/run_policies.py

@@ -0,0 +1,43 @@
+
+import os
+from ast import literal_eval
+
+policies_in_action = ['ebs_unattached', 'ip_unattached', 'zombie_snapshots', 'unused_nat_gateway', 's3_inactive', 'empty_roles']
+policies_not_in_action = ['ec2_stop', 'ec2_idle', 'zombie_cluster_resource']
+
+access_key = os.environ['access_key']
+secret_key = os.environ['secret_key']
+s3_bucket = os.environ['s3_bucket']
+account_name = os.environ['account_name']
+days_to_delete_resource = os.environ.get('days_to_delete_resource', 14)
+LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME']
+LOGS = os.environ.get('LOGS', 'logs')
+ES_HOST = os.environ['ES_HOST']
+ES_PORT = os.environ['ES_PORT']
+GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS']
+SPREADSHEET_ID = os.environ['AWS_IAM_USER_SPREADSHEET_ID']
+
+regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1',
+           'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1']
+
+es_doc_type = '_doc'
+
+os.system(f"""echo Running the cloud_governance policies with dry_run=yes""")
+os.system(f"echo Polices list: {policies_not_in_action}")
+for region in regions:
+    for policy in policies_not_in_action:
+        os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="yes" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
+        if policy == 'zombie_cluster_resource':
+            os.system(f"""podman run --rm --name cloud-governance-poc-haim -e upload_data_es="upload_data_es" -e account="{account_name}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e es_doc_type="{es_doc_type}" -e bucket="{s3_bucket}" -e policy="{policy}" -e AWS_DEFAULT_REGION="{region}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
+
+os.system('echo "Running the CloudGovernance policies with dry_run=no" ')
+os.system(f"echo Polices list: {policies_in_action}")
+for region in regions:
+    for policy in policies_in_action:
+        if policy in ('empty_roles', 's3_inactive') and region == 'us-east-1':
+            os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
+        elif policy not in ('empty_roles', 's3_inactive'):
+            os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e MANAGER_EMAIL_ALERT="False" -e EMAIL_ALERT="False" -e account="{account_name}" -e policy="{policy}" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e dry_run="no" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}" -e es_host="{ES_HOST}" -e es_port="{ES_PORT}" -e policy_output="s3://{s3_bucket}/{LOGS}/{region}" -e DAYS_TO_DELETE_RESOURCE="{days_to_delete_resource}" -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")
+
+os.system(f"""echo "Running the tag_iam_user" """)
+os.system(f"""podman run --rm --name cloud-governance-poc-haim --net="host" -e account="{account_name}" -e -e EMAIL_ALERT="False" -e policy="tag_iam_user" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e user_tag_operation="update" -e SPREADSHEET_ID="{SPREADSHEET_ID}" -e GOOGLE_APPLICATION_CREDENTIALS="{GOOGLE_APPLICATION_CREDENTIALS}" -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" -e LDAP_HOST_NAME="{LDAP_HOST_NAME}"  -e log_level="INFO" quay.io/ebattat/cloud-governance:latest""")

jenkins/poc/haim/common/run_tagging.py

@@ -0,0 +1,19 @@
+
+import os
+
+
+access_key = os.environ['access_key']
+secret_key = os.environ['secret_key']
+account_name = os.environ['account_name']
+LDAP_HOST_NAME = os.environ['LDAP_HOST_NAME']
+
+
+LOGS = os.environ.get('LOGS', 'logs')
+
+mandatory_tags_appeng = {'Budget': account_name}
+
+
+os.system(f"""echo "Running the tag_resources" """)
+regions = ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'eu-central-1', 'ap-south-1', 'eu-north-1', 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-3', 'sa-east-1']
+for region in regions:
+    os.system(f"""podman run --rm --name cloud-governance-poc-haim -e account="{account_name}" -e EMAIL_ALERT="False" -e policy="tag_resources" -e AWS_ACCESS_KEY_ID="{access_key}" -e AWS_SECRET_ACCESS_KEY="{secret_key}" -e AWS_DEFAULT_REGION="{region}" -e tag_operation="update" -e mandatory_tags="{mandatory_tags_appeng}" -e log_level="INFO" -v "/etc/localtime":"/etc/localtime" quay.io/ebattat/cloud-governance:latest""")