From 835921d2e97a5d4c535126d5b541212aacc9c527 Mon Sep 17 00:00:00 2001 From: Limmen Date: Sun, 10 Mar 2024 10:07:06 +0100 Subject: [PATCH] update ansible installation files --- README.md | 1 + ansible/README.md | 34 ++++-- ansible/group_vars/all/variables.yml | 12 +++ ansible/install.yml | 21 ++++ ...ystem.yml => install_emulation_system.yml} | 20 ++-- ...stem.yml => install_management_system.yml} | 12 +-- ...he_metastore.yml => install_metastore.yml} | 19 ++-- ...stallation_setup.yml => install_setup.yml} | 21 ++-- ansible/install_simulation_system.yml | 51 +++++++++ ansible/installing_simulation_system.yml | 102 ------------------ ansible/inventory | 4 + ...nt.txt => nginx_configuration_content.txt} | 0 ... => nginx_configuration_content_https.txt} | 0 ansible/single_command_test.yml | 23 ---- docs/_docs/contributing.md | 3 +- 15 files changed, 143 insertions(+), 180 deletions(-) create mode 100644 ansible/group_vars/all/variables.yml create mode 100644 ansible/install.yml rename ansible/{installing_emulation_system.yml => install_emulation_system.yml} (86%) rename ansible/{installing_the_management_system.yml => install_management_system.yml} (94%) rename ansible/{installing_the_metastore.yml => install_metastore.yml} (88%) rename ansible/{installation_setup.yml => install_setup.yml} (85%) create mode 100644 ansible/install_simulation_system.yml delete mode 100644 ansible/installing_simulation_system.yml rename ansible/{Nginx_configuration_content.txt => nginx_configuration_content.txt} (100%) rename ansible/{Nginx_configuration_content_https.txt => nginx_configuration_content_https.txt} (100%) delete mode 100644 ansible/single_command_test.yml diff --git a/README.md b/README.md index c8a92bc51..869cf2cd5 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ Thanks go to these people! Arvid Lagerqvist
Arvid Lagerqvist Nils Forsgren
Nils Forsgren Forough Shahab Samani
Forough Shahab Samani + Benth Roland Pappila
Benth Roland Pappila diff --git a/ansible/README.md b/ansible/README.md index 4f0e7979a..0ec132ce8 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -13,21 +13,43 @@ Ansible can be installed by running the command: pip install ansible ``` +## Setup SSH keys + +Before starting the installation, make sure to setup SSH key pairs on all of the servers where CSLE will be installed +and make sure that you can reach all of those servers from the workstation/laptop where the ansible playbooks will be run. +To generate an ssh key pair on a server, run the command `ssh-keygen`. Once keys have been created on all hosts, +copy the public key (e.g., `~/.ssh/id_rsa.pub`) of each host to the file `.ssh/authorized_keys` of the other hosts. +After creating the SSH keys and verifying the connections, define the path to the private key of the host from which +the installation will run in the file [ansible.cfg](ansible.cfg). + ## Configuring the installation -Before starting the CSLE installation, open the file XXX and configure the following variables: -- user -- todo +Before starting the CSLE installation, do the following. + +1. Specify the leader and the worker hosts in [inventory](inventory) +2. Configure installation variables (e.g., the user of the installation and the Python version) in [group_vars/all/variables.yml](group_vars/all/variables.yml). + ## Installing CSLE -To install CSLE with ansible, run the following commands: +To install CSLE with ansible, run the following playbook: + ```bash ansible-playbook --ask-become-pass installing_the_management_system.yml ``` +Alternatively, you can run each playbook manually by executing the commands: +```bash +ansible-playbook --ask-become-pass install_setup.yml +ansible-playbook --ask-become-pass install_metastore.yml +ansible-playbook --ask-become-pass install_simulation_system.yml +ansible-playbook --ask-become-pass install_emulation_system.yml +ansible-playbook --ask-become-pass install_management_system.yml +``` ### Debugging -If the installation fails at some step, you can debug the reason for the failure by adding the following -line to the Ansible playbook. First, we register a variable that holds a dictionary of the output for the module in that task. In the given example git_installation is this variable. In the next lines, we use debug to print the variable. + +If the installation fails at some step, you can debug the reason for the failure as follows. +Start by registering a variable that holds a dictionary of the output for the module in the task that failed (`git_installation` in the example below). +After registering the variable, add it to the debug group. ```bash - name: Installation of git diff --git a/ansible/group_vars/all/variables.yml b/ansible/group_vars/all/variables.yml new file mode 100644 index 000000000..b1115252a --- /dev/null +++ b/ansible/group_vars/all/variables.yml @@ -0,0 +1,12 @@ +user: "shahab" +csle_git_repo_url: "https://github.com/Limmen/csle" +leader_ip: "172.31.212.83" +nginx_content_name: "nginx_configuration_content.txt" +web_port: 7777 +postgres_version: "15" +postgres_citus_version: "postgresql-15-citus-11.2" +postgresql_url: "https://install.citusdata.com/community/deb.sh" +anaconda_url: "https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh" +anaconda_install_script: "Anaconda3-2022.10-Linux-x86_64.sh" +conda_environment_name: "base" +python_version: "3.9" \ No newline at end of file diff --git a/ansible/install.yml b/ansible/install.yml new file mode 100644 index 000000000..93e7a6b08 --- /dev/null +++ b/ansible/install.yml @@ -0,0 +1,21 @@ +--- + +- hosts: all + become: true + + tasks: + + - name: Installation setup + ansible.builtin.import_playbook: install_setup.yaml + + - name: Install metastore + ansible.builtin.import_playbook: install_metastore.yaml + + - name: Install simulation system + ansible.builtin.import_playbook: install_simulation_system.yaml + + - name: Install emulation system + ansible.builtin.import_playbook: install_emulation_system.yaml + + - name: Install management system + ansible.builtin.import_playbook: install_management_system.yaml \ No newline at end of file diff --git a/ansible/installing_emulation_system.yml b/ansible/install_emulation_system.yml similarity index 86% rename from ansible/installing_emulation_system.yml rename to ansible/install_emulation_system.yml index 37e6638a9..3aaad0ae2 100644 --- a/ansible/installing_emulation_system.yml +++ b/ansible/install_emulation_system.yml @@ -3,18 +3,14 @@ - hosts: all become: true - vars: - user: "shahab" - leader_ip: "172.31.212.83" - tasks: - - name: Check if keyrings folder exist + - name: Check if apt keyrings directory exist stat: path: /etc/apt/keyrings register: keyrings_directory_exists - - name: Creates keyrings directory if it does not exist + - name: Create keyrings directory if it does not exist ansible.builtin.file: path: /etc/apt/keyrings state: directory @@ -30,7 +26,7 @@ interact ' - - name: Second command for installing keys + - name: Install docker apt keys shell: echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - name: Update package cache @@ -51,25 +47,25 @@ state: present - name: Add user to Docker group - shell: sudo usermod -aG docker $USER + shell: "sudo usermod -aG docker {{ user }}" - name: Initialize Docker Swarm command: docker swarm init --advertise-addr "{{ leader_ip }}" ignore_errors: yes - - name: Pulling base image + - name: Pulling base Docker images shell: "cd /home/{{ user }}/csle/emulation-system/base_images && make pull" - - name: Pulling derived image + - name: Pulling derived Docker images shell: | cd "/home/{{ user }}/csle/emulation-system/derived_images" && \ make pull args: executable: /bin/bash - - name: Installing the envs on the leader node + - name: Installing the emulation environments on the leader node shell: | - source "/home/{{ user }}/anaconda3/bin/activate" py39 && \ + source "/home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }}" && \ cd "/home/{{ user }}/csle/emulation-system/envs" && \ make install args: diff --git a/ansible/installing_the_management_system.yml b/ansible/install_management_system.yml similarity index 94% rename from ansible/installing_the_management_system.yml rename to ansible/install_management_system.yml index 47ebece1f..d28e3b07f 100644 --- a/ansible/installing_the_management_system.yml +++ b/ansible/install_management_system.yml @@ -1,18 +1,10 @@ --- - hosts: all - #become: true - - vars: - user: "shahab" - leader_ip: "172.31.212.83" - nginx_content_name: "Nginx_configuration_content.txt" - web_port: 7777 tasks: - - name: Check if nvm is installed shell: | source "/home/{{ user }}/.nvm/nvm.sh" && \ @@ -119,7 +111,7 @@ ignore_errors: true register: port_check_result - - name: install and start pgadmin + - name: Install and start pgadmin become: true shell: | docker pull dpage/pgadmin4 @@ -196,7 +188,7 @@ when: variable_exists.rc != 0 - name: CSLE init - shell: "source /home/{{ user }}/anaconda3/bin/activate py39 && echo $CSLE_HOME && csle init" + shell: "source /home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }} && echo $CSLE_HOME && csle init" args: executable: /bin/bash environment: diff --git a/ansible/installing_the_metastore.yml b/ansible/install_metastore.yml similarity index 88% rename from ansible/installing_the_metastore.yml rename to ansible/install_metastore.yml index ab366f2f3..852f6bea6 100644 --- a/ansible/installing_the_metastore.yml +++ b/ansible/install_metastore.yml @@ -3,13 +3,6 @@ - hosts: all become: true - vars: - csle_git_repo_url: "https://github.com/Limmen/csle" - postgresql_url: "https://install.citusdata.com/community/deb.sh" - user: "shahab" - postgres_version: "15" - leader_IP: "172.31.212.90" - tasks: - name: Check if PostgreSQL is installed, if you see error here just ignore it! @@ -20,7 +13,7 @@ - name: Download PostgreSQL ansible.builtin.get_url: url: "{{ postgresql_url }}" - dest: /home/shahab + dest: "/home/{{ user }}" mode: '0770' when: postgresql_installed.rc != 0 @@ -30,7 +23,7 @@ - name: Install PostgreSQL Citus extension apt: - name: postgresql-15-citus-11.2 + name: "{{ postgres_citus_version }}" state: present update_cache: yes when: postgresql_installed.rc != 0 @@ -58,10 +51,10 @@ state: present - name: Set shared_preload_libraries using pg_conftool - shell: "pg_conftool 15 main set shared_preload_libraries citus" + shell: "pg_conftool {{ postgres_version }} main set shared_preload_libraries citus" - name: Set listen address using pg_conftool - shell: "pg_conftool 15 main set listen_addresses '*'" + shell: "pg_conftool {{ postgres_version }} main set listen_addresses '*'" - name: Run psql and set password for postgres user @@ -123,7 +116,7 @@ lineinfile: path: "/home/{{ user }}/csle/metastore/create_cluster.sql" regexp: '^SELECT citus_set_coordinator_host' - line: "SELECT citus_set_coordinator_host('{{ leader_IP }}', 5432);" + line: "SELECT citus_set_coordinator_host('{{ leader_ip }}', 5432);" - name: Create PostgreSQL cluster expect: @@ -149,5 +142,5 @@ file: path: /var/log/postgresql owner: "{{ user }}" - mode: "0775" # Specify the desired permissions here + mode: "0775" become: true diff --git a/ansible/installation_setup.yml b/ansible/install_setup.yml similarity index 85% rename from ansible/installation_setup.yml rename to ansible/install_setup.yml index 5a8973c9d..51060bddf 100644 --- a/ansible/installation_setup.yml +++ b/ansible/install_setup.yml @@ -3,11 +3,6 @@ - hosts: all become: true - vars: - csle_git_repo_url: "https://github.com/Limmen/csle" - anaconda_url: "https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh" - user: "shahab" - tasks: - name: Installation of build-essential @@ -35,22 +30,22 @@ register: anaconda_installed ignore_errors: true - - name: Download anaconda + - name: Download Anaconda ansible.builtin.get_url: url: "{{ anaconda_url }}" dest: "/home/{{ user }}" mode: '0770' when: anaconda_installed.rc != 0 - - name: Install anaconda - shell: "/home/{{ user }}/Anaconda3-2022.10-Linux-x86_64.sh -b -u -p /home/{{ user }}/anaconda3" + - name: Install Anaconda + shell: "/home/{{ user }}/{{ anaconda_install_script }} -b -u -p /home/{{ user }}/anaconda3" when: anaconda_installed.rc != 0 - - name: Add anaconda bin to path + - name: Add Anaconda bin to path shell: "echo export PATH=/home/{{ user }}/anaconda3/bin:$PATH >> /etc/profile" when: anaconda_installed.rc != 0 - - name: Conda - read permission for all + - name: Set read permission for all on the Anaconda folder file: path: "/home/{{ user }}/anaconda3" mode: +r @@ -93,7 +88,7 @@ path: /var/log/csle register: log_directory_exists - - name: Creates CSLE log directory + - name: Create the CSLE log directory ansible.builtin.file: path: /var/log/csle state: directory @@ -106,7 +101,7 @@ path: /tmp/csle register: tmp_directory_exists - - name: Creates CSLE tmp directory + - name: Create the CSLE tmp directory ansible.builtin.file: path: /tmp/csle state: directory @@ -114,7 +109,7 @@ owner: "{{ user }}" when: not tmp_directory_exists.stat.exists - - name: Add or modify sudoers configuration + - name: Add or modify the sudoers configuration lineinfile: path: /etc/sudoers line: "{{ user }} ALL= NOPASSWD: /usr/sbin/service docker stop, /usr/sbin/service docker start, /usr/sbin/service docker restart, /usr/sbin/service nginx stop, /usr/sbin/service nginx start, /usr/sbin/service nginx restart, /usr/sbin/service postgresql start, /usr/sbin/service postgresql stop, /usr/sbin/service postgresql restart, /bin/kill, /usr/bin/journalctl -u docker.service -n 100 --no-pager -e" diff --git a/ansible/install_simulation_system.yml b/ansible/install_simulation_system.yml new file mode 100644 index 000000000..4adc4e31c --- /dev/null +++ b/ansible/install_simulation_system.yml @@ -0,0 +1,51 @@ +--- + +- hosts: all + become: true + + tasks: + + - name: List all Conda environments + shell: "/home/{{ user }}/anaconda3/bin/conda env list" + register: conda_env_list + changed_when: false + + - name: Check if the conda environment exists + set_fact: + conda_env_exists: "{{ '{{ conda_environment_name }}' in conda_env_list.stdout }}" + + - name: Create and activate the conda environment if it does not exist + shell: | + "/home/{{ user }}/anaconda3/bin/conda create -y -n {{ conda_environment_name }} python={{ python_version }}" && \ + source "/home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }}" && \ + "/home/{{ user }}/anaconda3/bin/conda" install -y pip + args: + executable: /bin/bash + when: not conda_env_exists + + - name: Add execute permissions on Python install script + file: + path: "/home/{{ user }}/csle/simulation-system/libs/local_install.sh" + mode: +rwx + recurse: yes + + - name: Add execute permissions on Python dev-install script + file: + path: "/home/{{ user }}/csle/simulation-system/libs/local_install_dev.sh" + mode: +rwx + recurse: yes + + - name: Activate the environment if it exist and install CSLE Python libraries + shell: | + source "/home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }}" && \ + cd "/home/{{ user }}/csle/simulation-system/libs" && \ + ./local_install.sh && \ + ./local_install_dev.sh + + - name: Install CSLE simulation environments on the leader node + shell: | + source "/home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }}" && \ + cd "/home/{{ user }}/csle/simulation-system/envs" && \ + make install + args: + executable: /bin/bash diff --git a/ansible/installing_simulation_system.yml b/ansible/installing_simulation_system.yml deleted file mode 100644 index 04e5188a8..000000000 --- a/ansible/installing_simulation_system.yml +++ /dev/null @@ -1,102 +0,0 @@ ---- - -- hosts: all - become: true - - vars: - user: "shahab" - - tasks: - - - name: Check if Conda environment py39 exists - shell: "/home/{{ user }}/anaconda3/bin/conda env list" - register: conda_env_list - changed_when: false - - - name: Check if py39 environment is in the list - set_fact: - py39_env_exists: "{{ 'py39' in conda_env_list.stdout }}" - - - name: Create and activate py39 environment if it does not exist - shell: | - "/home/{{ user }}/anaconda3/bin/conda" create -y -n py39 python=3.9 && \ - source "/home/{{ user }}/anaconda3/bin/activate" py39 && \ - "/home/{{ user }}/anaconda3/bin/conda" install -y pip - args: - executable: /bin/bash - when: not py39_env_exists - - - name: Activate the environment if it exist and install csle-base - shell: | - source "/home/{{ user }}/anaconda3/bin/activate" py39 - "/home/{{ user }}/anaconda3/envs/py39/bin/pip" install csle-base -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-collector - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-collector" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-ryu - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-ryu" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-common - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-common" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-attacker - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-attacker" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-defender - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-defender" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-system-identification - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-system-identification" -# when: py39_env_exists - - - name: Activate the environment if it exist and install gym-csle-stopping-game - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install gym-csle-stopping-game" -# when: py39_env_exists - - - name: Activate the environment if it exist and install gym-csle-apt-game - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install gym-csle-apt-game" -# when: py39_env_exists - - - name: Activate the environment if it exist and install gym-csle-cyborg - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install gym-csle-cyborg" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-agents - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-agents" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-rest-api - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-rest-api" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-cli - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-cli" -# when: py39_env_exists - - - name: Activate the environment if it existi and install csle-cluster - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-cluster" -# when: py39_env_exists - - - name: Activate the environment if it exist and install gym-csle-intrusion-response-game - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install gym-csle-intrusion-response-game" -# when: py39_env_exists - - - name: Activate the environment if it exist and install csle-tolerance - shell: "/home/{{ user }}/anaconda3/envs/py39/bin/pip install csle-tolerance" -# when: py39_env_exists - - - name: Activate the environment if it exist and run last command on the leader node - shell: | - source "/home/{{ user }}/anaconda3/bin/activate" py39 && \ - cd "/home/{{ user }}/csle/simulation-system/envs" && \ - make install - args: - executable: /bin/bash -# when: py39_env_exists diff --git a/ansible/inventory b/ansible/inventory index 9a895550f..53127d9f6 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -1 +1,5 @@ +[leader] 172.31.212.90 + +[worker] + diff --git a/ansible/Nginx_configuration_content.txt b/ansible/nginx_configuration_content.txt similarity index 100% rename from ansible/Nginx_configuration_content.txt rename to ansible/nginx_configuration_content.txt diff --git a/ansible/Nginx_configuration_content_https.txt b/ansible/nginx_configuration_content_https.txt similarity index 100% rename from ansible/Nginx_configuration_content_https.txt rename to ansible/nginx_configuration_content_https.txt diff --git a/ansible/single_command_test.yml b/ansible/single_command_test.yml deleted file mode 100644 index 894e0f47d..000000000 --- a/ansible/single_command_test.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- - -- hosts: all - become: true - - vars: - csle_git_repo_url: "https://github.com/Limmen/csle" - postgresql_url: "https://install.citusdata.com/community/deb.sh" - user: "shahab" - postgres_version: "15" - leader_IP: "172.31.212.90" - - tasks: - - - name: Create clsuter - expect: - command: /bin/bash -c "cd /home/{{ user }}/csle/metastore && make cluster" - responses: - ".*Password.*": "postgres\r" - register: make_cluster_result - - - debug: - var: make_cluster_result diff --git a/docs/_docs/contributing.md b/docs/_docs/contributing.md index cac5b6c2e..216f0456c 100644 --- a/docs/_docs/contributing.md +++ b/docs/_docs/contributing.md @@ -29,8 +29,9 @@ should be on the list but is not): - Kim Hammar, creator and main developer. - Rolf Stadler, technical advisor. - Pontus Johnson, technical advisor. +- Forough Shahab Samani, software development, testing, and documentation. - Antonio Frederico Nesti Lopes, software development. - Jakob Stymne, software development. - Arvid Lagerqvist, software development. - Nils Forsgren, software development. -- Forough Shahab Samani, software development and documentation. \ No newline at end of file +- Bength Roland Pappila, software development. \ No newline at end of file