Merge branch 'master' into import

.github/workflows/python-csle-rest-api-build.yml

@@ -0,0 +1,35 @@
+name: python-csle-rest-api-build
+run-name: ${{ github.actor }} python-csle-rest-api-build
+on:
+  push:
+    branches:
+      - "master"
+#  pull_request:
+#    branches:
+#      - "master"
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9']
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install python build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox tox-gh-actions
+      - name: Tox tests csle-rest-api
+        run: cd simulation-system/libs/csle-rest-api; tox
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          files: simulation-system/libs/csle-rest-api/coverage.yml

README.md

@@ -131,6 +131,10 @@ features. We currently support each release for a window of 6 months.
 <img src="https://upload.wikimedia.org/wikipedia/commons/3/35/Tux.svg" width="7%" height="7%" style="margin-left:70px;"/>
 </p>
 
+## Datasets
+
+A dataset of 6400 intrusion traces can be found [here](https://zenodo.org/records/10234379).
+
 ## Maintainer
 
 <table>
@@ -168,6 +172,7 @@ Thanks go to these people!
     </tr>
     <tr>
        <td align="center" valign="top" width="14.28%"><a href="https://github.com/kingxiaofire"><img src="https://github.com/kingxiaofire.png" width="100px;" alt="Yan Wang"/><br /><sub><b>Yan Wang</b></sub></a></td>
+       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Awsnaser"><img src="https://github.com/Awsnaser.png" width="100px;" alt="Aws Jaber"/><br /><sub><b>Aws Jaber</b></sub></a></td>
     </tr>
   </tbody>
 </table>

docs/_docs/contributing.md

@@ -36,4 +36,5 @@ should be on the list but is not):
 - Nils Forsgren, software development.
 - Bength Roland Pappila, software development.
 - Yu Hu, software development.
-- Yan Wang, software development.
+- Yan Wang, software development.
+- Aws Jaber, software development.

examples/eval/cyborg_scenario_two/eval_on_base_env.py

@@ -13,22 +13,29 @@
         maximum_steps=100, red_agent_distribution=[1.0], reduced_action_space=True, decoy_state=True,
         scanned_state=True, decoy_optimization=False, cache_visited_states=False)
     csle_cyborg_env = CyborgScenarioTwoDefender(config=config)
-    num_evaluations = 10000
-    max_horizon = 100
+    num_evaluations = 1
+    max_horizon = 20
     returns = []
     seed = 215125
     random.seed(seed)
     np.random.seed(seed)
     torch.manual_seed(seed)
-    print("Starting policy evaluation")
+    # print(csle_cyborg_env.action_id_to_type_and_host)
+    # import sys
+    # sys.exit(0)
+    # print("Starting policy evaluation")
     for i in range(num_evaluations):
         o, _ = csle_cyborg_env.reset()
         R = 0
         t = 0
         while t < max_horizon:
-            a = ppo_policy.action(o=o)
+            # a = ppo_policy.action(o=o)
+            a = 4
             o, r, done, _, info = csle_cyborg_env.step(a)
+            table = csle_cyborg_env.get_true_table()
+            print(table)
+            print(r)
             R += r
             t += 1
         returns.append(R)
-        print(f"{i}/{num_evaluations}, avg R: {np.mean(returns)}, R: {R}")
+        # print(f"{i}/{num_evaluations}, avg R: {np.mean(returns)}, R: {R}")

examples/eval/cyborg_scenario_two/evaluate_on_wrapper_env.py

@@ -3,12 +3,14 @@
 import random
 from csle_common.metastore.metastore_facade import MetastoreFacade
 from gym_csle_cyborg.envs.cyborg_scenario_two_wrapper import CyborgScenarioTwoWrapper
+from gym_csle_cyborg.dao.red_agent_type import RedAgentType
 from gym_csle_cyborg.dao.csle_cyborg_wrapper_config import CSLECyborgWrapperConfig
 
 if __name__ == '__main__':
     ppo_policy = MetastoreFacade.get_ppo_policy(id=58)
     config = CSLECyborgWrapperConfig(maximum_steps=100, gym_env_name="",
-                                     save_trace=False, reward_shaping=False, scenario=2)
+                                     save_trace=False, reward_shaping=False, scenario=2,
+                                     red_agent_type=RedAgentType.B_LINE_AGENT)
     env = CyborgScenarioTwoWrapper(config=config)
     num_evaluations = 10000
     max_horizon = 100
@@ -25,6 +27,7 @@
         while t < max_horizon:
             a = ppo_policy.action(o=o)
             o, r, done, _, info = env.step(a)
+            env.show
             R += r
             t += 1
         returns.append(R)

management-system/csle-mgmt-webapp/package.json

@@ -62,6 +62,7 @@
   "devDependencies": {
     "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
     "@testing-library/react": "^14.0.0",
+    "ajv": "^7.2.4",
     "eslint": "^8.28.0",
     "eslint-config-prettier": "^8.5.0",
     "eslint-config-standard": "^17.0.0",

simulation-system/libs/csle-common/tests/test_cluster_util.py

@@ -0,0 +1,140 @@
+import os
+from unittest.mock import patch, MagicMock
+import csle_common.constants.constants as constants
+from csle_common.util.cluster_util import ClusterUtil
+
+
+class TestClusterUtilSuite:
+    """
+    Test suite for cluster util
+    """
+
+    def test_am_i_leader(self) -> None:
+        """
+        Test function that checks if a given IP is a leader or not
+        """
+        config = MagicMock()
+        config.cluster_config.cluster_nodes.ip = "192.168.1.1"
+        config.cluster_config.cluster_nodes.leader = False
+        config.cluster_config.cluster_nodes.cpus = 1
+        config.cluster_config.cluster_nodes.gpus = 2
+        config.cluster_config.cluster_nodes.RAM = 8
+        assert not ClusterUtil.am_i_leader("192.168.1.1", config)
+
+    @patch("psycopg.connect")
+    @patch("csle_common.util.cluster_util.ClusterUtil.get_config")
+    @patch.dict(os.environ, {"CSLE_HOME": "/mock/csle/home"})
+    def test_get_config(self, mock_get_config, mock_connect) -> None:
+        """
+        Test function that gets the current cluster config from the metastore or from disk
+        depending on if it is the leader node or not
+
+        :param mock_get_config: mock_get_config
+        :param mock_connect: mock_connect
+
+        :return: None
+        """
+        mock_config = MagicMock()
+        mock_get_config.return_value = mock_config
+        from csle_common.util.cluster_util import ClusterUtil
+
+        result = ClusterUtil.get_config()
+        assert result == mock_config
+
+    @patch.dict(os.environ, {"CSLE_HOME": "/mock/csle/home"})
+    @patch("csle_common.util.cluster_util.ClusterUtil.get_config")
+    def test_set_config_parameters_from_config_file(self, mock_get_config) -> None:
+        """
+        Test function that reads the config file from $CSLE_HOME/config.json and initializes certain config parameters
+
+        :param mock_get_config: mock_get_config
+
+        :return: None
+        """
+        mock_config = MagicMock()
+        mock_config.management_admin_username_default = "admin"
+        mock_config.management_admin_password_default = "admin_pw"
+        mock_config.management_admin_email_default = "[email protected]"
+        mock_config.management_admin_organization_default = "Example Org"
+        mock_config.management_admin_first_name_default = "Admin"
+        mock_config.management_admin_last_name_default = "User"
+        mock_config.ssh_admin_username = "ssh_admin"
+        mock_config.ssh_admin_password = "ssh_pw"
+        mock_config.management_guest_username_default = "guest"
+        mock_config.management_guest_password_default = "guest_pw"
+        mock_config.management_guest_email_default = "[email protected]"
+        mock_config.management_guest_organization_default = "Example Org"
+        mock_config.management_guest_first_name_default = "Guest"
+        mock_config.management_guest_last_name_default = "User"
+        mock_config.ssh_agent_username = "agent"
+        mock_config.ssh_agent_password = "agent_pw"
+        mock_config.metastore_user = "metastore_user"
+        mock_config.metastore_password = "metastore_pw"
+        mock_config.metastore_database_name = "metastore_db"
+        mock_config.pgadmin_username = "pgadmin"
+        mock_config.pgadmin_password = "pgadmin_pw"
+        mock_config.grafana_username = "grafana"
+        mock_config.grafana_password = "grafana_pw"
+        mock_config.node_exporter_port = 9100
+        mock_config.grafana_port = 3000
+        mock_config.management_system_port = 8000
+        mock_config.cadvisor_port = 8080
+        mock_config.pgadmin_port = 5050
+        mock_config.prometheus_port = 9090
+        mock_config.node_exporter_pid_file = "/var/run/node_exporter.pid"
+        mock_config.csle_mgmt_webapp_pid_file = "/var/run/csle_mgmt_webapp.pid"
+        mock_config.node_exporter_log_file = "/var/log/node_exporter.log"
+        mock_config.docker_stats_manager_outfile = "/var/log/docker_stats.out"
+        mock_config.docker_stats_manager_pidfile = "/var/run/docker_stats.pid"
+        mock_config.prometheus_pid_file = "/var/run/prometheus.pid"
+        mock_config.prometheus_log_file = "/var/log/prometheus.log"
+        mock_config.postgresql_log_dir = "/var/log/postgresql"
+        mock_config.nginx_log_dir = "/var/log/nginx"
+        mock_config.flask_log_file = "/var/log/flask.log"
+        mock_config.default_log_dir = "/var/log"
+
+        mock_get_config.return_value = mock_config
+
+        ClusterUtil.set_config_parameters_from_config_file()
+
+        assert constants.CONFIG_FILE.PARSED_CONFIG == mock_config
+        assert constants.CSLE_ADMIN.MANAGEMENT_USER == "admin"
+        assert constants.CSLE_ADMIN.MANAGEMENT_PW == "admin_pw"
+        assert constants.CSLE_ADMIN.MANAGEMENT_EMAIL == "[email protected]"
+        assert constants.CSLE_ADMIN.MANAGEMENT_ORGANIZATION == "Example Org"
+        assert constants.CSLE_ADMIN.MANAGEMENT_FIRST_NAME == "Admin"
+        assert constants.CSLE_ADMIN.MANAGEMENT_LAST_NAME == "User"
+        assert constants.CSLE_ADMIN.SSH_USER == "ssh_admin"
+        assert constants.CSLE_ADMIN.SSH_PW == "ssh_pw"
+        assert constants.CSLE_GUEST.MANAGEMENT_USER == "guest"
+        assert constants.CSLE_GUEST.MANAGEMENT_PW == "guest_pw"
+        assert constants.CSLE_GUEST.MANAGEMENT_EMAIL == "[email protected]"
+        assert constants.CSLE_GUEST.MANAGEMENT_ORGANIZATION == "Example Org"
+        assert constants.CSLE_GUEST.MANAGEMENT_FIRST_NAME == "Guest"
+        assert constants.CSLE_GUEST.MANAGEMENT_LAST_NAME == "User"
+        assert constants.AGENT.USER == "agent"
+        assert constants.AGENT.PW == "agent_pw"
+        assert constants.METADATA_STORE.USER == "metastore_user"
+        assert constants.METADATA_STORE.PASSWORD == "metastore_pw"
+        assert constants.METADATA_STORE.DBNAME == "metastore_db"
+        assert constants.COMMANDS.PGADMIN_USERNAME == "pgadmin"
+        assert constants.COMMANDS.PGADMIN_PW == "pgadmin_pw"
+        assert constants.COMMANDS.GRAFANA_USERNAME == "grafana"
+        assert constants.COMMANDS.GRAFANA_PW == "grafana_pw"
+        assert constants.COMMANDS.NODE_EXPORTER_PORT == 9100
+        assert constants.COMMANDS.GRAFANA_PORT == 3000
+        assert constants.COMMANDS.MANAGEMENT_SYSTEM_PORT == 8000
+        assert constants.COMMANDS.CADVISOR_PORT == 8080
+        assert constants.COMMANDS.PGADMIN_PORT == 5050
+        assert constants.COMMANDS.PROMETHEUS_PORT == 9090
+        assert constants.COMMANDS.NODE_EXPORTER_PID_FILE == "/var/run/node_exporter.pid"
+        assert constants.COMMANDS.CSLE_MGMT_WEBAPP_PID_FILE == "/var/run/csle_mgmt_webapp.pid"
+        assert constants.COMMANDS.NODE_EXPORTER_LOG_FILE == "/var/log/node_exporter.log"
+        assert constants.COMMANDS.DOCKER_STATS_MANAGER_OUTFILE == "/var/log/docker_stats.out"
+        assert constants.COMMANDS.DOCKER_STATS_MANAGER_PIDFILE == "/var/run/docker_stats.pid"
+        assert constants.COMMANDS.PROMETHEUS_PID_FILE == "/var/run/prometheus.pid"
+        assert constants.COMMANDS.PROMETHEUS_LOG_FILE == "/var/log/prometheus.log"
+        assert constants.COMMANDS.POSTGRESQL_LOG_DIR == "/var/log/postgresql"
+        assert constants.COMMANDS.NGINX_LOG_DIR == "/var/log/nginx"
+        assert constants.COMMANDS.FLASK_LOG_FILE == "/var/log/flask.log"
+        assert constants.LOGGING.DEFAULT_LOG_DIR == "/var/log"