Merge pull request #2368 from tpurschke/develop

Develop various small improvements
CactuseSecurity · Apr 7, 2024 · 7742e15 · 7742e15
2 parents 2e98871 + ef8c6c1
commit 7742e15
Show file tree

Hide file tree

Showing 195 changed files with 1,675 additions and 255 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -82,7 +82,7 @@
                 "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}"
             },
             "args": [
-                "-m24",
+                "-m5",
                 "-d1",
                 "-f",
                 "-s",
@@ -98,15 +98,17 @@
             ]
         },
         {
-            "name": "py-write-config",
+            "name": "py-customize-FWO-sample",
             "type": "debugpy",
             "request": "launch",
-            "program": "${workspaceFolder}/scripts/customizing/sample-config-change.py",
+            "program": "${workspaceFolder}/scripts/customizing/customizeFwoSampleScript.py",
             "console": "integratedTerminal",
             "env": {
                 "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}"
             },
-            "args": []
+            "args": [
+                "-c${workspaceFolder}/scripts/customizing/sampleCustomSettings.json"
+            ]
         },
         {
             "name": "py-acquire-lock",
@@ -327,13 +329,13 @@
             "name": "c#-FWO Test",
             "type": "coreclr",
             "request": "launch",
-            "preLaunchTask": "build_test",
+            // "preLaunchTask": "build_test",
             "program": "/usr/bin/dotnet",
             "args": [
                 "test"
             ],
             "cwd": "${workspaceFolder}/roles/test/files/FWO.Test",
-            "stopAtEntry": true,
+            "stopAtEntry": false,
             "console": "integratedTerminal"
         },
         {

diff --git a/documentation/revision-history-develop.md b/documentation/revision-history-develop.md
@@ -182,3 +182,14 @@ bugfix release:
 
 # 8.0.2 - 11.03.2024 DEVELOP
 - first version of NSX import module
+
+# 8.0.3 - 08.04.2024 DEVELOP
+- add maintenance page during upgrade
+- sample customizing py script with sample data, closes  Installer customizable config (settings) #2275
+- remove log locking from importer due to stalling importer stops
+- credentials encryption, closes encrypt passwords and keys #1508
+  - breaking change for developer debugging: add the following local file when using -e testkeys=true:
+    /etc/fworch/secrets/main_key with content "not4production..not4production.."
+- add custom (user-defined) fields to import
+  - cp only so far, other fw types missing
+  - user-defined fields are not part of reports yet
diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml
@@ -1,5 +1,5 @@
 ### general settings
-product_version: "8.0.2"
+product_version: "8.0.3"
 ansible_user: "{{ lookup('env', 'USER') }}"
 ansible_become_method: sudo
 ansible_python_interpreter: /usr/bin/python3
@@ -31,6 +31,7 @@ force_install: false
 user_id: 60320
 fworch_user: "{{ product_name }}"
 fworch_group: "{{ fworch_user }}"
+postgres_group: "postgres"
 fworch_parent_dir: "/usr/local"
 fworch_home: "{{ fworch_parent_dir }}/{{ product_name }}"
 fworch_conf_dir: "{{ fworch_home }}/etc"

diff --git a/inventory/group_vars/apiserver.yml b/inventory/group_vars/apiserver.yml
@@ -8,7 +8,7 @@ api_hasura_admin_test_password: "not4production"
 api_user_email: "{{ api_user }}@{{ api_network_listening_ip_address }}"
 api_home: "{{ fworch_home }}/api"
 api_hasura_cli_bin: "{{ fworch_home }}/api/bin/hasura"
-api_hasura_version: "v2.37.0"
+api_hasura_version: "v2.38.1"
 api_project_name: api
 api_no_metadata: false
 api_rollback_is_running: false

diff --git a/inventory/group_vars/databaseserver.yml b/inventory/group_vars/databaseserver.yml
@@ -26,6 +26,7 @@ database_idempotent_files:
   - fworch-grants.sql
   - fworch-import.sql
   - fworch-import-main.sql
+  - fworch-encryption.sql
   - fworch-obj-import.sql
   - fworch-obj-refs.sql
   - fworch-path-analysis.sql

diff --git a/roles/FWO.sln b/roles/FWO.sln
@@ -33,6 +33,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FWO.Mail", "lib\files\FWO.M
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FWO.Recert", "lib\files\FWO.Recert\FWO.Recert.csproj", "{520779B1-20EB-45D9-8A02-D0C4DFEC9302}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "files", "files", "{B48F8BD5-1056-4670-BEFA-F4A260293B6F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FWO.Encryption", "lib\files\FWO.Encryption\FWO.Encryption.csproj", "{6EBEBF57-3399-4008-BA10-0D21F6827244}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FWO.GlobalConstants", "lib\files\FWO.GlobalConstants\FWO.GlobalConstants.csproj", "{0CBD4CC5-3E39-4134-A0E1-4DB8999619F3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -95,6 +101,18 @@ Global
 		{520779B1-20EB-45D9-8A02-D0C4DFEC9302}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{520779B1-20EB-45D9-8A02-D0C4DFEC9302}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{520779B1-20EB-45D9-8A02-D0C4DFEC9302}.Release|Any CPU.Build.0 = Release|Any CPU
+		{14B56819-F629-4CE9-BBC0-7579392EFC83}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{14B56819-F629-4CE9-BBC0-7579392EFC83}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{14B56819-F629-4CE9-BBC0-7579392EFC83}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{14B56819-F629-4CE9-BBC0-7579392EFC83}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6EBEBF57-3399-4008-BA10-0D21F6827244}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6EBEBF57-3399-4008-BA10-0D21F6827244}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6EBEBF57-3399-4008-BA10-0D21F6827244}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6EBEBF57-3399-4008-BA10-0D21F6827244}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0CBD4CC5-3E39-4134-A0E1-4DB8999619F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0CBD4CC5-3E39-4134-A0E1-4DB8999619F3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0CBD4CC5-3E39-4134-A0E1-4DB8999619F3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0CBD4CC5-3E39-4134-A0E1-4DB8999619F3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -111,6 +129,9 @@ Global
 		{C1D1FE54-4CDD-41C0-AABC-415950AA24D5} = {CE55F125-0CD2-4789-A3C1-045DEF33ABA5}
 		{1E7CA417-C64A-4BD9-98D2-5A0A2DD94726} = {CE55F125-0CD2-4789-A3C1-045DEF33ABA5}
 		{520779B1-20EB-45D9-8A02-D0C4DFEC9302} = {CE55F125-0CD2-4789-A3C1-045DEF33ABA5}
+		{B48F8BD5-1056-4670-BEFA-F4A260293B6F} = {CE55F125-0CD2-4789-A3C1-045DEF33ABA5}
+		{6EBEBF57-3399-4008-BA10-0D21F6827244} = {B48F8BD5-1056-4670-BEFA-F4A260293B6F}
+		{0CBD4CC5-3E39-4134-A0E1-4DB8999619F3} = {B48F8BD5-1056-4670-BEFA-F4A260293B6F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {68364621-1011-4D44-9CF5-518F0DC3F459}

diff --git a/roles/api/files/replace_metadata.json b/roles/api/files/replace_metadata.json
@@ -12559,6 +12559,7 @@
                       "rule_num",
                       "rule_to_zone",
                       "track_id",
+                      "rule_custom_fields",
                       "rule_num_numeric",
                       "parent_rule_type",
                       "rule_action",
@@ -12609,6 +12610,7 @@
                       "rule_num",
                       "rule_to_zone",
                       "track_id",
+                      "rule_custom_fields",
                       "rule_num_numeric",
                       "parent_rule_type",
                       "rule_action",
@@ -12659,6 +12661,7 @@
                       "rule_num",
                       "rule_to_zone",
                       "track_id",
+                      "rule_custom_fields",
                       "rule_num_numeric",
                       "parent_rule_type",
                       "rule_action",
@@ -12673,6 +12676,9 @@
                       "rule_track",
                       "rule_uid"
                     ],
+                    "computed_fields": [
+                      "rule_relevant_for_tenant"
+                    ],
                     "filter": {},
                     "allow_aggregations": true
                   }
@@ -12706,6 +12712,7 @@
                       "rule_num",
                       "rule_to_zone",
                       "track_id",
+                      "rule_custom_fields",
                       "rule_num_numeric",
                       "parent_rule_type",
                       "rule_action",
@@ -12774,6 +12781,7 @@
                       "rule_num",
                       "rule_to_zone",
                       "track_id",
+                      "rule_custom_fields",
                       "rule_num_numeric",
                       "parent_rule_type",
                       "rule_action",
@@ -12842,6 +12850,7 @@
                       "rule_num",
                       "rule_to_zone",
                       "track_id",
+                      "rule_custom_fields",
                       "rule_num_numeric",
                       "parent_rule_type",
                       "rule_action",

diff --git a/roles/api/tasks/hasura-install.yml b/roles/api/tasks/hasura-install.yml
@@ -34,9 +34,9 @@
   file:
     path: "{{ fworch_secrets_dir }}"
     state: directory
-    mode: "0700"
+    mode: "0750"
     owner: "{{ fworch_user }}"
-    group: "{{ fworch_group }}"
+    group: "{{ postgres_group }}"
   become: true
 
 - name: set static hasura admin pwd for test purposes only

diff --git a/roles/api/templates/fworch-hasura-docker-api.service.j2 b/roles/api/templates/fworch-hasura-docker-api.service.j2
@@ -1,18 +1,19 @@
 [Unit]
 Description={{ product_name }} API
+Requires=docker.service
 {% if 'databaseserver' in group_names -%}
-After=network.target remote-fs.target nss-lookup.target postgresql.service
+After=network.target remote-fs.target nss-lookup.target postgresql.service docker.service
 {% else -%}
-After=network.target remote-fs.target nss-lookup.target
+After=network.target remote-fs.target nss-lookup.target docker.service
 {%- endif %}
 [Service]
+Restart=on-failure
 WorkingDirectory={{ fworch_home }}
-ExecStartPre=/bin/sleep 10
-ExecStart=/usr/bin/docker start {{ api_container_name }}
+ExecStart=/usr/bin/docker start -a {{ api_container_name }}
+ExecStop=/usr/bin/docker stop -t 2 {{ api_container_name }}
 StandardOutput=journal
 StandardError=journal
 SyslogIdentifier={{ product_name }}-api
 User={{ fworch_user }}
-Environment=
 [Install]
-WantedBy=multi-user.target
+WantedBy=default.target
diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml
@@ -186,8 +186,37 @@
         path: "{{ fworch_home }}/etc/secrets"
         state: directory
         owner: "{{ fworch_user }}"
+        group: "{{ fworch_user }}"
+        # group: "{{ postgres_group }}" # group does not exist yet during install, created in role database
+        mode: '0750'
+      when: "installation_mode == 'new'"
+
+    - name: generate main key
+      set_fact:
+        main_key: "{{ randomly_generated_pwd }}"  # 32 bytes
+        main_key_file: "{{ fworch_secrets_dir }}/main_key"
+      when: testkeys is not defined or testkeys|bool is false
+
+    - name: set static main key for test purposes only
+      set_fact:
+        main_key: "{{ api_hasura_admin_test_password }}..{{ api_hasura_admin_test_password }}.." # to have 32 bytes
+        main_key_file: "{{ fworch_secrets_dir }}/main_key"
+      when: testkeys is defined and testkeys|bool
+
+    - name: Check if main key file exists
+      stat:
+        path: "{{ main_key_file }}"
+      register: stat_result
+
+    - name: write main key to secrets directory
+      copy:
+        content: "{{ main_key }}\n"
+        dest: "{{ main_key_file }}"
+        mode: '0640'
+        owner: "{{ fworch_user }}"
         group: "{{ fworch_group }}"
-        mode: '0700'
+      become: true
+      when: not stat_result.stat.exists
 
     - set_fact:
         wsgi_package_name: "{{ wsgi_package_name }}-py3"

diff --git a/roles/database/files/sql/creation/fworch-create-tables.sql b/roles/database/files/sql/creation/fworch-create-tables.sql
@@ -174,6 +174,7 @@ Create table "rule"
 	"rule_create" BIGINT NOT NULL,
 	"rule_last_seen" BIGINT NOT NULL,
 	"dev_id" Integer,
+	"rule_custom_fields" jsonb,
 	"access_rule" BOOLEAN Default TRUE,
 	"nat_rule" BOOLEAN Default FALSE,
 	"xlate_rule" BIGINT,
@@ -730,6 +731,7 @@ Create table "import_rule"
 	"parent_rule_uid" Text,
 	"rule_type" Varchar Default 'access',
 	"last_hit" Timestamp,
+	"rule_custom_fields" JSONB,
  primary key ("control_id","rule_id")
 );
 

diff --git a/roles/database/files/sql/idempotent/fworch-basic-procs.sql b/roles/database/files/sql/idempotent/fworch-basic-procs.sql
@@ -36,6 +36,19 @@ END;
 $$
 LANGUAGE plpgsql;
 
+CREATE OR REPLACE FUNCTION are_equal (jsonb, jsonb)
+    RETURNS boolean
+    AS $$
+BEGIN
+    IF (($1 IS NULL AND $2 IS NULL) OR $1 = $2) THEN
+        RETURN TRUE;
+    ELSE
+        RETURN FALSE;
+    END IF;
+END;
+$$
+LANGUAGE plpgsql;
+
 CREATE OR REPLACE FUNCTION are_equal (varchar, varchar)
     RETURNS boolean
     AS $$