Merge branch 'develop' into fix_#2535

documentation/installer/install-for-testing.md

@@ -12,7 +12,7 @@ This includes:
 Note: the relevant secrets are displayed at the very end of the installation. They can also be found in the etc/secrets directory.
 
 ```console
-ansible-playbook/ site.yml -e "testkeys=yes" -K
+ansible-playbook site.yml -e "testkeys=yes" -K
 ```
 
 A static jwt key helps with debugging c# code in visual studio (code) - you can use a static backend (ldap & api) with these keys.

inventory/hosts.yml

@@ -1,20 +1,20 @@
+# If you want to use distributed installation, follow the steps on localhost
 # 1. ssh-keygen -t rsa
 # Press enter for each line 
 # 2. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
 # 3. chmod og-wx ~/.ssh/authorized_keys
-# 4. change ansible_connection to ssh and ansible_host to your host
+# 4. ssh-copy-id <user>@<remote-server>
+# 5. add remote servers below
 
 all:
   hosts:
     localhost:
       ip_address: 127.0.0.1
       ansible_connection: local
-      # ansible_connection: local
-      # ansible_host: localhost
 #   Add servers for distributed installation here. Use these servers in the children dictionary below.
 #    ui-srv:
 #      ansible_connection: ssh
-#      ansible_host: 192.168.121.2
+#      ansible_host: 192.168.2.3
   children:
 
     frontends:

roles/database/files/sql/creation/fworch-fill-stm.sql

@@ -269,6 +269,7 @@ insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (16,'gsn_handover_group
 insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (17,'voip_sip');
 insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (18,'simple-gateway');
 insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (19,'external-gateway');
+insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (20,'voip');   -- general voip object replacing old specific ones and including CpmiVoipSipDomain
 
 insert into stm_action (action_id,action_name) VALUES (1,'accept'); -- cp, fortinet
 insert into stm_action (action_id,action_name) VALUES (2,'drop'); -- cp

roles/finalize/tasks/main.yml

@@ -77,22 +77,23 @@
   register: demo_data_present
   become: true
   become_user: postgres
+  when: "'sampleserver' in group_names"
 
 - name: find cron jobs in case of missing demo data
   find:
     paths: /etc/cron.d
     patterns: "{{ product_name }}_sample_data_*"
   register: files_to_delete
-  when: demo_data_present.query_result == []
   become: true
+  when: "demo_data_present.query_result == [] and 'sampleserver' in group_names"
 
 - name: delete cron jobs in case of missing demo data
   file:
     path: "{{ item.path }}"
     state: absent
   with_items: "{{ files_to_delete.files }}"
-  when: demo_data_present.query_result == []
   become: true
+  when: "demo_data_present.query_result == [] and 'sampleserver' in group_names"
 
 - name: remove temp importer_password from install host
   file:

roles/importer/files/importer/checkpointR8x/cp_const.py

@@ -27,7 +27,9 @@
 svc_obj_table_names = group_svc_obj_types + simple_svc_obj_types + [ 'CpmiAnyObject' ]
 # usr_obj_table_names : do not exist yet - not fetchable via API
 
-api_obj_types = nw_obj_table_names + svc_obj_table_names  # all obj table names to look at during import
+simple_user_obj_types = ['users']
+
+api_obj_types = nw_obj_table_names + svc_obj_table_names + simple_user_obj_types # all obj table names to look at during import
 
 cp_specific_object_types = [    # used for fetching enrichment data via "get object" separately (no specific API call)
     'simple-gateway', 'simple-cluster', 'CpmiVsClusterNetobj', 'CpmiVsxClusterNetobj', 'CpmiVsxClusterMember', 'CpmiVsNetobj',

roles/importer/files/importer/checkpointR8x/cp_getter.py

@@ -224,7 +224,7 @@ def get_layer_from_api_as_dict (api_v_url, sid, show_params_rules, layerUid=None
                 raise Exception ( "get_nat_rules_from_api - rulebase does not contain to field, get_rulebase_chunk_from_api found garbled json " + str(rulebase))
 
     # adding inline and domain layers (if they exist)
-    add_inline_layers (current_layer_json, api_v_url, sid, show_params_rules)    
+    add_inline_layers (current_layer_json, api_v_url, sid, show_params_rules, nativeConfig=nativeConfig)
 
     return current_layer_json
 
@@ -235,7 +235,7 @@ def add_inline_layers (rulebase, api_v_url, sid, show_params_rules, access_type=
         for chunk in rulebase['layerchunks']:
             if 'rulebase' in chunk:
                 for rules_chunk in chunk['rulebase']:
-                    add_inline_layers(rules_chunk, api_v_url, sid, show_params_rules)
+                    add_inline_layers(rules_chunk, api_v_url, sid, show_params_rules, nativeConfig=nativeConfig)
     else:
         if 'rulebase' in rulebase:
             rulebase_idx = 0

roles/importer/files/importer/checkpointR8x/cp_rule.py

@@ -171,15 +171,10 @@ def parse_single_rule(nativeRule, rulebase, layer_name, import_id, rule_num, par
             rule_time = list_delimiter.join(timeObjects.values())   # only considering the first time object
 
             # starting with the non-chunk objects
-            if 'name' in nativeRule and nativeRule['name'] != '':
-                rule_name = nativeRule['name']
-            else:
-                rule_name = None
+            rule_name = nativeRule.get('name', None)
 
             # new in v8.0.3:
-            rule_custom_fields = None
-            if 'custom-fields' in nativeRule:
-                rule_custom_fields = nativeRule['custom-fields']
+            rule_custom_fields = nativeRule.get('custom-fields', None)
 
             if 'meta-info' in nativeRule and 'last-modifier' in nativeRule['meta-info']:
                 rule_last_change_admin = nativeRule['meta-info']['last-modifier']
@@ -197,17 +192,10 @@ def parse_single_rule(nativeRule, rulebase, layer_name, import_id, rule_num, par
                 parent_rule_uid = None
 
             # new in v5.5.1:
-            if 'rule_type' in nativeRule:
-                rule_type = nativeRule['rule_type']
-            else:
-                rule_type = 'access'
+            rule_type = nativeRule.get('rule_type', 'access')
 
-            if 'comments' in nativeRule:
-                if nativeRule['comments'] == '':
-                    comments = None
-                else:
-                    comments = nativeRule['comments']
-            else:
+            comments = nativeRule.get('comments', None)
+            if comments == '':
                 comments = None
 
             if 'hits' in nativeRule and 'last-date' in nativeRule['hits'] and 'iso-8601' in nativeRule['hits']['last-date']:

roles/importer/files/importer/checkpointR8x/cp_user.py

@@ -3,7 +3,7 @@
 import json
 # from checkpointR8x.cp_getter import ParseUidToName
 
-def collect_users_from_rule(rule, users, objDict):
+def collect_users_from_rule(rule, users): #, objDict):
     if 'rule-number' in rule:  # standard rule
         logger = getFwoLogger()
         if 'type' in rule and rule['type'] != 'place-holder':
@@ -17,18 +17,18 @@ def collect_users_from_rule(rule, users, objDict):
                             user_name = src['name']
                             user_uid = src['uid']
                             user_typ = 'group'
-                            user_comment = src['comments']
+                            user_comment = src.get('comments', None)
                             user_color = src['color']
                             if 'users' in src:
                                 user_typ = 'simple'
                         elif src['type'] == 'LegacyUserAtLocation':
                             user_str = src["name"]
                             user_ar = user_str.split('@')
                             user_name = user_ar[0]
-                            user_uid = src["userGroup"]
+                            user_uid = src.get('userGroup', None)
                             user_typ = 'group'
-                            user_comment = src['comments']
-                            user_color = src['color']
+                            user_comment = src.get('comments', None)
+                            user_color = src.get('color', None)
                         else:
                             break
                         if user_comment == '':
@@ -60,4 +60,15 @@ def collect_users_from_rulebase(rulebase, users):
 def parse_user_objects_from_rulebase(rulebase, users, import_id):
     collect_users_from_rulebase(rulebase, users)
     for user_name in users.keys():
+        # TODO: get user info via API
+        userUid = getUserUidFromCpApi(user_name)
+        # finally add the import id
         users[user_name]['control_id'] = import_id
+
+
+
+def getUserUidFromCpApi (userName):
+    # show-object with UID
+    # dummy implementation returning the name as uid
+    return userName
+

roles/importer/files/importer/checkpointR8x/fwcommon.py

@@ -77,7 +77,7 @@ def get_config(config2import, full_config, current_import_id, mgm_details, limit
 
     cp_network.normalize_network_objects(full_config, config2import, current_import_id, mgm_id=mgm_details['id'])
     cp_service.normalize_service_objects(full_config, config2import, current_import_id)
-    #parse_users_from_rulebases(full_config, full_config['rulebases'], full_config['users'], config2import, current_import_id)
+    parse_users_from_rulebases(full_config, full_config['rulebases'], full_config['users'], config2import, current_import_id)
     config2import.update({'rules':  cp_rule.normalize_rulebases_top_level(full_config, current_import_id, config2import) })
     if not parsing_config_only: # get config from cp fw mgr
         logout_cp("https://" + mgm_details['hostname'] + ":" + str(mgm_details['port']) + "/web_api/", sid)

roles/importer/files/importer/common.py

@@ -365,14 +365,14 @@ def get_config_from_api(importState, full_config_json, config2import, import_tmp
         importState.ErrorString += "  login failed: mgm_id=" + str(importState.MgmDetails.Id) + ", mgm_name=" + importState.MgmDetails.Name + ", " + e.message
         importState.ErrorCount += 1
         logger.error(importState.ErrorString)
-        fwo_api.delete_import(importState.FwoConfig['fwo_api_base_url'], importState) # deleting trace of not even begun import
+        fwo_api.delete_import(importState) # deleting trace of not even begun import
         importState.ErrorCount = fwo_api.complete_import(importState)
         raise FwLoginFailed(e.message)
     except ImportRecursionLimitReached as e:
         importState.ErrorString += "  recursion limit reached: mgm_id=" + str(importState.MgmDetails.Id) + ", mgm_name=" + importState.MgmDetails.Name + ", " + e.message
         importState.ErrorCount += 1
         logger.error(importState.ErrorString)
-        fwo_api.delete_import(importState.FwoConfig['fwo_api_base_url'], importState.Jwt, importState.ImportId) # deleting trace of not even begun import
+        fwo_api.delete_import(importState.Jwt) # deleting trace of not even begun import
         importState.ErrorCount = fwo_api.complete_import(importState)
         raise ImportRecursionLimitReached(e.message)
     except:

roles/importer/files/importer/fwo_api.py

@@ -214,6 +214,7 @@ def readMainKey(filePath=fwo_const.mainKeyFile):
     return mainKey
 
 
+# this mgm field is used by mw dailycheck scheduler
 def log_import_attempt(fwo_api_base_url, jwt, mgm_id, successful=False):
     now = datetime.datetime.now().isoformat()
     query_variables = { "mgmId": mgm_id, "timeStamp": now, "success": successful }

roles/importer/files/importer/import-main-loop.py

@@ -176,7 +176,7 @@ def stop(self):
                             if not skipping and mgm_details["deviceType"]["id"] in (9, 11, 17, 22, 23, 24):  # only handle CPR8x Manager, fortiManager, Cisco MgmCenter, Palo Panorama, Palo FW, FortiOS REST
                                 logger.debug("import-main-loop: starting import of mgm_id=" + id)
                                 try:
-                                    import_result = import_management(mgm_id=id, debug_level_in=debug_level, 
+                                    import_result = import_management(mgmId=id, debug_level_in=debug_level, 
                                         clearManagementData=args.clear, force=args.force, limit=str(api_fetch_limit))
                                 except (FwoApiFailedLockImport, FwLoginFailed):
                                     pass # minor errors for a single mgm, go to next one

roles/lib/files/FWO.Api.Client/Data/WfReqTaskBase.cs

@@ -16,6 +16,7 @@ public struct AdditionalInfoKeys
     {
         public const string ConnId = "ConnId";
         public const string ReqOwner = "ReqOwner";
+        public const string GrpName = "GrpName";
     }
 
     public class WfReqTaskBase : WfTaskBase

roles/lib/files/FWO.Report/ReportAppRules.cs

@@ -132,6 +132,7 @@ private async Task GetAppServers(ApiConnection apiConnection)
                             {
                                 relevantObjects.Add(obj);
                                 found = true;
+                                break;
                             }
                         }
                     }
@@ -220,6 +221,7 @@ private void PrepareFilter(ManagementReport mgt)
                     }
                 }
             }
+            mgt.RelevantObjectIds = mgt.RelevantObjectIds.Distinct().ToList();
             mgt.HighlightedObjectIds = mgt.HighlightedObjectIds.Distinct().ToList();
         }
 

roles/lib/tasks/main.yml

@@ -40,13 +40,23 @@
     - name: install wkhtml pdf library needed for both ui and middleware servers
       include_tasks: install_wkhtml_pdf.yml
 
-    - name: copy {{ product_name }} dotnet lib files to lib target
+    - name: copy {{ product_name }} dotnet lib files to lib target (localhost)
       synchronize:
         src: "./"
         dest: "{{ lib_dir }}"
         rsync_opts:
           - "--chown={{ fworch_user }}:{{ fworch_group }}" 
       tags: [ 'test' ]
+      when: inventory_hostname == 'localhost'
+
+    - name: copy {{ product_name }} dotnet lib files to lib target (remote)
+      copy:
+        src: "./"
+        dest: "{{ lib_dir }}"
+        owner: "{{ fworch_user }}"
+        group: "{{ fworch_group }}"
+      tags: [ 'test' ]
+      when: inventory_hostname != 'localhost'
 
     - name: finalize handler for datarecovery
       set_fact:

roles/test/tasks/main.yml

@@ -56,18 +56,22 @@
 
 - name: database testing
   import_tasks: test-database.yml
+  when: "'databaseserver' in group_names"
 
 - name: create config with test user creds
   import_tasks: write-config-test-user-creds.yml
 
 - name: auth testing
   import_tasks: test-auth.yml
+  when: "'middlewareserver' in group_names"
 
 - name: api testing
   import_tasks: test-api.yml
-
+  when: "'apiserver' in group_names"
+
 - name: csharp testing
   import_tasks: test-csharp.yml
+  when: "'frontends' in group_names"
 
 - name: imorter testing
   import_tasks: test-importer.yml

roles/test/tasks/test-web.yml

@@ -8,9 +8,9 @@
   become: true
   when: "'frontends' in group_names"
 
-- name: test web server availability
+- name: test middleware web server availability
   uri:
-    url: "{{ loop_url }}"
+    url: "{{ middleware_uri }}/swagger/"
     method: GET
     headers:
       Content-Type: html/text
@@ -23,15 +23,53 @@
   until: web_call_result.status == 200
   retries: 10 # 10 * 5 seconds
   delay: 5 # Every 5 seconds
-  loop:
-    - "{{ middleware_uri }}/swagger/"
-    - https://{{ ui_hostname }}/
-    - https://{{ api_network_listening_ip_address }}:9443/api/
-  loop_control:
-    loop_var: loop_url
+  when: "'middlewareserver' in group_names"
 
-- name: show webserver test results
+- name: show middleware webserver test results
   fail:
-    msg: "url: {{ item.url }}, status: {{ item.status }}"
-  loop: "{{ web_call_result.results | from_yaml | list }}"
-  when: item.status!=200
+    msg: "url: {{ web_call_result.url }}, status: {{ web_call_result.status }}"
+  when: "'middlewareserver' in group_names and web_call_result.status!=200"
+
+- name: test api web server availability
+  uri:
+    url: "https://{{ api_network_listening_ip_address }}:9443/api/"
+    method: GET
+    headers:
+      Content-Type: html/text
+    body:
+    validate_certs: false
+    return_content: true
+  register: web_call_result
+  changed_when: false
+  failed_when: false
+  until: web_call_result.status == 200
+  retries: 10 # 10 * 5 seconds
+  delay: 5 # Every 5 seconds
+  when: "'apiserver' in group_names"
+
+- name: show api webserver test results
+  fail:
+    msg: "url: {{ web_call_result.url }}, status: {{ web_call_result.status }}"
+  when: "'apiserver' in group_names and web_call_result.status!=200"
+
+- name: test ui web server availability
+  uri:
+    url: "https://{{ ui_hostname }}/"
+    method: GET
+    headers:
+      Content-Type: html/text
+    body:
+    validate_certs: false
+    return_content: true
+  register: web_call_result
+  changed_when: false
+  failed_when: false
+  until: web_call_result.status == 200
+  retries: 10 # 10 * 5 seconds
+  delay: 5 # Every 5 seconds
+  when: "'frontends' in group_names"
+
+- name: show ui webserver test results
+  fail:
+    msg: "url: {{ web_call_result.url }}, status: {{ web_call_result.status }}"
+  when: "'frontends' in group_names and web_call_result.status!=200"