Merge pull request #2533 from tpurschke/cactus-develop

various cp importer hotfixes

roles/database/files/sql/creation/fworch-fill-stm.sql

@@ -269,6 +269,7 @@ insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (16,'gsn_handover_group
 insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (17,'voip_sip');
 insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (18,'simple-gateway');
 insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (19,'external-gateway');
+insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (20,'voip');   -- general voip object replacing old specific ones and including CpmiVoipSipDomain
 
 insert into stm_action (action_id,action_name) VALUES (1,'accept'); -- cp, fortinet
 insert into stm_action (action_id,action_name) VALUES (2,'drop'); -- cp

roles/importer/files/importer/checkpointR8x/cp_const.py

@@ -27,7 +27,9 @@
 svc_obj_table_names = group_svc_obj_types + simple_svc_obj_types + [ 'CpmiAnyObject' ]
 # usr_obj_table_names : do not exist yet - not fetchable via API
 
-api_obj_types = nw_obj_table_names + svc_obj_table_names  # all obj table names to look at during import
+simple_user_obj_types = ['users']
+
+api_obj_types = nw_obj_table_names + svc_obj_table_names + simple_user_obj_types # all obj table names to look at during import
 
 cp_specific_object_types = [    # used for fetching enrichment data via "get object" separately (no specific API call)
     'simple-gateway', 'simple-cluster', 'CpmiVsClusterNetobj', 'CpmiVsxClusterNetobj', 'CpmiVsxClusterMember', 'CpmiVsNetobj',

roles/importer/files/importer/checkpointR8x/cp_getter.py

@@ -224,7 +224,7 @@ def get_layer_from_api_as_dict (api_v_url, sid, show_params_rules, layerUid=None
                 raise Exception ( "get_nat_rules_from_api - rulebase does not contain to field, get_rulebase_chunk_from_api found garbled json " + str(rulebase))
 
     # adding inline and domain layers (if they exist)
-    add_inline_layers (current_layer_json, api_v_url, sid, show_params_rules)    
+    add_inline_layers (current_layer_json, api_v_url, sid, show_params_rules, nativeConfig=nativeConfig)
 
     return current_layer_json
 
@@ -235,7 +235,7 @@ def add_inline_layers (rulebase, api_v_url, sid, show_params_rules, access_type=
         for chunk in rulebase['layerchunks']:
             if 'rulebase' in chunk:
                 for rules_chunk in chunk['rulebase']:
-                    add_inline_layers(rules_chunk, api_v_url, sid, show_params_rules)
+                    add_inline_layers(rules_chunk, api_v_url, sid, show_params_rules, nativeConfig=nativeConfig)
     else:
         if 'rulebase' in rulebase:
             rulebase_idx = 0

roles/importer/files/importer/checkpointR8x/cp_rule.py

@@ -171,15 +171,10 @@ def parse_single_rule(nativeRule, rulebase, layer_name, import_id, rule_num, par
             rule_time = list_delimiter.join(timeObjects.values())   # only considering the first time object
 
             # starting with the non-chunk objects
-            if 'name' in nativeRule and nativeRule['name'] != '':
-                rule_name = nativeRule['name']
-            else:
-                rule_name = None
+            rule_name = nativeRule.get('name', None)
 
             # new in v8.0.3:
-            rule_custom_fields = None
-            if 'custom-fields' in nativeRule:
-                rule_custom_fields = nativeRule['custom-fields']
+            rule_custom_fields = nativeRule.get('custom-fields', None)
 
             if 'meta-info' in nativeRule and 'last-modifier' in nativeRule['meta-info']:
                 rule_last_change_admin = nativeRule['meta-info']['last-modifier']
@@ -197,17 +192,10 @@ def parse_single_rule(nativeRule, rulebase, layer_name, import_id, rule_num, par
                 parent_rule_uid = None
 
             # new in v5.5.1:
-            if 'rule_type' in nativeRule:
-                rule_type = nativeRule['rule_type']
-            else:
-                rule_type = 'access'
+            rule_type = nativeRule.get('rule_type', 'access')
 
-            if 'comments' in nativeRule:
-                if nativeRule['comments'] == '':
-                    comments = None
-                else:
-                    comments = nativeRule['comments']
-            else:
+            comments = nativeRule.get('comments', None)
+            if comments == '':
                 comments = None
 
             if 'hits' in nativeRule and 'last-date' in nativeRule['hits'] and 'iso-8601' in nativeRule['hits']['last-date']:

roles/importer/files/importer/checkpointR8x/cp_user.py

@@ -3,7 +3,7 @@
 import json
 # from checkpointR8x.cp_getter import ParseUidToName
 
-def collect_users_from_rule(rule, users, objDict):
+def collect_users_from_rule(rule, users): #, objDict):
     if 'rule-number' in rule:  # standard rule
         logger = getFwoLogger()
         if 'type' in rule and rule['type'] != 'place-holder':
@@ -17,18 +17,18 @@ def collect_users_from_rule(rule, users, objDict):
                             user_name = src['name']
                             user_uid = src['uid']
                             user_typ = 'group'
-                            user_comment = src['comments']
+                            user_comment = src.get('comments', None)
                             user_color = src['color']
                             if 'users' in src:
                                 user_typ = 'simple'
                         elif src['type'] == 'LegacyUserAtLocation':
                             user_str = src["name"]
                             user_ar = user_str.split('@')
                             user_name = user_ar[0]
-                            user_uid = src["userGroup"]
+                            user_uid = src.get('userGroup', None)
                             user_typ = 'group'
-                            user_comment = src['comments']
-                            user_color = src['color']
+                            user_comment = src.get('comments', None)
+                            user_color = src.get('color', None)
                         else:
                             break
                         if user_comment == '':
@@ -60,4 +60,15 @@ def collect_users_from_rulebase(rulebase, users):
 def parse_user_objects_from_rulebase(rulebase, users, import_id):
     collect_users_from_rulebase(rulebase, users)
     for user_name in users.keys():
+        # TODO: get user info via API
+        userUid = getUserUidFromCpApi(user_name)
+        # finally add the import id
         users[user_name]['control_id'] = import_id
+
+
+
+def getUserUidFromCpApi (userName):
+    # show-object with UID
+    # dummy implementation returning the name as uid
+    return userName
+

roles/importer/files/importer/checkpointR8x/fwcommon.py

@@ -77,7 +77,7 @@ def get_config(config2import, full_config, current_import_id, mgm_details, limit
 
     cp_network.normalize_network_objects(full_config, config2import, current_import_id, mgm_id=mgm_details['id'])
     cp_service.normalize_service_objects(full_config, config2import, current_import_id)
-    #parse_users_from_rulebases(full_config, full_config['rulebases'], full_config['users'], config2import, current_import_id)
+    parse_users_from_rulebases(full_config, full_config['rulebases'], full_config['users'], config2import, current_import_id)
     config2import.update({'rules':  cp_rule.normalize_rulebases_top_level(full_config, current_import_id, config2import) })
     if not parsing_config_only: # get config from cp fw mgr
         logout_cp("https://" + mgm_details['hostname'] + ":" + str(mgm_details['port']) + "/web_api/", sid)

roles/importer/files/importer/common.py

@@ -365,14 +365,14 @@ def get_config_from_api(importState, full_config_json, config2import, import_tmp
         importState.ErrorString += "  login failed: mgm_id=" + str(importState.MgmDetails.Id) + ", mgm_name=" + importState.MgmDetails.Name + ", " + e.message
         importState.ErrorCount += 1
         logger.error(importState.ErrorString)
-        fwo_api.delete_import(importState.FwoConfig['fwo_api_base_url'], importState) # deleting trace of not even begun import
+        fwo_api.delete_import(importState) # deleting trace of not even begun import
         importState.ErrorCount = fwo_api.complete_import(importState)
         raise FwLoginFailed(e.message)
     except ImportRecursionLimitReached as e:
         importState.ErrorString += "  recursion limit reached: mgm_id=" + str(importState.MgmDetails.Id) + ", mgm_name=" + importState.MgmDetails.Name + ", " + e.message
         importState.ErrorCount += 1
         logger.error(importState.ErrorString)
-        fwo_api.delete_import(importState.FwoConfig['fwo_api_base_url'], importState.Jwt, importState.ImportId) # deleting trace of not even begun import
+        fwo_api.delete_import(importState.Jwt) # deleting trace of not even begun import
         importState.ErrorCount = fwo_api.complete_import(importState)
         raise ImportRecursionLimitReached(e.message)
     except:

roles/importer/files/importer/fwo_api.py

@@ -214,6 +214,7 @@ def readMainKey(filePath=fwo_const.mainKeyFile):
     return mainKey
 
 
+# this mgm field is used by mw dailycheck scheduler
 def log_import_attempt(fwo_api_base_url, jwt, mgm_id, successful=False):
     now = datetime.datetime.now().isoformat()
     query_variables = { "mgmId": mgm_id, "timeStamp": now, "success": successful }