From 090fb863ca2779b685f34c12fa8472671a7cdb3e Mon Sep 17 00:00:00 2001 From: Tim Purschke Date: Fri, 13 Sep 2024 16:13:42 +0200 Subject: [PATCH 1/2] fix voip domain handling in cp parser --- roles/importer/files/importer/checkpointR8x/cp_getter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/importer/files/importer/checkpointR8x/cp_getter.py b/roles/importer/files/importer/checkpointR8x/cp_getter.py index b11c78bd1..19e834822 100644 --- a/roles/importer/files/importer/checkpointR8x/cp_getter.py +++ b/roles/importer/files/importer/checkpointR8x/cp_getter.py @@ -224,7 +224,7 @@ def get_layer_from_api_as_dict (api_v_url, sid, show_params_rules, layerUid=None raise Exception ( "get_nat_rules_from_api - rulebase does not contain to field, get_rulebase_chunk_from_api found garbled json " + str(rulebase)) # adding inline and domain layers (if they exist) - add_inline_layers (current_layer_json, api_v_url, sid, show_params_rules) + add_inline_layers (current_layer_json, api_v_url, sid, show_params_rules, nativeConfig=nativeConfig) return current_layer_json @@ -235,7 +235,7 @@ def add_inline_layers (rulebase, api_v_url, sid, show_params_rules, access_type= for chunk in rulebase['layerchunks']: if 'rulebase' in chunk: for rules_chunk in chunk['rulebase']: - add_inline_layers(rules_chunk, api_v_url, sid, show_params_rules) + add_inline_layers(rules_chunk, api_v_url, sid, show_params_rules, nativeConfig=nativeConfig) else: if 'rulebase' in rulebase: rulebase_idx = 0 From e93edff3b6457f5f43185bff1c3f2ad018070f91 Mon Sep 17 00:00:00 2001 From: Tim Purschke Date: Sat, 14 Sep 2024 12:16:21 +0200 Subject: [PATCH 2/2] user parsing re-added (not includling legacy user@localtion) --- .../files/sql/creation/fworch-fill-stm.sql | 1 + .../files/importer/checkpointR8x/cp_const.py | 4 +++- .../files/importer/checkpointR8x/cp_rule.py | 22 +++++-------------- .../files/importer/checkpointR8x/cp_user.py | 21 +++++++++++++----- .../files/importer/checkpointR8x/fwcommon.py | 2 +- roles/importer/files/importer/common.py | 4 ++-- roles/importer/files/importer/fwo_api.py | 1 + 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/roles/database/files/sql/creation/fworch-fill-stm.sql b/roles/database/files/sql/creation/fworch-fill-stm.sql index 412c45ea8..79fd1c728 100644 --- a/roles/database/files/sql/creation/fworch-fill-stm.sql +++ b/roles/database/files/sql/creation/fworch-fill-stm.sql @@ -269,6 +269,7 @@ insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (16,'gsn_handover_group insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (17,'voip_sip'); insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (18,'simple-gateway'); insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (19,'external-gateway'); +insert into stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (20,'voip'); -- general voip object replacing old specific ones and including CpmiVoipSipDomain insert into stm_action (action_id,action_name) VALUES (1,'accept'); -- cp, fortinet insert into stm_action (action_id,action_name) VALUES (2,'drop'); -- cp diff --git a/roles/importer/files/importer/checkpointR8x/cp_const.py b/roles/importer/files/importer/checkpointR8x/cp_const.py index 07898be7e..013be4012 100644 --- a/roles/importer/files/importer/checkpointR8x/cp_const.py +++ b/roles/importer/files/importer/checkpointR8x/cp_const.py @@ -27,7 +27,9 @@ svc_obj_table_names = group_svc_obj_types + simple_svc_obj_types + [ 'CpmiAnyObject' ] # usr_obj_table_names : do not exist yet - not fetchable via API -api_obj_types = nw_obj_table_names + svc_obj_table_names # all obj table names to look at during import +simple_user_obj_types = ['users'] + +api_obj_types = nw_obj_table_names + svc_obj_table_names + simple_user_obj_types # all obj table names to look at during import cp_specific_object_types = [ # used for fetching enrichment data via "get object" separately (no specific API call) 'simple-gateway', 'simple-cluster', 'CpmiVsClusterNetobj', 'CpmiVsxClusterNetobj', 'CpmiVsxClusterMember', 'CpmiVsNetobj', diff --git a/roles/importer/files/importer/checkpointR8x/cp_rule.py b/roles/importer/files/importer/checkpointR8x/cp_rule.py index 9c7f2b585..d34b0ded8 100644 --- a/roles/importer/files/importer/checkpointR8x/cp_rule.py +++ b/roles/importer/files/importer/checkpointR8x/cp_rule.py @@ -171,15 +171,10 @@ def parse_single_rule(nativeRule, rulebase, layer_name, import_id, rule_num, par rule_time = list_delimiter.join(timeObjects.values()) # only considering the first time object # starting with the non-chunk objects - if 'name' in nativeRule and nativeRule['name'] != '': - rule_name = nativeRule['name'] - else: - rule_name = None + rule_name = nativeRule.get('name', None) # new in v8.0.3: - rule_custom_fields = None - if 'custom-fields' in nativeRule: - rule_custom_fields = nativeRule['custom-fields'] + rule_custom_fields = nativeRule.get('custom-fields', None) if 'meta-info' in nativeRule and 'last-modifier' in nativeRule['meta-info']: rule_last_change_admin = nativeRule['meta-info']['last-modifier'] @@ -197,17 +192,10 @@ def parse_single_rule(nativeRule, rulebase, layer_name, import_id, rule_num, par parent_rule_uid = None # new in v5.5.1: - if 'rule_type' in nativeRule: - rule_type = nativeRule['rule_type'] - else: - rule_type = 'access' + rule_type = nativeRule.get('rule_type', 'access') - if 'comments' in nativeRule: - if nativeRule['comments'] == '': - comments = None - else: - comments = nativeRule['comments'] - else: + comments = nativeRule.get('comments', None) + if comments == '': comments = None if 'hits' in nativeRule and 'last-date' in nativeRule['hits'] and 'iso-8601' in nativeRule['hits']['last-date']: diff --git a/roles/importer/files/importer/checkpointR8x/cp_user.py b/roles/importer/files/importer/checkpointR8x/cp_user.py index 9180d5745..099a47991 100644 --- a/roles/importer/files/importer/checkpointR8x/cp_user.py +++ b/roles/importer/files/importer/checkpointR8x/cp_user.py @@ -3,7 +3,7 @@ import json # from checkpointR8x.cp_getter import ParseUidToName -def collect_users_from_rule(rule, users, objDict): +def collect_users_from_rule(rule, users): #, objDict): if 'rule-number' in rule: # standard rule logger = getFwoLogger() if 'type' in rule and rule['type'] != 'place-holder': @@ -17,7 +17,7 @@ def collect_users_from_rule(rule, users, objDict): user_name = src['name'] user_uid = src['uid'] user_typ = 'group' - user_comment = src['comments'] + user_comment = src.get('comments', None) user_color = src['color'] if 'users' in src: user_typ = 'simple' @@ -25,10 +25,10 @@ def collect_users_from_rule(rule, users, objDict): user_str = src["name"] user_ar = user_str.split('@') user_name = user_ar[0] - user_uid = src["userGroup"] + user_uid = src.get('userGroup', None) user_typ = 'group' - user_comment = src['comments'] - user_color = src['color'] + user_comment = src.get('comments', None) + user_color = src.get('color', None) else: break if user_comment == '': @@ -60,4 +60,15 @@ def collect_users_from_rulebase(rulebase, users): def parse_user_objects_from_rulebase(rulebase, users, import_id): collect_users_from_rulebase(rulebase, users) for user_name in users.keys(): + # TODO: get user info via API + userUid = getUserUidFromCpApi(user_name) + # finally add the import id users[user_name]['control_id'] = import_id + + + +def getUserUidFromCpApi (userName): + # show-object with UID + # dummy implementation returning the name as uid + return userName + diff --git a/roles/importer/files/importer/checkpointR8x/fwcommon.py b/roles/importer/files/importer/checkpointR8x/fwcommon.py index d7311fe4f..ff6c4981b 100644 --- a/roles/importer/files/importer/checkpointR8x/fwcommon.py +++ b/roles/importer/files/importer/checkpointR8x/fwcommon.py @@ -77,7 +77,7 @@ def get_config(config2import, full_config, current_import_id, mgm_details, limit cp_network.normalize_network_objects(full_config, config2import, current_import_id, mgm_id=mgm_details['id']) cp_service.normalize_service_objects(full_config, config2import, current_import_id) - #parse_users_from_rulebases(full_config, full_config['rulebases'], full_config['users'], config2import, current_import_id) + parse_users_from_rulebases(full_config, full_config['rulebases'], full_config['users'], config2import, current_import_id) config2import.update({'rules': cp_rule.normalize_rulebases_top_level(full_config, current_import_id, config2import) }) if not parsing_config_only: # get config from cp fw mgr logout_cp("https://" + mgm_details['hostname'] + ":" + str(mgm_details['port']) + "/web_api/", sid) diff --git a/roles/importer/files/importer/common.py b/roles/importer/files/importer/common.py index 22afeddeb..207d67369 100644 --- a/roles/importer/files/importer/common.py +++ b/roles/importer/files/importer/common.py @@ -365,14 +365,14 @@ def get_config_from_api(importState, full_config_json, config2import, import_tmp importState.ErrorString += " login failed: mgm_id=" + str(importState.MgmDetails.Id) + ", mgm_name=" + importState.MgmDetails.Name + ", " + e.message importState.ErrorCount += 1 logger.error(importState.ErrorString) - fwo_api.delete_import(importState.FwoConfig['fwo_api_base_url'], importState) # deleting trace of not even begun import + fwo_api.delete_import(importState) # deleting trace of not even begun import importState.ErrorCount = fwo_api.complete_import(importState) raise FwLoginFailed(e.message) except ImportRecursionLimitReached as e: importState.ErrorString += " recursion limit reached: mgm_id=" + str(importState.MgmDetails.Id) + ", mgm_name=" + importState.MgmDetails.Name + ", " + e.message importState.ErrorCount += 1 logger.error(importState.ErrorString) - fwo_api.delete_import(importState.FwoConfig['fwo_api_base_url'], importState.Jwt, importState.ImportId) # deleting trace of not even begun import + fwo_api.delete_import(importState.Jwt) # deleting trace of not even begun import importState.ErrorCount = fwo_api.complete_import(importState) raise ImportRecursionLimitReached(e.message) except: diff --git a/roles/importer/files/importer/fwo_api.py b/roles/importer/files/importer/fwo_api.py index 2273a6885..618d79e49 100644 --- a/roles/importer/files/importer/fwo_api.py +++ b/roles/importer/files/importer/fwo_api.py @@ -214,6 +214,7 @@ def readMainKey(filePath=fwo_const.mainKeyFile): return mainKey +# this mgm field is used by mw dailycheck scheduler def log_import_attempt(fwo_api_base_url, jwt, mgm_id, successful=False): now = datetime.datetime.now().isoformat() query_variables = { "mgmId": mgm_id, "timeStamp": now, "success": successful }