Merge pull request #2076 from CactuseSecurity/develop

Develop to Main - adding FortiGate internet service support

documentation/revision-history.md

@@ -305,3 +305,9 @@ adding report template format fk and permissions
 - Reporting: new lean export format JSON for resolved and tech reports
 - hotfix FortiGate FortiOS REST importer: removing reference to gw_networking
 - hotfix CPR8x importer: handling of empty section headers
+
+### 6.4.1 02.06.2023 MAIN
+- FortiOS importer: add support for internet services
+
+### 6.4.2 xx.06.2023 DEVELOP
+- CPR8x importer: basic support for inline layers

inventory/group_vars/all.yml

@@ -1,5 +1,5 @@
 ### general settings
-product_version: "6.4"
+product_version: "6.4.1"
 ansible_user: "{{ lookup('env', 'USER') }}"
 ansible_become_method: sudo
 ansible_python_interpreter: /usr/bin/python3

inventory/group_vars/apiserver.yml

@@ -8,7 +8,7 @@ api_hasura_admin_test_password: "not4production"
 api_user_email: "{{ api_user }}@{{ api_network_listening_ip_address }}"
 api_home: "{{ fworch_home }}/api"
 api_hasura_cli_bin: "{{ fworch_home }}/api/bin/hasura"
-api_hasura_version: "v2.24.1"
+api_hasura_version: "v2.26.0"
 api_project_name: api
 api_no_metadata: false
 api_rollback_is_running: false

roles/importer/files/importer/fortiosmanagementREST/fOS_getter.py

@@ -50,14 +50,16 @@ def update_config_with_fortiOS_api_call(config_json, api_url, result_name, show_
     limit = int(limit)
     returned_new_objects = True
     full_result = []
-    while returned_new_objects:
-        range = [offset, limit]        
-        result = fortiOS_api_call(api_url)
-        full_result.extend(result)
-        offset += limit
-        if len(result)<limit:
-            returned_new_objects = False
-
+    result = fortiOS_api_call(api_url)
+    full_result.extend(result)
+    # removing loop for api gets (no limit option in FortiOS API)
+    # while returned_new_objects:
+    #     range = [offset, limit]        
+    # result = fortiOS_api_call(api_url)
+    # full_result.extend(result)
+    #     offset += limit
+    #     if len(result)<limit:
+    #         returned_new_objects = False
     if result_name in config_json:  # data already exists - extend
         config_json[result_name].extend(full_result)
     else:

roles/importer/files/importer/fortiosmanagementREST/fOS_network.py

@@ -120,9 +120,10 @@ def normalize_nwobjects(full_config, config2import, import_id, nw_obj_types, jwt
     original_obj_uid = 'Original'
     orig_obj = create_network_object(import_id=import_id, name=original_obj_name, type='network', ip='0.0.0.0/0',\
         uid=original_obj_uid, zone='global', color='black', comment='"original" network object created by FWO importer for NAT purposes')
-
     full_config['nw_obj_lookup_dict'][original_obj_name] = original_obj_uid
     nw_objects.append(orig_obj)
+
+    resolve_nw_groups(nw_objects)
     config2import.update({'network_objects': nw_objects})
 
 
@@ -145,21 +146,37 @@ def resolve_nw_uid_to_name(uid, nw_objects):
     return 'ERROR: uid "' + uid + '" not found'
 
 
-def add_member_names_for_nw_group(idx, nw_objects):
-    group = nw_objects.pop(idx)
-    if group['obj_member_refs'] == '' or group['obj_member_refs'] == None:
-        #member_names = None
-        #obj_member_refs = None
-        group['obj_member_names'] = None
-        group['obj_member_refs'] = None
-    else:
-        member_names = ''
-        obj_member_refs = group['obj_member_refs'].split(list_delimiter)
-        for ref in obj_member_refs:
-            member_name = resolve_nw_uid_to_name(ref, nw_objects)
-            member_names += member_name + list_delimiter
-        group['obj_member_names'] = member_names[:-1]
-    nw_objects.insert(idx, group)
+def resolve_nw_groups(nw_objects):
+    # add uids (if possible)
+
+    # build helper dict with idx = name
+    helper_dict = {}
+    for obj in nw_objects:
+        helper_dict[obj['obj_name']] = obj['obj_uid']
+
+    for obj in nw_objects:
+        if obj['obj_typ'] == 'group':
+            member_ref_ar = []
+            for member_name in obj['obj_member_names'].split(list_delimiter):
+                member_ref_ar.append(helper_dict[member_name])
+            obj['obj_member_refs'] = list_delimiter.join(member_ref_ar)
+
+
+# def add_member_names_for_nw_group(idx, nw_objects):
+#     group = nw_objects.pop(idx)
+#     if group['obj_member_refs'] == '' or group['obj_member_refs'] == None:
+#         #member_names = None
+#         #obj_member_refs = None
+#         group['obj_member_names'] = None
+#         group['obj_member_refs'] = None
+#     else:
+#         member_names = ''
+#         obj_member_refs = group['obj_member_refs'].split(list_delimiter)
+#         for ref in obj_member_refs:
+#             member_name = resolve_nw_uid_to_name(ref, nw_objects)
+#             member_names += member_name + list_delimiter
+#         group['obj_member_names'] = member_names[:-1]
+#     nw_objects.insert(idx, group)
 
 
 def create_network_object(import_id, name, type, ip, uid, color, comment, zone):

roles/importer/files/importer/fortiosmanagementREST/fOS_rule.py

@@ -14,6 +14,8 @@
 
 
 rule_access_scope_v4 = ['rules']
+rule_access_scope_v6 = []
+
 rule_access_scope = ['rules']
 rule_nat_scope = ['rules_nat']
 rule_scope = rule_access_scope + rule_nat_scope
@@ -45,6 +47,7 @@ def getNatPolicy(sid, fm_api_url, raw_config, adom_name, device, limit):
 
 
 def normalize_access_rules(full_config, config2import, import_id, mgm_details={}, jwt=None):
+    logger = getFwoLogger()
     rules = []
     rule_number = 0
     # rule_number, first_v4, first_v6 = insert_headers(rule_table, first_v6, first_v4, full_config, rules, import_id, localPkgName,src_ref_all,dst_ref_all,rule_number)
@@ -90,10 +93,29 @@ def normalize_access_rules(full_config, config2import, import_id, mgm_details={}
         rule['rule_dst'] = list_delimiter.join([d['name'] for d in rule_orig['dstaddr']])
         rule['rule_svc'] = list_delimiter.join([d['name'] for d in rule_orig['service']])
 
-        # TODO: add ipv6 addresses
-        # rule['rule_src'] = extend_string_list(rule['rule_src'], rule_orig, 'srcaddr6', list_delimiter, jwt=jwt, import_id=import_id)
-        # rule['rule_dst'] = extend_string_list(rule['rule_dst'], rule_orig, 'dstaddr6', list_delimiter, jwt=jwt, import_id=import_id)
+        # handling internet-service rules - no mixed mode between (src/dst) and internet service (src), so overwriting)
+        if 'internet-service-src-name' in rule_orig and len(rule_orig['internet-service-src-name'])>0:
+            rule['rule_src'] = list_delimiter.join([d['name'] for d in rule_orig['internet-service-src-name']])
+            set_service_field_internet_service(rule, config2import, import_id)
+        if 'internet-service-name' in rule_orig and len(rule_orig['internet-service-name'])>0:
+            rule['rule_dst'] = list_delimiter.join([d['name'] for d in rule_orig['internet-service-name']])
+            set_service_field_internet_service(rule, config2import, import_id)
+
+        # add ipv6 addresses
+        rule_src_v6 = [d['name'] for d in rule_orig['srcaddr6']]
+        rule_dst_v6 = [d['name'] for d in rule_orig['dstaddr6']]
+        if len(rule_src_v6)>0:
+            if len(rule['rule_src'])>0:
+                rule['rule_src'] = list_delimiter.join(rule['rule_src'].split(list_delimiter) + rule_src_v6)
+            else:
+                rule['rule_src'] = list_delimiter.join(rule_src_v6)
+        if len(rule_dst_v6)>0:
+            if len(rule['rule_dst'])>0:
+                rule['rule_dst'] = list_delimiter.join(rule['rule_dst'].split(list_delimiter) + rule_dst_v6)
+            else:
+                rule['rule_dst'] = list_delimiter.join(rule_dst_v6)
 
+        # add zone information
         if len(rule_orig['srcintf'])>0:
             src_obj_zone = fOS_zone.add_zone_if_missing (config2import, rule_orig['srcintf'][0]['name'], import_id)
             rule.update({ 'rule_from_zone': src_obj_zone }) # todo: currently only using the first zone
@@ -118,6 +140,18 @@ def normalize_access_rules(full_config, config2import, import_id, mgm_details={}
         rule_number += 1    # nat rules have their own numbering
     config2import.update({'rules': rules})
 
+def set_service_field_internet_service(rule, config2import, import_id):
+    # check if dummy service "Internet Service" already exists and create if not
+    found_internet_service_obj = next((item for item in config2import['service_objects'] if item["svc_name"] == "Internet Service"), None)
+    if found_internet_service_obj is None:
+        config2import['service_objects'].append({
+            'svc_name': 'Internet Service', 'svc_typ': 'group', 'svc_uid': 'Internet Service', 'control_id': import_id
+            })
+
+    # set service to "Internet Service"
+    rule['rule_svc'] = 'Internet Service'
+    rule['rule_svc_refs'] = 'Internet Service'
+
 
 # pure nat rules 
 def normalize_nat_rules(full_config, config2import, import_id, jwt=None):

roles/importer/files/importer/fortiosmanagementREST/fOS_service.py

@@ -84,7 +84,7 @@ def normalize_svcobjects(full_config, config2import, import_id, scope):
                     addObject(svc_objects, 'simple', name, color, 1, None, None, None, import_id, full_config=full_config)
                     added_svc_obj += 1
                 else:
-                    logger.warning("Unknown service found: " + obj_orig['name'] +', proto: ' + obj_orig['protocol'])
+                    logger.warning("Unknown service protocol found: " + obj_orig['name'] +', proto: ' + obj_orig['protocol'])
             elif type == 'group':
                 addObject(svc_objects, type, name, color, 0, None, member_names, session_timeout, import_id, full_config=full_config)
             else:

roles/importer/files/importer/fortiosmanagementREST/fwcommon.py

@@ -18,7 +18,9 @@
 
 
 nw_obj_types = ['firewall/address', 'firewall/address6', 'firewall/addrgrp',
-                'firewall/addrgrp6', 'firewall/ippool', 'firewall/vip']
+                'firewall/addrgrp6', 'firewall/ippool', 'firewall/vip', 
+                'firewall/internet-service', 'firewall/internet-service-group'] 
+                # internet-service is not a service as such but is used as dest (mainly)
 svc_obj_types = ['application/list', 'application/group',
                  # 'application/categories',
                  #'application/custom',