Merge pull request #2355 from tpurschke/nsx

Nsx - first draft NSX import module

.vscode/launch.json

@@ -82,7 +82,7 @@
                 "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}"
             },
             "args": [
-                "-m7",
+                "-m24",
                 "-d1",
                 "-f",
                 "-s",

documentation/installer/basic-installation.md

@@ -33,13 +33,14 @@ git clone https://github.com/CactuseSecurity/firewall-orchestrator.git
 3) Ansible installation
 
 Make sure you have ansible version 2.13 or above installed on your system (check with "ansible --version"). 
-If this is not the case, install a newer ansible. One possible way is to run the four commands of the following script (and entering your sudo password) - run them separately if the script :
+If this is not the case, install a newer ansible. One possible way is to run the following script:
+
         cd firewall-orchestrator
         source scripts/install-ansible-from-venv.sh
 
 Note that if your server is behind a proxy, you will have to set the proxy for pip as follows (to allow for ansible venv download):
 
-         pip config set global.proxy http://proxy:3128
+         pip config set global.proxy http://YOUR-PROXY-NAME:YOUR-PROXY-PORT
 
 4) Firewall Orchestrator installation
 

documentation/revision-history-develop.md

@@ -179,3 +179,6 @@ bugfix release:
 # 8.0.1 - 20.02.2024 DEVELOP
 - iconify modelling
 - add missing config values
+
+# 8.0.2 - 29.02.2024 DEVELOP
+- adding firewall importer support for NSX

inventory/group_vars/all.yml

@@ -1,5 +1,5 @@
 ### general settings
-product_version: "8.0.1"
+product_version: "8.0.2"
 ansible_user: "{{ lookup('env', 'USER') }}"
 ansible_become_method: sudo
 ansible_python_interpreter: /usr/bin/python3

roles/database/files/sql/creation/fworch-fill-stm.sql

@@ -356,6 +356,11 @@ insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufac
     VALUES (24,'FortiOS Management','REST','Fortinet','',false,true,false) ON CONFLICT DO NOTHING;
 insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
     VALUES (25,'Fortinet FortiOS Gateway','REST','Fortinet','',false,false,false) ON CONFLICT DO NOTHING;
+insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
+    VALUES (26,'NSX','REST','VMWare','',false,true,false) ON CONFLICT DO NOTHING;
+insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
+    VALUES (27,'NSX DFW Gateway','REST','VMWare','',false,false,false) ON CONFLICT DO NOTHING;
+
 
 update stm_dev_typ set dev_typ_predef_svc=
 'ANY;0;0;65535;1;other;simple

roles/database/files/upgrade/8.0.2.sql

@@ -0,0 +1,4 @@
+insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
+    VALUES (26,'NSX','4ff','VMWare','',false,true,false) ON CONFLICT DO NOTHING;
+insert into stm_dev_typ (dev_typ_id,dev_typ_name,dev_typ_version,dev_typ_manufacturer,dev_typ_predef_svc,dev_typ_is_multi_mgmt,dev_typ_is_mgmt,is_pure_routing_device)
+    VALUES (27,'NSX DFW Gateway','4ff','VMWare','',false,false,false) ON CONFLICT DO NOTHING;

roles/importer/files/importer/common.py

@@ -129,7 +129,7 @@ def import_management(mgm_id=None, ssl_verification=None, debug_level_in=0,
                 config_changed_since_last_import, error_string, error_count, change_count = get_config_from_api(mgm_details, full_config_json, config2import, jwt, current_import_id, start_time,
                     in_file=in_file, import_tmp_path=import_tmp_path, error_string=error_string, error_count=error_count, change_count=change_count, 
                     limit=limit, force=force)
-                if (debug_level>7):  # dump full native config read from fw API
+                if (debug_level>8):  # dump full native config read from fw API
                     logger.info(json.dumps(full_config_json, indent=2))
 
         time_get_config = int(time.time()) - start_time
@@ -183,7 +183,7 @@ def import_management(mgm_id=None, ssl_verification=None, debug_level_in=0,
         else: # if no changes were found, we skip everything else without errors
             pass
 
-        if (debug_level>8): # dump normalized config for debugging purposes
+        if (debug_level>7): # dump normalized config for debugging purposes
             logger.info(json.dumps(config2import, indent=2))
 
         error_count = complete_import(current_import_id, error_string, start_time, mgm_details, change_count, error_count, jwt)

roles/importer/files/importer/nsx4ff/discovery_logging.conf

@@ -0,0 +1,41 @@
+[loggers]
+keys=root,discoveryDebugLogger
+#keys=root,__main__
+
+[handlers]
+keys=consoleHandler,debugFileHandler
+
+[formatters]
+keys=defaultFormatter,debugFileFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_discoveryDebugLogger]
+#[logger___main__]
+level=DEBUG
+handlers=debugFileHandler
+qualname=discoveryDebugLogger
+#qualname=__main__
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=DEBUG
+formatter=defaultFormatter
+args=(sys.stderr,)
+
+[handler_debugFileHandler]
+class=FileHandler
+level=DEBUG
+formatter=debugFileFormatter
+args=('/tmp/fworch_discovery.log',)
+# args=('/var/log/fworch/discovery.log',)
+
+[formatter_defaultFormatter]
+format=%(levelname)s:%(name)s:%(message)s
+
+[formatter_debugFileFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+

roles/importer/files/importer/nsx4ff/fwcommon.py

@@ -0,0 +1,105 @@
+import sys
+import base64
+from common import importer_base_dir
+sys.path.append(importer_base_dir + "/nsx4ff")
+from nsx_service import normalize_svcobjects
+# from nsx_application import normalize_application_objects
+from nsx_rule import normalize_access_rules
+from nsx_network import normalize_nwobjects
+# from nsx_zone import normalize_zones
+from nsx_getter import update_config_with_nsxdcfw_api_call
+from fwo_log import getFwoLogger
+from nsx_base import api_version_str
+
+def has_config_changed(full_config, mgm_details, force=False):
+    # dummy - may be filled with real check later on
+    return True
+
+
+def get_config(config2import, full_config, current_import_id, mgm_details, limit=1000, force=False, jwt=''):
+    logger = getFwoLogger()
+    if full_config == {}:   # no native config was passed in, so getting it from Azzure
+        parsing_config_only = False
+    else:
+        parsing_config_only = True
+
+    if not parsing_config_only: # no native config was passed in, so getting it from Palo Firewall
+        apipwd = mgm_details["import_credential"]['secret']
+        apiuser = mgm_details["import_credential"]['user']
+        apihost = mgm_details["hostname"]
+        domain = mgm_details["configPath"]
+
+        vsys_base_objects   = ["/infra/services"]
+        vsys_object_groups   = ["/infra/domains/{domain}/groups".format(domain=domain)]
+        vsys_objects   = vsys_object_groups + vsys_base_objects
+
+        #predef_objects = ["/Objects/Applications"]
+        rulebase_names = ["security-policies"] # , "/Policies/NATRules"]
+
+        for obj_path in vsys_objects:
+            full_config[obj_path] = []
+
+        # for obj_path in predef_objects:
+        #     full_config[obj_path] = []
+
+        credentials = base64.b64encode((apiuser + ":" + apipwd).encode())
+
+        ## get objects:
+        # base_url =  "https://{apihost}/policy/api/v1/infra/domains/{domain}/security-policies/[policy name]".format(apihost=apihost, api_version_str=api_version_str)
+
+        # vsys_name = "vsys1" # TODO - automate this hard-coded name
+        # location = "vsys"       # alternative: panorama-pushed
+
+
+        for obj_path in vsys_objects:
+            base_url =  "https://{apihost}/policy/api/v1{path}".format(apihost=apihost, path=obj_path)
+            update_config_with_nsxdcfw_api_call(base_url, full_config, obj_path, obj_type=obj_path, credentials=credentials)
+
+        # for obj_path in predef_objects:
+        #     update_config_with_nsxdcfw_api_call(key, base_url, full_config, obj_path + "?location={location}".format(location="predefined"), obj_type=obj_path)
+
+        # users
+
+        # get rules
+        full_config.update({'devices': {}})
+        for device in mgm_details["devices"]:
+            dev_id = device['id']
+            dev_name = device['local_rulebase_name']
+            full_config['devices'].update({ dev_id: {} })
+
+            for obj_path in rulebase_names:
+                base_url =  "https://{apihost}/policy/api/v1/infra/domains/{domain}/{rulebase_name}/{policy_name}".format(apihost=apihost, domain=domain, policy_name=dev_name, rulebase_name=obj_path)
+                update_config_with_nsxdcfw_api_call(
+                        base_url, full_config['devices'][device['id']], 
+                        obj_path,
+                        obj_type=obj_path, credentials=credentials)
+
+    ##################
+    # now we normalize relevant parts of the raw config and write the results to config2import dict
+
+    normalize_nwobjects(full_config, config2import, current_import_id, jwt=jwt, mgm_id=mgm_details['id'], domain=domain)
+    normalize_svcobjects(full_config, config2import, current_import_id)
+    # normalize_application_objects(full_config, config2import, current_import_id)
+    # normalize_users(full_config, config2import, current_import_id, user_scope)
+
+    # adding default any and predefined objects
+    any_nw_svc = {"svc_uid": "any_svc_placeholder", "svc_name": "any", "svc_comment": "Placeholder service.", 
+        "svc_typ": "simple", "ip_proto": -1, "svc_port": 0, "svc_port_end": 65535, "control_id": current_import_id}
+    http_svc = {"svc_uid": "http_predefined_svc", "svc_name": "service-http", "svc_comment": "Predefined service", 
+        "svc_typ": "simple", "ip_proto": 6, "svc_port": 80, "control_id": current_import_id}
+    https_svc = {"svc_uid": "https_predefined_svc", "svc_name": "service-https", "svc_comment": "Predefined service", 
+        "svc_typ": "simple", "ip_proto": 6, "svc_port": 443, "control_id": current_import_id}
+
+    config2import["service_objects"].append(any_nw_svc)
+    config2import["service_objects"].append(http_svc)
+    config2import["service_objects"].append(https_svc)
+
+    any_nw_object = {"obj_uid": "any_obj_placeholder", "obj_name": "any", "obj_comment": "Placeholder object.",
+        "obj_typ": "network", "obj_ip": "0.0.0.0/0", "control_id": current_import_id}
+    config2import["network_objects"].append(any_nw_object)
+
+    # normalize_zones(full_config, config2import, current_import_id)
+    normalize_access_rules(full_config, config2import, current_import_id, mgm_details=mgm_details)
+    # normalize_nat_rules(full_config, config2import, current_import_id, jwt=jwt)
+
+    return 0

roles/importer/files/importer/nsx4ff/nsx_application.py

@@ -0,0 +1,37 @@
+from fwo_const import list_delimiter
+from fwo_log import getFwoLogger
+
+
+# def normalize_application_objects(full_config, config2import, import_id):
+#     app_objects = []
+#     for app_orig in full_config["/Objects/Applications"]:
+#         app_objects.append(parse_app(app_orig, import_id,config2import))
+#     config2import['service_objects'] += app_objects
+
+
+def extract_base_app_infos(app_orig, import_id):
+    app = {}
+    if "@name" in app_orig:
+        app["svc_uid"] = app_orig["@name"]
+        app["svc_name"] = app_orig["@name"]
+    if "comment" in app_orig:
+        app["svc_comment"] = app_orig["comment"]
+    app["control_id"] = import_id 
+    app["svc_typ"] = 'simple' 
+    return app
+
+
+def parse_app(app_orig, import_id,config2import):
+    svc = extract_base_app_infos(app_orig, import_id)
+    app_comment = ''
+    if 'category' in app_orig:
+        app_comment = "category: " + app_orig['category']
+        if 'subcategory' in app_orig:
+            app_comment += ", " +  "subcategory: " + app_orig['subcategory']
+            if 'technology' in app_orig:
+                app_comment += ", " + "technology: " + app_orig['technology']
+    if 'svc_comment' in svc:
+        svc['svc_comment'] += "; " + app_comment
+    else:
+        svc['svc_comment'] = app_comment
+    return svc

roles/importer/files/importer/nsx4ff/nsx_base.py

@@ -0,0 +1,2 @@
+
+api_version_str="9.1"

roles/importer/files/importer/nsx4ff/nsx_getter.py

@@ -0,0 +1,95 @@
+# library for API get functions
+import base64
+from typing import Dict
+from fwo_log import getFwoLogger
+import requests.packages
+import requests
+import xmltodict, json
+import fwo_globals
+from fwo_exception import FwLoginFailed
+
+
+def api_call(url, params = {}, headers = {}, data = {}, credentials = '', show_progress=False, method='get'):
+    logger = getFwoLogger()
+    result_type='json'
+    request_headers = {'Content-Type': 'application/json'}
+    for header_key in headers:
+        request_headers[header_key] = headers[header_key]
+    if credentials != '':
+        request_headers["Authorization"] = 'Basic {credentials}'.format(credentials=credentials.decode("utf-8"))
+        result_type='json'
+
+    if method == "post":
+        response = requests.post(url, params=params, data=data, headers=request_headers, verify=fwo_globals.verify_certs)
+    elif method == "get":
+        response = requests.get(url, params=params, headers=request_headers, verify=fwo_globals.verify_certs)
+    else:
+        raise Exception("unknown HTTP method found in nsx_getter")
+
+    # error handling:
+    exception_text = ''
+    if response is None:
+        if 'password' in json.dumps(data):
+            exception_text = "error while sending api_call containing credential information to url '" + \
+                str(url)
+        else:
+            exception_text = "error while sending api_call to url '" + str(url) + "' with payload '" + json.dumps(
+                data, indent=2) + "' and  headers: '" + json.dumps(request_headers, indent=2)
+    if not response.ok:
+        exception_text = 'error code: {error_code}, error={error}'.format(error_code=response.status_code, error=response.content)
+        #logger.error(response.content)
+    if (len(response.content) == 0):
+        exception_text = 'empty response content'
+
+    if exception_text != '':
+        raise Exception(exception_text)
+
+    # no errors found
+    if result_type=='xml':
+        r = xmltodict.parse(response.content)
+        body_json = json.loads(json.dumps(r))
+    elif result_type=='json':
+        body_json = json.loads(response.content)
+        if 'result' in body_json:
+            body_json = body_json['result']
+
+    else:
+        body_json = None
+
+    return body_json
+
+
+# def login(apiuser, apipwd, apihost):
+#     base_url = "https://{apihost}/api/?type=keygen&user={apiuser}&password={apipwd}".format(apihost=apihost, apiuser=apiuser, apipwd=apipwd)
+#     try:
+#         body = api_call(base_url, method="get", headers={}, data={})
+#     except Exception as e:
+#         raise FwLoginFailed("Palo FW login to firewall=" + str(apihost) + " failed; Message: " + str(e)) from None
+
+#     if 'response' in body and 'result' in body['response'] and 'key' in body['response']['result'] and not body['response']['result']['key'] == None:
+#         key = body['response']['result']['key']
+#     else:
+#         raise FwLoginFailed("Palo FW login to firewall=" + str(apihost) + " failed") from None
+
+#     if fwo_globals.debug_level > 2:
+#         logger = getFwoLogger()
+#         logger.debug("Login successful. Received key: " + key)
+
+#     return key
+
+
+def update_config_with_nsxdcfw_api_call(api_base_url, config, api_path, credentials='', obj_type='generic', parameters={}, payload={}, show_progress=False, limit: int=1000, method="get"):
+    returned_new_data = True
+
+    full_result = []
+    result = api_call(api_base_url, credentials=credentials, params=parameters, data=payload, show_progress=show_progress, method=method)
+    # if "entry" in result:
+    #     returned_new_data = len(result['entry'])>0
+    # else:
+    #     returned_new_data = False
+    if returned_new_data:
+        if 'results' in result:
+            config.update({obj_type: result['results']})
+        else:
+        # full_result.extend(result)           
+            config.update({obj_type: result})