Merge pull request #2456 from tpurschke/develop

Develop customization changes for app data import

.vscode/launch.json

@@ -1,6 +1,25 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": ".NET Core SSH Attach ubu20",
+            "type": "coreclr",
+            "request": "attach",
+            "processId": "${command:pickRemoteProcess}",
+            "pipeTransport": {
+                "pipeProgram": "ssh",
+                "pipeArgs": [ "-T", "-p 22006", "tim@localhost" ],
+                "debuggerPath": "~/vsdbg-root/vsdbg",
+                "pipeCwd": "${workspaceRoot}",
+                "quoteArgs": true
+            },
+            "sourceFileMap": {
+                "/home/tim/firewall-orchestrator/roles": "${workspaceRoot}"
+            },
+            "logging": {
+              "logging.diagnosticsLog.protocolMessages": true
+            }
+        },
         {
             "name": "c#-MiddlewareServer",
             "type": "coreclr",
@@ -82,7 +101,7 @@
                 "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}"
             },
             "args": [
-                "-m5",
+                "-m23",
                 "-d1",
                 "-f",
                 "-s",

inventory/group_vars/apiserver.yml

@@ -8,7 +8,7 @@ api_hasura_admin_test_password: "not4production"
 api_user_email: "{{ api_user }}@{{ api_network_listening_ip_address }}"
 api_home: "{{ fworch_home }}/api"
 api_hasura_cli_bin: "{{ fworch_home }}/api/bin/hasura"
-api_hasura_version: "v2.39.2"
+api_hasura_version: "v2.40.0"
 api_project_name: api
 api_no_metadata: false
 api_rollback_is_running: false

roles/database/files/sql/idempotent/fworch-texts.sql

@@ -644,6 +644,8 @@ INSERT INTO txt VALUES ('destination_zone',		'German', 	'Zielzone');
 INSERT INTO txt VALUES ('destination_zone',		'English', 	'Destination Zone');
 INSERT INTO txt VALUES ('enabled',		        'German', 	'Aktiviert');
 INSERT INTO txt VALUES ('enabled',		        'English', 	'Enabled');
+INSERT INTO txt VALUES ('install_on',		    'German', 	'Installiere auf');
+INSERT INTO txt VALUES ('install_on',		    'English', 	'Install On');
 INSERT INTO txt VALUES ('uid',		            'German', 	'UID');
 INSERT INTO txt VALUES ('uid',		            'English', 	'UID');
 INSERT INTO txt VALUES ('created',		        'German', 	'Angelegt');

scripts/customizing/modelling/getOwnersFromTufinRlm.py → scripts/customizing/modelling/getOwnersFromMultipleSources.py

@@ -1,6 +1,22 @@
 #!/usr/bin/python3
+# reads the main app data from a git repo
+# and renriches the data with csv files containing users and server ip addresses
+
+# dependencies: 
+#   a) package python3-git must be installed
+#   b) requires the following config items in /usr/local/orch/etc/secrets/customizingConfig.json
+#       Tufin RLM
+#           username
+#           password
+#           apiBaseUri      # Tufin API, e.g. "https://tufin.domain.com/"
+#       git
+#           gitRepoUrl
+#           gitusername
+#           gitpassword
+#       csvFiles # array of file basenames containing the app data
+#       ldapPath # full ldap user path (used for building DN from user basename)
+
 
-# library for Tufin STRACK API calls
 from asyncio.log import logger
 import traceback
 from textwrap import indent
@@ -15,11 +31,21 @@
 import os
 import socket
 from pathlib import Path
+import git  # apt install python3-git # or: pip install git
+import csv
+
+
 
+baseDir = "/usr/local/fworch/"
+baseDirEtc = baseDir + "etc/"
+repoTargetDir = baseDirEtc + "cmdb-repo"
+defaultConfigFileName = baseDirEtc + "secrets/customizingConfig.json"
+defaultRlmImportFileName = baseDir + "scripts/customizing/modelling/getOwnersFromTufinRlm.json"
+importSourceString = "tufinRlm"
 
+# TUFIN settings:
 api_url_path_rlm_login = 'apps/public/rlm/oauth/token'
 api_url_path_rlm_apps = 'apps/public/rlm/api/owners'
-defaultConfigFileName = "/usr/local/fworch/etc/secrets/customizingConfig.json"
 
 class ApiLoginFailed(Exception):
     """Raised when login to API failed"""
@@ -52,22 +78,34 @@ def __init__(self, message="API unavailable"):
 
 def readConfig(configFilename):
     try:
-
         with open(configFilename, "r") as customConfigFH:
             customConfig = json.loads(customConfigFH.read())
-        return (customConfig['username'], customConfig['password'], customConfig['ldapPath'], customConfig['apiBaseUri'])
-
+        return (customConfig['username'], customConfig['password'], customConfig['apiBaseUri'],
+                customConfig['ldapPath'],
+                customConfig['gitRepoUrl'], customConfig['gitusername'], customConfig['gitpassword'], customConfig['csvFiles'])
     except:
         logger.error("could not read config file " + configFilename + ", Exception: " + str(traceback.format_exc()))
         sys.exit(1)
 
 
+# read owners from json file on disk which where imported from RLM
+def getExistingOwnerIds(ownersIn):
+    rlmOwners = []
+    # convert owners into list of owner ids
+    for o in ownersIn:
+        if 'app_id_external' in o and not o['app_id_external'] in rlmOwners:
+            rlmOwners.append(o['app_id_external'])
+    return rlmOwners
+
+
 def buildDN(userId, ldapPath):
-    if '{USERID}' in ldapPath:
-        return ldapPath.replace('{USERID}', userId)
-    else:
-        logger.error("could not find {USERID} parameter in ldapPath " + ldapPath)
-        sys.exit(1)
+    dn = ""
+    if len(userId)>0:
+        if '{USERID}' in ldapPath:
+            dn = ldapPath.replace('{USERID}', userId)
+        else:
+            logger.error("could not find {USERID} parameter in ldapPath " + ldapPath)
+    return dn
 
 
 def getNetworkBorders(ip):
@@ -111,7 +149,7 @@ def reverse_dns_lookup(ip_address):
 
 def extractSocketInfo(asset, services):
     # ignoring services for the moment
-    sockets =[]
+    sockets = []
 
     if 'assets' in asset and 'values' in asset['assets']:
         for ip in asset['assets']['values']:
@@ -149,11 +187,11 @@ def getLogger(debug_level_in=0):
     logging.basicConfig(format=logformat, datefmt="%Y-%m-%dT%H:%M:%S%z", level=llevel)
     logger.setLevel(llevel)
 
-    #set log level for noisy requests/connectionpool module to WARNING: 
+    #set log level for noisy requests/connectionpool module to WARNING:
     connection_log = logging.getLogger("urllib3.connectionpool")
     connection_log.setLevel(logging.WARNING)
     connection_log.propagate = True
-    
+
     if debug_level>8:
         logger.debug ("debug_level=" + str(debug_level) )
     return logger
@@ -204,26 +242,74 @@ def rlmGetOwners(token, api_url):
                             ", status code: " + str(response))
 
 
-if __name__ == "__main__": 
+if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description='Read configuration from FW management via API calls')
     parser.add_argument('-c', '--config', default=defaultConfigFileName,
                         help='Filename of custom config file for modelling imports')
-    parser.add_argument('-s', "--suppress_certificate_warnings", action='store_true', default = True, 
+    parser.add_argument('-s', "--suppress_certificate_warnings", action='store_true', default = True,
                         help = "suppress certificate warnings")
     parser.add_argument('-l', '--limit', metavar='api_limit', default='150',
                         help='The maximal number of returned results per HTTPS Connection; default=50')
 
     args = parser.parse_args()
-    owners = {}
+
+    ownersById = {}
 
     if args.suppress_certificate_warnings:
         requests.packages.urllib3.disable_warnings()
 
     logger = getLogger(debug_level_in=2)
 
     # read config
-    rlmUsername, rlmPassword, ldapPath, rlmApiUrl = readConfig(args.config)
+    rlmUsername, rlmPassword, rlmApiUrl, ldapPath, gitRepoUrl, gitUsername, gitPassword, csvFiles = readConfig(args.config)
+
+    ######################################################
+    # 1. get all owners
+    # get cmdb repo
+    repoUrl = "https://" + gitUsername + ":" + gitPassword + "@" + gitRepoUrl
+    if os.path.exists(repoTargetDir):
+        # If the repository already exists, open it and perform a pull
+        repo = git.Repo(repoTargetDir)
+        origin = repo.remotes.origin
+        origin.pull()
+    else:
+        repo = git.Repo.clone_from(repoUrl, repoTargetDir)
+
+    dfAllApps = []
+    for csvFile in csvFiles:
+        csvFile = repoTargetDir + '/' + csvFile # add directory to csv files
+        with open(csvFile, newline='') as csvFile:
+            reader = csv.reader(csvFile)
+            dfAllApps += list(reader)[1:]# Skip headers in first line
+
+    logger.info("#total aps: " + str(len(dfAllApps)))
+
+    # append all owners from CSV
+    for owner in dfAllApps:
+        appId = owner[1]
+        if appId not in ownersById.keys():
+            if appId.lower().startswith('app-') or appId.lower().startswith('com-'):
+                mainUserDn = buildDN(owner[3], ldapPath)
+                if mainUserDn=='':
+                    logger.warning('adding app without main user: ' + appId)
+
+                ownersById.update(
+                    {
+                    owner[1]:
+                        {
+                            "app_id_external": appId,
+                            "name": owner[0],
+                            "main_user": mainUserDn,
+                            "modellers": [],
+                            "import_source": importSourceString,
+                            "app_servers": [],
+                        }
+                    }
+                )
+
+    ######################################################
+    # 2. now add data from RLM (add. users, server data)
 
     if not rlmApiUrl.startswith("http"):
         # assuming config file instead of direct API access
@@ -238,35 +324,41 @@ def rlmGetOwners(token, api_url):
         try:
             oauthToken = rlmLogin(rlmUsername, rlmPassword, rlmApiUrl + api_url_path_rlm_login)
             # logger.debug("token for RLM: " + oauthToken)
-            ownerData = rlmGetOwners(oauthToken, rlmApiUrl + api_url_path_rlm_apps)
+            rlmOwnerData = rlmGetOwners(oauthToken, rlmApiUrl + api_url_path_rlm_apps)
 
         except:
             logger.error("error while getting owner data from RLM API: " + str(traceback.format_exc()))
             sys.exit(1)
 
-    # normalizing owners config from Tufin RLM
+    for rlmOwner in rlmOwnerData['owners']:
+        # collect modeller users
+        users = []
+        appId = rlmOwner['owner']['name']
+        for uid in rlmOwner['owner']['members']:
+            dn = buildDN(uid, ldapPath)
+            if appId in ownersById:
+                if not dn == ownersById[appId]["main_user"]:  # leave out main owner
+                    users.append(dn)
+
+        # enrich modeller users and servers
+        if appId in ownersById:
+            ownersById[appId]['modellers'] += users
+            ownersById[appId]['app_servers'] += extractSocketInfo(rlmOwner['asset'], rlmOwner['services'])
+        else:
+            logger.warning('got app-id from RLM which is not in main app export: ' + appId)
+
+    # 3. convert to normalized struct
     normOwners = { "owners": [] }
-    for owner in ownerData['owners']:
-        # logger.info("dealing with owner " + str(owner))
+    for o in ownersById:
+        normOwners['owners'].append(ownersById[o])
+
+    ###################################################################################################
+    # 4. write owners to json file
 
-        users = []
-        for uid in owner['owner']['members']:
-            users.append(buildDN(uid, ldapPath))
-
-        ownNorm = {
-            "app_id_external": owner['owner']['name'],
-            "name": owner['description'],
-            "main_user": None,
-            "modellers": users,
-            "import_source": "tufinRlm",
-            "app_servers": extractSocketInfo(owner['asset'], owner['services']),
-        }
-        normOwners['owners'].append(ownNorm)
-
-    # logger.info("normOwners = " + json.dumps(normOwners, indent=3))
     path = os.path.dirname(__file__)
     fileOut = path + '/' + Path(os.path.basename(__file__)).stem + ".json"
     logger.info("dumping into file " + fileOut)
+
     with open(fileOut, "w") as outFH:
         json.dump(normOwners, outFH, indent=3)
     sys.exit(0)