diff --git a/.vscode/launch.json b/.vscode/launch.json index 3dc23a3c0..4d2e539b5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,25 @@ { "version": "0.2.0", "configurations": [ + { + "name": ".NET Core SSH Attach ubu20", + "type": "coreclr", + "request": "attach", + "processId": "${command:pickRemoteProcess}", + "pipeTransport": { + "pipeProgram": "ssh", + "pipeArgs": [ "-T", "-p 22006", "tim@localhost" ], + "debuggerPath": "~/vsdbg-root/vsdbg", + "pipeCwd": "${workspaceRoot}", + "quoteArgs": true + }, + "sourceFileMap": { + "/home/tim/firewall-orchestrator/roles": "${workspaceRoot}" + }, + "logging": { + "logging.diagnosticsLog.protocolMessages": true + } + }, { "name": "c#-MiddlewareServer", "type": "coreclr", @@ -82,7 +101,7 @@ "PYTHONPATH": "${PYTHONPATH}:${workspaceRoot}" }, "args": [ - "-m5", + "-m23", "-d1", "-f", "-s", diff --git a/inventory/group_vars/apiserver.yml b/inventory/group_vars/apiserver.yml index f9fa8de72..a5dbadd27 100644 --- a/inventory/group_vars/apiserver.yml +++ b/inventory/group_vars/apiserver.yml @@ -8,7 +8,7 @@ api_hasura_admin_test_password: "not4production" api_user_email: "{{ api_user }}@{{ api_network_listening_ip_address }}" api_home: "{{ fworch_home }}/api" api_hasura_cli_bin: "{{ fworch_home }}/api/bin/hasura" -api_hasura_version: "v2.39.2" +api_hasura_version: "v2.40.0" api_project_name: api api_no_metadata: false api_rollback_is_running: false diff --git a/roles/database/files/sql/idempotent/fworch-texts.sql b/roles/database/files/sql/idempotent/fworch-texts.sql index 5932835ae..acc3085be 100644 --- a/roles/database/files/sql/idempotent/fworch-texts.sql +++ b/roles/database/files/sql/idempotent/fworch-texts.sql @@ -644,6 +644,8 @@ INSERT INTO txt VALUES ('destination_zone', 'German', 'Zielzone'); INSERT INTO txt VALUES ('destination_zone', 'English', 'Destination Zone'); INSERT INTO txt VALUES ('enabled', 'German', 'Aktiviert'); INSERT INTO txt VALUES ('enabled', 'English', 'Enabled'); +INSERT INTO txt VALUES ('install_on', 'German', 'Installiere auf'); +INSERT INTO txt VALUES ('install_on', 'English', 'Install On'); INSERT INTO txt VALUES ('uid', 'German', 'UID'); INSERT INTO txt VALUES ('uid', 'English', 'UID'); INSERT INTO txt VALUES ('created', 'German', 'Angelegt'); diff --git a/scripts/customizing/modelling/getOwnersFromTufinRlm.py b/scripts/customizing/modelling/getOwnersFromMultipleSources.py similarity index 64% rename from scripts/customizing/modelling/getOwnersFromTufinRlm.py rename to scripts/customizing/modelling/getOwnersFromMultipleSources.py index b605895d2..071c35912 100644 --- a/scripts/customizing/modelling/getOwnersFromTufinRlm.py +++ b/scripts/customizing/modelling/getOwnersFromMultipleSources.py @@ -1,6 +1,22 @@ #!/usr/bin/python3 +# reads the main app data from a git repo +# and renriches the data with csv files containing users and server ip addresses + +# dependencies: +# a) package python3-git must be installed +# b) requires the following config items in /usr/local/orch/etc/secrets/customizingConfig.json +# Tufin RLM +# username +# password +# apiBaseUri # Tufin API, e.g. "https://tufin.domain.com/" +# git +# gitRepoUrl +# gitusername +# gitpassword +# csvFiles # array of file basenames containing the app data +# ldapPath # full ldap user path (used for building DN from user basename) + -# library for Tufin STRACK API calls from asyncio.log import logger import traceback from textwrap import indent @@ -15,11 +31,21 @@ import os import socket from pathlib import Path +import git # apt install python3-git # or: pip install git +import csv + + +baseDir = "/usr/local/fworch/" +baseDirEtc = baseDir + "etc/" +repoTargetDir = baseDirEtc + "cmdb-repo" +defaultConfigFileName = baseDirEtc + "secrets/customizingConfig.json" +defaultRlmImportFileName = baseDir + "scripts/customizing/modelling/getOwnersFromTufinRlm.json" +importSourceString = "tufinRlm" +# TUFIN settings: api_url_path_rlm_login = 'apps/public/rlm/oauth/token' api_url_path_rlm_apps = 'apps/public/rlm/api/owners' -defaultConfigFileName = "/usr/local/fworch/etc/secrets/customizingConfig.json" class ApiLoginFailed(Exception): """Raised when login to API failed""" @@ -52,22 +78,34 @@ def __init__(self, message="API unavailable"): def readConfig(configFilename): try: - with open(configFilename, "r") as customConfigFH: customConfig = json.loads(customConfigFH.read()) - return (customConfig['username'], customConfig['password'], customConfig['ldapPath'], customConfig['apiBaseUri']) - + return (customConfig['username'], customConfig['password'], customConfig['apiBaseUri'], + customConfig['ldapPath'], + customConfig['gitRepoUrl'], customConfig['gitusername'], customConfig['gitpassword'], customConfig['csvFiles']) except: logger.error("could not read config file " + configFilename + ", Exception: " + str(traceback.format_exc())) sys.exit(1) +# read owners from json file on disk which where imported from RLM +def getExistingOwnerIds(ownersIn): + rlmOwners = [] + # convert owners into list of owner ids + for o in ownersIn: + if 'app_id_external' in o and not o['app_id_external'] in rlmOwners: + rlmOwners.append(o['app_id_external']) + return rlmOwners + + def buildDN(userId, ldapPath): - if '{USERID}' in ldapPath: - return ldapPath.replace('{USERID}', userId) - else: - logger.error("could not find {USERID} parameter in ldapPath " + ldapPath) - sys.exit(1) + dn = "" + if len(userId)>0: + if '{USERID}' in ldapPath: + dn = ldapPath.replace('{USERID}', userId) + else: + logger.error("could not find {USERID} parameter in ldapPath " + ldapPath) + return dn def getNetworkBorders(ip): @@ -111,7 +149,7 @@ def reverse_dns_lookup(ip_address): def extractSocketInfo(asset, services): # ignoring services for the moment - sockets =[] + sockets = [] if 'assets' in asset and 'values' in asset['assets']: for ip in asset['assets']['values']: @@ -149,11 +187,11 @@ def getLogger(debug_level_in=0): logging.basicConfig(format=logformat, datefmt="%Y-%m-%dT%H:%M:%S%z", level=llevel) logger.setLevel(llevel) - #set log level for noisy requests/connectionpool module to WARNING: + #set log level for noisy requests/connectionpool module to WARNING: connection_log = logging.getLogger("urllib3.connectionpool") connection_log.setLevel(logging.WARNING) connection_log.propagate = True - + if debug_level>8: logger.debug ("debug_level=" + str(debug_level) ) return logger @@ -204,18 +242,19 @@ def rlmGetOwners(token, api_url): ", status code: " + str(response)) -if __name__ == "__main__": +if __name__ == "__main__": parser = argparse.ArgumentParser( description='Read configuration from FW management via API calls') parser.add_argument('-c', '--config', default=defaultConfigFileName, help='Filename of custom config file for modelling imports') - parser.add_argument('-s', "--suppress_certificate_warnings", action='store_true', default = True, + parser.add_argument('-s', "--suppress_certificate_warnings", action='store_true', default = True, help = "suppress certificate warnings") parser.add_argument('-l', '--limit', metavar='api_limit', default='150', help='The maximal number of returned results per HTTPS Connection; default=50') args = parser.parse_args() - owners = {} + + ownersById = {} if args.suppress_certificate_warnings: requests.packages.urllib3.disable_warnings() @@ -223,7 +262,54 @@ def rlmGetOwners(token, api_url): logger = getLogger(debug_level_in=2) # read config - rlmUsername, rlmPassword, ldapPath, rlmApiUrl = readConfig(args.config) + rlmUsername, rlmPassword, rlmApiUrl, ldapPath, gitRepoUrl, gitUsername, gitPassword, csvFiles = readConfig(args.config) + + ###################################################### + # 1. get all owners + # get cmdb repo + repoUrl = "https://" + gitUsername + ":" + gitPassword + "@" + gitRepoUrl + if os.path.exists(repoTargetDir): + # If the repository already exists, open it and perform a pull + repo = git.Repo(repoTargetDir) + origin = repo.remotes.origin + origin.pull() + else: + repo = git.Repo.clone_from(repoUrl, repoTargetDir) + + dfAllApps = [] + for csvFile in csvFiles: + csvFile = repoTargetDir + '/' + csvFile # add directory to csv files + with open(csvFile, newline='') as csvFile: + reader = csv.reader(csvFile) + dfAllApps += list(reader)[1:]# Skip headers in first line + + logger.info("#total aps: " + str(len(dfAllApps))) + + # append all owners from CSV + for owner in dfAllApps: + appId = owner[1] + if appId not in ownersById.keys(): + if appId.lower().startswith('app-') or appId.lower().startswith('com-'): + mainUserDn = buildDN(owner[3], ldapPath) + if mainUserDn=='': + logger.warning('adding app without main user: ' + appId) + + ownersById.update( + { + owner[1]: + { + "app_id_external": appId, + "name": owner[0], + "main_user": mainUserDn, + "modellers": [], + "import_source": importSourceString, + "app_servers": [], + } + } + ) + + ###################################################### + # 2. now add data from RLM (add. users, server data) if not rlmApiUrl.startswith("http"): # assuming config file instead of direct API access @@ -238,35 +324,41 @@ def rlmGetOwners(token, api_url): try: oauthToken = rlmLogin(rlmUsername, rlmPassword, rlmApiUrl + api_url_path_rlm_login) # logger.debug("token for RLM: " + oauthToken) - ownerData = rlmGetOwners(oauthToken, rlmApiUrl + api_url_path_rlm_apps) + rlmOwnerData = rlmGetOwners(oauthToken, rlmApiUrl + api_url_path_rlm_apps) except: logger.error("error while getting owner data from RLM API: " + str(traceback.format_exc())) sys.exit(1) - # normalizing owners config from Tufin RLM + for rlmOwner in rlmOwnerData['owners']: + # collect modeller users + users = [] + appId = rlmOwner['owner']['name'] + for uid in rlmOwner['owner']['members']: + dn = buildDN(uid, ldapPath) + if appId in ownersById: + if not dn == ownersById[appId]["main_user"]: # leave out main owner + users.append(dn) + + # enrich modeller users and servers + if appId in ownersById: + ownersById[appId]['modellers'] += users + ownersById[appId]['app_servers'] += extractSocketInfo(rlmOwner['asset'], rlmOwner['services']) + else: + logger.warning('got app-id from RLM which is not in main app export: ' + appId) + + # 3. convert to normalized struct normOwners = { "owners": [] } - for owner in ownerData['owners']: - # logger.info("dealing with owner " + str(owner)) + for o in ownersById: + normOwners['owners'].append(ownersById[o]) + + ################################################################################################### + # 4. write owners to json file - users = [] - for uid in owner['owner']['members']: - users.append(buildDN(uid, ldapPath)) - - ownNorm = { - "app_id_external": owner['owner']['name'], - "name": owner['description'], - "main_user": None, - "modellers": users, - "import_source": "tufinRlm", - "app_servers": extractSocketInfo(owner['asset'], owner['services']), - } - normOwners['owners'].append(ownNorm) - - # logger.info("normOwners = " + json.dumps(normOwners, indent=3)) path = os.path.dirname(__file__) fileOut = path + '/' + Path(os.path.basename(__file__)).stem + ".json" logger.info("dumping into file " + fileOut) + with open(fileOut, "w") as outFH: json.dump(normOwners, outFH, indent=3) sys.exit(0)