From 90173afd7dc979ee36eb652362756e96c8b1fa06 Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Tue, 20 Jun 2023 22:33:46 +0530 Subject: [PATCH 1/4] Fix queries and add changes information to see what exactly changed --- TA-cyences/bin/sudousers.sh | 28 ------------------- TA-cyences/default/inputs.conf | 7 ----- .../default/savedsearches.conf | 27 ++++++++++++------ docs/data_onboarding/linux.md | 10 +++++-- 4 files changed, 27 insertions(+), 45 deletions(-) delete mode 100755 TA-cyences/bin/sudousers.sh diff --git a/TA-cyences/bin/sudousers.sh b/TA-cyences/bin/sudousers.sh deleted file mode 100755 index 4e481e1b..00000000 --- a/TA-cyences/bin/sudousers.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -test=$(cat /etc/sudoers | grep "#includedir /etc/sudoers.d") -if [ -z "$test" ]; -then -groups=$(cat /etc/sudoers | grep "ALL\s*=\s*(ALL" | awk '{print $1}' | grep % | grep -v '#') -else -groups=$(cat /etc/sudoers /etc/sudoers.d/* | grep "ALL\s*=\s*(ALL" | awk '{print $1}' | grep % | grep -v '#') -fi -for i in $groups -do -group_name=$(echo $i | cut -d "%" -f 2) -users=$(getent group $group_name| cut -d ":" -f 4) -IFS=',' read -ra user_array <<< "$users" -for i in "${user_array[@]}" -do - echo sudouser=$i -done -done -if [ -z "$test" ]; -then -users=$(cat /etc/sudoers | grep "ALL\s*=\s*(ALL" | awk '{print $1}' | grep -v % | grep -v '#') -else -users=$(cat /etc/sudoers /etc/sudoers.d/* | grep "ALL\s*=\s*(ALL" | awk '{print $1}' | grep -v % | grep -v '#') -fi -for i in $users -do -echo sudouser=$i -done \ No newline at end of file diff --git a/TA-cyences/default/inputs.conf b/TA-cyences/default/inputs.conf index 3849de34..efdfa414 100644 --- a/TA-cyences/default/inputs.conf +++ b/TA-cyences/default/inputs.conf @@ -1,13 +1,6 @@ ################################################################# ### Input to collect sudo user information from linux systems ### ################################################################# -[script://./bin/sudousers.sh] -disabled = 1 -index = linux -# default index for sudo user information collection index -interval = 3600 -sourcetype = sudousers -source = sudousers [script://./bin/users.sh] disabled = 1 diff --git a/cyences_app_for_splunk/default/savedsearches.conf b/cyences_app_for_splunk/default/savedsearches.conf index c201668d..23660167 100644 --- a/cyences_app_for_splunk/default/savedsearches.conf +++ b/cyences_app_for_splunk/default/savedsearches.conf @@ -3962,7 +3962,9 @@ action.cyences_notable_event_action.attacker_search = | stats count by USERNAME action.cyences_notable_event_action.attacker_drilldown = `cs_linux_sudousers` $row.USERNAME$ | rex field=_raw "sudouser=(?.*)" max_match=0 action.cyences_send_email_action = 1 action.cyences_notable_event_action.products = Linux - +action.cyences_notable_event_action.deprecated = 1 +action.cyences_notable_event_action.deprecated_from_version = 4.1.0 +action.cyences_notable_event_action.deprecated_replacement = Linux - User Added/Updated/Deleted [Linux - cs_linux_groups Lookup Gen] disabled = 0 @@ -3987,7 +3989,7 @@ search = `cs_linux_groups` \ [| inputlookup cs_linux_groups \ | eval old_users=users \ | stats latest(old_*) as old_* latest(status) as status latest(_time) as old_time by host group_name \ - | search status!="removed" \ + | search status!="group_removed" \ | fields - status ] \ | stats values(*) as * by host group_name \ | eventstats values(host_status) as host_status , max(new_time) as host_latest by host \ @@ -4022,12 +4024,19 @@ search = `cs_linux_users` UID!=""\ [| inputlookup cs_linux_users \ | eval old_COMMAND_SHELL = COMMAND_SHELL, old_GID=GID, old_HOME_DIR=HOME_DIR,old_SUDOACCESS=SUDOACCESS,old_USER_INFO=USER_INFO,old_USERNAME=USERNAME\ | stats latest(old_*) as old_* latest(status) as status latest(_time) as old_time by host UID \ - | search status!="removed" \ + | search status!="user_removed" \ | fields - status ] \ | stats values(*) as * by host UID \ | eventstats values(host_status) as host_status , max(new_time) as host_latest by host \ | eval status=case(isnull(old_USERNAME) AND host_status=="reporting","user_added",isnull(new_USERNAME) AND host_status=="reporting","user_removed",(new_COMMAND_SHELL!=old_COMMAND_SHELL OR new_HOME_DIR!=old_HOME_DIR OR new_SUDOACCESS!=old_SUDOACCESS OR new_USER_INFO!=old_USER_INFO OR new_GID!=old_GID OR new_USERNAME!=old_USERNAME ) AND host_status=="reporting","user_updated",1=1,"no change") \ | search status!="no change" \ +| eval changes1 = if(status="user_updated" and new_COMMAND_SHELL!=old_COMMAND_SHELL, "COMMAND_SHELL:: ".old_COMMAND_SHELL. " => ".new_COMMAND_SHELL, "")\ +| eval changes2 = if(status="user_updated" and new_HOME_DIR!=old_HOME_DIR, "HOME_DIR:: ".old_HOME_DIR. " => ".new_HOME_DIR, "")\ +| eval changes3 = if(status="user_updated" and new_SUDOACCESS!=old_SUDOACCESS, "SUDOACCESS:: ".old_SUDOACCESS. " => ".new_SUDOACCESS, "")\ +| eval changes4 = if(status="user_updated" and new_USER_INFO!=old_USER_INFO, "USER_INFO:: ".old_USER_INFO. " => ".new_USER_INFO, "")\ +| eval changes5 = if(status="user_updated" and new_GID!=old_GID, "GID:: ".old_GID. " => ".new_GID, "")\ +| eval changes6 = if(status="user_updated" and new_USERNAME!=old_USERNAME, "USERNAME:: ".old_USERNAME. " => ".new_USERNAME, "")\ +| eval changes=mvappend(changes1, changes2, changes3, changes4, changes5, changes6)\ | eval _time=case(status=="user_added",new_time,status=="user_removed",host_latest,status=="user_updated",new_time) \ | eval USERNAME=case(status=="user_added",new_USERNAME,status=="user_removed",old_USERNAME,status=="user_updated",new_USERNAME) \ | eval COMMAND_SHELL=case(status=="user_added",new_COMMAND_SHELL,status=="user_removed",old_COMMAND_SHELL,status=="user_updated",new_COMMAND_SHELL) \ @@ -4035,7 +4044,7 @@ search = `cs_linux_users` UID!=""\ | eval SUDOACCESS=case(status=="user_added",new_SUDOACCESS,status=="user_removed",old_SUDOACCESS,status=="user_updated",new_SUDOACCESS) \ | eval USER_INFO=case(status=="user_added",new_USER_INFO,status=="user_removed",old_USER_INFO,status=="user_updated",new_USER_INFO) \ | eval GID=case(status=="user_added",new_GID,status=="user_removed",old_GID,status=="user_updated",new_GID) \ -| table host UID _time USERNAME COMMAND_SHELL HOME_DIR SUDOACCESS USER_INFO GID status \ +| table host UID _time USERNAME COMMAND_SHELL HOME_DIR SUDOACCESS USER_INFO GID status changes \ | outputlookup cs_linux_users append=true action.cyences_notable_event_action.products = Linux @@ -4059,8 +4068,9 @@ display.page.search.tab = statistics display.page.search.mode = fast request.ui_dispatch_app = cyences_app_for_splunk request.ui_dispatch_view = search -search = | inputlookup cs_linux_users\ -| where _time>relative_time(now(),"-h") | table host UID _time USERNAME COMMAND_SHELL HOME_DIR SUDOACCESS USER_INFO GID status \ +search = | inputlookup cs_linux_users \ +| addinfo | where _time>=info_min_time and _time relative_time(now(),"-h") | table _time host group_name users status \ +search = | inputlookup cs_linux_groups \ +| addinfo | where _time>=info_min_time and _time Configuration**). From 12f0277796bdc00e7da9f753dadacdef42ef4bd6 Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Tue, 27 Jun 2023 22:54:28 +0530 Subject: [PATCH 2/4] group change improvement --- cyences_app_for_splunk/bin/cyences_mvdiff.py | 57 +++++++++++++++++++ cyences_app_for_splunk/default/commands.conf | 6 ++ .../data/ui/views/cs_linux_reports.xml | 3 +- .../default/savedsearches.conf | 7 ++- 4 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 cyences_app_for_splunk/bin/cyences_mvdiff.py diff --git a/cyences_app_for_splunk/bin/cyences_mvdiff.py b/cyences_app_for_splunk/bin/cyences_mvdiff.py new file mode 100644 index 00000000..74d48b33 --- /dev/null +++ b/cyences_app_for_splunk/bin/cyences_mvdiff.py @@ -0,0 +1,57 @@ +import sys + +from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option + +import logging +import logger_manager + +logger = logger_manager.setup_logging("mvdiff", logging.INFO) + + +@Configuration() +class CyencesMVDiff(StreamingCommand): + + old_values_name = Option(name="oldfield", require=True) + new_values_name = Option(name="newfield", require=True) + ignore_values = Option(name="ignore", require=False, default='') + + def build_set(self, values, ignore_values): + if isinstance(values, str): + new_values = {values} - ignore_values + else: + new_values = set(values) - ignore_values + + return {i for i in new_values if i.strip() } + + def stream(self, records): + try: + for record in records: + old_values = record.get(self.old_values_name, []) + new_values = record.get(self.new_values_name, []) + ignore_values = set(self.ignore_values.split(',')) + + old_values = self.build_set(old_values, ignore_values) + new_values = self.build_set(new_values, ignore_values) + + logger.debug("old_values={}".format(old_values)) + logger.debug("new_values={}".format(new_values)) + + added = list(new_values - old_values) + removed = list(old_values - new_values) + + record['added'] = added if len(added) > 0 else None + record['removed'] = removed if len(removed) > 0 else None + + logger.debug("added={}".format(record['added'])) + logger.debug("removed={}".format(record['removed'])) + + yield record + + except Exception as err: + msg = "Error occurred in CyencesMVDiff command: {}".format(err) + self.write_error(msg) + logger.exception(msg) + + +if __name__ == "__main__": + dispatch(CyencesMVDiff, sys.argv, sys.stdin, sys.stdout, __name__) diff --git a/cyences_app_for_splunk/default/commands.conf b/cyences_app_for_splunk/default/commands.conf index 787fb324..c1aeac24 100644 --- a/cyences_app_for_splunk/default/commands.conf +++ b/cyences_app_for_splunk/default/commands.conf @@ -52,6 +52,11 @@ filename = timestamp_based_correlation.py chunked = true python.version = python3 +[cyencesmvdiff] +filename = cyences_mvdiff.py +chunked = true +python.version = python3 + [cyencesnotablegenerateid] filename = notable_generate_id.py chunked = true @@ -66,3 +71,4 @@ python.version = python3 filename = cyences_upgrade.py chunked = true python.version = python3 + diff --git a/cyences_app_for_splunk/default/data/ui/views/cs_linux_reports.xml b/cyences_app_for_splunk/default/data/ui/views/cs_linux_reports.xml index fac082a0..20fba96d 100644 --- a/cyences_app_for_splunk/default/data/ui/views/cs_linux_reports.xml +++ b/cyences_app_for_splunk/default/data/ui/views/cs_linux_reports.xml @@ -45,7 +45,8 @@ | append [| tstats count where `cs_linux_users_with_previledge` | eval label="`cs_linux` sourcetype=\"usersWithLoginPrivs\"" ] | append [| search index=* `cs_linux_failed_login` | head 1 | stats count | eval label="`cs_linux` (sourcetype=linux_secure eventtype=*failed_login*) OR (source=/var/log/audit.log user=* process=sshd action=failure)"] | append [| search index=* `cs_linux_success_login` | head 1 | stats count | eval label="`cs_linux` (source=/var/log/secure app=sshd) OR (source=/var/log/audit.log user=* process=sshd ) action=success"] -| append [| tstats count where `cs_linux_sudousers` | eval label="`cs_linux` sourcetype=\"sudousers\""] +| append [| tstats count where `cs_linux_users` | eval label="`cs_linux` sourcetype=\"cyences:linux:users\""] +| append [| tstats count where `cs_linux_groups` | eval label="`cs_linux` sourcetype=\"cyences:linux:groups\""] | append [| tstats count where `cs_linux_unix_version` | eval label="`cs_linux` sourcetype=\"Unix:Version\""] | append [| tstats count where `cs_linux_unix_uptime` | eval label="`cs_linux` sourcetype=\"Unix:Uptime\""] | append [| tstats count where `cs_linux_hardware` | eval label="`cs_linux` sourcetype=\"hardware\""] diff --git a/cyences_app_for_splunk/default/savedsearches.conf b/cyences_app_for_splunk/default/savedsearches.conf index 23660167..9b216e83 100644 --- a/cyences_app_for_splunk/default/savedsearches.conf +++ b/cyences_app_for_splunk/default/savedsearches.conf @@ -3995,9 +3995,14 @@ search = `cs_linux_groups` \ | eventstats values(host_status) as host_status , max(new_time) as host_latest by host \ | eval status=case(isnull(old_users) AND host_status=="reporting","group_added",isnull(new_users) AND host_status=="reporting","group_removed",(new_users!=old_users) AND host_status=="reporting","group_updated",1=1,"no change") \ | search status!="no change" \ +| eval mv_old_users = split(old_users, ","), mv_new_users = split(new_users, ",") \ +| cyencesmvdiff oldfield="mv_old_users" newfield="mv_new_users" ignore="NOUSERS" \ +| eval changes1 = if(isnotnull(added), "USER_ADDED:: ". mvjoin(added, ","), "") \ +| eval changes2 = if(isnotnull(removed), "USER_REMOVED:: ". mvjoin(removed, ","), "") \ +| eval changes=mvappend(changes1, changes2) \ | eval _time=case(status=="group_added",new_time,status=="group_removed",host_latest,status=="group_updated",new_time) \ | eval users=case(status=="group_added",new_users,status=="group_removed",old_users,status=="group_updated",new_users) \ -| table host group_name _time users status \ +| table host group_name _time users status changes \ | outputlookup cs_linux_groups append=true action.cyences_notable_event_action.products = Linux From 7c230cd72019dae15d3a5f5e87d99105392533d5 Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Wed, 28 Jun 2023 15:25:43 +0530 Subject: [PATCH 3/4] remove comments --- TA-cyences/default/inputs.conf | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/TA-cyences/default/inputs.conf b/TA-cyences/default/inputs.conf index efdfa414..9121264c 100644 --- a/TA-cyences/default/inputs.conf +++ b/TA-cyences/default/inputs.conf @@ -1,19 +1,14 @@ -################################################################# -### Input to collect sudo user information from linux systems ### -################################################################# [script://./bin/users.sh] disabled = 1 -index = linux -# default index for sudo user information collection index +index = linux interval = 3600 sourcetype = cyences:linux:users source = TA-cyences:users.sh [script://./bin/groups.sh] disabled = 1 -index = linux -# default index for sudo user information collection index +index = linux interval = 3600 sourcetype = cyences:linux:groups source = TA-cyences:groups.sh From dbde2911d8bd3cb5de30a918be0391ec63f89bf6 Mon Sep 17 00:00:00 2001 From: Mahir Chavda Date: Wed, 28 Jun 2023 18:55:21 +0530 Subject: [PATCH 4/4] add changes field into collection definition --- cyences_app_for_splunk/default/transforms.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cyences_app_for_splunk/default/transforms.conf b/cyences_app_for_splunk/default/transforms.conf index 24fed6cb..c2550324 100644 --- a/cyences_app_for_splunk/default/transforms.conf +++ b/cyences_app_for_splunk/default/transforms.conf @@ -137,13 +137,13 @@ case_sensitive_match = false [cs_linux_users] external_type = kvstore collection = cs_linux_users_colllections -fields_list = _key,host,UID,_time,USERNAME,COMMAND_SHELL,HOME_DIR,SUDOACCESS,USER_INFO,GID,status +fields_list = _key,host,UID,_time,USERNAME,COMMAND_SHELL,HOME_DIR,SUDOACCESS,USER_INFO,GID,status,changes case_sensitive_match = false [cs_linux_groups] external_type = kvstore collection = cs_linux_groups_colllections -fields_list = _key,host,group_name,_time,users,status +fields_list = _key,host,group_name,_time,users,status,changes case_sensitive_match = false