Merge pull request #334 from CrossRealms/linux-user-changes

CY-485: Linux user changes

TA-cyences/default/inputs.conf

@@ -1,26 +1,14 @@
-#################################################################
-### Input to collect sudo user information from linux systems ###
-#################################################################
-[script://./bin/sudousers.sh]
-disabled = 1
-index = linux   
-# default index for sudo user information collection index
-interval = 3600
-sourcetype = sudousers
-source = sudousers
 
 [script://./bin/users.sh]
 disabled = 1
-index = linux   
-# default index for sudo user information collection index
+index = linux
 interval = 3600
 sourcetype = cyences:linux:users
 source = TA-cyences:users.sh
 
 [script://./bin/groups.sh]
 disabled = 1
-index = linux   
-# default index for sudo user information collection index
+index = linux
 interval = 3600
 sourcetype = cyences:linux:groups
 source = TA-cyences:groups.sh

cyences_app_for_splunk/bin/cyences_mvdiff.py

@@ -0,0 +1,57 @@
+import sys
+
+from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option
+
+import logging
+import logger_manager
+
+logger = logger_manager.setup_logging("mvdiff", logging.INFO)
+
+
+@Configuration()
+class CyencesMVDiff(StreamingCommand):
+
+    old_values_name = Option(name="oldfield", require=True)
+    new_values_name = Option(name="newfield", require=True)
+    ignore_values = Option(name="ignore", require=False, default='')
+
+    def build_set(self, values, ignore_values):
+        if isinstance(values, str):
+            new_values =  {values} - ignore_values
+        else:
+            new_values = set(values) - ignore_values
+
+        return {i for i in new_values if i.strip() }
+
+    def stream(self, records):
+        try:
+            for record in records:
+                old_values = record.get(self.old_values_name, [])
+                new_values = record.get(self.new_values_name, [])
+                ignore_values = set(self.ignore_values.split(','))
+
+                old_values = self.build_set(old_values, ignore_values)
+                new_values = self.build_set(new_values, ignore_values)
+
+                logger.debug("old_values={}".format(old_values))
+                logger.debug("new_values={}".format(new_values))
+
+                added = list(new_values - old_values)
+                removed = list(old_values - new_values)
+
+                record['added'] = added if len(added) > 0 else None
+                record['removed'] = removed if len(removed) > 0 else None
+
+                logger.debug("added={}".format(record['added']))
+                logger.debug("removed={}".format(record['removed']))
+
+                yield record
+
+        except Exception as err:
+            msg = "Error occurred in CyencesMVDiff command: {}".format(err)
+            self.write_error(msg)
+            logger.exception(msg)
+
+
+if __name__ == "__main__":
+    dispatch(CyencesMVDiff, sys.argv, sys.stdin, sys.stdout, __name__)

cyences_app_for_splunk/default/commands.conf

@@ -52,6 +52,11 @@ filename = timestamp_based_correlation.py
 chunked = true
 python.version = python3
 
+[cyencesmvdiff]
+filename = cyences_mvdiff.py
+chunked = true
+python.version = python3
+
 [cyencesnotablegenerateid]
 filename = notable_generate_id.py
 chunked = true
@@ -66,3 +71,4 @@ python.version = python3
 filename = cyences_upgrade.py
 chunked = true
 python.version = python3
+

cyences_app_for_splunk/default/data/ui/views/cs_linux_reports.xml

@@ -45,7 +45,8 @@
 | append [| tstats count where `cs_linux_users_with_previledge` | eval label="`cs_linux` sourcetype=\"usersWithLoginPrivs\"" ]
 | append [| search index=* `cs_linux_failed_login` | head 1 | stats count  | eval label="`cs_linux` (sourcetype=linux_secure eventtype=*failed_login*) OR (source=/var/log/audit.log user=* process=sshd action=failure)"]
 | append [| search index=* `cs_linux_success_login` | head 1 | stats count | eval label="`cs_linux` (source=/var/log/secure app=sshd) OR (source=/var/log/audit.log user=* process=sshd ) action=success"]
-| append [| tstats count where `cs_linux_sudousers`  | eval label="`cs_linux` sourcetype=\"sudousers\""]
+| append [| tstats count where `cs_linux_users`  | eval label="`cs_linux` sourcetype=\"cyences:linux:users\""]
+| append [| tstats count where `cs_linux_groups`  | eval label="`cs_linux` sourcetype=\"cyences:linux:groups\""]
 | append [| tstats count where `cs_linux_unix_version`  | eval label="`cs_linux` sourcetype=\"Unix:Version\""]
 | append [| tstats count where `cs_linux_unix_uptime`  | eval label="`cs_linux` sourcetype=\"Unix:Uptime\""]
 | append [| tstats count where `cs_linux_hardware`  | eval label="`cs_linux` sourcetype=\"hardware\""]

cyences_app_for_splunk/default/savedsearches.conf

@@ -4051,7 +4051,9 @@ action.cyences_notable_event_action.attacker_search = | stats count by USERNAME
 action.cyences_notable_event_action.attacker_drilldown = `cs_linux_sudousers` $row.USERNAME$ | rex field=_raw "sudouser=(?<USERNAME>.*)" max_match=0
 action.cyences_send_email_action = 1
 action.cyences_notable_event_action.products = Linux
-
+action.cyences_notable_event_action.deprecated = 1
+action.cyences_notable_event_action.deprecated_from_version = 4.1.0
+action.cyences_notable_event_action.deprecated_replacement = Linux - User Added/Updated/Deleted
 
 [Linux - cs_linux_groups Lookup Gen]
 disabled = 0
@@ -4076,15 +4078,20 @@ search = `cs_linux_groups` \
     [| inputlookup cs_linux_groups \
     | eval old_users=users \
     | stats latest(old_*) as old_* latest(status) as status latest(_time) as old_time by host group_name \
-    | search status!="removed" \
+    | search status!="group_removed" \
     | fields - status ] \
 | stats values(*) as * by host group_name \
 | eventstats values(host_status) as host_status , max(new_time) as host_latest by host \
 | eval status=case(isnull(old_users) AND host_status=="reporting","group_added",isnull(new_users) AND host_status=="reporting","group_removed",(new_users!=old_users) AND host_status=="reporting","group_updated",1=1,"no change") \
 | search status!="no change" \
+| eval mv_old_users = split(old_users, ","), mv_new_users = split(new_users, ",") \
+| cyencesmvdiff oldfield="mv_old_users" newfield="mv_new_users" ignore="NOUSERS" \
+| eval changes1 = if(isnotnull(added), "USER_ADDED:: ". mvjoin(added, ","), "") \
+| eval changes2 = if(isnotnull(removed), "USER_REMOVED:: ". mvjoin(removed, ","), "") \
+| eval changes=mvappend(changes1, changes2) \
 | eval _time=case(status=="group_added",new_time,status=="group_removed",host_latest,status=="group_updated",new_time) \
 | eval users=case(status=="group_added",new_users,status=="group_removed",old_users,status=="group_updated",new_users) \
-| table host group_name _time users status \
+| table host group_name _time users status changes \
 | outputlookup cs_linux_groups append=true
 action.cyences_notable_event_action.products = Linux
 
@@ -4111,20 +4118,27 @@ search = `cs_linux_users` UID!=""\
     [| inputlookup cs_linux_users \
     | eval old_COMMAND_SHELL = COMMAND_SHELL, old_GID=GID, old_HOME_DIR=HOME_DIR,old_SUDOACCESS=SUDOACCESS,old_USER_INFO=USER_INFO,old_USERNAME=USERNAME\
     | stats latest(old_*) as old_* latest(status) as status latest(_time) as old_time by host UID \
-    | search status!="removed" \
+    | search status!="user_removed" \
     | fields - status ] \
 | stats values(*) as * by  host UID \
 | eventstats values(host_status) as host_status , max(new_time) as host_latest by host \
 | eval status=case(isnull(old_USERNAME) AND host_status=="reporting","user_added",isnull(new_USERNAME) AND host_status=="reporting","user_removed",(new_COMMAND_SHELL!=old_COMMAND_SHELL OR new_HOME_DIR!=old_HOME_DIR OR new_SUDOACCESS!=old_SUDOACCESS OR new_USER_INFO!=old_USER_INFO OR new_GID!=old_GID OR new_USERNAME!=old_USERNAME ) AND host_status=="reporting","user_updated",1=1,"no change") \
 | search status!="no change" \
+| eval changes1 = if(status="user_updated" and new_COMMAND_SHELL!=old_COMMAND_SHELL, "COMMAND_SHELL:: ".old_COMMAND_SHELL. " => ".new_COMMAND_SHELL, "")\
+| eval changes2 = if(status="user_updated" and new_HOME_DIR!=old_HOME_DIR, "HOME_DIR:: ".old_HOME_DIR. " => ".new_HOME_DIR, "")\
+| eval changes3 = if(status="user_updated" and new_SUDOACCESS!=old_SUDOACCESS, "SUDOACCESS:: ".old_SUDOACCESS. " => ".new_SUDOACCESS, "")\
+| eval changes4 = if(status="user_updated" and new_USER_INFO!=old_USER_INFO, "USER_INFO:: ".old_USER_INFO. " => ".new_USER_INFO, "")\
+| eval changes5 = if(status="user_updated" and new_GID!=old_GID, "GID:: ".old_GID. " => ".new_GID, "")\
+| eval changes6 = if(status="user_updated" and new_USERNAME!=old_USERNAME, "USERNAME:: ".old_USERNAME. " => ".new_USERNAME, "")\
+| eval changes=mvappend(changes1, changes2, changes3, changes4, changes5, changes6)\
 | eval _time=case(status=="user_added",new_time,status=="user_removed",host_latest,status=="user_updated",new_time) \
 | eval USERNAME=case(status=="user_added",new_USERNAME,status=="user_removed",old_USERNAME,status=="user_updated",new_USERNAME) \
 | eval COMMAND_SHELL=case(status=="user_added",new_COMMAND_SHELL,status=="user_removed",old_COMMAND_SHELL,status=="user_updated",new_COMMAND_SHELL) \
 | eval HOME_DIR=case(status=="user_added",new_HOME_DIR,status=="user_removed",old_HOME_DIR,status=="user_updated",new_HOME_DIR) \
 | eval SUDOACCESS=case(status=="user_added",new_SUDOACCESS,status=="user_removed",old_SUDOACCESS,status=="user_updated",new_SUDOACCESS) \
 | eval USER_INFO=case(status=="user_added",new_USER_INFO,status=="user_removed",old_USER_INFO,status=="user_updated",new_USER_INFO) \
 | eval GID=case(status=="user_added",new_GID,status=="user_removed",old_GID,status=="user_updated",new_GID) \
-| table host UID _time USERNAME COMMAND_SHELL HOME_DIR SUDOACCESS USER_INFO GID status \
+| table host UID _time USERNAME COMMAND_SHELL HOME_DIR SUDOACCESS USER_INFO GID status changes \
 | outputlookup cs_linux_users append=true
 action.cyences_notable_event_action.products = Linux
 
@@ -4148,8 +4162,9 @@ display.page.search.tab = statistics
 display.page.search.mode = fast
 request.ui_dispatch_app = cyences_app_for_splunk
 request.ui_dispatch_view = search
-search = | inputlookup cs_linux_users\
-| where _time>relative_time(now(),"-h") | table host UID _time USERNAME COMMAND_SHELL HOME_DIR SUDOACCESS USER_INFO GID status \
+search = | inputlookup cs_linux_users \
+| addinfo | where _time>=info_min_time and _time <info_max_time \
+| table host UID _time USERNAME COMMAND_SHELL HOME_DIR SUDOACCESS USER_INFO GID status \
 | eval cyences_severity = case(UID=0, "high",1=1, "medium") \
 | `cs_change_in_user_linux_filter`
 action.cyences_notable_event_action = 1
@@ -4181,8 +4196,9 @@ display.page.search.tab = statistics
 display.page.search.mode = fast
 request.ui_dispatch_app = cyences_app_for_splunk
 request.ui_dispatch_view = search
-search = | inputlookup cs_linux_groups\
-| where _time>relative_time(now(),"-h") | table _time host group_name users status \
+search = | inputlookup cs_linux_groups \
+| addinfo | where _time>=info_min_time and _time <info_max_time \
+| table _time host group_name users status \
 | eval cyences_severity = case(group_name="root", "high",1=1, "medium") \
 | `cs_change_in_group_of_linux_filter`
 action.cyences_notable_event_action = 1

cyences_app_for_splunk/default/transforms.conf

@@ -137,13 +137,13 @@ case_sensitive_match = false
 [cs_linux_users]
 external_type = kvstore
 collection = cs_linux_users_colllections
-fields_list = _key,host,UID,_time,USERNAME,COMMAND_SHELL,HOME_DIR,SUDOACCESS,USER_INFO,GID,status
+fields_list = _key,host,UID,_time,USERNAME,COMMAND_SHELL,HOME_DIR,SUDOACCESS,USER_INFO,GID,status,changes
 case_sensitive_match = false
 
 [cs_linux_groups]
 external_type = kvstore
 collection = cs_linux_groups_colllections
-fields_list = _key,host,group_name,_time,users,status
+fields_list = _key,host,group_name,_time,users,status,changes
 case_sensitive_match = false
 
 

docs/data_onboarding/linux.md

@@ -88,8 +88,14 @@ Splunkbase Download:
 
 Insert the input stanza below for the Cyences Add-on for Splunk. The stanza should be placed in the inputs.conf file (create a local directory if necessary): 
 
-    [script://./bin/sudousers.sh] 
-    disabled = 0 
+    [script://./bin/users.sh]
+    disabled = 0
+    index = linux   
+
+    [script://./bin/groups.sh]
+    disabled = 0
+    index = linux   
+
 
 **Note:** Use index=**linux** or index=**os**, for data collection or update the macro definition for Linux Data (**Settings > Configuration**).