Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add host to the device inventory v2 from internal logs #382

Merged
merged 5 commits into from
Aug 29, 2023
Merged

Add host to the device inventory v2 from internal logs #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
859e747
Added the table border of the expanded tables
hardikhdholariya Aug 28, 2023
c18825d
Added the savedsearch to add the forwders available in splunk env.
hardikhdholariya Aug 28, 2023
111225a
Added macro to the cyences configuration
hardikhdholariya Aug 29, 2023
2aab449
Added index_time field to device inventory v2 and updated the cleanup…
hardikhdholariya Aug 29, 2023
86a7b16
Used the _indextime instead of _time
hardikhdholariya Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cyences_app_for_splunk/appserver/static/js/build/cs_configuration.js
View file
Open in desktop
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mahirchavda - I think this needs to be changed outside minified JS. Please work with @mahirchavda do make that change.

Large diffs are not rendered by default.

25 changes: 17 additions & 8 deletions cyences_app_for_splunk/appserver/static/notable_event_editor.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,28 +30,37 @@
/* Hidden Columns */
#notable_event_main_tbl .notable_event_id,
#notable_event_main_tbl [data\-sort\-key='notable_event_id'] {
display:none;
display: none;
}


/* Hide Table Headers */
#notable_event_main_tbl [data\-sort\-key='notable_event_selector'],
#notable_event_main_tbl [data\-sort\-key='notable_event_edit'],
#notable_event_main_tbl [data\-sort\-key='notable_event_assignee'],
#notable_event_main_tbl [data\-sort\-key='notable_event_quick_assign_to_me']
{
#notable_event_main_tbl [data\-sort\-key='notable_event_quick_assign_to_me'] {
min-width: 20px !important;
max-width: 20px !important;
width: 20px !important;
padding:0px;
margin:0px;
padding: 0px;
margin: 0px;
}

#notable_event_main_tbl [data\-sort\-key='notable_event_selector'] a,
#notable_event_main_tbl [data\-sort\-key='notable_event_edit'] a,
#notable_event_main_tbl [data\-sort\-key='notable_event_assignee'] a,
#notable_event_main_tbl [data\-sort\-key='notable_event_quick_assign_to_me'] a
{
#notable_event_main_tbl [data\-sort\-key='notable_event_quick_assign_to_me'] a {
/* color:transparent !important; */
display:none;
display: none;
}

/* raw expansion table borders */
#notable_event_history_table table tbody tr td {
border: 1px solid;
border-color: black;
}

#notable_event_result_table table tbody tr td {
border: 1px solid;
border-color: black;
}
27 changes: 14 additions & 13 deletions cyences_app_for_splunk/bin/cyences_device_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ class CyencesDeviceManagerCommand(EventingCommand):
# I prefer to also put ".local" at the end as well to ensure proper hostname matching

products_to_cleanup = Option(name="products_to_cleanup", require=False, default="*")
cleanup_mintime = Option(name="mintime", require=False, default=None, validate=validators.Float()) # default past 1 years
cleanup_maxtime = Option(name="maxtime", require=False, default=None, validate=validators.Float()) # default forseeable future
cleanup_minindextime = Option(name="minindextime", require=False, default=None, validate=validators.Float()) # default past 1 years
cleanup_maxindextime = Option(name="maxindextime", require=False, default=None, validate=validators.Float()) # default forseeable future
target_device = Option(name="target_device", require=False, default="")
devices_to_merge = Option(name="devices_to_merge", require=False, default=None)
# cleanup_ip_mintime = Option(name="ipmintime", require=False, default=None, validate=validators.Float()) # default past 30 days
Expand All @@ -57,24 +57,24 @@ def validate_param_value_and_type(command_options):
else:
raise Exception("{} value is not as expected.".format(command_options))
return command_options

def validate_inputs(self):
if self.operation not in ["getdevices", "addentries", "cleanup", "merge", "manualmerge"]:
raise Exception("operation - allowed values: getdevices, addentries, cleanup, merge, manualmerge")

if self.operation == "cleanup":
timenow = time.time()

if self.cleanup_mintime is None:
self.cleanup_mintime = timenow - YEAR_IN_SECOND
if self.cleanup_maxtime is None:
self.cleanup_maxtime = MAX_TIME_EPOCH
if self.cleanup_minindextime is None:
self.cleanup_minindextime = timenow - YEAR_IN_SECOND
if self.cleanup_maxindextime is None:
self.cleanup_maxindextime = MAX_TIME_EPOCH

if self.cleanup_minindextime >= self.cleanup_maxindextime:
raise Exception("minindextime should be less than maxindextime.")

if self.cleanup_mintime >= self.cleanup_maxtime:
raise Exception("mintime should be less than maxtime.")

self.products_to_cleanup = self.validate_param_value_and_type(self.products_to_cleanup)

elif self.operation == "manualmerge":
self.devices_to_merge = self.validate_param_value_and_type(self.devices_to_merge)

Expand Down Expand Up @@ -103,20 +103,21 @@ def transform(self, records):
with DeviceManager(session_key, logger, DEVICE_INVENTORY_LOOKUP_COLLECTION, hostname_postfixes) as dm:
for record in records:
other_fields = copy.deepcopy(record)
del other_fields["indextime"]
del other_fields["time"]
del other_fields["product_name"]
del other_fields["product_uuid"]
del other_fields["ip"]
del other_fields["mac_address"]
del other_fields["hostname"]
entry = DeviceEntry(record["product_name"], record["time"], record["product_uuid"], record["ip"], record["mac_address"], record["hostname"], other_fields)
entry = DeviceEntry(record["product_name"], record["time"], record["indextime"], record["product_uuid"], record["ip"], record["mac_address"], record["hostname"], other_fields)
device_id = dm.add_device_entry(entry)
record["device_id"] = device_id
yield record

elif self.operation == "cleanup":
with DeviceManager(session_key, logger, DEVICE_INVENTORY_LOOKUP_COLLECTION) as dm:
messages = dm.cleanup_devices(self.cleanup_mintime, self.cleanup_maxtime, self.products_to_cleanup)
messages = dm.cleanup_devices(self.cleanup_minindextime, self.cleanup_maxindextime, self.products_to_cleanup)
for m in messages:
yield {"message": m}

Expand Down
13 changes: 8 additions & 5 deletions cyences_app_for_splunk/bin/device_inventory_v2_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ def __init__(
self,
product_name: str,
time: int,
indextime: int,
product_uuid: str,
ips,
mac_addresses,
Expand All @@ -97,6 +98,7 @@ def __init__(
) -> None:
self.product_name = product_name
self.time = time
self.indextime = indextime
self.product_uuid = product_uuid
self.ips = self._internal_check_list_field_format(ips)
self.mac_addresses = self._internal_check_list_field_format(mac_addresses)
Expand Down Expand Up @@ -475,7 +477,7 @@ def add_device_entry(self, new_entry: DeviceEntry):
return new_device.get("uuid")

def cleanup(
self, device, min_time, max_time=MAX_TIME_EPOCH, products_to_cleanup=None
self, device, min_indextime, max_indextime=MAX_TIME_EPOCH, products_to_cleanup=None
):
"""
products_to_cleanup is None meaning cleanup all products
Expand All @@ -487,8 +489,8 @@ def cleanup(

for product_uuid, entry_details in product_items.items():
if (
int(float(entry_details["time"])) < min_time
or int(float(entry_details["time"])) > max_time
int(float(entry_details["indextime"])) < min_indextime
or int(float(entry_details["indextime"])) > max_indextime
):
self._remove_entry_content(
product_name, product_uuid, entry_details, device
Expand Down Expand Up @@ -544,7 +546,7 @@ def reorganize_device_list(self):
return messages

def cleanup_devices(
self, min_time, max_time=MAX_TIME_EPOCH, products_to_cleanup=None
self, min_indextime, max_indextime=MAX_TIME_EPOCH, products_to_cleanup=None
):
"""
products_to_cleanup is None meaning cleanup all products
Expand All @@ -554,7 +556,7 @@ def cleanup_devices(
idx = 0
while idx < len(self.devices):
is_device_still_valid = self.cleanup(
self.devices[idx], min_time, max_time, products_to_cleanup
self.devices[idx], min_indextime, max_indextime, products_to_cleanup
)
if not is_device_still_valid:
messages.append(
Expand Down Expand Up @@ -602,6 +604,7 @@ def _convert_device_to_deviceentry_obj(self, device_obj):
DeviceEntry(
product_name=product_name,
time=element_details["time"],
indextime=element_details["indextime"],
product_uuid=product_uuid,
ips=element_details["ips"],
mac_addresses=element_details["mac_addresses"],
Expand Down
2 changes: 1 addition & 1 deletion cyences_app_for_splunk/default/data/ui/views/cs_device_inventory_table_v2.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
</html>
<table>
<search>
<query>| rest /servicesNS/-/cyences_app_for_splunk/saved/searches count=0 splunk_server=local | search "eai:acl.app"="cyences_app_for_splunk" title IN ("Device Inventory - Lansweeper - v2", "Device Inventory - Tenable - v2", "Device Inventory - Tenable Vuln - v2", "Device Inventory - Qualys - v2", "Device Inventory - Sophos - v2", "Device Inventory - Windows Defender - v2", "Device Inventory - CrowdStrike - v2", "Device Inventory - Kaspersky - v2", "Device Inventory Backfill - V2", "Device Inventory Lookup CleanUp - V2", "Device Inventory Merge Similar Devices - V2") | table title, disabled | rename title as label | eval status=if(disabled=1, "Report is disabled", "Report is enabled") | table label, status</query>
<query>| rest /servicesNS/-/cyences_app_for_splunk/saved/searches count=0 splunk_server=local | search "eai:acl.app"="cyences_app_for_splunk" title IN ("Device Inventory - Lansweeper - v2", "Device Inventory - Tenable - v2", "Device Inventory - Tenable Vuln - v2", "Device Inventory - Qualys - v2", "Device Inventory - Sophos - v2", "Device Inventory - Windows Defender - v2", "Device Inventory - CrowdStrike - v2", "Device Inventory - Kaspersky - v2", "Device Inventory - Splunk Internal - V2", "Device Inventory Backfill - V2", "Device Inventory Lookup CleanUp - V2", "Device Inventory Merge Similar Devices - V2") | table title, disabled | rename title as label | eval status=if(disabled=1, "Report is disabled", "Report is enabled") | table label, status</query>
<earliest>0</earliest>
<latest></latest>
</search>
Expand Down
4 changes: 2 additions & 2 deletions cyences_app_for_splunk/default/macros.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,9 +275,9 @@ definition = dedup host \

[cs_windows_defender_inventory_fill_search_v2]
definition = dedup host \
| eval hostname=lower(host), ip="", mac_address="", product_name="Windows Defender", product_uuid=lower(host), Last_full_scan_start_time=if(Last_full_scan_start_time="1/1/1601 12:00:00 AM", "-", Last_full_scan_start_time), Last_full_scan_end_time=if(Last_full_scan_end_time="1/1/1601 12:00:00 AM", "-", Last_full_scan_end_time), Last_quick_scan_start_time=if(Last_quick_scan_start_time="1/1/1601 12:00:00 AM", "-", Last_quick_scan_start_time), Last_quick_scan_end_time=if(Last_quick_scan_end_time="1/1/1601 12:00:00 AM", "-", Last_quick_scan_end_time), time=_time \
| eval hostname=lower(host), ip="", mac_address="", product_name="Windows Defender", product_uuid=lower(host), Last_full_scan_start_time=if(Last_full_scan_start_time="1/1/1601 12:00:00 AM", "-", Last_full_scan_start_time), Last_full_scan_end_time=if(Last_full_scan_end_time="1/1/1601 12:00:00 AM", "-", Last_full_scan_end_time), Last_quick_scan_start_time=if(Last_quick_scan_start_time="1/1/1601 12:00:00 AM", "-", Last_quick_scan_start_time), Last_quick_scan_end_time=if(Last_quick_scan_end_time="1/1/1601 12:00:00 AM", "-", Last_quick_scan_end_time), time=_time, indextime=_indextime \
| rename host as windows_defender_host \
| table time, product_name, product_uuid, windows_defender_host, hostname, ip, mac_address, RTP_state, Platform_version, Engine_version, AVSignature_version, BM_state, IOAV_state, OA_state, Last_full_scan_start_time, Last_full_scan_end_time, Last_quick_scan_start_time, Last_quick_scan_end_time \
| table time, indextime, product_name, product_uuid, windows_defender_host, hostname, ip, mac_address, RTP_state, Platform_version, Engine_version, AVSignature_version, BM_state, IOAV_state, OA_state, Last_full_scan_start_time, Last_full_scan_end_time, Last_quick_scan_start_time, Last_quick_scan_end_time \
| cyencesdevicemanager operation="addentries" \
| append \
[| inputlookup cs_windows_defender_inventory_v2 ] \
Expand Down
Loading
Loading