diff --git a/scripts/__load__.zeek b/scripts/__load__.zeek index 9b6b1c2..886c422 100644 --- a/scripts/__load__.zeek +++ b/scripts/__load__.zeek @@ -1 +1,5 @@ -@load ./enrich.zeek +@load ./new_conn.zeek +@load ./main.zeek +@load ./known.zeek +@load ./conn.zeek +@load ./id-logs.zeek diff --git a/scripts/conn.zeek b/scripts/conn.zeek new file mode 100644 index 0000000..75ef426 --- /dev/null +++ b/scripts/conn.zeek @@ -0,0 +1,52 @@ +module EndpointEnrichment; + +## Enrich Conn.log ## +export { + ## Enables the logging of endpoint details to the conn log. + option extra_logging_conn = F; + option extra_logging_conn_cid = F; +} + +redef record Conn::Info += { + orig_ep_status: string &log &optional; + orig_ep_uid: string &log &optional; + orig_ep_cid: string &log &optional; + orig_ep_source: string &log &optional; + resp_ep_status: string &log &optional; + resp_ep_uid: string &log &optional; + resp_ep_cid: string &log &optional; + resp_ep_source: string &log &optional; +}; + + +event new_connection(c: connection) { + if (extra_logging_conn) { + if ( !c$conn?$local_orig && !c$conn?$local_resp ) { + return; + } + + # If the orig IP is local and in the list, update the conn log. + if ( c$conn?$local_orig && c$id$orig_h in hosts_data ) { + local orig_data = hosts_data[c$id$orig_h]; + if ( orig_data ?$ status) + c$conn$orig_ep_status = orig_data$status; + if ( orig_data ?$ uid) + c$conn$orig_ep_uid = orig_data$uid; + if ( orig_data ?$ cid && extra_logging_conn_cid) + c$conn$orig_ep_cid = orig_data$cid; + c$conn$orig_ep_source = orig_data$source; + } + + # If the resp IP is local and in the list, update the conn log. + if ( c$conn?$local_resp && c$id$resp_h in hosts_data ) { + local resp_data = hosts_data[c$id$resp_h]; + if ( resp_data ?$ status) + c$conn$resp_ep_status = resp_data$status; + if ( resp_data ?$ uid) + c$conn$resp_ep_uid = resp_data$uid; + if ( resp_data ?$ cid && extra_logging_conn_cid) + c$conn$resp_ep_cid = resp_data$cid; + c$conn$resp_ep_source = resp_data$source; + } + } +} diff --git a/scripts/enrich.zeek b/scripts/enrich.zeek deleted file mode 100644 index 6178a5e..0000000 --- a/scripts/enrich.zeek +++ /dev/null @@ -1,173 +0,0 @@ -module EndpointEnrichment; - -type Idx: record { - ip: addr; -}; - -type Val: record { - ## The description of the endpoint. - desc: string &log &optional; - ## The status of the endpoint host. - status: string &log &optional; - ## The unique identifier, assigned by the source, of the endpoint host. - host_uid: string &log &optional; - ## The customer ID the host belongs to. - cid: string &log &optional; - ## The Operating System version of the endpoint host. - os_version: string &log &optional; - ## The source of the endpoint information. - source: string &log &optional; - ## The criticality of the endpoint host. - criticality: string &log &optional; - ## The MAC address of the endpoint host. - mac: string &optional; - ## The hostname of the vulnerable host. - hostname: string &optional; - ## The machine domain of the endpoint host. - machine_domain: string &optional; -}; - -global hosts_data: table[addr] of Val = table(); -# # source to use for all unknown IPs -# global unknownSource: string; - -event zeek_init() { - Input::add_table([ - $source="hosts_data.tsv", - $name="hosts_data", - $idx=Idx, - $val=Val, - $destination=hosts_data, - $mode=Input::REREAD - ]); -} - -## Enrich Conn.log ## -export { - ## Enables the logging of endpoint details to the conn log. - option extra_logging_conn = F; -} - -redef record Conn::Info += { - orig_endpoint_status: string &log &optional; - orig_endpoint_host_uid: string &log &optional; - orig_endpoint_cid: string &log &optional; - orig_endpoint_source: string &log &optional; - resp_endpoint_status: string &log &optional; - resp_endpoint_host_uid: string &log &optional; - resp_endpoint_cid: string &log &optional; - resp_endpoint_source: string &log &optional; -}; - -# priority of -5 is too long for enriching the conn.log, -# the connection has already been removed from memory -event connection_state_remove(c: connection) -{ - if (extra_logging_conn) { - if ( !c$conn?$local_orig && !c$conn?$local_resp ) { - return; - } - - # If the orig IP is local and in the list, update the conn log. - if ( c$conn?$local_orig && c$id$orig_h in hosts_data ) { - local orig_data = hosts_data[c$id$orig_h]; - if ( orig_data ?$ status) - c$conn$orig_endpoint_status = orig_data$status; - if ( orig_data ?$ host_uid) - c$conn$orig_endpoint_host_uid = orig_data$host_uid; - if ( orig_data ?$ cid) - c$conn$orig_endpoint_cid = orig_data$cid; - c$conn$orig_endpoint_source = orig_data$source; - } - - # If the resp IP is local and in the list, update the conn log. - if ( c$conn?$local_resp && c$id$resp_h in hosts_data ) { - local resp_data = hosts_data[c$id$resp_h]; - if ( resp_data ?$ status) - c$conn$resp_endpoint_status = resp_data$status; - if ( resp_data ?$ host_uid) - c$conn$resp_endpoint_host_uid = resp_data$host_uid; - if ( resp_data ?$ cid) - c$conn$resp_endpoint_cid = resp_data$cid; - c$conn$resp_endpoint_source = resp_data$source; - } - } -} - - - - -## Enrich known_hosts ## -redef record Known::HostDetails += { - endpoint: Val &log &optional; -}; - -hook Known::add_host_details(h: Known::HostDetails, d: Known::HostDetails){ - #d is from worker - #h is the internal table - if (d ?$ endpoint){ - h$endpoint = d$endpoint; - } -} - -# update logs -function knownEndpoint (ip: addr) { - local data = hosts_data[ip]; - # Reporter::info (cat(data)); - if ( data ?$ hostname) { - # add source to protocol field - Known::get_name_details(ip, data$hostname)$protocols+=set(data$source); - # # add source to annotation field - Known::add_name_annotation(ip, data$hostname, set(data$source+"/"+data$status)); - } - if ( data ?$ mac) { - # some MAC's have "-" and should have ":", normalize to ":" - local mac = subst_string(data$mac, "-", ":"); - # add source to protocol field - Known::get_device_details(ip, mac)$protocols+=set(data$source); - # # add source to annotation field - Known::add_device_annotation(ip, mac, set(data$source+"/"+data$status)); - } - if ( data ?$ machine_domain) { - # add source to protocol field - Known::get_domain_details(ip, data$machine_domain)$protocols+=set(data$source); - # # add source to annotation field - Known::add_domain_annotation(ip, data$machine_domain, set(data$source+"/"+data$status)); - } - # add new fields to hosts log - Known::get_host_details(ip)$endpoint = data; -} -function unknownEndpoint (ip: addr) { - # TODO: create a list of all possible sources from the input file, or don't include a source with unknown hosts - # local data: Val = [$status = "unknown", $source = unknownSource]; - local data: Val = [$status = "unknown"]; - Known::get_host_details(ip)$endpoint = data; -} - -# priority of -5 to make sure the Known-entities creates an entry first -event connection_state_remove(c: connection) &priority=-5 -{ - if ( !c$conn?$local_orig && !c$conn?$local_resp ) { - return; - } - - # If the orig IP is local, check the list, update the following logs. - if ( c$conn?$local_orig ) { - # If it's in the list, update the fields, else flag it as unknown - if ( c$id$orig_h in hosts_data ) { - knownEndpoint(c$id$orig_h); - } else { - unknownEndpoint(c$id$orig_h); - } - } - - # If the resp IP is local, check the list, update the following logs. - if ( c$conn?$local_resp ) { - # If it's in the list, update the fields, else flag it as unknown - if ( c$id$resp_h in hosts_data ) { - knownEndpoint(c$id$resp_h); - } else { - unknownEndpoint(c$id$resp_h); - } - } -} diff --git a/scripts/files.zeek b/scripts/files.zeek new file mode 100644 index 0000000..577b26e --- /dev/null +++ b/scripts/files.zeek @@ -0,0 +1,32 @@ +module EndpointEnrichment; + +## Add VLAN to all logs with an "id" field. +export { + ## Enables the logging of endpoint details to the conn log. + option extra_logging_files = F; + option extra_logging_files_cid = F; +} + +redef record Files::Info += { + orig_ep_status: string &log &optional; + orig_ep_uid: string &log &optional; + orig_ep_cid: string &log &optional; + orig_ep_source: string &log &optional; + resp_ep_status: string &log &optional; + resp_ep_uid: string &log &optional; + resp_ep_cid: string &log &optional; + resp_ep_source: string &log &optional; +}; + +# event file_sniff(f: fa_file, meta: fa_metadata) { +# if (extra_logging_files) { +# for ( tx in f$tx_hosts ) { +# if ( id?$vlan ) +# f$info$vlan = id$vlan; +# if ( id?$vlan_inner ) +# f$info$vlan_inner = id$vlan_inner; +# # just grab the first one +# break; +# } +# } +# } diff --git a/scripts/id-logs.zeek b/scripts/id-logs.zeek new file mode 100644 index 0000000..1ddab27 --- /dev/null +++ b/scripts/id-logs.zeek @@ -0,0 +1,52 @@ +module EndpointEnrichment; + +## Add VLAN to all logs with an "id" field. +export { + ## Enables the logging of endpoint details to the conn log. + option extra_logging_all = F; + option extra_logging_all_cid = F; +} + +redef record conn_id += { + orig_ep_status: string &log &optional; + orig_ep_uid: string &log &optional; + orig_ep_cid: string &log &optional; + orig_ep_source: string &log &optional; + resp_ep_status: string &log &optional; + resp_ep_uid: string &log &optional; + resp_ep_cid: string &log &optional; + resp_ep_source: string &log &optional; +}; + + +event new_connection(c: connection) { + if (extra_logging_all) { + if ( !c$conn?$local_orig && !c$conn?$local_resp ) { + return; + } + + # If the orig IP is local and in the list, update the conn log. + if ( c$conn?$local_orig && c$id$orig_h in hosts_data ) { + local orig_data = hosts_data[c$id$orig_h]; + if ( orig_data ?$ status) + c$id$orig_ep_status = orig_data$status; + if ( orig_data ?$ uid) + c$id$orig_ep_uid = orig_data$uid; + if ( orig_data ?$ cid && extra_logging_all_cid) + c$id$orig_ep_cid = orig_data$cid; + c$id$orig_ep_source = orig_data$source; + } + + # If the resp IP is local and in the list, update the conn log. + if ( c$conn?$local_resp && c$id$resp_h in hosts_data ) { + local resp_data = hosts_data[c$id$resp_h]; + if ( resp_data ?$ status) + c$id$resp_ep_status = resp_data$status; + if ( resp_data ?$ uid) + c$id$resp_ep_uid = resp_data$uid; + if ( resp_data ?$ cid && extra_logging_all_cid) + c$id$resp_ep_cid = resp_data$cid; + c$id$resp_ep_source = resp_data$source; + } + } +} diff --git a/scripts/known.zeek b/scripts/known.zeek new file mode 100644 index 0000000..84fac8c --- /dev/null +++ b/scripts/known.zeek @@ -0,0 +1,76 @@ +module EndpointEnrichment; + +## Enrich known_hosts ## +redef record Known::HostDetails += { + ep: Val &log &optional; +}; + +hook Known::add_host_details(h: Known::HostDetails, d: Known::HostDetails){ + #d is from worker + #h is the internal table + if (d ?$ ep){ + h$ep = d$ep; + } +} + +# update logs +function knownEndpoint (ip: addr) { + local data = hosts_data[ip]; + # Reporter::info (cat(data)); + if ( data ?$ hostname) { + # add source to protocol field + Known::get_name_details(ip, data$hostname)$protocols+=set(data$source); + # # add source to annotation field + Known::add_name_annotation(ip, data$hostname, set(data$source+"/"+data$status)); + } + if ( data ?$ mac) { + # some MAC's have "-" and should have ":", normalize to ":" + local mac = subst_string(data$mac, "-", ":"); + # add source to protocol field + Known::get_device_details(ip, mac)$protocols+=set(data$source); + # # add source to annotation field + Known::add_device_annotation(ip, mac, set(data$source+"/"+data$status)); + } + if ( data ?$ machine_domain) { + # add source to protocol field + Known::get_domain_details(ip, data$machine_domain)$protocols+=set(data$source); + # # add source to annotation field + Known::add_domain_annotation(ip, data$machine_domain, set(data$source+"/"+data$status)); + } + # add new fields to hosts log + Known::get_host_details(ip)$ep = data; +} +function unknownEndpoint (ip: addr) { + # TODO: create a list of all possible sources from the input file, or don't include a source with unknown hosts + # local data: Val = [$status = "unknown", $source = unknownSource]; + local data: Val = [$status = "unknown"]; + Known::get_host_details(ip)$ep = data; +} + +# priority of -5 to make sure the Known-entities creates an entry first +# note: priority of -5, the connection will already be removed from memory +event connection_state_remove(c: connection) &priority=-5 { + if ( !c$conn?$local_orig && !c$conn?$local_resp ) { + return; + } + + # If the orig IP is local, check the list, update the following logs. + if ( c$conn?$local_orig ) { + # If it's in the list, update the fields, else flag it as unknown + if ( c$id$orig_h in hosts_data ) { + knownEndpoint(c$id$orig_h); + } else { + unknownEndpoint(c$id$orig_h); + } + } + + # If the resp IP is local, check the list, update the following logs. + if ( c$conn?$local_resp ) { + # If it's in the list, update the fields, else flag it as unknown + if ( c$id$resp_h in hosts_data ) { + knownEndpoint(c$id$resp_h); + } else { + unknownEndpoint(c$id$resp_h); + } + } +} diff --git a/scripts/main.zeek b/scripts/main.zeek new file mode 100644 index 0000000..a174e59 --- /dev/null +++ b/scripts/main.zeek @@ -0,0 +1,43 @@ +module EndpointEnrichment; + +type Idx: record { + ip: addr; +}; + +type Val: record { + ## The description of the endpoint. + desc: string &log &optional; + ## The status of the endpoint host. + status: string &log &optional; + ## The unique identifier, assigned by the source, of the endpoint host. + uid: string &log &optional; + ## The customer ID the host belongs to. + cid: string &log &optional; + ## The Operating System version of the endpoint host. + os_version: string &log &optional; + ## The source of the endpoint information. + source: string &log &optional; + ## The criticality of the endpoint host. + criticality: string &log &optional; + ## The MAC address of the endpoint host. + mac: string &optional; + ## The hostname of the vulnerable host. + hostname: string &optional; + ## The machine domain of the endpoint host. + machine_domain: string &optional; +}; + +global hosts_data: table[addr] of Val = table(); +# # source to use for all unknown IPs +# global unknownSource: string; + +event zeek_init() { + Input::add_table([ + $source="hosts_data.tsv", + $name="hosts_data", + $idx=Idx, + $val=Val, + $destination=hosts_data, + $mode=Input::REREAD + ]); +} diff --git a/scripts/new_conn.zeek b/scripts/new_conn.zeek new file mode 100644 index 0000000..9700d2e --- /dev/null +++ b/scripts/new_conn.zeek @@ -0,0 +1,6 @@ +module Conn; + +event new_connection(c: connection) &priority=10 +{ + set_conn(c, F); +} diff --git a/scripts/x509.zeek b/scripts/x509.zeek new file mode 100644 index 0000000..8e2e42d --- /dev/null +++ b/scripts/x509.zeek @@ -0,0 +1,32 @@ +module EndpointEnrichment; + +## Add VLAN to all logs with an "id" field. +export { + ## Enables the logging of endpoint details to the conn log. + option extra_logging_x509 = F; + option extra_logging_x509_cid = F; +} + +redef record X509::Info += { + orig_ep_status: string &log &optional; + orig_ep_uid: string &log &optional; + orig_ep_cid: string &log &optional; + orig_ep_source: string &log &optional; + resp_ep_status: string &log &optional; + resp_ep_uid: string &log &optional; + resp_ep_cid: string &log &optional; + resp_ep_source: string &log &optional; +}; + +# event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) +# { +# for ( id in f$conns ) +# { +# if ( id?$vlan ) +# f$info$x509$vlan = id$vlan; +# if ( id?$vlan_inner ) +# f$info$x509$vlan_inner = id$vlan_inner; +# # just grab the first one +# break; +# } +# } diff --git a/zkg.meta b/zkg.meta index 2f28474..2519e47 100644 --- a/zkg.meta +++ b/zkg.meta @@ -1,5 +1,5 @@ [package] -description = updated 23Dec20231014 - This package enriches the Known-Entities by adding information from a vulnerability scanner or endpoint agent. +description = updated 12Jan20241200 - This package enriches multiple logs by adding information from a vulnerability scanner or endpoint agent. script_dir = scripts [input hosts_data.tsv]