Merge pull request #2 from corelight/all-logs

All logs

scripts/__load__.zeek

@@ -1 +1,5 @@
-@load ./enrich.zeek
+@load ./new_conn.zeek
+@load ./main.zeek
+@load ./known.zeek
+@load ./conn.zeek
+@load ./id-logs.zeek

scripts/conn.zeek

@@ -0,0 +1,52 @@
+module EndpointEnrichment;
+
+## Enrich Conn.log ##
+export {
+    ## Enables the logging of endpoint details to the conn log.
+    option extra_logging_conn = F;
+    option extra_logging_conn_cid = F;
+}
+
+redef record Conn::Info += {
+    orig_ep_status: string &log &optional;
+    orig_ep_uid: string &log &optional;
+    orig_ep_cid: string &log &optional;
+    orig_ep_source: string &log &optional;
+    resp_ep_status: string &log &optional;
+    resp_ep_uid: string &log &optional;
+    resp_ep_cid: string &log &optional;
+    resp_ep_source: string &log &optional;
+};
+
+
+event new_connection(c: connection) {
+    if (extra_logging_conn) {
+        if ( !c$conn?$local_orig && !c$conn?$local_resp ) {
+            return;
+        }
+
+        # If the orig IP is local and in the list, update the conn log.
+        if ( c$conn?$local_orig && c$id$orig_h in hosts_data ) {
+            local orig_data = hosts_data[c$id$orig_h];
+            if ( orig_data ?$ status)
+                c$conn$orig_ep_status = orig_data$status;
+            if ( orig_data ?$ uid)
+                c$conn$orig_ep_uid = orig_data$uid;
+            if ( orig_data ?$ cid && extra_logging_conn_cid)
+                c$conn$orig_ep_cid = orig_data$cid;
+            c$conn$orig_ep_source = orig_data$source;
+        }
+
+        # If the resp IP is local and in the list, update the conn log.
+        if ( c$conn?$local_resp && c$id$resp_h in hosts_data ) {
+            local resp_data = hosts_data[c$id$resp_h];
+            if ( resp_data ?$ status)
+                c$conn$resp_ep_status = resp_data$status;
+            if ( resp_data ?$ uid)
+                c$conn$resp_ep_uid = resp_data$uid;
+            if ( resp_data ?$ cid && extra_logging_conn_cid)
+                c$conn$resp_ep_cid = resp_data$cid;
+            c$conn$resp_ep_source = resp_data$source;
+        }
+    }
+}

scripts/files.zeek

@@ -0,0 +1,32 @@
+module EndpointEnrichment;
+
+## Add VLAN to all logs with an "id" field.
+export {
+    ## Enables the logging of endpoint details to the conn log.
+    option extra_logging_files = F;
+    option extra_logging_files_cid = F;
+}
+
+redef record Files::Info += {
+    orig_ep_status: string &log &optional;
+    orig_ep_uid: string &log &optional;
+    orig_ep_cid: string &log &optional;
+    orig_ep_source: string &log &optional;
+    resp_ep_status: string &log &optional;
+    resp_ep_uid: string &log &optional;
+    resp_ep_cid: string &log &optional;
+    resp_ep_source: string &log &optional;
+};
+
+# event file_sniff(f: fa_file, meta: fa_metadata) 	{
+#     if (extra_logging_files) {
+#         for ( tx in f$tx_hosts ) {
+#             if ( id?$vlan )
+#                 f$info$vlan = id$vlan;
+#             if ( id?$vlan_inner )
+#                 f$info$vlan_inner = id$vlan_inner;
+#             # just grab the first one
+#             break;
+#         }
+#     }
+# }

scripts/id-logs.zeek

@@ -0,0 +1,52 @@
+module EndpointEnrichment;
+
+## Add VLAN to all logs with an "id" field.
+export {
+    ## Enables the logging of endpoint details to the conn log.
+    option extra_logging_all = F;
+    option extra_logging_all_cid = F;
+}
+
+redef record conn_id += {
+    orig_ep_status: string &log &optional;
+    orig_ep_uid: string &log &optional;
+    orig_ep_cid: string &log &optional;
+    orig_ep_source: string &log &optional;
+    resp_ep_status: string &log &optional;
+    resp_ep_uid: string &log &optional;
+    resp_ep_cid: string &log &optional;
+    resp_ep_source: string &log &optional;
+};
+
+
+event new_connection(c: connection) {
+    if (extra_logging_all) {
+        if ( !c$conn?$local_orig && !c$conn?$local_resp ) {
+            return;
+        }
+
+        # If the orig IP is local and in the list, update the conn log.
+        if ( c$conn?$local_orig && c$id$orig_h in hosts_data ) {
+            local orig_data = hosts_data[c$id$orig_h];
+            if ( orig_data ?$ status)
+                c$id$orig_ep_status = orig_data$status;
+            if ( orig_data ?$ uid)
+                c$id$orig_ep_uid = orig_data$uid;
+            if ( orig_data ?$ cid && extra_logging_all_cid)
+                c$id$orig_ep_cid = orig_data$cid;
+            c$id$orig_ep_source = orig_data$source;
+        }
+
+        # If the resp IP is local and in the list, update the conn log.
+        if ( c$conn?$local_resp && c$id$resp_h in hosts_data ) {
+            local resp_data = hosts_data[c$id$resp_h];
+            if ( resp_data ?$ status)
+                c$id$resp_ep_status = resp_data$status;
+            if ( resp_data ?$ uid)
+                c$id$resp_ep_uid = resp_data$uid;
+            if ( resp_data ?$ cid && extra_logging_all_cid)
+                c$id$resp_ep_cid = resp_data$cid;
+            c$id$resp_ep_source = resp_data$source;
+        }
+    }
+}

scripts/known.zeek

@@ -0,0 +1,76 @@
+module EndpointEnrichment;
+
+## Enrich known_hosts ##
+redef record Known::HostDetails += {
+    ep: Val &log &optional;
+};
+
+hook Known::add_host_details(h: Known::HostDetails, d: Known::HostDetails){
+    #d is from worker
+    #h is the internal table
+    if (d ?$ ep){
+        h$ep = d$ep;
+    }
+}
+
+# update logs
+function knownEndpoint (ip: addr) {
+    local data = hosts_data[ip];
+    # Reporter::info (cat(data));
+    if ( data ?$ hostname) {
+        # add source to protocol field
+        Known::get_name_details(ip, data$hostname)$protocols+=set(data$source);
+        # # add source to annotation field
+        Known::add_name_annotation(ip, data$hostname, set(data$source+"/"+data$status));
+    }
+    if ( data ?$ mac) {
+        # some MAC's have "-" and should have ":", normalize to ":"
+        local mac = subst_string(data$mac, "-", ":");
+        # add source to protocol field
+        Known::get_device_details(ip, mac)$protocols+=set(data$source);
+        # # add source to annotation field
+        Known::add_device_annotation(ip, mac, set(data$source+"/"+data$status));
+    }
+    if ( data ?$ machine_domain) {
+        # add source to protocol field
+        Known::get_domain_details(ip, data$machine_domain)$protocols+=set(data$source);
+        # # add source to annotation field
+        Known::add_domain_annotation(ip, data$machine_domain, set(data$source+"/"+data$status));
+    }
+    # add new fields to hosts log
+    Known::get_host_details(ip)$ep = data;
+}
+function unknownEndpoint (ip: addr) {
+    # TODO: create a list of all possible sources from the input file, or don't include a source with unknown hosts
+    # local data: Val = [$status = "unknown", $source = unknownSource];
+    local data: Val = [$status = "unknown"];
+    Known::get_host_details(ip)$ep = data;
+}
+
+# priority of -5 to make sure the Known-entities creates an entry first
+# note: priority of -5, the connection will already be removed from memory
+event connection_state_remove(c: connection) &priority=-5 {
+    if ( !c$conn?$local_orig && !c$conn?$local_resp ) {
+        return;
+    }
+
+    # If the orig IP is local, check the list, update the following logs.
+    if ( c$conn?$local_orig ) {
+        # If it's in the list, update the fields, else flag it as unknown
+        if ( c$id$orig_h in hosts_data ) {
+            knownEndpoint(c$id$orig_h);
+        } else {
+            unknownEndpoint(c$id$orig_h);
+        }
+    }
+
+    # If the resp IP is local, check the list, update the following logs.
+    if ( c$conn?$local_resp ) {
+        # If it's in the list, update the fields, else flag it as unknown
+        if ( c$id$resp_h in hosts_data ) {
+            knownEndpoint(c$id$resp_h);
+        } else {
+            unknownEndpoint(c$id$resp_h);
+        }
+    }
+}