Merge remote-tracking branch 'origin/topic/awelzel/17-nfq-rx-threads'

* origin/topic/awelzel/17-nfq-rx-threads:
  Add support for RX threads.
  Bump dependencies, fix tests

go.mod

@@ -3,15 +3,16 @@ module github.com/corelight/suricata_exporter
 go 1.22
 
 require (
-	github.com/prometheus/client_golang v1.18.0
-	github.com/prometheus/client_model v0.5.0
+	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/client_model v0.6.1
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/prometheus/common v0.46.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )

go.sum

@@ -1,20 +1,22 @@
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
-github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
-github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
-github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
-github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y=
-github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
-github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
-github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
+github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=

main.go

@@ -202,6 +202,19 @@ var (
 		newPerThreadCounterMetric("flow_bypassed", "local_capture_bytes_total", "", "local_capture_bytes"),
 	}
 
+	perThreadIpsMetrics = []metricInfo{
+		newPerThreadCounterMetric("ips", "accepted_packets_total", "", "accepted"),
+		newPerThreadCounterMetric("ips", "blocked_packets_total", "", "blocked"),
+		newPerThreadCounterMetric("ips", "rejected_packets_total", "", "rejected"),
+		newPerThreadCounterMetric("ips", "replaced_packets_total", "", "replaced"),
+	}
+
+	perThreadTcpMetricsReceive = []metricInfo{
+		newPerThreadCounterMetric("tcp", "syn_packets_total", "", "syn"),
+		newPerThreadCounterMetric("tcp", "synack_packets_total", "", "synack"),
+		newPerThreadCounterMetric("tcp", "rst_packets_total", "", "rst"),
+	}
+
 	// From .thread.tcp
 	perThreadTcpMetrics = []metricInfo{
 		// New in 7.0.0
@@ -221,9 +234,6 @@ var (
 		newPerThreadCounterMetric("tcp", "invalid_checksum_packets_total", "", "invalid_checksum"),
 		// Removed in 7.0.0: 0360cb654293c333e3be70204705fa7ec328512e
 		newPerThreadCounterMetric("tcp", "no_flow_total", "", "no_flow").Optional(),
-		newPerThreadCounterMetric("tcp", "syn_packets_total", "", "syn"),
-		newPerThreadCounterMetric("tcp", "synack_packets_total", "", "synack"),
-		newPerThreadCounterMetric("tcp", "rst_packets_total", "", "rst"),
 		newPerThreadCounterMetric("tcp", "midstream_pickups_total", "", "midstream_pickups"),
 		newPerThreadCounterMetric("tcp", "pkt_on_wrong_thread_total", "", "pkt_on_wrong_thread"),
 		newPerThreadCounterMetric("tcp", "segment_memcap_drop_total", "", "segment_memcap_drop"),
@@ -546,7 +556,12 @@ func handleNapatechMetrics(ch chan<- prometheus.Metric, message map[string]any)
 	}
 }
 
-func handleWorkerThread(ch chan<- prometheus.Metric, threadName string, thread map[string]any) {
+// Handle the shared RX and worker thread portions.
+//
+// Depending on autofp or workers runmode, the "capture" entry
+// is in the RX threads.
+func handleReceiveCommon(ch chan<- prometheus.Metric, threadName string, thread map[string]any) {
+
 	if capture, ok := thread["capture"].(map[string]any); ok {
 		for _, m := range perThreadCaptureMetrics {
 			if cm := newConstMetric(m, capture, threadName); cm != nil {
@@ -577,58 +592,95 @@ func handleWorkerThread(ch chan<- prometheus.Metric, threadName string, thread m
 		}
 	}
 
-	tcp := thread["tcp"].(map[string]any)
-	for _, m := range perThreadTcpMetrics {
-		if cm := newConstMetric(m, tcp, threadName); cm != nil {
-			ch <- cm
-		}
-	}
-
-	flow := thread["flow"].(map[string]any)
-	for _, m := range perThreadFlowMetrics {
-		if cm := newConstMetric(m, flow, threadName); cm != nil {
-			ch <- cm
-		}
-	}
-
-	wrk := flow["wrk"].(map[string]any)
-	for _, m := range perThreadFlowWrkMetrics {
-		if cm := newConstMetric(m, wrk, threadName); cm != nil {
+	// Convert all decoder entries that look like numbers
+	// as perThreadDecoder metric with a "kind" label.
+	decoder := thread["decoder"].(map[string]any)
+	for _, m := range perThreadDecoderMetrics {
+		if cm := newConstMetric(m, decoder, threadName); cm != nil {
 			ch <- cm
 		}
 	}
 
+	// Defrag stats from worker and receive threads.
 	defrag := thread["defrag"].(map[string]any)
 	defragIpv4 := defrag["ipv4"].(map[string]any)
 	defragIpv6 := defrag["ipv6"].(map[string]any)
+
 	for _, m := range perThreadDefragIpv4Metrics {
 		if cm := newConstMetric(m, defragIpv4, threadName); cm != nil {
 			ch <- cm
 		}
 	}
+
 	for _, m := range perThreadDefragIpv6Metrics {
 		if cm := newConstMetric(m, defragIpv6, threadName); cm != nil {
 			ch <- cm
 		}
 	}
+
 	for _, m := range perThreadDefragMetrics {
 		if cm := newConstMetric(m, defrag, threadName); cm != nil {
 			ch <- cm
 		}
 	}
 
-	detect := thread["detect"].(map[string]any)
-	for _, m := range perThreadDetectMetrics {
-		if cm := newConstMetric(m, detect, threadName); cm != nil {
+	tcp := thread["tcp"].(map[string]any)
+	for _, m := range perThreadTcpMetricsReceive {
+		if cm := newConstMetric(m, tcp, threadName); cm != nil {
 			ch <- cm
 		}
 	}
+}
 
-	// Convert all decoder entries that look like numbers
-	// as perThreadDecoder metric with a "kind" label.
-	decoder := thread["decoder"].(map[string]any)
-	for _, m := range perThreadDecoderMetrics {
-		if cm := newConstMetric(m, decoder, threadName); cm != nil {
+func handleIps(ch chan<- prometheus.Metric, threadName string, thread map[string]any) {
+	// Extract basic IPS metrics if they exist.
+	if ips, ok := thread["ips"].(map[string]any); ok {
+		for _, m := range perThreadIpsMetrics {
+			if cm := newConstMetric(m, ips, threadName); cm != nil {
+				ch <- cm
+			}
+		}
+	}
+}
+
+// Receive threads have the same layout as worker threads.
+func handleReceiveThread(ch chan<- prometheus.Metric, threadName string, thread map[string]any) {
+	handleReceiveCommon(ch, threadName, thread)
+	handleIps(ch, threadName, thread)
+}
+
+func handleTransmitThread(ch chan<- prometheus.Metric, threadName string, thread map[string]any) {
+	handleIps(ch, threadName, thread)
+}
+
+func handleWorkerThread(ch chan<- prometheus.Metric, threadName string, thread map[string]any) {
+	handleReceiveCommon(ch, threadName, thread)
+	handleIps(ch, threadName, thread)
+
+	tcp := thread["tcp"].(map[string]any)
+	for _, m := range perThreadTcpMetrics {
+		if cm := newConstMetric(m, tcp, threadName); cm != nil {
+			ch <- cm
+		}
+	}
+
+	flow := thread["flow"].(map[string]any)
+	for _, m := range perThreadFlowMetrics {
+		if cm := newConstMetric(m, flow, threadName); cm != nil {
+			ch <- cm
+		}
+	}
+
+	wrk := flow["wrk"].(map[string]any)
+	for _, m := range perThreadFlowWrkMetrics {
+		if cm := newConstMetric(m, wrk, threadName); cm != nil {
+			ch <- cm
+		}
+	}
+
+	detect := thread["detect"].(map[string]any)
+	for _, m := range perThreadDetectMetrics {
+		if cm := newConstMetric(m, detect, threadName); cm != nil {
 			ch <- cm
 		}
 	}
@@ -766,8 +818,12 @@ func produceMetrics(ch chan<- prometheus.Metric, counters map[string]any) {
 	// Produce per thread metrics
 	for threadName, thread_ := range message["threads"].(map[string]any) {
 		if thread, ok := thread_.(map[string]any); ok {
-			if strings.HasPrefix(threadName, "W#") {
+			if strings.HasPrefix(threadName, "W#") || strings.HasPrefix(threadName, "W-") {
 				handleWorkerThread(ch, threadName, thread)
+			} else if strings.HasPrefix(threadName, "RX") {
+				handleReceiveThread(ch, threadName, thread)
+			} else if strings.HasPrefix(threadName, "TX") {
+				handleTransmitThread(ch, threadName, thread)
 			} else if strings.HasPrefix(threadName, "FM") {
 				handleFlowManagerThread(ch, threadName, thread)
 			} else if strings.HasPrefix(threadName, "FR") {