Merge pull request #733 from mlibrary/monitor-etcd

Collect etcd metrics

manifests/profile/kubernetes/destination_port/etcd.pp

@@ -11,4 +11,10 @@
     content => "  server ${::hostname} ${::ipaddress}:2379 check\n",
     tag     => "${cluster_name}_haproxy_kubernetes_etcd",
   }
+
+  @@concat_fragment { "prometheus etcd service ${::hostname}":
+    tag     => "${::datacenter}_prometheus_etcd_service_list",
+    target  => '/etc/prometheus/etcd.yml',
+    content => template('nebula/profile/prometheus/exporter/etcd/target.yaml.erb')
+  }
 }

manifests/profile/kubernetes/kubelet.pp

@@ -70,7 +70,7 @@
     ;
 
     '200 Cluster etcd':
-      dport => ['2379', '2380'],
+      dport => ['2379', '2380', '2381'],
     ;
 
     '200 Cluster kubelet':

manifests/profile/prometheus.pp

@@ -38,6 +38,7 @@
       '/etc/prometheus/haproxy.yml:/etc/prometheus/haproxy.yml',
       '/etc/prometheus/mysql.yml:/etc/prometheus/mysql.yml',
       '/etc/prometheus/ipmi.yml:/etc/prometheus/ipmi.yml',
+      '/etc/prometheus/etcd.yml:/etc/prometheus/etcd.yml',
       '/etc/prometheus/tls:/tls',
       '/opt/prometheus:/prometheus',
     ],
@@ -105,6 +106,13 @@
 
   Concat_fragment <<| tag == "${::datacenter}_prometheus_ipmi_exporter" |>>
 
+  concat_file { '/etc/prometheus/etcd.yml':
+    notify  => Docker::Run['prometheus'],
+    require => File['/etc/prometheus'],
+  }
+
+  Concat_fragment <<| tag == "${::datacenter}_prometheus_etcd_service_list" |>>
+
   file { '/etc/prometheus':
     ensure => 'directory',
   }

spec/classes/profile/kubernetes/destination_port_spec.rb

@@ -31,6 +31,14 @@
               .with_content("  server #{facts[:hostname]} #{facts[:ipaddress]}:#{port} #{options}\n")
               .with_tag("first_cluster_haproxy_kubernetes_#{service}")
           end
+
+          if service == 'etcd'
+            it do
+              is_expected.to contain_concat_fragment("prometheus #{service.tr('_', ' ')} service #{facts[:hostname]}")
+                .with_target("/etc/prometheus/#{service}.yml")
+                .with_tag("mydatacenter_prometheus_etcd_service_list")
+            end
+          end
         end
       end
     end

spec/classes/profile/kubernetes/kubelet_spec.rb

@@ -87,14 +87,14 @@
         end
 
         [
-          [22,            'ssh',            'tcp'],
-          [179,           'BGP',            'tcp'],
-          [4789,          'VXLAN',          'udp'],
-          [%w[2379 2380], 'etcd',           'tcp'],
-          [10250,         'kubelet',        'tcp'],
-          [6443,          'kubernetes API', 'tcp'],
-          ['30000-32767', 'NodePorts',      'tcp'],
-          [9100,          'Prometheus',     'tcp'],
+          [22,                 'ssh',            'tcp'],
+          [179,                'BGP',            'tcp'],
+          [4789,               'VXLAN',          'udp'],
+          [%w[2379 2380 2381], 'etcd',           'tcp'],
+          [10250,              'kubelet',        'tcp'],
+          [6443,               'kubernetes API', 'tcp'],
+          ['30000-32767',      'NodePorts',      'tcp'],
+          [9100,               'Prometheus',     'tcp'],
         ].each do |ports, purpose, proto|
           it do
             is_expected.to contain_firewall("200 Cluster #{purpose}")

spec/classes/profile/prometheus_spec.rb

@@ -23,6 +23,7 @@
                          '/etc/prometheus/haproxy.yml:/etc/prometheus/haproxy.yml',
                          '/etc/prometheus/mysql.yml:/etc/prometheus/mysql.yml',
                          '/etc/prometheus/ipmi.yml:/etc/prometheus/ipmi.yml',
+                         '/etc/prometheus/etcd.yml:/etc/prometheus/etcd.yml',
                          '/etc/prometheus/tls:/tls',
                          '/opt/prometheus:/prometheus'])
           .that_requires('File[/opt/prometheus]')

templates/profile/prometheus/config.yml.erb

@@ -50,5 +50,8 @@ scrape_configs:
 <% end -%>
 <% end -%>
 <% end -%>
+- job_name: etcd
+  file_sd_configs:
+  - files: [ etcd.yml ]
 scrape_config_files:
 - "ipmi.yml"

templates/profile/prometheus/exporter/etcd/target.yaml.erb

@@ -0,0 +1,4 @@
+- targets: [ '<%= @ipaddress %>:2381' ]
+  labels:
+    datacenter: '<%= @datacenter %>'
+    hostname: '<%= @fqdn %>'