Merge pull request #47 from richm/collect-ovirt

no _source or _all for collectd metrics

namespaces/_default_.yml

@@ -1,4 +1,4 @@
-version: 2017.05.23.0
+version: 2017.06.06.0
 
 field_defaults:
   type: string
@@ -14,7 +14,7 @@ _default_:
   fields:
     - name: "@timestamp"
       type: date
-      format: yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime
+      format: yyyy-MM-dd HH:mm:ss,SSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime
       example: 2015-01-24T14:06:05.071Z
       description: >
         UTC value marking when the log payload was created, or when log payload was first collected if the creation time is not known;

namespaces/collectd.yml

@@ -6,36 +6,43 @@ namespace:
   fields:
   - name: interval
     type: float
+    include_in_all: false
     description: >
       Collectd's interval.
 
   - name: plugin
     type: string
+    include_in_all: false
     description: >
       Collectd's plugin.
 
   - name: plugin_instance
     type: string
+    include_in_all: false
     description: >
       Collectd's plugin_instance.
 
   - name: type_instance
     type: string
+    include_in_all: false
     description: >
       Collectd's type_instance.
 
   - name: type
     type: string
+    include_in_all: false
     description: >
       Collectd's type.
 
   - name: dstypes
     type: string
+    include_in_all: false
     description: >
       Collectd's dstypes.
 
   - name: processes
     type: group
+    include_in_all: false
     description: >
       Corresponds to collectd's processes plugin.
     fields:
@@ -154,6 +161,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's disk plugin.
+    include_in_all: false
     fields:
     - name: disk_merged
       type: group
@@ -239,6 +247,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's interface plugin.
+    include_in_all: false
     fields:
     - name: if_octets
       type: group
@@ -302,6 +311,7 @@ namespace:
 
   - name: virt
     type: group
+    include_in_all: false
     description: >
       Corresponds to collectd's virt plugin.
     fields:
@@ -412,6 +422,7 @@ namespace:
 
   - name: CPU
     type: group
+    include_in_all: false
     description: >
       Corresponds to collectd's CPU plugin.
     fields:
@@ -422,6 +433,7 @@ namespace:
 
   - name: df
     type: group
+    include_in_all: false
     description: >
       Corresponds to collectd's df plugin.
     fields:
@@ -438,6 +450,7 @@ namespace:
 
   - name: entropy
     type: group
+    include_in_all: false
     description: >
       Corresponds to collectd's entropy plugin.
     fields:
@@ -451,6 +464,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's nfs plugin.
+    include_in_all: false
     fields:
 
     - name: nfs_procedure
@@ -462,6 +476,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's memory plugin.
+    include_in_all: false
     fields:
 
     - name: memory
@@ -478,6 +493,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's swap plugin.
+    include_in_all: false
     fields:
 
     - name: swap
@@ -494,6 +510,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's load plugin.
+    include_in_all: false
     fields:
     - name: load
       type: group
@@ -519,6 +536,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's aggregation plugin.
+    include_in_all: false
     fields:
 
     - name: percent
@@ -530,6 +548,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's statsd plugin.
+    include_in_all: false
     fields:
 
     - name: host_cpu
@@ -731,6 +750,7 @@ namespace:
     type: group
     description: >
       Corresponds to collectd's postgresql plugin.
+    include_in_all: false
     fields:
 
     - name: pg_n_tup_g

namespaces/kubernetes.yml

@@ -7,6 +7,8 @@ namespace:
   fields:
   - name: pod_name
     type: string
+    norms:
+      enabled: true
     description: >
       The name of the pod
 
@@ -20,21 +22,29 @@ namespace:
 
   - name: pod_id
     type: string
+    norms:
+      enabled: true
     description: >
       Kubernetes ID of the pod.
 
   - name: namespace_name
     type: string
+    norms:
+      enabled: true
     description: >
       The name of the namespace in Kubernetes.
 
   - name: namespace_id
     type: string
+    norms:
+      enabled: true
     description: >
       ID of the namespace in Kubernetes.
 
   - name: host
     type: string
+    norms:
+      enabled: true
     description: >
       Kubernetes node name
 

scripts/concat-index-pattern-fields.py

@@ -4,14 +4,26 @@
 import json
 
 doc = {}
+names = {}
+
+def removedupnames(fieldary):
+    ret = []
+    for field in fieldary:
+        if not field['name'] in names:
+            names[field['name']] = True
+            ret.append(field)
+    return ret
+
 for filename in sys.argv[1:]:
     with open(filename, "r") as ff:
         if doc:
             d2 = json.load(ff)
-            d2fieldary = json.loads(d2['fields'])
+            d2fieldary = removedupnames(json.loads(d2['fields']))
             docfieldary = json.loads(doc['fields'])
             docfieldary.extend(d2fieldary)
             doc['fields'] = json.dumps(docfieldary)
         else:
             doc = json.load(ff)
-sys.stdout.write(json.dumps(doc, indent=4, sort_keys=True))
+            docfieldary = removedupnames(json.loads(doc['fields']))
+            doc['fields'] = json.dumps(docfieldary)
+sys.stdout.write(json.dumps(doc, indent=4, sort_keys=True, separators=(',', ': ')))

scripts/generate_template.py

@@ -77,6 +77,7 @@ def object_types_to_template(template_definition, output, output_index_pattern,
 
     json.dump(
         skeleton, output, indent=2, separators=(',', ': '), sort_keys=True)
+    output.write('\n')
 
     # index pattern stuff
     time_field_name = "time"
@@ -92,6 +93,7 @@ def object_types_to_template(template_definition, output, output_index_pattern,
     skeleton_index_pattern["fields"] = json.dumps(index_pattern_fields)
     json.dump(
         skeleton_index_pattern, output_index_pattern, indent=2, separators=(',', ': '), sort_keys=True)
+    output_index_pattern.write('\n')
 
 
 def add_mapping_to_skeleton(map_type, skeleton):

templates/collectd_metrics/template.yml

@@ -1,10 +1,14 @@
 skeleton_path: ../skeleton.json
 skeleton_index_pattern_path: ../skeleton-index-pattern.json
 
+# exclude all collectd stats fields from _source
 elasticsearch_template:
   name: org.ovirt.viaq-collectd
   index_pattern: "project.ovirt-metrics-*"
-  order: 10
+  order: 20
+  _source:
+    excludes:
+    - collectd.*
 
 namespaces:
   - collectd.yml

templates/openshift/Makefile

@@ -1,9 +1,8 @@
-TEMPLATE_NAME = com.redhat.viaq-openshift
-
 .PHONY: all clean
 
 all:
-	python ../../scripts/generate_template.py template.yml ../../namespaces/
+	python ../../scripts/generate_template.py template-project.yml ../../namespaces/
+	python ../../scripts/generate_template.py template-operations.yml ../../namespaces/
 
 clean:
 	rm *.template.json

templates/openshift/template-operations.yml

@@ -0,0 +1,18 @@
+skeleton_path: ../skeleton.json
+skeleton_index_pattern_path: ../skeleton-index-pattern.json
+
+elasticsearch_template:
+  name: com.redhat.viaq-openshift-operations
+  index_pattern: ".operations.*"
+  order: 10
+
+namespaces:
+  - rsyslog.yml
+  - systemd.yml
+  - kubernetes.yml
+  - docker.yml
+  - pipeline_metadata.yml
+  - ovirt.yml
+  - aushape.yml
+  - tlog.yml
+#  - openshift/app_log.yml

templates/openshift/template.yml → templates/openshift/template-project.yml

@@ -2,8 +2,8 @@ skeleton_path: ../skeleton.json
 skeleton_index_pattern_path: ../skeleton-index-pattern.json
 
 elasticsearch_template:
-  name: com.redhat.viaq-openshift
-  index_pattern: "*"
+  name: com.redhat.viaq-openshift-project
+  index_pattern: "project.*"
   order: 10
 
 namespaces: