Merge pull request #36 from richm/add-support-for-index-patterns

add support for Kibana index patterns

scripts/generate_template.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 
 """
-This script generates the ES template file (*.template.json) from
-the fields.yml file and skeleton.json.
+This script generates both the Elasticsearch template file (*.template.json)
+and the Kibana index pattern setting file (*.index-pattern.json) from the
+fields.yml file and skeleton.json.
 The script is built upon the similar script from libbeats.
 
 Example usage:
@@ -15,7 +16,7 @@
 import json
 
 
-def object_types_to_template(template_definition, output, namespaces_dir):
+def object_types_to_template(template_definition, output, output_index_pattern, namespaces_dir):
     """
     Assemble objects for the particular template.
     """
@@ -33,6 +34,14 @@ def object_types_to_template(template_definition, output, namespaces_dir):
     with open(template_definition['skeleton_path'], 'r') as f:
         skeleton = yaml.load(f)
 
+    if 'skeleton_index_pattern_path' not in template_definition:
+        print "skeleton_index_pattern_path is not defined. Cannot generate template."
+        return
+
+    # Load skeleton of the template
+    with open(template_definition['skeleton_index_pattern_path'], 'r') as f:
+        skeleton_index_pattern = yaml.load(f)
+
     # Load object_type files
     with open(namespaces_dir + '/_default_.yml', 'r') as f:
         default_mapping_yml = yaml.load(f)
@@ -48,7 +57,7 @@ def object_types_to_template(template_definition, output, namespaces_dir):
         default_mapping['fields'].append(cur_ns_yml['namespace'])
 
     skeleton['mappings']['_default_']['properties'] = (traverse_group_section(
-        default_mapping, default_mapping_yml['field_defaults']))
+        default_mapping, default_mapping_yml['field_defaults'], process_leaf))
 
     add_type_version(default_mapping_yml["version"],
                      skeleton['mappings']['_default_'])
@@ -62,6 +71,20 @@ def object_types_to_template(template_definition, output, namespaces_dir):
     json.dump(
         skeleton, output, indent=2, separators=(',', ': '), sort_keys=True)
 
+    # index pattern stuff
+    time_field_name = "time"
+    for ii in default_mapping["fields"]:
+        if ii['type'] == 'date':
+            time_field_name = ii['name']
+            break
+    skeleton_index_pattern["timeFieldName"] = time_field_name
+    skeleton_index_pattern["description"] = skeleton_index_pattern["description"].replace("<the_index_type>", template_definition['elasticsearch_template']['index_pattern'])
+    # get fields
+    index_pattern_fields = (traverse_group_section_index_pattern(
+        default_mapping, default_mapping_yml['field_defaults'], process_leaf_index_pattern))
+    skeleton_index_pattern["fields"] = json.dumps(index_pattern_fields)
+    json.dump(
+        skeleton_index_pattern, output_index_pattern, indent=2, separators=(',', ': '), sort_keys=True)
 
 def add_mapping_to_skeleton(map_type, skeleton):
     """Add mapping type to the skeleton by cloning '_default_' section.
@@ -75,12 +98,12 @@ def add_mapping_to_skeleton(map_type, skeleton):
         del skeleton['mappings'][map_type]['dynamic_templates']
 
 
-def traverse_group_section(group, defaults):
+def traverse_group_section(group, defaults, leaf_handler, groupname=None):
     """
     Traverse the sections tree and fill in the properties
     map.
     Args:
-        group(dict): field of type group, that has mutiple subfields under
+        group(dict): field of type group, that has multiple subfields under
     'fields' key.
         defaults(dict): dict with the defaults for all fields
     Returns:
@@ -91,15 +114,51 @@ def traverse_group_section(group, defaults):
     # print "Trying to fill section properties of section %s" % (group)
     try:
         for field in group["fields"]:
-            prop = traverse_field(field, defaults)
+            if groupname:
+                subgroupname = groupname + "." + group["name"]
+            else:
+                subgroupname = group.get("name", None)
+            prop = traverse_field(field, defaults, leaf_handler, subgroupname)
             properties.update(prop)
     except KeyError:
         print "Skipping empty section %s" % (group)
     # print "Section filled with properties: %s" % (properties)
     return properties
 
 
-def traverse_field(field, defaults):
+def traverse_group_section_index_pattern(group, defaults, leaf_handler, groupname=None):
+    """
+    Traverse the sections tree and fill in the index pattern fields
+    map.
+    Args:
+        group(dict): field of type group, that has multiple subfields under
+    'fields' key.
+        defaults(dict): dict with the defaults for all fields
+    Returns:
+        array of field definitions.
+    """
+    fields = []
+
+    # print "Trying to fill section properties of section %s" % (group)
+    try:
+        for field in group["fields"]:
+            if groupname:
+                subgroupname = groupname + "." + group["name"]
+            else:
+                subgroupname = group.get("name", None)
+            if field.get("type") == "group":
+                more_fields = traverse_group_section_index_pattern(field, defaults, leaf_handler, subgroupname)
+                fields.extend(more_fields)
+            else:
+                out_field = leaf_handler(field, defaults, subgroupname)
+                fields.append(out_field)
+    except KeyError:
+        print "Skipping empty section %s" % (group)
+    # print "Section filled with properties: %s" % (properties)
+    return fields
+
+
+def traverse_field(field, defaults, leaf_handler, groupname):
     """
     Add data about a particular field in the properties
     map.
@@ -113,23 +172,23 @@ def traverse_field(field, defaults):
     properties = {}
     # print "current field is %s" % (field)
 
-    # TODO: Make this more dyanmic
+    # TODO: Make this more dynamic
 
     if field.get("type") == "group":
-        prop = traverse_group_section(field, defaults)
+        prop = traverse_group_section(field, defaults, leaf_handler, groupname)
 
         # Only add properties if they have a content
         if len(prop) is not 0:
             properties[field.get("name")] = {"properties": {}}
             properties[field.get("name")]["properties"] = prop
     else:
-        properties = process_leaf(field, defaults)
+        properties = leaf_handler(field, defaults, groupname)
 
     # print "Result of traversing field is : %s" % (properties)
     return properties
 
 
-def process_leaf(field, defaults):
+def process_leaf(field, defaults, groupname=None):
     """Process field that is not a group. Fill the template copy with the actual
     data.
     Args:
@@ -184,12 +243,54 @@ def process_subleaf(field, defaults):
     """
     properties = {}
     if field.get("fields"):
-        prop = traverse_group_section(field, defaults)
+        prop = traverse_group_section(field, defaults, process_leaf)
         if len(prop) is not 0:
             properties["fields"] = prop
     return properties
 
 
+def process_leaf_index_pattern(field, defaults, groupname):
+    """Process field that is not a group. Fill the template copy with the actual
+    data.
+    Args:
+        field(dict): contents of the field.
+        defaults(dict): default values.
+        groupname(string): name of group this field belongs to e.g. "systemd.u"
+    Returns:
+        dict corresponding to the data in the particular field.
+    """
+    if groupname:
+        fieldname = groupname + "." + field["name"]
+    else:
+        fieldname = field["name"]
+    # Kibana field types:
+    # https://github.com/elastic/kibana/blob/master/src/ui/public/index_patterns/_field_types.js
+    if field.get("type") in ["string", "date", "ip", "boolean"]:
+        fieldtype = field.get("type")
+    elif field.get("type") in ["integer", "long", "float"]:
+        fieldtype = "number"
+    elif field.get("type") == "object":
+        if "geo_point" == field.get("object_struct", {}).get("properties", {}).get("location", {}).get("type", ''):
+            fieldtype = "geo_point"
+        else:
+            fieldtype = "string"
+    elif field.get("type") == "nested":
+        fieldtype = "string"
+    else:
+        print "Unknown field type. Skipped adding field %s" % (field)
+    analyzed = field.get("index", "") == "analyzed"
+    res = {
+        "name": fieldname,
+        "type": fieldtype,
+        "count": 0,
+        "scripted": False,
+        "indexed": True,
+        "analyzed": analyzed,
+        "doc_values": field.get("doc_values", True)
+    }
+    return res
+
+
 def add_type_version(version, obj_type):
     """replaces <version> placeholder in the template(index name and _meta)
     with the actual version number
@@ -237,4 +338,6 @@ def parse_args():
 
     with open('{0[elasticsearch_template][name]}.template.json'.format(
             template_definition), 'w') as output:
-        object_types_to_template(template_definition, output, args.namespaces_dir)
+        with open('{0[elasticsearch_template][name]}.index-pattern.json'.format(
+                template_definition), 'w') as output_index_pattern:
+            object_types_to_template(template_definition, output, output_index_pattern, args.namespaces_dir)

templates/openshift/README.md

@@ -1,4 +1,4 @@
-viaq openshift index templates for ElasticSearch
+viaq openshift index templates for ElasticSearch and index patterns for Kibana
 =================================
 
 The template files are automatically generated.
@@ -9,7 +9,7 @@ In order to edit the template please modify [objects.yml](objects.yml) and the r
 To rebuild the template, run:
 > python ../scripts/generate_template.py . ../../objects_dir
 
-For details about the mapping please see [ElasticSearch reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html)
+For details about the mapping please see [ElasticSearch reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html) and [Kibana reference](https://www.elastic.co/guide/en/kibana/current/index-patterns.html)
 
 skeleton.json
 -------------
@@ -24,7 +24,24 @@ by sections:
   `properties`: empty section that is populated with the content from [fields.yml](fields.yml)  
 `order`: order of the template. lower order templates are applied first.  
 `settings`: various settings  
-`template`: indices that will be matched by this template  
+`template`: indices that will be matched by this template
+
+skeleton-index-pattern.json
+---------------------------
+This file the skeleton of the index pattern file.
+
+by sections:
+`title`: Filled in by the openshift-elasticseach-plugin [index pattern loader](https://github.com/fabric8io/openshift-elasticsearch-plugin/blob/master/src/main/java/io/fabric8/elasticsearch/plugin/kibana/KibanaSeed.java#L371)
+`timeFieldName`: Name of the time field - the script will look for the first field in the `default` namespace that has `type: date`
+`description`: The script will fill in the type of pattern
+`fields`: The script will fill this in based on the namespace
+* `name`: The name of the field from the namespace
+* `type`: Field data type (string, date, etc.) - this is the `type` parameter from the namespace
+* `count`: Always has value of 0
+* `scripted`: `true` or `false` - all of our fields are not scripted, so `false`
+* `indexed`: `true` or `false` - all of our fields are indexed, so `true`
+* `analyzed`: `true` or `false` - `true` if the namespace field has `index: analyzed`, `false` otherwise
+* `doc_values`: `true` or `false` - taken from the namespace `doc_values` field
 
 template.yml
 ----------

templates/openshift/template.yml

@@ -1,4 +1,5 @@
 skeleton_path: ../skeleton.json
+skeleton_index_pattern_path: ../skeleton-index-pattern.json
 
 elasticsearch_template:
   name: com.redhat.viaq-openshift