Merge pull request #51 from vparfonov/log2819-fix2

LOG-2819: move code for extracting structured level to the separate method

fluentd/lib/fluent-plugin-viaq_data_model/lib/fluent/plugin/filter_viaq_data_model.rb

@@ -131,7 +131,7 @@ class ViaqDataModelFilter < Filter
     config_param :dest_time_name, :string, default: '@timestamp'
 
     desc 'Take log level from structured and set them to the root level'
-    config_param :extract_structured_loglevel, :bool, default: false
+    config_param :extract_structured_loglevel, :bool, default: true
 
 
     # <formatter>
@@ -454,7 +454,7 @@ def rename_time_field(record)
     end
 
     def extract_structured_loglevel_field(record)
-      normalize_level!(record, nil, true) 
+      extract_level_from_struct!(record) 
     end
 
 

fluentd/lib/fluent-plugin-viaq_data_model/lib/fluent/plugin/viaq_data_model_log_level_normalizer.rb

@@ -61,14 +61,13 @@ module LogLevelNormalizer
 
         # normalize_level! attempts to convert all level values into a common format
         # on the output side.  It optionally takes a block for further processing as needed
-        def normalize_level!(record, priority=nil, dig_structured=true)
+        def normalize_level!(record, priority=nil)
 
             level = record['level']
             # if the record already has a level field, and it looks like one of our well
             # known values, convert it to the canonical normalized form - otherwise,
             # preserve the value in string format
             retlevel = nil
-            struct_level = nil
             if !level.nil?
                 unless (retlevel = NORMAL_LEVELS[level]) ||
                         (level.respond_to?(:downcase) && (retlevel = NORMAL_LEVELS[level.downcase]))
@@ -77,8 +76,6 @@ def normalize_level!(record, priority=nil, dig_structured=true)
             elsif !priority.nil?
               retlevel = PRIORITY_LEVELS[priority]
             end
-            struct_level = record.dig('structured','level') if dig_structured
-            retlevel = struct_level if !struct_level.nil?  
             if record['message'] && retlevel.nil? 
                 retlevel = extract_level_from_message(record['message'], @level_matcher)
             end
@@ -99,5 +96,14 @@ def extract_level_from_message(message, check)
             return nil
         end
 
+        # extract_level_from_struct extracting log level from structured field if exits 
+        # and checking is it match with log level from root field. 
+        # Replace log level from root field if not match
+        def extract_level_from_struct!(record)
+            struct_level = record.dig('structured','level')
+            level = record['level']
+            record['level'] = struct_level if !struct_level.nil? && struct_level != level
+        end
+
     end
 end

fluentd/lib/fluent-plugin-viaq_data_model/test/test_filter_viaq_data_model.rb

@@ -316,6 +316,12 @@ def emit_with_tag(tag, msg={}, conf='')
       assert_equal("info", rec['level'])
       assert_equal({ "level" => "info"}, rec['structured'])
     end
+    test 'when enabled extruct structured log level shoul keep root log level as is is structured is empty' do
+      rec = emit_with_tag('tag', {"level" => "warning"}, '
+        extract_structured_loglevel true
+      ')
+      assert_equal("warning", rec['level'])
+    end
     test 'when disabled extruct structured log level do nothing' do
       rec = emit_with_tag('tag', {"structured" => { "level" => "info"}}, '
         extract_structured_loglevel false

fluentd/lib/fluent-plugin-viaq_data_model/test/test_viaq_data_model_log_level_normalizer.rb

@@ -51,33 +51,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
                 assert_equal('warn', newrecord['level'])
             end
         end
-
-        sub_test_case 'when take level from structured field' do
-
-            test "should add level field to the record" do
-                record = {"message" => "20210909T12:15:09 Warning Some Warning message", "structured" => { "level" => "info" }}
-                normalize_level!(record, nil, true)
-                assert_equal('info',record['level'])
-            end
-
-            test "should replace level from root with structured.level " do
-                record = {"level" => "error", "structured" => { "level" => "info" }}
-                normalize_level!(record, nil, true)
-                assert_equal('info',record['level'])
-            end
-
-            test "should keep log level as is if config" do
-                record = {"level" => "debug", "structured" => { "level" => "error" }}
-                normalize_level!(record, nil, false)
-                assert_equal('debug',record['level'])
-            end
-
-            test "should keep log level as if not structured.level field" do
-                record = {"level" => "info", "structured" => { "foo" => "error" }}
-                normalize_level!(record, nil, true)
-                assert_equal('info',record['level'])
-            end
-        end
+
     end
 
     sub_test_case '#extract_level_from_message' do
@@ -110,4 +84,26 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
         end
 
     end
+
+    sub_test_case 'when take level from structured field' do
+
+        test "should add level field to the record" do
+            record = {"message" => "20210909T12:15:09 Warning Some Warning message", "structured" => { "level" => "info" }}
+            extract_level_from_struct!(record)
+            assert_equal('info',record['level'])
+        end
+
+        test "should replace level from root with structured.level " do
+            record = {"level" => "error", "structured" => { "level" => "info" }}
+            extract_level_from_struct!(record)
+            assert_equal('info',record['level'])
+        end
+
+        test "should keep log level as if not structured.level field" do
+            record = {"level" => "info", "structured" => { "foo" => "error" }}
+            extract_level_from_struct!(record)
+            assert_equal('info',record['level'])
+        end
+
+    end
 end