feat(ConfigProvider): Modify config merge to do replace instead of update for Specific tags

.changelog/1604.breaking.txt

@@ -0,0 +1 @@
+feat(ConfigProvider)!: Modify config merge to do replace instead of update for Specific tags

docs/upgrading.md

@@ -2,6 +2,8 @@
 
 - [Upgrading to v0.104.0-sumo-0](#upgrading-to-v01040-sumo-0)
   - [`sumologic` exporter: remove `compress_encoding`](#sumologic-exporter-remove-compress_encoding)
+- [Upgrading to v0.103.0-sumo-0](#upgrading-to-v01030-sumo-0)
+  - [`sumologic` configuration: modified the `configuration files` merge behaviour](#sumologic-configuration-modified-the-configuration-files-merge-behaviour)
 - [Upgrading to v0.96.0-sumo-0](#upgrading-to-v0960-sumo-0)
   - [`sumologic` exporter: remove `json_logs`](#sumologic-exporter-remove-json_logs)
   - [`sumologic` exporter: remove `clear_logs_timestamp`](#sumologic-exporter-remove-clear_logs_timestamp)
@@ -81,6 +83,79 @@ to this:
       compression: ""
   ```
 
+## Upgrading to v0.103.0-sumo-0
+
+### `sumologic` configuration: modified the `configuration files` merge behaviour
+
+Modified the configuration merge behaviour to perform overwrite instead of update for `collector_fields` key of [Sumo Logic Extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/sumologicextension#configuration).
+
+For example, if two configuration files(say `conf1.yaml` and `conf2.yaml`) define the `collector_fields`,
+
+In previous flow, the values of `collector_fields` from `conf2.yaml` will be added to `conf1.yaml` and the effective configuration will have vaules from both the configurations.
+
+Starting from `v0.103.0-sumo-0`, the values of `collector_fields` tag from `conf1.yaml` will be overwritten by values from `conf2.yaml` and the effective configuration will have `collector_fields` value of `conf2.yaml` only.
+
+For example:
+
+`conf1.yaml`:
+
+```
+extensions:
+  sumologic:
+    collector_description: "My OpenTelemetry Collector"
+    collector_fields:
+      cluster: "cluster-1"
+    some_list:
+      - element 1
+      - element 2
+```
+
+`conf2.yaml`:
+
+```
+extensions:
+  sumologic:
+    collector_fields:
+      zone: "eu"
+    some_list:
+      - element 3
+      - element 4
+```
+
+effective configuration (`old behaviour`)
+
+```
+extensions:
+  sumologic:
+    collector_description: "My OpenTelemetry Collector"
+    collector_fields:
+      cluster: "cluster-1"
+      zone: "eu"
+    some_list:
+      - element 3
+      - element 4
+```
+
+effective configuration (`new behaviour`)
+
+```
+extensions:
+  sumologic:
+    collector_description: "My OpenTelemetry Collector"
+    collector_fields:
+      zone: "eu"
+    some_list:
+      - element 3
+      - element 4
+```
+
+If you have multiple config files with `collector_fields` key specified, only the value from the last file(**alphabetically** sorted order) will be present in effective configuration.
+Due to above, avoid maintaining `collector_fields` in multiple configuration files and move them to a single file.
+
+Note: This applies only to `collector_fields` key, all other key behaviour will remain the same.
+
+For more details regarding configuration structure and merge behaviour, see https://help.sumologic.com/docs/send-data/opentelemetry-collector/data-source-configurations/overview/#configuration-structure
+
 ## Upgrading to v0.96.0-sumo-0
 
 ### `sumologic` exporter: remove `json_logs`

otelcolbuilder/.otelcol-builder.yaml

@@ -245,3 +245,4 @@ replaces:
   - github.com/SumoLogic/sumologic-otel-collector/pkg/extension/opampextension => ../../pkg/extension/opampextension
   - github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/globprovider => ../../pkg/configprovider/globprovider
   - github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/opampprovider => ../../pkg/configprovider/opampprovider
+  - github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/providerutil => ../../pkg/configprovider/providerutil

pkg/configprovider/globprovider/go.mod

@@ -5,6 +5,7 @@ go 1.21.0
 toolchain go1.22.3
 
 require (
+	github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/providerutil v0.0.0-00010101000000-000000000000
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/confmap v0.104.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -24,3 +25,5 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 )
+
+replace github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/providerutil => ../providerutil

pkg/configprovider/globprovider/provider.go

@@ -22,9 +22,9 @@ import (
 	"sort"
 	"strings"
 
-	"gopkg.in/yaml.v3"
-
+	"github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/providerutil"
 	"go.opentelemetry.io/collector/confmap"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -68,11 +68,12 @@ func (fmp *provider) Retrieve(ctx context.Context, uri string, _ confmap.Watcher
 			return &confmap.Retrieved{}, err
 		}
 		pathConf := confmap.NewFromStringMap(rawConf)
+		providerutil.PrepareForReplaceBehavior(conf, pathConf)
 		if err := conf.Merge(pathConf); err != nil {
 			return &confmap.Retrieved{}, err
 		}
-	}
 
+	}
 	return confmap.NewRetrieved(conf.ToStringMap())
 }
 

pkg/configprovider/opampprovider/go.mod

@@ -6,6 +6,7 @@ toolchain go1.22.3
 
 require (
 	github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/globprovider v0.0.0-00010101000000-000000000000
+	github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/providerutil v0.0.0-00010101000000-000000000000
 	github.com/google/go-cmp v0.5.9
 	go.opentelemetry.io/collector/confmap v0.104.0
 	go.opentelemetry.io/collector/confmap/provider/fileprovider v0.104.0
@@ -27,3 +28,5 @@ require (
 )
 
 replace github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/globprovider => ../globprovider
+
+replace github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/providerutil => ../providerutil

pkg/configprovider/opampprovider/provider.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/globprovider"
+	"github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/providerutil"
 	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/confmap/provider/fileprovider"
 	"gopkg.in/yaml.v2"
@@ -100,6 +101,9 @@ func (p *Provider) Retrieve(ctx context.Context, configPath string, fn confmap.W
 	if err != nil {
 		return nil, err
 	}
+	// Order of conf parameters is important, see method comments
+	providerutil.PrepareForReplaceBehavior(addlConf, retConf)
+
 	// merge the file config in
 	if err := conf.Merge(addlConf); err != nil {
 		return nil, err

pkg/configprovider/providerutil/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

pkg/configprovider/providerutil/go.mod

@@ -0,0 +1,24 @@
+module github.com/SumoLogic/sumologic-otel-collector/pkg/configprovider/providerutil
+
+go 1.21.0
+
+toolchain go1.22.3
+
+require (
+	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/collector/confmap v0.102.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
+	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
+	github.com/knadh/koanf/v2 v2.1.1 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

pkg/configprovider/providerutil/go.sum

@@ -0,0 +1,30 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c=
+github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs=
+github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
+github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU=
+github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU=
+github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM=
+github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.opentelemetry.io/collector/confmap v0.102.1 h1:wZuH+d/P11Suz8wbp+xQCJ0BPE9m5pybtUe74c+rU7E=
+go.opentelemetry.io/collector/confmap v0.102.1/go.mod h1:KgpS7UxH5rkd69CzAzlY2I1heH8Z7eNCZlHmwQBMxNg=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/configprovider/providerutil/utils.go

@@ -0,0 +1,44 @@
+package providerutil
+
+import (
+	"go.opentelemetry.io/collector/confmap"
+)
+
+// Removes the key(path passed in keys array) from sourceMap if same key path is present in MergeMap
+func removeMatchingKeyFromSrcMap(srcMap map[string]interface{}, mergeMap map[string]interface{}, keys []string) map[string]interface{} {
+
+	if len(keys) == 0 {
+		return srcMap
+	}
+
+	currentKey := keys[0]
+	if len(keys) == 1 { // Got Leaf key, if leaf key exists in both maps, remove it from source map
+		_, existInSrc := srcMap[currentKey]
+		_, existInMerge := mergeMap[currentKey]
+		// If leaf key exists in both maps, remove from source map
+		if existInSrc && existInMerge {
+			delete(srcMap, currentKey)
+		}
+		return srcMap
+	}
+	// More levels to go, descend into child if current key present in both maps
+	srcNestedMap, isCurrKeyInSrcMap := srcMap[currentKey].(map[string]interface{})
+	mergeNestedMap, isCurrKeyInMergeMap := mergeMap[currentKey].(map[string]interface{})
+	if isCurrKeyInSrcMap && isCurrKeyInMergeMap {
+		removeMatchingKeyFromSrcMap(srcNestedMap, mergeNestedMap, keys[1:])
+	}
+	return srcMap
+}
+
+// Checks and prepares scrConf for replace behavior instead of map merge
+// Hack for confmap.Conf.Merge method to replace specific fields instead of merging them
+// Merge method merges field values from source and mergeConf, so by removing existing values from
+// source map, we can achieve replace behavior
+func PrepareForReplaceBehavior(srcConf *confmap.Conf, mergeConf *confmap.Conf) {
+	keyPathsWithReplaceBehavior := [][]string{
+		{"extensions", "sumologic", "collector_fields"},
+	}
+	for _, keyPath := range keyPathsWithReplaceBehavior {
+		*srcConf = *confmap.NewFromStringMap(removeMatchingKeyFromSrcMap(srcConf.ToStringMap(), mergeConf.ToStringMap(), keyPath))
+	}
+}

pkg/configprovider/providerutil/utils_test.go

@@ -0,0 +1,91 @@
+package providerutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/confmap"
+)
+
+func TestPrepareForReplaceBehavior(t *testing.T) {
+	tests := []struct {
+		name         string
+		srcConf      *confmap.Conf
+		mergeConf    *confmap.Conf
+		expectedConf *confmap.Conf
+	}{
+		{
+			name: "Remove matching key path from source map",
+			srcConf: confmap.NewFromStringMap(map[string]interface{}{
+				"extensions": map[string]interface{}{
+					"sumologic": map[string]interface{}{
+						"childKey2": "value2",
+						"collector_fields": map[string]interface{}{
+							"cluster": "cluster-1",
+						},
+					},
+				},
+				"anotherKey": "anotherValue",
+			}),
+			mergeConf: confmap.NewFromStringMap(map[string]interface{}{
+				"extensions": map[string]interface{}{
+					"sumologic": map[string]interface{}{
+						"collector_fields": map[string]interface{}{
+							"zone": "eu",
+						},
+						"childKey2": "value2",
+					},
+				},
+				"anotherKey": "anotherValue",
+			}),
+			expectedConf: confmap.NewFromStringMap(map[string]interface{}{
+				"extensions": map[string]interface{}{
+					"sumologic": map[string]interface{}{
+						"childKey2": "value2",
+					},
+				},
+				"anotherKey": "anotherValue",
+			}),
+		},
+		{
+			name: "No matching key paths to remove, source map remains unaffected",
+			srcConf: confmap.NewFromStringMap(map[string]interface{}{
+				"extensions": map[string]interface{}{
+					"sumologic": map[string]interface{}{
+						"collector_fields1": map[string]interface{}{
+							"cluster": "cluster-1",
+						},
+					},
+				},
+				"anotherKey": "anotherValue",
+			}),
+			mergeConf: confmap.NewFromStringMap(map[string]interface{}{
+				"extensions": map[string]interface{}{
+					"sumologic": map[string]interface{}{
+						"collector_fields2": map[string]interface{}{
+							"zone": "eu",
+						},
+					},
+				},
+				"anotherKey": "anotherValue",
+			}),
+			expectedConf: confmap.NewFromStringMap(map[string]interface{}{
+				"extensions": map[string]interface{}{
+					"sumologic": map[string]interface{}{
+						"collector_fields1": map[string]interface{}{
+							"cluster": "cluster-1",
+						},
+					},
+				},
+				"anotherKey": "anotherValue",
+			}),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			PrepareForReplaceBehavior(tt.srcConf, tt.mergeConf)
+			assert.Equal(t, tt.expectedConf, tt.srcConf)
+		})
+	}
+}