[service] add support for views

.chloggen/codeboten_configure-views.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: service
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: add support for metric views configuration
+
+# One or more tracking issues or pull requests related to the change
+issues: [8237]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

docs/observability.md

@@ -76,9 +76,55 @@ For begin/stop events we need to define an appropriate hysteresis to avoid gener
 
 The service should collect host resource metrics in addition to service's own process metrics. This may help to understand that the problem that we observe in the service is induced by a different process on the same host.
 
-## How We Expose Metrics/Traces
-
-Collector configuration must allow specifying the target for own metrics/traces (which can be different from the target of collected data). The metrics and traces must be clearly tagged to indicate that they are service’s own metrics (to avoid conflating with collected data in the backend).
+## How We Expose Telemetry
+
+By default, the Collector exposes service telemetry in two ways currently:
+
+- internal metrics are exposed via a Prometheus interface which defaults to port `8888`
+- logs are emitted to stdout
+
+Traces are not exposed by default. There is an effort underway to [change this][issue7532]. The work includes supporting
+configuration of the OpenTelemetry SDK used to produce the Collector's internal telemetry. This feature is
+currently behind two feature gates:
+
+```bash
+  --feature-gates=telemetry.useOtelForInternalMetrics
+  --feature-gates=telemetry.useOtelWithSDKConfigurationForInternalTelemetry
+```
+
+The `useOtelForInternalMetrics` feature gate changes the internal telemetry to use OpenTelemetry rather
+than OpenCensus. This will become the default at some point [in the future][issue7454]. The second gate,
+`useOtelWithSDKConfigurationForInternalTelemetry` enables the Collector to parse configuration
+that aligns with the [OpenTelemetry Configuration] schema. The support for this schema is still
+experimental, but it does allow telemetry to be exported via OTLP.
+
+The following configuration can be used in combination with the feature gates aforementioned
+to emit internal metrics and traces from the Collector to an OTLP backend:
+
+```yaml
+service:
+ telemetry:
+   metrics:
+     readers:
+       - periodic:
+           interval: 5000
+           exporter:
+             otlp:
+               protocol: grpc/protobuf
+               endpoint: https://backend:4317
+   traces:
+     processors:
+       - batch:
+           exporter:
+             otlp:
+               protocol: grpc/protobuf
+               endpoint: https://backend2:4317
+```
+
+See the configuration's [example][kitchen-sink] for additional configuration options.
+
+Note that this configuration does not support emitting logs as there is no support for [logs] in
+OpenTelemetry Go SDK at this time.
 
 ### Impact
 
@@ -89,3 +135,9 @@ We need to be able to assess the impact of these observability improvements on t
 Some of the metrics/traces can be high volume and may not be desirable to always observe. We should consider adding an observability verboseness “level” that allows configuring the Collector to send more or less observability data (or even finer granularity to allow turning on/off specific metrics).
 
 The default level of observability must be defined in a way that has insignificant performance impact on the service.
+
+[issue7532]: https://github.com/open-telemetry/opentelemetry-collector/issues/7532
+[issue7454]: https://github.com/open-telemetry/opentelemetry-collector/issues/7454
+[logs]: https://github.com/open-telemetry/opentelemetry-go/issues/3827
+[OpenTelemetry Configuration]: https://github.com/open-telemetry/opentelemetry-configuration
+[kitchen-sink]: https://github.com/open-telemetry/opentelemetry-configuration/blob/main/examples/kitchen-sink.yaml

extension/zpagesextension/go.mod

@@ -35,6 +35,7 @@ require (
 	go.opentelemetry.io/collector/service v0.0.0-20230915215502-07938f20fcc7 // indirect
 	go.opentelemetry.io/otel v1.18.0 // indirect
 	go.opentelemetry.io/otel/metric v1.18.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v0.41.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect

extension/zpagesextension/go.sum

@@ -82,6 +82,8 @@ go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvz
 go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k=
 go.opentelemetry.io/otel/sdk v1.18.0 h1:e3bAB0wB3MljH38sHzpV/qWrOTCFrdZF2ct9F8rBkcY=
 go.opentelemetry.io/otel/sdk v1.18.0/go.mod h1:1RCygWV7plY2KmdskZEDDBs4tJeHG92MdHZIluiYs/M=
+go.opentelemetry.io/otel/sdk/metric v0.41.0 h1:c3sAt9/pQ5fSIUfl0gPtClV3HhE18DCVzByD33R/zsk=
+go.opentelemetry.io/otel/sdk/metric v0.41.0/go.mod h1:PmOmSt+iOklKtIg5O4Vz9H/ttcRFSNTgii+E1KGyn1w=
 go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10=
 go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=

service/telemetry.go

@@ -163,6 +163,8 @@ func (tel *telemetryInitializer) initMetrics(res *resource.Resource, logger *zap
 		opts = append(opts, sdkmetric.WithReader(r))
 	}
 
+	opts = append(opts, telemetry.ViewOptionsFromConfig(cfg.Metrics.Views)...)
+
 	mp, err := proctelemetry.InitOpenTelemetry(res, opts, tel.disableHighCardinality)
 	if err != nil {
 		return err

service/telemetry/config.go

@@ -111,6 +111,9 @@
 	// Readers allow configuration of metric readers to emit metrics to
 	// any number of supported backends.
 	Readers []MetricReader `mapstructure:"readers"`
+
+	// Views allow configuration of metric views.
+	Views []View `mapstructure:"views"`
 }
 
 // TracesConfig exposes the common Telemetry configuration for collector's internal spans.
@@ -202,3 +205,83 @@
 	}
 	return nil
 }
+
+func (v *View) Unmarshal(conf *confmap.Conf) error {
+	if !obsreportconfig.UseOtelWithSDKConfigurationForInternalTelemetryFeatureGate.IsEnabled() {
+		// only unmarshal if feature gate is enabled
+		return nil
+	}
+
+	if conf == nil {
+		return nil
+	}
+
+	if err := conf.Unmarshal(v); err != nil {
+		return fmt.Errorf("invalid view configuration: %w", err)
+	}
+
+	return v.Validate()
+}
+
+func (v *View) Validate() error {
+	if v.Selector == nil || v.Stream == nil {
+		return fmt.Errorf("invalid view configuration")
+	}
+	return nil
+}
+
+func (s *ViewSelector) InstrumentNameStr() string {
+	if s.InstrumentName == nil {
+		return ""
+	}
+	return *s.InstrumentName
+}
+
+func (s *ViewSelector) InstrumentTypeStr() string {
+	if s.InstrumentType == nil {
+		return ""
+	}
+	return *s.InstrumentType
+}
+
+func (s *ViewSelector) MeterNameStr() string {
+	if s.MeterName == nil {
+		return ""
+	}
+	return *s.MeterName
+}
+
+func (s *ViewSelector) MeterVersionStr() string {
+	if s.MeterVersion == nil {
+		return ""
+	}
+	return *s.MeterVersion
+}
+
+func (s *ViewSelector) MeterSchemaURLStr() string {
+	if s.MeterSchemaUrl == nil {
+		return ""
+	}
+	return *s.MeterSchemaUrl
+}
+
+func (s *ViewStream) NameStr() string {
+	if s.Name == nil {
+		return ""
+	}
+	return *s.Name
+}
+
+func (s *ViewStream) DescriptionStr() string {
+	if s.Description == nil {
+		return ""
+	}
+	return *s.Description
+}
+
+func (e *ViewStreamAggregationExplicitBucketHistogram) RecordMinMaxBool() bool {
+	if e.RecordMinMax == nil {
+		return false
+	}
+	return *e.RecordMinMax
+}

service/telemetry/metric_view.go

@@ -0,0 +1,95 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package telemetry // import "go.opentelemetry.io/collector/service/telemetry"
+
+import (
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/instrumentation"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+)
+
+func ViewOptionsFromConfig(views []View) []sdkmetric.Option {
+	opts := []sdkmetric.Option{}
+	for _, view := range views {
+		if view.Selector == nil || view.Stream == nil {
+			continue
+		}
+		opts = append(opts, sdkmetric.WithView(
+			sdkmetric.NewView(
+				sdkmetric.Instrument{
+					Name: view.Selector.InstrumentNameStr(),
+					Kind: instrumentTypeToKind(view.Selector.InstrumentTypeStr()),
+					// TODO: add unit once https://github.com/open-telemetry/opentelemetry-configuration/pull/38
+					//       is merged
+					// Unit: *view.Selector.Unit,
+					Scope: instrumentation.Scope{
+						Name:      view.Selector.MeterNameStr(),
+						Version:   view.Selector.MeterVersionStr(),
+						SchemaURL: view.Selector.MeterSchemaURLStr(),
+					},
+				},
+				sdkmetric.Stream{
+					Name:            view.Stream.NameStr(),
+					Description:     view.Stream.DescriptionStr(),
+					Aggregation:     viewStreamAggregationToAggregation(view.Stream.Aggregation),
+					AttributeFilter: attributeKeysToAttributeFilter(view.Stream.AttributeKeys),
+				},
+			),
+		))
+	}
+	return opts
+}
+
+var invalidInstrumentKind = sdkmetric.InstrumentKind(0)
+
+func instrumentTypeToKind(instrument string) sdkmetric.InstrumentKind {
+	switch instrument {
+	case "counter":
+		return sdkmetric.InstrumentKindCounter
+	case "histogram":
+		return sdkmetric.InstrumentKindHistogram
+	case "observable_counter":
+		return sdkmetric.InstrumentKindObservableCounter
+	case "observable_gauge":
+		return sdkmetric.InstrumentKindObservableGauge
+	case "observable_updown_counter":
+		return sdkmetric.InstrumentKindObservableUpDownCounter
+	case "updown_counter":
+		return sdkmetric.InstrumentKindUpDownCounter
+	}
+	return invalidInstrumentKind
+}
+
+func attributeKeysToAttributeFilter(keys []string) attribute.Filter {
+	kvs := make([]attribute.KeyValue, len(keys))
+	for i, key := range keys {
+		kvs[i] = attribute.Bool(key, true)
+	}
+	filter := attribute.NewSet(kvs...)
+	return func(kv attribute.KeyValue) bool {
+		return !filter.HasValue(kv.Key)
+	}
+}
+
+func viewStreamAggregationToAggregation(agg *ViewStreamAggregation) sdkmetric.Aggregation {
+	if agg == nil {
+		return sdkmetric.AggregationDefault{}
+	}
+	if agg.Sum != nil {
+		return sdkmetric.AggregationSum{}
+	}
+	if agg.Drop != nil {
+		return sdkmetric.AggregationDrop{}
+	}
+	if agg.LastValue != nil {
+		return sdkmetric.AggregationLastValue{}
+	}
+	if agg.ExplicitBucketHistogram != nil {
+		return sdkmetric.AggregationExplicitBucketHistogram{
+			Boundaries: agg.ExplicitBucketHistogram.Boundaries,
+			NoMinMax:   !agg.ExplicitBucketHistogram.RecordMinMaxBool(),
+		}
+	}
+	return sdkmetric.AggregationDefault{}
+}