From 1b33efc67184aa3a72b7bed46de73afc4037fce2 Mon Sep 17 00:00:00 2001 From: "gtn3010@gmail.com" Date: Mon, 8 Jul 2024 18:46:37 +0700 Subject: [PATCH] feat: Add support for multi-account query --- pkg/clients/cloudwatch/client.go | 6 ++-- pkg/clients/cloudwatch/v1/client.go | 46 ++++++++++++++++++++++------- pkg/clients/cloudwatch/v2/client.go | 46 ++++++++++++++++++++++------- pkg/clients/tagging/v1/client.go | 7 +++++ pkg/clients/tagging/v2/client.go | 7 +++++ pkg/config/config.go | 4 +++ pkg/job/custom.go | 3 +- pkg/job/discovery.go | 3 +- pkg/model/model.go | 28 +++++++++++++----- pkg/promutil/migrate.go | 5 ++++ 10 files changed, 121 insertions(+), 34 deletions(-) diff --git a/pkg/clients/cloudwatch/client.go b/pkg/clients/cloudwatch/client.go index 1839ff552..afaed098b 100644 --- a/pkg/clients/cloudwatch/client.go +++ b/pkg/clients/cloudwatch/client.go @@ -18,7 +18,7 @@ type Client interface { // ListMetrics returns the list of metrics and dimensions for a given namespace // and metric name. Results pagination is handled automatically: the caller can // optionally pass a non-nil func in order to handle results pages. - ListMetrics(ctx context.Context, namespace string, metric *model.MetricConfig, recentlyActiveOnly bool, fn func(page []*model.Metric)) error + ListMetrics(ctx context.Context, namespace string, metric *model.MetricConfig, includeLinkedAccounts []string, recentlyActiveOnly bool, fn func(page []*model.Metric)) error // GetMetricData returns the output of the GetMetricData CloudWatch API. // Results pagination is handled automatically. @@ -75,9 +75,9 @@ func (c limitedConcurrencyClient) GetMetricData(ctx context.Context, getMetricDa return res } -func (c limitedConcurrencyClient) ListMetrics(ctx context.Context, namespace string, metric *model.MetricConfig, recentlyActiveOnly bool, fn func(page []*model.Metric)) error { +func (c limitedConcurrencyClient) ListMetrics(ctx context.Context, namespace string, metric *model.MetricConfig, includeLinkedAccounts []string, recentlyActiveOnly bool, fn func(page []*model.Metric)) error { c.limiter.Acquire(listMetricsCall) - err := c.client.ListMetrics(ctx, namespace, metric, recentlyActiveOnly, fn) + err := c.client.ListMetrics(ctx, namespace, metric, includeLinkedAccounts, recentlyActiveOnly, fn) c.limiter.Release(listMetricsCall) return err } diff --git a/pkg/clients/cloudwatch/v1/client.go b/pkg/clients/cloudwatch/v1/client.go index 5a8e3ace2..7c1b7d07b 100644 --- a/pkg/clients/cloudwatch/v1/client.go +++ b/pkg/clients/cloudwatch/v1/client.go @@ -2,6 +2,7 @@ package v1 import ( "context" + "slices" "time" "github.com/aws/aws-sdk-go/aws" @@ -26,11 +27,14 @@ func NewClient(logger logging.Logger, cloudwatchAPI cloudwatchiface.CloudWatchAP } } -func (c client) ListMetrics(ctx context.Context, namespace string, metric *model.MetricConfig, recentlyActiveOnly bool, fn func(page []*model.Metric)) error { +func (c client) ListMetrics(ctx context.Context, namespace string, metric *model.MetricConfig, includeLinkedAccounts []string, recentlyActiveOnly bool, fn func(page []*model.Metric)) error { filter := &cloudwatch.ListMetricsInput{ MetricName: aws.String(metric.Name), Namespace: aws.String(namespace), } + if len(includeLinkedAccounts) > 0 { + filter.IncludeLinkedAccounts = aws.Bool(true) + } if recentlyActiveOnly { filter.RecentlyActive = aws.String("PT3H") } @@ -42,7 +46,7 @@ func (c client) ListMetrics(ctx context.Context, namespace string, metric *model err := c.cloudwatchAPI.ListMetricsPagesWithContext(ctx, filter, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { promutil.CloudwatchAPICounter.WithLabelValues("ListMetrics").Inc() - metricsPage := toModelMetric(page) + metricsPage := toModelMetric(page, includeLinkedAccounts) if c.logger.IsDebugEnabled() { c.logger.Debug("ListMetrics", "output", metricsPage, "last_page", lastPage) @@ -60,15 +64,31 @@ func (c client) ListMetrics(ctx context.Context, namespace string, metric *model return nil } -func toModelMetric(page *cloudwatch.ListMetricsOutput) []*model.Metric { +func toModelMetric(page *cloudwatch.ListMetricsOutput, includeLinkedAccounts []string) []*model.Metric { modelMetrics := make([]*model.Metric, 0, len(page.Metrics)) - for _, cloudwatchMetric := range page.Metrics { - modelMetric := &model.Metric{ - MetricName: *cloudwatchMetric.MetricName, - Namespace: *cloudwatchMetric.Namespace, - Dimensions: toModelDimensions(cloudwatchMetric.Dimensions), + if len(includeLinkedAccounts) > 0 { + for i := 0; i < len(page.Metrics); i++ { + linkedAccountId := *page.OwningAccounts[i] + if !slices.Contains(includeLinkedAccounts, "*") && !slices.Contains(includeLinkedAccounts, linkedAccountId) { + continue + } + modelMetric := &model.Metric{ + MetricName: *page.Metrics[i].MetricName, + Namespace: *page.Metrics[i].Namespace, + Dimensions: toModelDimensions(page.Metrics[i].Dimensions), + LinkedAccountId: linkedAccountId, + } + modelMetrics = append(modelMetrics, modelMetric) + } + } else { + for _, cloudwatchMetric := range page.Metrics { + modelMetric := &model.Metric{ + MetricName: *cloudwatchMetric.MetricName, + Namespace: *cloudwatchMetric.Namespace, + Dimensions: toModelDimensions(cloudwatchMetric.Dimensions), + } + modelMetrics = append(modelMetrics, modelMetric) } - modelMetrics = append(modelMetrics, modelMetric) } return modelMetrics } @@ -97,11 +117,15 @@ func (c client) GetMetricData(ctx context.Context, getMetricData []*model.Cloudw Period: &data.GetMetricDataProcessingParams.Period, Stat: &data.GetMetricDataProcessingParams.Statistic, } - metricDataQueries = append(metricDataQueries, &cloudwatch.MetricDataQuery{ + metricDataQuery := &cloudwatch.MetricDataQuery{ Id: &data.GetMetricDataProcessingParams.QueryID, MetricStat: metricStat, ReturnData: aws.Bool(true), - }) + } + if data.LinkedAccountId != "" { + metricDataQuery.AccountId = aws.String(data.LinkedAccountId) + } + metricDataQueries = append(metricDataQueries, metricDataQuery) } input := &cloudwatch.GetMetricDataInput{ EndTime: &endTime, diff --git a/pkg/clients/cloudwatch/v2/client.go b/pkg/clients/cloudwatch/v2/client.go index 33ea14fde..330488515 100644 --- a/pkg/clients/cloudwatch/v2/client.go +++ b/pkg/clients/cloudwatch/v2/client.go @@ -2,6 +2,7 @@ package v2 import ( "context" + "slices" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -26,11 +27,14 @@ func NewClient(logger logging.Logger, cloudwatchAPI *cloudwatch.Client) cloudwat } } -func (c client) ListMetrics(ctx context.Context, namespace string, metric *model.MetricConfig, recentlyActiveOnly bool, fn func(page []*model.Metric)) error { +func (c client) ListMetrics(ctx context.Context, namespace string, metric *model.MetricConfig, includeLinkedAccounts []string, recentlyActiveOnly bool, fn func(page []*model.Metric)) error { filter := &cloudwatch.ListMetricsInput{ MetricName: aws.String(metric.Name), Namespace: aws.String(namespace), } + if len(includeLinkedAccounts) > 0 { + filter.IncludeLinkedAccounts = aws.Bool(true) + } if recentlyActiveOnly { filter.RecentlyActive = types.RecentlyActivePt3h } @@ -52,7 +56,7 @@ func (c client) ListMetrics(ctx context.Context, namespace string, metric *model return err } - metricsPage := toModelMetric(page) + metricsPage := toModelMetric(page, includeLinkedAccounts) if c.logger.IsDebugEnabled() { c.logger.Debug("ListMetrics", "output", metricsPage) } @@ -63,15 +67,31 @@ func (c client) ListMetrics(ctx context.Context, namespace string, metric *model return nil } -func toModelMetric(page *cloudwatch.ListMetricsOutput) []*model.Metric { +func toModelMetric(page *cloudwatch.ListMetricsOutput, includeLinkedAccounts []string) []*model.Metric { modelMetrics := make([]*model.Metric, 0, len(page.Metrics)) - for _, cloudwatchMetric := range page.Metrics { - modelMetric := &model.Metric{ - MetricName: *cloudwatchMetric.MetricName, - Namespace: *cloudwatchMetric.Namespace, - Dimensions: toModelDimensions(cloudwatchMetric.Dimensions), + if len(includeLinkedAccounts) > 0 { + for i := 0; i < len(page.Metrics); i++ { + linkedAccountId := page.OwningAccounts[i] + if !slices.Contains(includeLinkedAccounts, "*") && !slices.Contains(includeLinkedAccounts, linkedAccountId) { + continue + } + modelMetric := &model.Metric{ + MetricName: *page.Metrics[i].MetricName, + Namespace: *page.Metrics[i].Namespace, + Dimensions: toModelDimensions(page.Metrics[i].Dimensions), + LinkedAccountId: linkedAccountId, + } + modelMetrics = append(modelMetrics, modelMetric) + } + } else { + for _, cloudwatchMetric := range page.Metrics { + modelMetric := &model.Metric{ + MetricName: *cloudwatchMetric.MetricName, + Namespace: *cloudwatchMetric.Namespace, + Dimensions: toModelDimensions(cloudwatchMetric.Dimensions), + } + modelMetrics = append(modelMetrics, modelMetric) } - modelMetrics = append(modelMetrics, modelMetric) } return modelMetrics } @@ -100,11 +120,15 @@ func (c client) GetMetricData(ctx context.Context, getMetricData []*model.Cloudw Period: aws.Int32(int32(data.GetMetricDataProcessingParams.Period)), Stat: &data.GetMetricDataProcessingParams.Statistic, } - metricDataQueries = append(metricDataQueries, types.MetricDataQuery{ + metricDataQuery := types.MetricDataQuery{ Id: &data.GetMetricDataProcessingParams.QueryID, MetricStat: metricStat, ReturnData: aws.Bool(true), - }) + } + if data.LinkedAccountId != "" { + metricDataQuery.AccountId = aws.String(data.LinkedAccountId) + } + metricDataQueries = append(metricDataQueries, metricDataQuery) } input := &cloudwatch.GetMetricDataInput{ diff --git a/pkg/clients/tagging/v1/client.go b/pkg/clients/tagging/v1/client.go index 06b3edfe9..5d392f602 100644 --- a/pkg/clients/tagging/v1/client.go +++ b/pkg/clients/tagging/v1/client.go @@ -67,6 +67,13 @@ func (c client) GetResources(ctx context.Context, job model.DiscoveryJob, region var resources []*model.TaggedResource shouldHaveDiscoveredResources := false + if len(job.IncludeLinkedAccounts) > 0 { + // when setting `includeLinkedAccounts`, don't get resources in cross accounts (because we need more permissions in cross accounts) + c.logger.Debug("Return empty resources when enable includeLinkedAccounts") + resources = []*model.TaggedResource{} + return resources, nil + } + if len(svc.ResourceFilters) > 0 { shouldHaveDiscoveredResources = true diff --git a/pkg/clients/tagging/v2/client.go b/pkg/clients/tagging/v2/client.go index 9e2d6a3e2..ed8b12da0 100644 --- a/pkg/clients/tagging/v2/client.go +++ b/pkg/clients/tagging/v2/client.go @@ -67,6 +67,13 @@ func (c client) GetResources(ctx context.Context, job model.DiscoveryJob, region var resources []*model.TaggedResource shouldHaveDiscoveredResources := false + if len(job.IncludeLinkedAccounts) > 0 { + // when setting `includeLinkedAccounts`, don't get resources in cross accounts (because we need more permissions in cross accounts) + c.logger.Debug("Return empty resources when enable includeLinkedAccounts") + resources = []*model.TaggedResource{} + return resources, nil + } + if len(svc.ResourceFilters) > 0 { shouldHaveDiscoveredResources = true filters := make([]string, 0, len(svc.ResourceFilters)) diff --git a/pkg/config/config.go b/pkg/config/config.go index 6ca776042..5b2f8c5ae 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -53,6 +53,7 @@ type Job struct { RoundingPeriod *int64 `yaml:"roundingPeriod"` RecentlyActiveOnly bool `yaml:"recentlyActiveOnly"` IncludeContextOnInfoMetrics bool `yaml:"includeContextOnInfoMetrics"` + IncludeLinkedAccounts []string `yaml:"includeLinkedAccounts"` JobLevelMetricFields `yaml:",inline"` } @@ -76,6 +77,7 @@ type CustomNamespace struct { CustomTags []Tag `yaml:"customTags"` DimensionNameRequirements []string `yaml:"dimensionNameRequirements"` RoundingPeriod *int64 `yaml:"roundingPeriod"` + IncludeLinkedAccounts []string `yaml:"includeLinkedAccounts"` JobLevelMetricFields `yaml:",inline"` } @@ -409,6 +411,7 @@ func (c *ScrapeConf) toModelConfig() model.JobsConfig { job.CustomTags = toModelTags(discoveryJob.CustomTags) job.Metrics = toModelMetricConfig(discoveryJob.Metrics) job.IncludeContextOnInfoMetrics = discoveryJob.IncludeContextOnInfoMetrics + job.IncludeLinkedAccounts = discoveryJob.IncludeLinkedAccounts job.DimensionsRegexps = svc.ToModelDimensionsRegexp() job.ExportedTagsOnMetrics = []string{} @@ -444,6 +447,7 @@ func (c *ScrapeConf) toModelConfig() model.JobsConfig { job.Roles = toModelRoles(customNamespaceJob.Roles) job.CustomTags = toModelTags(customNamespaceJob.CustomTags) job.Metrics = toModelMetricConfig(customNamespaceJob.Metrics) + job.IncludeLinkedAccounts = customNamespaceJob.IncludeLinkedAccounts jobsCfg.CustomNamespaceJobs = append(jobsCfg.CustomNamespaceJobs, job) } diff --git a/pkg/job/custom.go b/pkg/job/custom.go index b1355c6d7..e9f4ad40a 100644 --- a/pkg/job/custom.go +++ b/pkg/job/custom.go @@ -51,7 +51,7 @@ func getMetricDataForQueriesForCustomNamespace( go func(metric *model.MetricConfig) { defer wg.Done() - err := clientCloudwatch.ListMetrics(ctx, customNamespaceJob.Namespace, metric, customNamespaceJob.RecentlyActiveOnly, func(page []*model.Metric) { + err := clientCloudwatch.ListMetrics(ctx, customNamespaceJob.Namespace, metric, customNamespaceJob.IncludeLinkedAccounts, customNamespaceJob.RecentlyActiveOnly, func(page []*model.Metric) { var data []*model.CloudwatchData for _, cwMetric := range page { @@ -63,6 +63,7 @@ func getMetricDataForQueriesForCustomNamespace( data = append(data, &model.CloudwatchData{ MetricName: metric.Name, ResourceName: customNamespaceJob.Name, + LinkedAccountId: cwMetric.LinkedAccountId, Namespace: customNamespaceJob.Namespace, Dimensions: cwMetric.Dimensions, GetMetricDataProcessingParams: &model.GetMetricDataProcessingParams{ diff --git a/pkg/job/discovery.go b/pkg/job/discovery.go index da3b46d20..5a6e299c2 100644 --- a/pkg/job/discovery.go +++ b/pkg/job/discovery.go @@ -93,7 +93,7 @@ func getMetricDataForQueries( go func(metric *model.MetricConfig) { defer wg.Done() - err := clientCloudwatch.ListMetrics(ctx, svc.Namespace, metric, discoveryJob.RecentlyActiveOnly, func(page []*model.Metric) { + err := clientCloudwatch.ListMetrics(ctx, svc.Namespace, metric, discoveryJob.IncludeLinkedAccounts, discoveryJob.RecentlyActiveOnly, func(page []*model.Metric) { data := getFilteredMetricDatas(logger, discoveryJob.Type, discoveryJob.ExportedTagsOnMetrics, page, discoveryJob.DimensionNameRequirements, metric, assoc) mux.Lock() @@ -157,6 +157,7 @@ func getFilteredMetricDatas( getMetricsData = append(getMetricsData, &model.CloudwatchData{ MetricName: m.Name, ResourceName: resource.ARN, + LinkedAccountId: cwMetric.LinkedAccountId, Namespace: namespace, Dimensions: cwMetric.Dimensions, GetMetricDataProcessingParams: &model.GetMetricDataProcessingParams{ diff --git a/pkg/model/model.go b/pkg/model/model.go index 04d6bcfbd..e552b4479 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -30,6 +30,7 @@ type DiscoveryJob struct { RecentlyActiveOnly bool ExportedTagsOnMetrics []string IncludeContextOnInfoMetrics bool + IncludeLinkedAccounts []string DimensionsRegexps []DimensionsRegexp } @@ -53,6 +54,17 @@ type CustomNamespaceJob struct { Metrics []*MetricConfig CustomTags []Tag DimensionNameRequirements []string + IncludeLinkedAccounts []string + JobLevelMetricFields +} + +type JobLevelMetricFields struct { + Statistics []string + Period int64 + Length int64 + Delay int64 + NilToZero *bool + AddCloudwatchTimestamp *bool } type Role struct { @@ -94,9 +106,10 @@ type Dimension struct { type Metric struct { // The dimensions for the metric. - Dimensions []Dimension - MetricName string - Namespace string + Dimensions []Dimension + MetricName string + Namespace string + LinkedAccountId string } type Datapoint struct { @@ -148,10 +161,11 @@ type CloudwatchData struct { // DiscoveryJob = Resource ARN associated with the metric or global when it could not be associated but shouldn't be dropped // StaticJob = Resource Name from static job config // CustomNamespace = Custom Namespace job name - ResourceName string - Namespace string - Tags []Tag - Dimensions []Dimension + ResourceName string + Namespace string + Tags []Tag + Dimensions []Dimension + LinkedAccountId string // GetMetricDataProcessingParams includes necessary fields to run GetMetricData GetMetricDataProcessingParams *GetMetricDataProcessingParams diff --git a/pkg/promutil/migrate.go b/pkg/promutil/migrate.go index 781159713..4145eb8b5 100644 --- a/pkg/promutil/migrate.go +++ b/pkg/promutil/migrate.go @@ -113,6 +113,11 @@ func BuildMetrics(results []model.CloudwatchMetricResult, labelsSnakeCase bool, name := BuildMetricName(metric.Namespace, metric.MetricName, statistic) promLabels := createPrometheusLabels(metric, labelsSnakeCase, contextLabels, logger) + if metric.LinkedAccountId != "" { + contextLabels["account_id"] = metric.LinkedAccountId + } + maps.Copy(promLabels, contextLabels) + observedMetricLabels = recordLabelsForMetric(name, promLabels, observedMetricLabels) output = append(output, &PrometheusMetric{