Use probelab prometheus server

probe-lab · Nov 9, 2023 · ed0971f · ed0971f
1 parent fbf82e0
commit ed0971f
Show file tree

Hide file tree

Showing 12 changed files with 42 additions and 32 deletions.
diff --git a/cmd/thunderdome/infra/base.go b/cmd/thunderdome/infra/base.go
@@ -23,7 +23,7 @@ type BaseInfra struct {
 	EcsExecutionRoleArn           string
 	EfsFileSystemID               string
 	ExperimentsTableName          string
-	GrafanaPushSecretArn          string
+	PrometheusSecretArn           string
 	IronbarAddr                   string
 	LogGroupName                  string
 	RequestSNSTopicArn            string

diff --git a/cmd/thunderdome/infra/dealgood.go b/cmd/thunderdome/infra/dealgood.go
@@ -334,12 +334,16 @@ func (d *Dealgood) createTaskDefinition() Task {
 						},
 						Secrets: []*ecs.Secret{
 							{
-								Name:      aws.String("GRAFANA_USER"),
-								ValueFrom: aws.String(d.base.GrafanaPushSecretArn + ":username::"),
+								Name:      aws.String("PROMETHEUS_URL"),
+								ValueFrom: aws.String(d.base.PrometheusSecretArn + ":url::"),
 							},
 							{
-								Name:      aws.String("GRAFANA_PASS"),
-								ValueFrom: aws.String(d.base.GrafanaPushSecretArn + ":password::"),
+								Name:      aws.String("PROMETHEUS_USER"),
+								ValueFrom: aws.String(d.base.PrometheusSecretArn + ":username::"),
+							},
+							{
+								Name:      aws.String("PROMETHEUS_PASS"),
+								ValueFrom: aws.String(d.base.PrometheusSecretArn + ":password::"),
 							},
 						},
 					},

diff --git a/cmd/thunderdome/infra/target.go b/cmd/thunderdome/infra/target.go
@@ -313,12 +313,16 @@ func (t *Target) createTaskDefinition() Task {
 						},
 						Secrets: []*ecs.Secret{
 							{
-								Name:      aws.String("GRAFANA_USER"),
-								ValueFrom: aws.String(t.base.GrafanaPushSecretArn + ":username::"),
+								Name:      aws.String("PROMETHEUS_URL"),
+								ValueFrom: aws.String(t.base.PrometheusSecretArn + ":url::"),
 							},
 							{
-								Name:      aws.String("GRAFANA_PASS"),
-								ValueFrom: aws.String(t.base.GrafanaPushSecretArn + ":password::"),
+								Name:      aws.String("PROMETHEUS_USER"),
+								ValueFrom: aws.String(t.base.PrometheusSecretArn + ":username::"),
+							},
+							{
+								Name:      aws.String("PROMETHEUS_PASS"),
+								ValueFrom: aws.String(t.base.PrometheusSecretArn + ":password::"),
 							},
 						},
 					},

diff --git a/tf/files/grafana-agent-config/dealgood.yaml b/tf/files/grafana-agent-config/dealgood.yaml
@@ -3,10 +3,10 @@ metrics:
     scrape_interval: 30s
     scrape_timeout: 30s
     remote_write:
-      - url: https://prometheus-us-central1.grafana.net/api/prom/push
+      - url: ${PROMETHEUS_URL}
         basic_auth:
-          username: ${GRAFANA_USER}
-          password: ${GRAFANA_PASS}
+          username: ${PROMETHEUS_USER}
+          password: ${PROMETHEUS_PASS}
   configs:
     - name: thunderdome
       host_filter: false
@@ -41,5 +41,5 @@ traces:
       - endpoint: tempo-eu-west-0.grafana.net:443
         basic_auth:
           username: 263936
-          password: ${GRAFANA_PASS}
+          password: ${PROMETHEUS_PASS}
 
diff --git a/tf/files/grafana-agent-config/ironbar.yaml b/tf/files/grafana-agent-config/ironbar.yaml
@@ -3,10 +3,10 @@ metrics:
     scrape_interval: 30s
     scrape_timeout: 30s
     remote_write:
-      - url: https://prometheus-us-central1.grafana.net/api/prom/push
+      - url: ${PROMETHEUS_URL}
         basic_auth:
-          username: ${GRAFANA_USER}
-          password: ${GRAFANA_PASS}
+          username: ${PROMETHEUS_USER}
+          password: ${PROMETHEUS_PASS}
   configs:
     - name: thunderdome
       host_filter: false

diff --git a/tf/files/grafana-agent-config/skyfish.yaml b/tf/files/grafana-agent-config/skyfish.yaml
@@ -3,10 +3,10 @@ metrics:
     scrape_interval: 30s
     scrape_timeout: 30s
     remote_write:
-      - url: https://prometheus-us-central1.grafana.net/api/prom/push
+      - url: ${PROMETHEUS_URL}
         basic_auth:
-          username: ${GRAFANA_USER}
-          password: ${GRAFANA_PASS}
+          username: ${PROMETHEUS_USER}
+          password: ${PROMETHEUS_PASS}
   configs:
     - name: thunderdome
       host_filter: false
@@ -39,5 +39,5 @@ traces:
       - endpoint: tempo-eu-west-0.grafana.net:443
         basic_auth:
           username: 263936
-          password: ${GRAFANA_PASS}
+          password: ${PROMETHEUS_PASS}
 
diff --git a/tf/files/grafana-agent-config/target.yaml b/tf/files/grafana-agent-config/target.yaml
@@ -3,10 +3,10 @@ metrics:
     scrape_interval: 30s
     scrape_timeout: 30s
     remote_write:
-      - url: https://prometheus-us-central1.grafana.net/api/prom/push
+      - url: ${PROMETHEUS_URL}
         basic_auth:
-          username: ${GRAFANA_USER}
-          password: ${GRAFANA_PASS}
+          username: ${PROMETHEUS_USER}
+          password: ${PROMETHEUS_PASS}
   configs:
     - name: thunderdome
       host_filter: false
@@ -64,4 +64,4 @@ traces:
       - endpoint: tempo-eu-west-0.grafana.net:443
         basic_auth:
           username: 263936
-          password: ${GRAFANA_PASS}
+          password: ${PROMETHEUS_PASS}
diff --git a/tf/iam.tf b/tf/iam.tf
@@ -22,7 +22,7 @@ data "aws_iam_policy_document" "ecsTaskExecutionRole_secretsmanager" {
   statement {
     actions = ["kms:Decrypt", "secretsmanager:GetSecretValue"]
     resources = [
-      data.aws_secretsmanager_secret.grafana-push-secret.arn,
+      data.aws_secretsmanager_secret.prometheus-secret.arn,
       data.aws_secretsmanager_secret.dealgood-loki-secret.arn,
       data.aws_kms_key.default_secretsmanager_key.arn,
     ]

diff --git a/tf/ironbar.tf b/tf/ironbar.tf
@@ -144,8 +144,9 @@ resource "aws_ecs_task_definition" "ironbar" {
       ]
       portMappings = []
       secrets      = [
-        { name = "GRAFANA_USER", valueFrom = "${data.aws_secretsmanager_secret.grafana-push-secret.arn}:username::" },
-        { name = "GRAFANA_PASS", valueFrom = "${data.aws_secretsmanager_secret.grafana-push-secret.arn}:password::" }
+        { name = "PROMETHEUS_URL", valueFrom = "${data.aws_secretsmanager_secret.prometheus-secret.arn}:url::" },
+        { name = "PROMETHEUS_USER", valueFrom = "${data.aws_secretsmanager_secret.prometheus-secret.arn}:username::" },
+        { name = "PROMETHEUS_PASS", valueFrom = "${data.aws_secretsmanager_secret.prometheus-secret.arn}:password::" }
       ]
       volumesFrom  = []
     }

diff --git a/tf/locals.tf b/tf/locals.tf
@@ -17,7 +17,7 @@ locals {
     EcsExecutionRoleArn             = aws_iam_role.ecsTaskExecutionRole.arn
     EfsFileSystemID                 = aws_efs_file_system.thunderdome.id
     ExperimentsTableName            = aws_dynamodb_table.experiments.name
-    GrafanaPushSecretArn            = data.aws_secretsmanager_secret.grafana-push-secret.arn
+    PrometheusSecretArn             = data.aws_secretsmanager_secret.prometheus-secret.arn
     IronbarAddr                     = "${aws_eip.ecs[0].public_ip}:${local.ironbar_port_number}"
     LogGroupName                    = aws_cloudwatch_log_group.logs.name
     RequestSNSTopicArn              = aws_sns_topic.gateway_requests.arn

diff --git a/tf/main.tf b/tf/main.tf
@@ -24,8 +24,8 @@ data "aws_kms_key" "default_secretsmanager_key" {
   key_id = "arn:aws:kms:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:key/d32eafb8-e9b3-44f2-9703-fd4663949020"
 }
 
-data "aws_secretsmanager_secret" "grafana-push-secret" {
-  arn = "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:grafana-push-MxjNiv"
+data "aws_secretsmanager_secret" "prometheus-secret" {
+  arn = "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:prometheus_credentials-ujqWDl"
 }
 
 data "aws_secretsmanager_secret" "dealgood-loki-secret" {

diff --git a/tf/skyfish.tf b/tf/skyfish.tf
@@ -126,8 +126,9 @@ resource "aws_ecs_task_definition" "skyfish" {
       ]
       portMappings = []
       secrets      = [
-        { name = "GRAFANA_USER", valueFrom = "${data.aws_secretsmanager_secret.grafana-push-secret.arn}:username::" },
-        { name = "GRAFANA_PASS", valueFrom = "${data.aws_secretsmanager_secret.grafana-push-secret.arn}:password::" }
+        { name = "PROMETHEUS_URL", valueFrom = "${data.aws_secretsmanager_secret.prometheus-secret.arn}:url::" },
+        { name = "PROMETHEUS_USER", valueFrom = "${data.aws_secretsmanager_secret.prometheus-secret.arn}:username::" },
+        { name = "PROMETHEUS_PASS", valueFrom = "${data.aws_secretsmanager_secret.prometheus-secret.arn}:password::" }
       ]
       volumesFrom  = []
     }