feat: generic api to run promql query for metrics

cmd/commands/server.go

@@ -40,6 +40,7 @@ func NewServerCommand() *cobra.Command {
 		corsAllowedOrigins   string
 		readOnly             bool
 		daemonClientProtocol string
+		prometheusServerUrl  string
 	)
 
 	command := &cobra.Command{
@@ -64,6 +65,7 @@ func NewServerCommand() *cobra.Command {
 				CorsAllowedOrigins:   corsAllowedOrigins,
 				ReadOnly:             readOnly,
 				DaemonClientProtocol: daemonClientProtocol,
+				PrometheusServerUrl:  prometheusServerUrl,
 			}
 			server := svrcmd.NewServer(opts)
 			log := logging.NewLogger().Named("server")
@@ -80,5 +82,7 @@ func NewServerCommand() *cobra.Command {
 	command.Flags().StringVar(&serverAddr, "server-addr", sharedutil.LookupEnvStringOr("NUMAFLOW_SERVER_ADDRESS", "https://localhost:8443"), "The external address of the Numaflow server.")
 	command.Flags().StringVar(&corsAllowedOrigins, "cors-allowed-origins", sharedutil.LookupEnvStringOr("NUMAFLOW_SERVER_CORS_ALLOWED_ORIGINS", ""), "The values for allowed cors AllowOrigins header field, separated by comma.")
 	command.Flags().StringVar(&daemonClientProtocol, "daemon-client-protocol", sharedutil.LookupEnvStringOr("NUMAFLOW_SERVER_DAEMON_CLIENT_PROTOCOL", "grpc"), "The protocol used to connect to the Pipeline daemon service from Numaflow UX server, defaults to 'grpc'.")
+	command.Flags().StringVar(&prometheusServerUrl, "prometheus-server-url", sharedutil.LookupEnvStringOr("NUMAFLOW_SERVER_PROMETHEUS_URL", ""), "The prometheus server url for querying metrics data.")
+
 	return command
 }

config/advanced-install/namespaced-numaflow-server.yaml

@@ -129,6 +129,12 @@ metadata:
   name: numaflow-cmd-params-config
 ---
 apiVersion: v1
+data: null
+kind: ConfigMap
+metadata:
+  name: numaflow-metric-server-config
+---
+apiVersion: v1
 data:
   admin.enabled: "true"
 kind: ConfigMap
@@ -268,6 +274,12 @@ spec:
               key: server.daemon.client.protocol
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+          valueFrom:
+            configMapKeyRef:
+              key: metric.server.url
+              name: numaflow-metric-server-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         livenessProbe:
@@ -323,6 +335,12 @@ spec:
               key: server.disable.auth
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+          valueFrom:
+            configMapKeyRef:
+              key: url
+              name: numaflow-metric-server-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         name: server-secrets-init

config/advanced-install/numaflow-server.yaml

@@ -135,6 +135,13 @@ metadata:
   namespace: numaflow-system
 ---
 apiVersion: v1
+data: null
+kind: ConfigMap
+metadata:
+  name: numaflow-metric-server-config
+  namespace: numaflow-system
+---
+apiVersion: v1
 data:
   admin.enabled: "true"
 kind: ConfigMap
@@ -279,6 +286,12 @@ spec:
               key: server.daemon.client.protocol
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+          valueFrom:
+            configMapKeyRef:
+              key: metric.server.url
+              name: numaflow-metric-server-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         livenessProbe:
@@ -334,6 +347,12 @@ spec:
               key: server.disable.auth
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+          valueFrom:
+            configMapKeyRef:
+              key: url
+              name: numaflow-metric-server-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         name: server-secrets-init

config/base/numaflow-server/kustomization.yaml

@@ -7,4 +7,5 @@ resources:
   - numaflow-server-deployment.yaml
   - numaflow-server-service.yaml
   - numaflow-server-local-user-config.yaml
-  - numaflow-server-secrets.yaml
+  - numaflow-server-secrets.yaml
+  - numaflow-metric-server-config.yaml

config/base/numaflow-server/numaflow-metric-server-config.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: numaflow-metric-server-config
+data:
+  ### The  address of the Prometheus server. This is required for querying metrics from the server.
+  # metric.server.url: "http://localhost:9090"

config/base/numaflow-server/numaflow-server-deployment.yaml

@@ -58,6 +58,12 @@ spec:
                 name: numaflow-cmd-params-config
                 key: server.disable.auth
                 optional: true
+          - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+            valueFrom:
+              configMapKeyRef:
+                name: numaflow-metric-server-config
+                key: url
+                optional: true
       containers:
         - name: main
           image: quay.io/numaproj/numaflow:latest
@@ -144,6 +150,12 @@ spec:
                 name: numaflow-cmd-params-config
                 key: server.daemon.client.protocol
                 optional: true
+          - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+            valueFrom:
+              configMapKeyRef:
+                name: numaflow-metric-server-config
+                key: metric.server.url
+                optional: true
           resources:
             limits:
               cpu: 500m

config/install.yaml

@@ -25692,6 +25692,13 @@ metadata:
   namespace: numaflow-system
 ---
 apiVersion: v1
+data: null
+kind: ConfigMap
+metadata:
+  name: numaflow-metric-server-config
+  namespace: numaflow-system
+---
+apiVersion: v1
 data:
   admin.enabled: "true"
 kind: ConfigMap
@@ -26049,6 +26056,12 @@ spec:
               key: server.daemon.client.protocol
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+          valueFrom:
+            configMapKeyRef:
+              key: metric.server.url
+              name: numaflow-metric-server-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         livenessProbe:
@@ -26104,6 +26117,12 @@ spec:
               key: server.disable.auth
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+          valueFrom:
+            configMapKeyRef:
+              key: url
+              name: numaflow-metric-server-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         name: server-secrets-init

config/namespace-install.yaml

@@ -25581,6 +25581,12 @@ metadata:
   name: numaflow-dex-server-config
 ---
 apiVersion: v1
+data: null
+kind: ConfigMap
+metadata:
+  name: numaflow-metric-server-config
+---
+apiVersion: v1
 data:
   admin.enabled: "true"
 kind: ConfigMap
@@ -25929,6 +25935,12 @@ spec:
               key: server.daemon.client.protocol
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+          valueFrom:
+            configMapKeyRef:
+              key: metric.server.url
+              name: numaflow-metric-server-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         livenessProbe:
@@ -25984,6 +25996,12 @@ spec:
               key: server.disable.auth
               name: numaflow-cmd-params-config
               optional: true
+        - name: NUMAFLOW_SERVER_PROMETHEUS_URL
+          valueFrom:
+            configMapKeyRef:
+              key: url
+              name: numaflow-metric-server-config
+              optional: true
         image: quay.io/numaproj/numaflow:latest
         imagePullPolicy: Always
         name: server-secrets-init

go.sum

@@ -397,6 +397,8 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@@ -472,6 +474,8 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/nats-io/jwt/v2 v2.5.8 h1:uvdSzwWiEGWGXf+0Q+70qv6AQdvcvxrv9hPM0RiPamE=

server/apis/interface.go

@@ -47,4 +47,5 @@ type Handler interface {
 	ListMonoVertexPods(c *gin.Context)
 	CreateMonoVertex(c *gin.Context)
 	GetMonoVertexMetrics(c *gin.Context)
+	GetMetricData(c *gin.Context)
 }