From 9d512f0fec583f9b19d4b854d264c5719db6a08d Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Wed, 27 Mar 2024 19:07:00 +0100 Subject: [PATCH 1/5] Revert "driver_lxc: Include running state in metrics" This reverts commit 44802497c9188f71ba8c2c838a76c2beab94823e. Signed-off-by: Gabriel Mougard --- lxd/instance/drivers/driver_lxc.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lxd/instance/drivers/driver_lxc.go b/lxd/instance/drivers/driver_lxc.go index 124a6b1f1790..980228da4b6d 100644 --- a/lxd/instance/drivers/driver_lxc.go +++ b/lxd/instance/drivers/driver_lxc.go @@ -8246,14 +8246,12 @@ func (d *lxc) Info() instance.Info { // Metrics returns the metric set for the LXC driver. It collects various metrics related to memory, CPU, disk, filesystem, and network usage. func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error) { - state := instance.PowerStateStopped isRunning := d.IsRunning() - - if isRunning { - state = instance.PowerStateRunning + if !isRunning { + return nil, ErrInstanceIsStopped } - out := metrics.NewMetricSet(map[string]string{"project": d.project.Name, "name": d.name, "type": instancetype.Container.String(), "state": state}) + out := metrics.NewMetricSet(map[string]string{"project": d.project.Name, "name": d.name, "type": instancetype.Container.String()}) cc, err := d.initLXC(false) if err != nil { From af3950d8bf2a87203a355c337a5ff0dc398a90e1 Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Tue, 19 Mar 2024 11:55:15 +0100 Subject: [PATCH 2/5] lxd/instance/drivers/lxc: default some metrics to `0` instead of `-1` * Some CGroup related metrics like `MemorySwapUsage`, `OOMKills`, and `CPUs` (effective CPUs) can return -1 in case of error (the CGroup controller might not be available). This value help the application logic but in regards to the metrics, we should expose them as `0` instead of a numerical total being negative which is misleading. Signed-off-by: Gabriel Mougard --- lxd/instance/drivers/driver_lxc.go | 39 +++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/lxd/instance/drivers/driver_lxc.go b/lxd/instance/drivers/driver_lxc.go index 980228da4b6d..03f01e9509ea 100644 --- a/lxd/instance/drivers/driver_lxc.go +++ b/lxd/instance/drivers/driver_lxc.go @@ -8246,8 +8246,7 @@ func (d *lxc) Info() instance.Info { // Metrics returns the metric set for the LXC driver. It collects various metrics related to memory, CPU, disk, filesystem, and network usage. func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error) { - isRunning := d.IsRunning() - if !isRunning { + if !d.IsRunning() { return nil, ErrInstanceIsStopped } @@ -8274,7 +8273,7 @@ func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error // Get memory stats. memStats, err := cg.GetMemoryStats() - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get memory stats", logger.Ctx{"err": err}) } else { for k, v := range memStats { @@ -8316,7 +8315,7 @@ func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error // Get memory usage. memoryUsage, err := cg.GetMemoryUsage() - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get memory usage", logger.Ctx{"err": err}) } @@ -8328,25 +8327,37 @@ func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error // Get oom kills. oomKills, err := cg.GetOOMKills() - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get oom kills", logger.Ctx{"err": err}) } + // If we failed to get OOM kills, because of a couple of reasons (instance stopped, cgroup controller not available, etc), + // we default to 0 instead of -1 for the MemoryOOMKillsTotal metric (a total of `-1` would be misleading). + if oomKills < 0 { + oomKills = 0 + } + out.AddSamples(metrics.MemoryOOMKillsTotal, metrics.Sample{Value: float64(oomKills)}) // Handle swap. if d.state.OS.CGInfo.Supports(cgroup.MemorySwapUsage, cg) { swapUsage, err := cg.GetMemorySwapUsage() - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get swap usage", logger.Ctx{"err": err}) } else { + // If we failed to get swap memory usage, because of a couple of reasons (instance stopped, cgroup controller not available, etc), + // we default to 0 instead of -1 for the MemorySwapBytes metric (`-1` bytes would be misleading). + if swapUsage < 0 { + swapUsage = 0 + } + out.AddSamples(metrics.MemorySwapBytes, metrics.Sample{Value: float64(swapUsage)}) } } // Get CPU stats usage, err := cg.GetCPUAcctUsageAll() - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get CPU usage", logger.Ctx{"err": err}) } else { for cpu, stats := range usage { @@ -8359,15 +8370,21 @@ func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error // Get CPUs. CPUs, err := cg.GetEffectiveCPUs() - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get CPUs", logger.Ctx{"err": err}) } else { + // If we failed to get the number of total effective CPUs, because of a couple of reasons (instance stopped, cgroup controller not available, etc), + // we default to 0 instead of -1 for the CPUs metric (a total of `-1` would be misleading). + if CPUs < 0 { + CPUs = 0 + } + out.AddSamples(metrics.CPUs, metrics.Sample{Value: float64(CPUs)}) } // Get disk stats diskStats, err := cg.GetIOStats() - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get disk stats", logger.Ctx{"err": err}) } else { for disk, stats := range diskStats { @@ -8382,7 +8399,7 @@ func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error // Get filesystem stats fsStats, err := d.getFSStats() - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get fs stats", logger.Ctx{"err": err}) } else { out.Merge(fsStats) @@ -8406,7 +8423,7 @@ func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error // Get number of processes pids, err := d.processesState(d.InitPID()) - if err != nil && isRunning { + if err != nil { d.logger.Warn("Failed to get total number of processes", logger.Ctx{"err": err}) } else { out.AddSamples(metrics.ProcsTotal, metrics.Sample{Value: float64(pids)}) From c0df1a09d442fdf78e02c4bcce6e305314e1138d Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Thu, 28 Mar 2024 11:24:26 +0100 Subject: [PATCH 3/5] lxd/metrics: Replace `lxd_containers` and `lxd_vms` metrics by `lxd_instances` We now differentiate the container and the vm counts using the `type` label in the `lxd_instances` count metric: ``` lxd_instances{project="default", type="container"} X lxd_instances{project="default", type="vm"} Y ``` Signed-off-by: Gabriel Mougard --- lxd/metrics/metrics.go | 3 +-- lxd/metrics/types.go | 12 ++++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/lxd/metrics/metrics.go b/lxd/metrics/metrics.go index 37307f6c0a4a..4162591f64b1 100644 --- a/lxd/metrics/metrics.go +++ b/lxd/metrics/metrics.go @@ -94,8 +94,7 @@ func (m *MetricSet) String() string { CPUs, GoGoroutines, GoHeapObjects, - Containers, - VMs, + Instances, } for _, metricType := range metricTypes { diff --git a/lxd/metrics/types.go b/lxd/metrics/types.go index ab0fc78a52d8..2402cadcc890 100644 --- a/lxd/metrics/types.go +++ b/lxd/metrics/types.go @@ -144,10 +144,8 @@ const ( GoOtherSysBytes // GoNextGCBytes represents the number of heap bytes when next garbage collection will take place. GoNextGCBytes - // Containers represents the container count. - Containers - // VMs represents the VM count. - VMs + // Instances represents the instance count. + Instances ) // MetricNames associates a metric type to its name. @@ -216,8 +214,7 @@ var MetricNames = map[MetricType]string{ ProcsTotal: "lxd_procs_total", UptimeSeconds: "lxd_uptime_seconds", WarningsTotal: "lxd_warnings_total", - Containers: "lxd_containers", - VMs: "lxd_vms", + Instances: "lxd_instances", } // MetricHeaders represents the metric headers which contain help messages as specified by OpenMetrics. @@ -286,6 +283,5 @@ var MetricHeaders = map[MetricType]string{ ProcsTotal: "# HELP lxd_procs_total The number of running processes.", UptimeSeconds: "# HELP lxd_uptime_seconds The daemon uptime in seconds.", WarningsTotal: "# HELP lxd_warnings_total The number of active warnings.", - Containers: "# HELP lxd_containers The number of containers.", - VMs: "# HELP lxd_vms The number of virtual machines.", + Instances: "# HELP lxd_instances The number of instances.", } From 786334739e6e3ccfa0400a1e9932d1568a8e5687 Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Wed, 27 Mar 2024 14:32:25 +0100 Subject: [PATCH 4/5] lxd/api_metrics: Make `lxd_instances` and internal metrics visible Signed-off-by: Gabriel Mougard --- lxd/api_metrics.go | 62 ++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/lxd/api_metrics.go b/lxd/api_metrics.go index ad85bcfdaa96..f88cf3a10d7b 100644 --- a/lxd/api_metrics.go +++ b/lxd/api_metrics.go @@ -109,7 +109,7 @@ func metricsGet(d *Daemon, r *http.Request) response.Response { metricSet := metrics.NewMetricSet(nil) var projectNames []string - + var intMetrics *metrics.MetricSet err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Figure out the projects to retrieve. if projectName != "" { @@ -127,9 +127,8 @@ func metricsGet(d *Daemon, r *http.Request) response.Response { } } - // Add internal metrics. - metricSet.Merge(internalMetrics(ctx, s.StartTime, tx)) - + // Register internal metrics. + intMetrics = internalMetrics(ctx, s.StartTime, tx) return nil }) if err != nil { @@ -228,23 +227,6 @@ func metricsGet(d *Daemon, r *http.Request) response.Response { allProjectInstances[instance.Project().Name][instance.Type()]++ } - for project, instanceCountMap := range allProjectInstances { - metricSet.AddSamples( - metrics.Containers, - metrics.Sample{ - Labels: map[string]string{"project": project}, - Value: float64(instanceCountMap[instancetype.Container]), - }, - ) - metricSet.AddSamples( - metrics.VMs, - metrics.Sample{ - Labels: map[string]string{"project": project}, - Value: float64(instanceCountMap[instancetype.VM]), - }, - ) - } - // Prepare temporary metrics storage. newMetrics := make(map[string]*metrics.MetricSet, len(projectsToFetch)) newMetricsLock := sync.Mutex{} @@ -268,6 +250,12 @@ func metricsGet(d *Daemon, r *http.Request) response.Response { // Ignore stopped instances. if !errors.Is(err, instanceDrivers.ErrInstanceIsStopped) { logger.Warn("Failed getting instance metrics", logger.Ctx{"instance": inst.Name(), "project": projectName, "err": err}) + } else { + // If the instance is stopped, we still need to add the project to the cache. + // to fetch associated counter metrics. + if newMetrics[projectName] == nil { + newMetrics[projectName] = metrics.NewMetricSet(nil) + } } } else { // Add the metrics. @@ -304,8 +292,40 @@ func metricsGet(d *Daemon, r *http.Request) response.Response { metricsCache = map[string]metricsCacheEntry{} } + // Add counter metrics for instances to the metric cache. + // We need to do create a distinct metric set for each project. + counterMetrics := make(map[string]*metrics.MetricSet, len(allProjectInstances)) + for project, instanceCountMap := range allProjectInstances { + counterMetricSetPerProject := metrics.NewMetricSet(nil) + counterMetricSetPerProject.AddSamples( + metrics.Instances, + metrics.Sample{ + Labels: map[string]string{"project": project, "type": instancetype.Container.String()}, + Value: float64(instanceCountMap[instancetype.Container]), + }, + ) + counterMetricSetPerProject.AddSamples( + metrics.Instances, + metrics.Sample{ + Labels: map[string]string{"project": project, "type": instancetype.VM.String()}, + Value: float64(instanceCountMap[instancetype.VM]), + }, + ) + + counterMetrics[project] = counterMetricSetPerProject + } + updatedProjects := []string{} for project, entries := range newMetrics { + if project == "default" { + entries.Merge(intMetrics) // internal metrics are always considered new. Add them to the default project. + } + + counterMetric, ok := counterMetrics[project] + if ok { + entries.Merge(counterMetric) + } + metricsCache[project] = metricsCacheEntry{ expiry: time.Now().Add(cacheDuration), metrics: entries, From 1878cfe981d6f4e6ab998f55b0f72ddb5c1a1c52 Mon Sep 17 00:00:00 2001 From: Gabriel Mougard Date: Wed, 27 Mar 2024 09:43:30 +0100 Subject: [PATCH 5/5] tests: Fix metrics tests Signed-off-by: Gabriel Mougard --- test/suites/metrics.sh | 57 +++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/test/suites/metrics.sh b/test/suites/metrics.sh index 514cabc7be34..827455e995e3 100644 --- a/test/suites/metrics.sh +++ b/test/suites/metrics.sh @@ -9,17 +9,22 @@ test_metrics() { # create another container in the non default project lxc project create foo -c features.images=false -c features.profiles=false - lxc init testimage c3 --project foo + lxc launch testimage c3 --project foo # c1 metrics should show as the container is running lxc query "/1.0/metrics" | grep "name=\"c1\"" lxc query "/1.0/metrics?project=default" | grep "name=\"c1\"" - # c2 metrics should show the container as stopped - lxc query "/1.0/metrics" | grep "name=\"c2\"" - lxc query "/1.0/metrics?project=default" | grep "name=\"c2\"" - lxc query "/1.0/metrics" | grep "name=\"c2\"" | grep "state=\"STOPPED\"" - lxc query "/1.0/metrics?project=default" | grep "name=\"c2\"" | grep "state=\"STOPPED\"" + # c2 metrics should not be shown as the container is stopped + ! lxc query "/1.0/metrics" | grep "name=\"c2\"" || false + ! lxc query "/1.0/metrics?project=default" | grep "name=\"c2\"" || false + + # Check that we can get the count of existing instances. + lxc query /1.0/metrics | grep -xF 'lxd_instances{project="default",type="container"} 2' + lxc query /1.0/metrics | grep -xF 'lxd_instances{project="foo",type="container"} 1' + # Ensure lxd_instances reports VM count properly (0) + lxc query /1.0/metrics | grep -xF 'lxd_instances{project="default",type="virtual-machine"} 0' + lxc query /1.0/metrics | grep -xF 'lxd_instances{project="foo",type="virtual-machine"} 0' # c3 metrics from another project also show up for non metrics unrestricted certificate lxc query "/1.0/metrics" | grep "name=\"c3\"" @@ -39,13 +44,13 @@ test_metrics() { curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics" | grep "name=\"c1\"" curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=default" | grep "name=\"c1\"" - # c2 metrics should show the container as stopped - curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics" | grep "name=\"c2\"" | grep "state=\"STOPPED\"" - curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=default" | grep "name=\"c2\"" | grep "state=\"STOPPED\"" + # c2 metrics should not be shown as the container is stopped + ! curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics" | grep "name=\"c2\"" || false + ! curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=default" | grep "name=\"c2\"" || false - # c3 metrics from another project should show the container as stopped for unrestricted certificate - curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics" | grep "name=\"c3\"" | grep "state=\"STOPPED\"" - curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=foo" | grep "name=\"c3\"" | grep "state=\"STOPPED\"" + # c3 metrics from another project should be shown for unrestricted certificate + curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics" | grep "name=\"c3\"" + curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=foo" | grep "name=\"c3\"" # internal server metrics should be shown as the certificate is not restricted curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics" | grep -E "^lxd_warnings_total [0-9]+$" @@ -58,17 +63,17 @@ test_metrics() { lxc config set core.metrics_address "${metrics_addr}" - # c1 metrics should show as the container is running + # c1 metrics should be shown as the container is running curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics" | grep "name=\"c1\"" curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics?project=default" | grep "name=\"c1\"" - # c2 metrics should show the container as stopped - curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics" | grep "name=\"c2\"" | grep "state=\"STOPPED\"" - curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics?project=default" | grep "name=\"c2\"" | grep "state=\"STOPPED\"" + # c2 metrics should not be shown as the container is stopped + ! curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics" | grep "name=\"c2\"" || false + ! curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics?project=default" | grep "name=\"c2\"" || false - # c3 metrics from another project should show the container as stopped for unrestricted metrics certificate - curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics" | grep "name=\"c3\"" | grep "state=\"STOPPED\"" - curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=foo" | grep "name=\"c3\"" | grep "state=\"STOPPED\"" + # c3 metrics from another project should be shown for unrestricted metrics certificate + curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics" | grep "name=\"c3\"" + curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=foo" | grep "name=\"c3\"" # internal server metrics should be shown as the certificate is not restricted curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics" | grep -E "^lxd_warnings_total [0-9]+$" @@ -84,10 +89,10 @@ test_metrics() { lxc config trust add "${TEST_DIR}/metrics-restricted.crt" --type=metrics --restricted --projects foo lxc config trust show "$(openssl x509 -in "${TEST_DIR}/metrics-restricted.crt" -outform der | sha256sum | head -c12)" | grep -xF "restricted: true" - # c3 metrics should show the container as stopped for restricted metrics certificate - curl -k -s --cert "${TEST_DIR}/metrics-restricted.crt" --key "${TEST_DIR}/metrics-restricted.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=foo" | grep "name=\"c3\"" | grep "state=\"STOPPED\"" + # c3 metrics should be showned + curl -k -s --cert "${TEST_DIR}/metrics-restricted.crt" --key "${TEST_DIR}/metrics-restricted.key" -X GET "https://${LXD_ADDR}/1.0/metrics?project=foo" | grep "name=\"c3\"" - # c3 metrics for the stopped container cannot be viewed via the generic metrics endpoint if the certificate is restricted + # c3 metrics cannot be viewed via the generic metrics endpoint if the certificate is restricted ! curl -k -s --cert "${TEST_DIR}/metrics-restricted.crt" --key "${TEST_DIR}/metrics-restricted.key" -X GET "https://${LXD_ADDR}/1.0/metrics" # other projects metrics aren't visible as they aren't allowed for the restricted certificate @@ -95,7 +100,13 @@ test_metrics() { # c1 and c2 metrics are not visible as they are in another project ! curl -k -s --cert "${TEST_DIR}/metrics-restricted.crt" --key "${TEST_DIR}/metrics-restricted.key" -X GET "https://${metrics_addr}/1.0/metrics?project=foo" | grep "name=\"c1\"" - ! curl -k -s --cert "${TEST_DIR}/metrics-restricted.crt" --key "${TEST_DIR}/metrics-restricted.key" -X GET "https://${metrics_addr}/1.0/metrics?project=foo" | grep "name=\"c2\"" | grep "state=\"STOPPED\"" + ! curl -k -s --cert "${TEST_DIR}/metrics-restricted.crt" --key "${TEST_DIR}/metrics-restricted.key" -X GET "https://${metrics_addr}/1.0/metrics?project=foo" | grep "name=\"c2\"" + + # Check that we can get the count of existing containers. There should be two in the default project: c1 (RUNNING) and c2 (STOPPED). + curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics" | grep -xF 'lxd_instances{project="default",type="container"} 2' + sleep 10 + # Try again after the metric cache has expired. We should still see two containers. + curl -k -s --cert "${TEST_DIR}/metrics.crt" --key "${TEST_DIR}/metrics.key" -X GET "https://${metrics_addr}/1.0/metrics" | grep -xF 'lxd_instances{project="default",type="container"} 2' # test unauthenticated connections ! curl -k -s -X GET "https://${metrics_addr}/1.0/metrics" | grep "name=\"c1\"" || false