Add metrics and Aspire app host (#1128)

- Implement metrics and OpenTelemetry tracing
- Use .NET aspire for local orchestration instead of Docker Compose

TODO:

- [x] Update self-hosting guidance as required env variables have
changed
- [x] Update Nomad deployment for the same reason
- [x] Block /metrics from reverse proxy

Aspire/Dawnshard.AppHost/Dawnshard.AppHost.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <Sdk Name="Aspire.AppHost.Sdk" Version="9.0.0-rc.1.24511.1" />
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <IsAspireHost>true</IsAspireHost>
+        <UserSecretsId>6c42e872-dfcb-405f-a064-2169c50f7123</UserSecretsId>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Aspire.Hosting.AppHost" />
+        <PackageReference Include="Aspire.Hosting.NodeJs" />
+        <PackageReference Include="Aspire.Hosting.PostgreSQL" />
+        <PackageReference Include="Aspire.Hosting.Redis" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\..\DragaliaAPI\DragaliaAPI\DragaliaAPI.csproj" />
+        <ProjectReference Include="..\..\PhotonStateManager\DragaliaAPI.Photon.StateManager\DragaliaAPI.Photon.StateManager.csproj" />
+    </ItemGroup>
+
+</Project>

Aspire/Dawnshard.AppHost/Program.cs

@@ -0,0 +1,39 @@
+using Microsoft.Extensions.Configuration;
+
+IDistributedApplicationBuilder builder = DistributedApplication.CreateBuilder(args);
+
+IResourceBuilder<PostgresServerResource> postgres = builder
+    .AddPostgres("postgres")
+    .WithImage("postgres", "16.4")
+    .WithDataVolume("dragalia-api-pgdata");
+
+IResourceBuilder<RedisResource> redis = builder
+    .AddRedis("redis")
+    .WithImage("redis/redis-stack", "7.4.0-v0");
+
+IResourceBuilder<ProjectResource> dragaliaApi = builder
+    .AddProject<Projects.DragaliaAPI>("dragalia-api")
+    .WithReference(postgres)
+    .WithReference(redis)
+    .WithExternalHttpEndpoints();
+
+if (builder.Configuration.GetValue<bool>("EnableStateManager"))
+{
+    IResourceBuilder<ProjectResource> stateManager = builder
+        .AddProject<Projects.DragaliaAPI_Photon_StateManager>("photon-state-manager")
+        .WithReference(redis)
+        .WithEndpoint("http", http => http.TargetHost = "0.0.0.0")
+        .WithExternalHttpEndpoints();
+
+    dragaliaApi.WithEnvironment("PhotonOptions__StateManagerUrl", stateManager.GetEndpoint("http"));
+}
+
+if (builder.Configuration.GetValue<bool>("EnableWebsite"))
+{
+    builder
+        .AddNpmApp("website", workingDirectory: "../Website", scriptName: "dev")
+        .WithEnvironment("PUBLIC_ENABLE_MSW", "false")
+        .WithEnvironment("DAWNSHARD_API_URL_SSR", dragaliaApi.GetEndpoint("http"));
+}
+
+builder.Build().Run();

Aspire/Dawnshard.AppHost/Properties/launchSettings.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "https://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "https": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "https://localhost:17267;http://localhost:15150",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "DOTNET_ENVIRONMENT": "Development",
+        "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21171",
+        "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22135"
+      }
+    },
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "applicationUrl": "http://localhost:15150",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "DOTNET_ENVIRONMENT": "Development",
+        "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19186",
+        "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20131"
+      }
+    }
+  }
+}

Aspire/Dawnshard.AppHost/appsettings.json

@@ -0,0 +1,12 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning",
+      "Aspire.Hosting.Dcp": "Warning"
+    }
+  },
+  "EnableStateManager": false,
+  "EnableWebsite": false,
+  "EnableGrafana": false
+}

Aspire/Dawnshard.ServiceDefaults/Dawnshard.ServiceDefaults.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <IsAspireSharedProject>true</IsAspireSharedProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
+    <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
+    <PackageReference Include="Serilog.Expressions" />
+    <PackageReference Include="Serilog.Extensions.Hosting" />
+    <PackageReference Include="Serilog.Settings.Configuration" />
+    <PackageReference Include="Serilog.Sinks.OpenTelemetry" />
+  </ItemGroup>
+
+</Project>

Aspire/Dawnshard.ServiceDefaults/Extensions.cs

@@ -0,0 +1,160 @@
+using Dawnshard.ServiceDefaults;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using OpenTelemetry.Logs;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
+using Serilog;
+
+// ReSharper disable once CheckNamespace
+namespace Microsoft.Extensions.Hosting;
+
+public static class Extensions
+{
+    public static WebApplicationBuilder AddServiceDefaults(this WebApplicationBuilder builder)
+    {
+        builder.ConfigureOpenTelemetry();
+        builder.AddDefaultHealthChecks();
+        builder.ConfigureLogging();
+
+        // Cannot add this as a transitive dependency upgrade breaks logging
+        // https://github.com/dotnet/extensions/issues/5336
+        // Re-evaluate when upgrading to .NET 9
+        //
+        // builder.Services.ConfigureHttpClientDefaults(http =>
+        // {
+        //     http.AddStandardResilienceHandler();
+        // });
+
+        return builder;
+    }
+
+    public static WebApplication MapDefaultEndpoints(this WebApplication app)
+    {
+        app.MapHealthChecks(
+            "/health",
+            new HealthCheckOptions() { ResponseWriter = HealthCheckWriter.WriteResponse }
+        );
+        app.MapHealthChecks(
+            "/ping",
+            new HealthCheckOptions { Predicate = r => r.Tags.Contains("live") }
+        );
+        app.MapPrometheusScrapingEndpoint();
+
+        return app;
+    }
+
+    private static WebApplicationBuilder ConfigureLogging(this WebApplicationBuilder builder)
+    {
+        builder.Host.UseSerilog(
+            static (context, config) =>
+            {
+                config.ReadFrom.Configuration(context.Configuration);
+                config.Enrich.FromLogContext();
+
+                config.Filter.ByExcluding(
+                    "EndsWith(RequestPath, '/health') and @l in ['verbose', 'debug', 'information'] ci"
+                );
+                config.Filter.ByExcluding(
+                    "EndsWith(RequestPath, '/ping') and @l in ['verbose', 'debug', 'information'] ci"
+                );
+                config.Filter.ByExcluding(
+                    "EndsWith(RequestPath, '/metrics') and @l in ['verbose', 'debug', 'information'] ci"
+                );
+
+                if (context.HasOtlpLogsEndpoint())
+                {
+                    config.WriteTo.OpenTelemetry();
+                }
+            }
+        );
+
+        return builder;
+    }
+
+    private static IHostApplicationBuilder ConfigureOpenTelemetry(
+        this IHostApplicationBuilder builder
+    )
+    {
+        builder
+            .Services.AddOpenTelemetry()
+            .WithMetrics(metrics =>
+            {
+                metrics
+                    .AddAspNetCoreInstrumentation()
+                    .AddHttpClientInstrumentation()
+                    .AddRuntimeInstrumentation()
+                    .AddPrometheusExporter();
+            });
+
+        if (builder.HasOtlpTracesEndpoint())
+        {
+            builder
+                .Services.AddOpenTelemetry()
+                .WithTracing(tracing =>
+                {
+                    tracing
+                        .AddAspNetCoreInstrumentation()
+                        .AddHttpClientInstrumentation()
+                        .AddProcessor<FilteringProcessor>();
+                });
+        }
+
+        builder.AddOpenTelemetryExporters();
+
+        return builder;
+    }
+
+    private static IHostApplicationBuilder AddOpenTelemetryExporters(
+        this IHostApplicationBuilder builder
+    )
+    {
+        if (builder.HasOtlpTracesEndpoint())
+        {
+            builder.Services.ConfigureOpenTelemetryTracerProvider(tracing =>
+                tracing.AddOtlpExporter()
+            );
+        }
+
+        if (builder.HasOtlpMetricsEndpoint())
+        {
+            builder.Services.ConfigureOpenTelemetryMeterProvider(metrics =>
+                metrics.AddOtlpExporter()
+            );
+        }
+
+        return builder;
+    }
+
+    private static IHostApplicationBuilder AddDefaultHealthChecks(
+        this IHostApplicationBuilder builder
+    )
+    {
+        builder
+            .Services.AddHealthChecks()
+            .AddCheck("self", () => HealthCheckResult.Healthy(), ["live"]);
+
+        return builder;
+    }
+
+    public static bool HasOtlpTracesEndpoint(this IHostApplicationBuilder builder) =>
+        !string.IsNullOrWhiteSpace(
+            builder.Configuration.GetOtlpEndpoint("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
+        );
+
+    private static bool HasOtlpMetricsEndpoint(this IHostApplicationBuilder builder) =>
+        !string.IsNullOrWhiteSpace(
+            builder.Configuration.GetOtlpEndpoint("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT")
+        );
+
+    private static bool HasOtlpLogsEndpoint(this HostBuilderContext context) =>
+        !string.IsNullOrWhiteSpace(
+            context.Configuration.GetOtlpEndpoint("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT")
+        );
+
+    private static string? GetOtlpEndpoint(this IConfiguration configuration, string envVarName) =>
+        configuration[envVarName] ?? configuration["OTEL_EXPORTER_OTLP_ENDPOINT"];
+}

Aspire/Dawnshard.ServiceDefaults/FilteringProcessor.cs

@@ -0,0 +1,33 @@
+using System.Diagnostics;
+using OpenTelemetry;
+
+namespace Dawnshard.ServiceDefaults;
+
+/// <summary>
+/// Custom processor to silence traces resulting from uninteresting activities such as metric scraping and healthchecks.
+/// </summary>
+internal sealed class FilteringProcessor : BaseProcessor<Activity>
+{
+    public override void OnEnd(Activity data)
+    {
+        Activity root = data;
+
+        while (root.Parent is not null)
+        {
+            root = root.Parent;
+        }
+
+        if (root.OperationName != "Microsoft.AspNetCore.Hosting.HttpRequestIn")
+        {
+            return;
+        }
+
+        foreach ((string key, string? value) in root.Tags)
+        {
+            if (key == "url.path" && value is "/metrics" or "/health" or "/ping")
+            {
+                root.ActivityTraceFlags &= ~ActivityTraceFlags.Recorded;
+            }
+        }
+    }
+}

DragaliaAPI/DragaliaAPI/Services/Health/HealthCheckWriter.cs → Aspire/Dawnshard.ServiceDefaults/HealthCheckWriter.cs

@@ -1,7 +1,9 @@
 using System.Text;
+using System.Text.Json;
+using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
 
-namespace DragaliaAPI.Services.Health;
+namespace Dawnshard.ServiceDefaults;
 
 public class HealthCheckWriter
 {

Aspire/Directory.Build.props

@@ -0,0 +1,9 @@
+<Project>
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

Directory.Packages.props

@@ -3,6 +3,10 @@
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
   </PropertyGroup>
   <ItemGroup>
+    <PackageVersion Include="Aspire.Hosting.AppHost" Version="9.0.0-rc.1.24511.1" />
+    <PackageVersion Include="Aspire.Hosting.NodeJs" Version="9.0.0-rc.1.24511.1" />
+    <PackageVersion Include="Aspire.Hosting.PostgreSQL" Version="9.0.0-rc.1.24511.1" />
+    <PackageVersion Include="Aspire.Hosting.Redis" Version="9.0.0-rc.1.24511.1" />
     <PackageVersion Include="AutoMapper" Version="13.0.1" />
     <PackageVersion Include="Basic.Reference.Assemblies" Version="1.7.9" />
     <PackageVersion Include="EntityGraphQL" Version="5.4.6" />
@@ -44,12 +48,22 @@
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.2" />
     <PackageVersion Include="Microsoft.IdentityModel.Tokens" Version="8.1.2" />
     <PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.9.0-beta.2" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.EntityFrameworkCore" Version="1.0.0-beta.12" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Process" Version="0.5.0-beta.7" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.StackExchangeRedis" Version="1.9.0-beta.1" />
     <PackageVersion Include="PolySharp" Version="1.14.1" />
     <PackageVersion Include="Riok.Mapperly" Version="4.0.0" />
     <PackageVersion Include="Serilog.Exceptions" Version="8.4.0" />
     <PackageVersion Include="Serilog.Expressions" Version="5.0.0" />
     <PackageVersion Include="Serilog.Settings.Configuration" Version="8.0.4" />
     <PackageVersion Include="Serilog.Sinks.Async" Version="2.0.0" />
+    <PackageVersion Include="Serilog.Sinks.OpenTelemetry" Version="4.1.0" />
     <PackageVersion Include="Serilog.Sinks.Seq" Version="8.0.0" />
     <PackageVersion Include="System.Text.Json" Version="8.0.5" />
     <PackageVersion Include="Serilog" Version="4.0.2" />