Merge branch 'develop' into 312-stabilize-client-side-otel-export

.github/workflows/integration-test.yaml

@@ -49,12 +49,6 @@ jobs:
           TEST_PROJECT_CODE: 'sena-3'
           TEST_DEFAULT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
         run: dotnet test --output ./bin --logger trx --results-directory ./test-results --filter Category=Integration
-      - name: Mask Playwright traces
-        if: always()
-        shell: pwsh
-        env:
-          PLAYWRIGHT_SECRET_1: ${{ secrets.TEST_USER_PASSWORD }}
-        run: pwsh backend/Testing/Browser/mask-playwright-traces.ps1 --traceDir ./bin/playwright-traces
       - name: Password protect Playwright traces
         if: always()
         shell: bash

backend/LexBoxApi/Auth/LexAuthService.cs

@@ -125,7 +125,7 @@ await context.SignInAsync(jwtUser.GetPrincipal("Refresh"),
         var identity = new ClaimsIdentity(JwtBearerDefaults.AuthenticationScheme);
         var id = Guid.NewGuid().ToString().GetHashCode().ToString("x", CultureInfo.InvariantCulture);
         identity.AddClaim(new Claim(JwtRegisteredClaimNames.Jti, id));
-        identity.AddClaims(user.GetClaims());
+        identity.AddClaims(user.GetClaims().Where(c => c.Type != JwtRegisteredClaimNames.Aud));
         var handler = new JwtSecurityTokenHandler();
         var jwt = handler.CreateJwtSecurityToken(
             audience: audience.ToString(),

backend/LexBoxApi/Config/TusConfig.cs

@@ -0,0 +1,11 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace LexBoxApi.Config;
+
+public class TusConfig
+{
+    [Required(AllowEmptyStrings = false)]
+    public required string TestUploadPath { get; set; }
+    [Required(AllowEmptyStrings = false)]
+    public required string ResetUploadPath { get; set; }
+}

backend/LexBoxApi/Controllers/TestingController.cs

@@ -1,6 +1,7 @@
 using LexBoxApi.Auth;
 using LexBoxApi.Services;
 using LexCore.Auth;
+using LexCore.Exceptions;
 using LexData;
 using LexData.Entities;
 using LexData.Redmine;
@@ -10,7 +11,6 @@
 
 namespace LexBoxApi.Controllers;
 
-#if DEBUG
 [ApiController]
 [Route("/api/[controller]")]
 public class TestingController : ControllerBase
@@ -31,8 +31,7 @@ public TestingController(LexAuthService lexAuthService,
         _redmineDbContext = redmineDbContext;
     }
 
-
-
+#if DEBUG
     [AllowAnonymous]
     [HttpGet("makeJwt")]
     [ProducesResponseType(StatusCodes.Status200OK)]
@@ -96,5 +95,17 @@ public async Task<ActionResult<RmUser[]>> ListRedmineUsernames()
     public record TestingControllerProject(Guid Id, string Name, string Code, List<TestingControllerProjectUser> Users);
 
     public record TestingControllerProjectUser(string? Username, string Role, string Email, Guid Id);
-}
+
 #endif
+    [HttpGet("throwsException")]
+    [AllowAnonymous]
+    public ActionResult ThrowsException()
+    {
+        throw new ExceptionWithCode("This is a test exception", "test-error");
+    }
+
+    [HttpGet("test500NoException")]
+    [AllowAnonymous]
+    [ProducesResponseType(StatusCodes.Status500InternalServerError)]
+    public ActionResult Test500NoError() => StatusCode(500);
+}

backend/LexBoxApi/ErrorHandling/AddExceptionFeatureDevExceptionFilter.cs

@@ -0,0 +1,24 @@
+using Microsoft.AspNetCore.Diagnostics;
+using Microsoft.AspNetCore.Http.Features;
+
+namespace LexBoxApi.ErrorHandling;
+
+public class AddExceptionFeatureDevExceptionFilter : IDeveloperPageExceptionFilter
+{
+    public async Task HandleExceptionAsync(ErrorContext errorContext, Func<ErrorContext, Task> next)
+    {
+        var httpContext = errorContext.HttpContext;
+        var features = httpContext.Features;
+        if (features.Get<IExceptionHandlerFeature>() is null)
+        {
+            features.Set<IExceptionHandlerFeature>(new ExceptionHandlerFeature
+            {
+                Error = errorContext.Exception,
+                Path = httpContext.Request.Path,
+                Endpoint = httpContext.GetEndpoint(),
+                RouteValues = features.Get<IRouteValuesFeature>()?.RouteValues
+            });
+        }
+        await next(errorContext);
+    }
+}

backend/LexBoxApi/LexBoxApi.csproj

@@ -38,6 +38,7 @@
         <PackageReference Include="OpenTelemetry.Instrumentation.Process" Version="0.5.0-beta.3" />
         <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.5.0" />
         <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
+        <PackageReference Include="tusdotnet" Version="2.7.1" />
     </ItemGroup>
 
     <ItemGroup>

backend/LexBoxApi/LexBoxKernel.cs

@@ -30,11 +30,16 @@ public static void AddLexBoxApi(this IServiceCollection services,
             .BindConfiguration("Email")
             .ValidateDataAnnotations()
             .ValidateOnStart();
+        services.AddOptions<TusConfig>()
+            .BindConfiguration("Tus")
+            .ValidateDataAnnotations()
+            .ValidateOnStart();
         services.AddHttpClient();
         services.AddScoped<LoggedInContext>();
         services.AddScoped<ProjectService>();
         services.AddScoped<UserService>();
         services.AddScoped<EmailService>();
+        services.AddScoped<TusService>();
         services.AddScoped<TurnstileService>();
         services.AddScoped<IHgService, HgService>();
         services.AddScoped<ILexProxyService, LexProxyService>();

backend/LexBoxApi/Program.cs

@@ -1,14 +1,18 @@
 using System.Text.Json.Serialization;
 using LexBoxApi;
 using LexBoxApi.Auth;
+using LexBoxApi.ErrorHandling;
 using LexBoxApi.Otel;
 using LexBoxApi.Services;
+using LexCore.Exceptions;
 using LexData;
 using LexSyncReverseProxy;
+using Microsoft.AspNetCore.Diagnostics;
 using Microsoft.AspNetCore.HttpLogging;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.Mvc.ModelBinding.Metadata;
 using Microsoft.OpenApi.Models;
+using tusdotnet;
 
 if (DbStartupService.IsMigrationRequest(args))
 {
@@ -57,6 +61,17 @@
     });
 });
 builder.Services.AddHealthChecks();
+//in prod the exception handler middleware adds the exception feature, but in dev we need to do it manually
+builder.Services.AddSingleton<IDeveloperPageExceptionFilter, AddExceptionFeatureDevExceptionFilter>();
+builder.Services.AddProblemDetails(o =>
+{
+    o.CustomizeProblemDetails = context =>
+    {
+        var exceptionHandlerFeature = context.HttpContext.Features.Get<IExceptionHandlerFeature>();
+        if (exceptionHandlerFeature?.Error is not IExceptionWithCode exceptionWithCode) return;
+        context.ProblemDetails.Extensions["app-error-code"] = exceptionWithCode.Code;
+    };
+});
 builder.Services.AddHttpLogging(options =>
 {
     options.LoggingFields = HttpLoggingFields.RequestPropertiesAndHeaders |
@@ -90,7 +105,9 @@
     context.Response.Headers.Add("lexbox-version", AppVersionService.Version);
     await next();
 });
-
+app.UseStatusCodePages();
+if (!app.Environment.IsDevelopment())
+    app.UseExceptionHandler();
 app.UseHealthChecks("/api/healthz");
 // Configure the HTTP request pipeline.
 //for now allow this to run in prod, maybe later we want to disable it.
@@ -117,6 +134,12 @@
     app.MapGraphQLSchema("/api/graphql/schema.graphql").AllowAnonymous();
 app.MapGraphQLHttp("/api/graphql");
 app.MapControllers();
+app.MapTus("/api/tus-test",
+        async context => await context.RequestServices.GetRequiredService<TusService>().GetTestConfig(context))
+    .RequireAuthorization(new AdminRequiredAttribute());
+app.MapTus($"/api/project/upload-zip/{{{ProxyConstants.HgProjectCodeRouteKey}}}",
+        async context => await context.RequestServices.GetRequiredService<TusService>().GetResetZipUploadConfig())
+    .RequireAuthorization(new AdminRequiredAttribute());
 // /api routes should never make it to this point, they should be handled by the controllers, so return 404
 app.Map("/api/{**catch-all}", () => Results.NotFound()).AllowAnonymous();
 app.MapSyncProxy(AuthKernel.DefaultScheme);

backend/LexBoxApi/Services/HgService.cs

@@ -48,6 +48,7 @@ private HttpClient GetClient(ProjectMigrationStatus migrationStatus, string code
         {
             client.DefaultRequestHeaders.Authorization = null;
         }
+
         return client;
     }
 
@@ -84,6 +85,7 @@ public async Task DeleteRepo(string code)
         {
             return null; // Which controller will turn into HTTP 404
         }
+
         string tempPath = Path.GetTempPath();
         string timestamp = FileUtils.ToTimestamp(DateTime.UtcNow);
         string baseName = $"backup-{code}-{timestamp}.zip";
@@ -98,7 +100,52 @@ public async Task ResetRepo(string code)
     {
         string timestamp = FileUtils.ToTimestamp(DateTimeOffset.UtcNow);
         await SoftDeleteRepo(code, $"{timestamp}__reset");
-        await InitRepo(code);
+        await InitRepo(code); // we don't want 404s
+    }
+
+    public async Task FinishReset(string code, Stream zipFile)
+    {
+        using var archive = new ZipArchive(zipFile, ZipArchiveMode.Read);
+        await DeleteRepo(code);
+        var repoPath = Path.Combine(_options.Value.RepoPath, code);
+        Directory.CreateDirectory(repoPath);
+        archive.ExtractToDirectory(repoPath);
+
+        var hgPath = Path.Join(repoPath, ".hg");
+        if (Directory.Exists(hgPath))
+        {
+            await CleanupRepoFolder(repoPath);
+            return;
+        }
+
+        var hgFolder = Directory.EnumerateDirectories(repoPath, ".hg", SearchOption.AllDirectories).FirstOrDefault();
+        if (hgFolder is null)
+        {
+            await DeleteRepo(code);
+            await InitRepo(code); // we don't want 404s
+            //not sure if this is the best way to handle this, might need to catch it further up to expose the error properly to tus
+            throw ProjectResetException.ZipMissingHgFolder();
+        }
+
+        Directory.Move(hgFolder, hgPath);
+        await CleanupRepoFolder(repoPath);
+    }
+
+    /// <summary>
+    /// deletes all files and folders in the repo folder except for .hg
+    /// </summary>
+    private async Task CleanupRepoFolder(string path)
+    {
+        var repoDir = new DirectoryInfo(path);
+        await Task.Run(() =>
+        {
+            foreach (var info in repoDir.EnumerateFileSystemInfos())
+            {
+                if (info.Name == ".hg") continue;
+                if (info is DirectoryInfo dir) dir.Delete(true);
+                else info.Delete();
+            }
+        });
     }
 
     public async Task<bool> MigrateRepo(Project project, CancellationToken cancellationToken)
@@ -126,8 +173,8 @@ public async Task<bool> MigrateRepo(Project project, CancellationToken cancellat
         if (process is null)
         {
             return false;
-
         }
+
         await process.WaitForExitAsync(cancellationToken);
         if (process.ExitCode == 0)
         {
@@ -138,7 +185,8 @@ public async Task<bool> MigrateRepo(Project project, CancellationToken cancellat
 
         var error = await process.StandardError.ReadToEndAsync(cancellationToken);
         var output = await process.StandardOutput.ReadToEndAsync(cancellationToken);
-        var description = $"rsync for project {project.Code} failed with exit code {process.ExitCode}. Error: {error}. Output: {output}";
+        var description =
+            $"rsync for project {project.Code} failed with exit code {process.ExitCode}. Error: {error}. Output: {output}";
         _logger.LogError(description);
         activity?.SetStatus(ActivityStatusCode.Error, description);
         return false;
@@ -168,8 +216,11 @@ await Task.Run(() =>
         });
     }
 
-    private const UnixFileMode Permissions = UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup |
-                                             UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.SetUser;
+    private const UnixFileMode Permissions = UnixFileMode.GroupRead | UnixFileMode.GroupWrite |
+                                             UnixFileMode.GroupExecute | UnixFileMode.SetGroup |
+                                             UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
+                                             UnixFileMode.SetUser;
+
     private static void CopyFilesRecursively(DirectoryInfo source, DirectoryInfo target)
     {
         foreach (DirectoryInfo dir in source.GetDirectories())
@@ -192,7 +243,8 @@ private static void CopyFilesRecursively(DirectoryInfo source, DirectoryInfo tar
     public async Task<DateTimeOffset?> GetLastCommitTimeFromHg(string projectCode,
         ProjectMigrationStatus migrationStatus)
     {
-        var response = await GetClient(migrationStatus, projectCode).GetAsync($"{projectCode}/log?style=json-lex&rev=tip");
+        var response = await GetClient(migrationStatus, projectCode)
+            .GetAsync($"{projectCode}/log?style=json-lex&rev=tip");
         response.EnsureSuccessStatusCode();
         var json = await response.Content.ReadFromJsonAsync<JsonObject>();
         //format is this: [1678687688, offset] offset is
@@ -216,6 +268,7 @@ public async Task<Changeset[]> GetChangesets(string projectCode, ProjectMigratio
     }
 
     private static readonly string[] InvalidRepoNames = { DELETED_REPO_FOLDER, "api" };
+
     private void AssertIsSafeRepoName(string name)
     {
         if (InvalidRepoNames.Contains(name, StringComparer.OrdinalIgnoreCase))
@@ -240,7 +293,8 @@ public static string DetermineProjectUrlPrefix(HgType type,
             //all resumable redmine go to the same place
             (HgType.resumable, ProjectMigrationStatus.PublicRedmine) => hgConfig.RedmineHgResumableUrl,
             (HgType.resumable, ProjectMigrationStatus.PrivateRedmine) => hgConfig.RedmineHgResumableUrl,
-            _ => throw new ArgumentException($"Unknown request, HG request type: {type}, migration status: {migrationStatus}")
+            _ => throw new ArgumentException(
+                $"Unknown request, HG request type: {type}, migration status: {migrationStatus}")
         };
     }
 }

backend/LexBoxApi/Services/ProjectService.cs

@@ -44,6 +44,11 @@ public async Task<Guid> CreateProject(CreateProjectInput input, Guid userId)
         return projectId;
     }
 
+    public async Task<bool> ProjectExists(string projectCode)
+    {
+        return await _dbContext.Projects.AnyAsync(p => p.Code == projectCode);
+    }
+
     public async Task<string?> BackupProject(ResetProjectByAdminInput input)
     {
         var backupFile = await _hgService.BackupRepo(input.Code);
@@ -52,9 +57,23 @@ public async Task<Guid> CreateProject(CreateProjectInput input, Guid userId)
 
     public async Task ResetProject(ResetProjectByAdminInput input)
     {
+        var rowsAffected = await _dbContext.Projects.Where(p => p.Code == input.Code && p.ResetStatus == ResetStatus.None)
+            .ExecuteUpdateAsync(u => u.SetProperty(p => p.ResetStatus, ResetStatus.InProgress));
+        if (rowsAffected == 0) throw new NotFoundException($"project {input.Code} not ready for reset, either already reset or not found");
         await _hgService.ResetRepo(input.Code);
     }
 
+    public async Task FinishReset(string code, Stream zipFile)
+    {
+        var project = await _dbContext.Projects.Where(p => p.Code == code).SingleOrDefaultAsync();
+        if (project is null) throw new NotFoundException($"project {code} not found");
+        if (project.ResetStatus != ResetStatus.InProgress) throw ProjectResetException.NotReadyForUpload(code);
+        await _hgService.FinishReset(code, zipFile);
+        project.ResetStatus = ResetStatus.None;
+        project.LastCommit = await _hgService.GetLastCommitTimeFromHg(project.Code, project.MigrationStatus);
+        await _dbContext.SaveChangesAsync();
+    }
+
     public async Task<DateTimeOffset?> UpdateLastCommit(string projectCode)
     {
         var project = await _dbContext.Projects.FirstOrDefaultAsync(p => p.Code == projectCode);