diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index cfac43b8f..10f027b03 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -49,12 +49,6 @@ jobs: TEST_PROJECT_CODE: 'sena-3' TEST_DEFAULT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} run: dotnet test --output ./bin --logger trx --results-directory ./test-results --filter Category=Integration - - name: Mask Playwright traces - if: always() - shell: pwsh - env: - PLAYWRIGHT_SECRET_1: ${{ secrets.TEST_USER_PASSWORD }} - run: pwsh backend/Testing/Browser/mask-playwright-traces.ps1 --traceDir ./bin/playwright-traces - name: Password protect Playwright traces if: always() shell: bash diff --git a/.idea/.idea.LexBox/.idea/jsLinters/eslint.xml b/.idea/.idea.LexBox/.idea/jsLinters/eslint.xml index 3708f45e5..7883a6967 100644 --- a/.idea/.idea.LexBox/.idea/jsLinters/eslint.xml +++ b/.idea/.idea.LexBox/.idea/jsLinters/eslint.xml @@ -1,6 +1,7 @@ - + + - + \ No newline at end of file diff --git a/backend/LexBoxApi/Auth/LexAuthService.cs b/backend/LexBoxApi/Auth/LexAuthService.cs index dbe670c5b..5b4ac7f8e 100644 --- a/backend/LexBoxApi/Auth/LexAuthService.cs +++ b/backend/LexBoxApi/Auth/LexAuthService.cs @@ -125,7 +125,7 @@ await context.SignInAsync(jwtUser.GetPrincipal("Refresh"), var identity = new ClaimsIdentity(JwtBearerDefaults.AuthenticationScheme); var id = Guid.NewGuid().ToString().GetHashCode().ToString("x", CultureInfo.InvariantCulture); identity.AddClaim(new Claim(JwtRegisteredClaimNames.Jti, id)); - identity.AddClaims(user.GetClaims()); + identity.AddClaims(user.GetClaims().Where(c => c.Type != JwtRegisteredClaimNames.Aud)); var handler = new JwtSecurityTokenHandler(); var jwt = handler.CreateJwtSecurityToken( audience: audience.ToString(), diff --git a/backend/LexBoxApi/Config/TusConfig.cs b/backend/LexBoxApi/Config/TusConfig.cs new file mode 100644 index 000000000..71c657cad --- /dev/null +++ b/backend/LexBoxApi/Config/TusConfig.cs @@ -0,0 +1,11 @@ +using System.ComponentModel.DataAnnotations; + +namespace LexBoxApi.Config; + +public class TusConfig +{ + [Required(AllowEmptyStrings = false)] + public required string TestUploadPath { get; set; } + [Required(AllowEmptyStrings = false)] + public required string ResetUploadPath { get; set; } +} diff --git a/backend/LexBoxApi/Controllers/TestingController.cs b/backend/LexBoxApi/Controllers/TestingController.cs index 3c60d9b2c..98c7cc8f0 100644 --- a/backend/LexBoxApi/Controllers/TestingController.cs +++ b/backend/LexBoxApi/Controllers/TestingController.cs @@ -1,6 +1,7 @@ using LexBoxApi.Auth; using LexBoxApi.Services; using LexCore.Auth; +using LexCore.Exceptions; using LexData; using LexData.Entities; using LexData.Redmine; @@ -10,7 +11,6 @@ namespace LexBoxApi.Controllers; -#if DEBUG [ApiController] [Route("/api/[controller]")] public class TestingController : ControllerBase @@ -31,8 +31,7 @@ public TestingController(LexAuthService lexAuthService, _redmineDbContext = redmineDbContext; } - - +#if DEBUG [AllowAnonymous] [HttpGet("makeJwt")] [ProducesResponseType(StatusCodes.Status200OK)] @@ -96,5 +95,17 @@ public async Task> ListRedmineUsernames() public record TestingControllerProject(Guid Id, string Name, string Code, List Users); public record TestingControllerProjectUser(string? Username, string Role, string Email, Guid Id); -} + #endif + [HttpGet("throwsException")] + [AllowAnonymous] + public ActionResult ThrowsException() + { + throw new ExceptionWithCode("This is a test exception", "test-error"); + } + + [HttpGet("test500NoException")] + [AllowAnonymous] + [ProducesResponseType(StatusCodes.Status500InternalServerError)] + public ActionResult Test500NoError() => StatusCode(500); +} diff --git a/backend/LexBoxApi/ErrorHandling/AddExceptionFeatureDevExceptionFilter.cs b/backend/LexBoxApi/ErrorHandling/AddExceptionFeatureDevExceptionFilter.cs new file mode 100644 index 000000000..e1c92fdbd --- /dev/null +++ b/backend/LexBoxApi/ErrorHandling/AddExceptionFeatureDevExceptionFilter.cs @@ -0,0 +1,24 @@ +using Microsoft.AspNetCore.Diagnostics; +using Microsoft.AspNetCore.Http.Features; + +namespace LexBoxApi.ErrorHandling; + +public class AddExceptionFeatureDevExceptionFilter : IDeveloperPageExceptionFilter +{ + public async Task HandleExceptionAsync(ErrorContext errorContext, Func next) + { + var httpContext = errorContext.HttpContext; + var features = httpContext.Features; + if (features.Get() is null) + { + features.Set(new ExceptionHandlerFeature + { + Error = errorContext.Exception, + Path = httpContext.Request.Path, + Endpoint = httpContext.GetEndpoint(), + RouteValues = features.Get()?.RouteValues + }); + } + await next(errorContext); + } +} diff --git a/backend/LexBoxApi/LexBoxApi.csproj b/backend/LexBoxApi/LexBoxApi.csproj index 9a0264ced..9e51fe404 100644 --- a/backend/LexBoxApi/LexBoxApi.csproj +++ b/backend/LexBoxApi/LexBoxApi.csproj @@ -38,6 +38,7 @@ + diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index c5e7eaff3..d06da125c 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -30,11 +30,16 @@ public static void AddLexBoxApi(this IServiceCollection services, .BindConfiguration("Email") .ValidateDataAnnotations() .ValidateOnStart(); + services.AddOptions() + .BindConfiguration("Tus") + .ValidateDataAnnotations() + .ValidateOnStart(); services.AddHttpClient(); services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/backend/LexBoxApi/Program.cs b/backend/LexBoxApi/Program.cs index 97c268ff8..939e40bb7 100644 --- a/backend/LexBoxApi/Program.cs +++ b/backend/LexBoxApi/Program.cs @@ -1,14 +1,18 @@ using System.Text.Json.Serialization; using LexBoxApi; using LexBoxApi.Auth; +using LexBoxApi.ErrorHandling; using LexBoxApi.Otel; using LexBoxApi.Services; +using LexCore.Exceptions; using LexData; using LexSyncReverseProxy; +using Microsoft.AspNetCore.Diagnostics; using Microsoft.AspNetCore.HttpLogging; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.Mvc.ModelBinding.Metadata; using Microsoft.OpenApi.Models; +using tusdotnet; if (DbStartupService.IsMigrationRequest(args)) { @@ -57,6 +61,17 @@ }); }); builder.Services.AddHealthChecks(); +//in prod the exception handler middleware adds the exception feature, but in dev we need to do it manually +builder.Services.AddSingleton(); +builder.Services.AddProblemDetails(o => +{ + o.CustomizeProblemDetails = context => + { + var exceptionHandlerFeature = context.HttpContext.Features.Get(); + if (exceptionHandlerFeature?.Error is not IExceptionWithCode exceptionWithCode) return; + context.ProblemDetails.Extensions["app-error-code"] = exceptionWithCode.Code; + }; +}); builder.Services.AddHttpLogging(options => { options.LoggingFields = HttpLoggingFields.RequestPropertiesAndHeaders | @@ -90,7 +105,9 @@ context.Response.Headers.Add("lexbox-version", AppVersionService.Version); await next(); }); - +app.UseStatusCodePages(); +if (!app.Environment.IsDevelopment()) + app.UseExceptionHandler(); app.UseHealthChecks("/api/healthz"); // Configure the HTTP request pipeline. //for now allow this to run in prod, maybe later we want to disable it. @@ -117,6 +134,12 @@ app.MapGraphQLSchema("/api/graphql/schema.graphql").AllowAnonymous(); app.MapGraphQLHttp("/api/graphql"); app.MapControllers(); +app.MapTus("/api/tus-test", + async context => await context.RequestServices.GetRequiredService().GetTestConfig(context)) + .RequireAuthorization(new AdminRequiredAttribute()); +app.MapTus($"/api/project/upload-zip/{{{ProxyConstants.HgProjectCodeRouteKey}}}", + async context => await context.RequestServices.GetRequiredService().GetResetZipUploadConfig()) + .RequireAuthorization(new AdminRequiredAttribute()); // /api routes should never make it to this point, they should be handled by the controllers, so return 404 app.Map("/api/{**catch-all}", () => Results.NotFound()).AllowAnonymous(); app.MapSyncProxy(AuthKernel.DefaultScheme); diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 843a733a0..0297df4ef 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -48,6 +48,7 @@ private HttpClient GetClient(ProjectMigrationStatus migrationStatus, string code { client.DefaultRequestHeaders.Authorization = null; } + return client; } @@ -84,6 +85,7 @@ public async Task DeleteRepo(string code) { return null; // Which controller will turn into HTTP 404 } + string tempPath = Path.GetTempPath(); string timestamp = FileUtils.ToTimestamp(DateTime.UtcNow); string baseName = $"backup-{code}-{timestamp}.zip"; @@ -98,7 +100,52 @@ public async Task ResetRepo(string code) { string timestamp = FileUtils.ToTimestamp(DateTimeOffset.UtcNow); await SoftDeleteRepo(code, $"{timestamp}__reset"); - await InitRepo(code); + await InitRepo(code); // we don't want 404s + } + + public async Task FinishReset(string code, Stream zipFile) + { + using var archive = new ZipArchive(zipFile, ZipArchiveMode.Read); + await DeleteRepo(code); + var repoPath = Path.Combine(_options.Value.RepoPath, code); + Directory.CreateDirectory(repoPath); + archive.ExtractToDirectory(repoPath); + + var hgPath = Path.Join(repoPath, ".hg"); + if (Directory.Exists(hgPath)) + { + await CleanupRepoFolder(repoPath); + return; + } + + var hgFolder = Directory.EnumerateDirectories(repoPath, ".hg", SearchOption.AllDirectories).FirstOrDefault(); + if (hgFolder is null) + { + await DeleteRepo(code); + await InitRepo(code); // we don't want 404s + //not sure if this is the best way to handle this, might need to catch it further up to expose the error properly to tus + throw ProjectResetException.ZipMissingHgFolder(); + } + + Directory.Move(hgFolder, hgPath); + await CleanupRepoFolder(repoPath); + } + + /// + /// deletes all files and folders in the repo folder except for .hg + /// + private async Task CleanupRepoFolder(string path) + { + var repoDir = new DirectoryInfo(path); + await Task.Run(() => + { + foreach (var info in repoDir.EnumerateFileSystemInfos()) + { + if (info.Name == ".hg") continue; + if (info is DirectoryInfo dir) dir.Delete(true); + else info.Delete(); + } + }); } public async Task MigrateRepo(Project project, CancellationToken cancellationToken) @@ -126,8 +173,8 @@ public async Task MigrateRepo(Project project, CancellationToken cancellat if (process is null) { return false; - } + await process.WaitForExitAsync(cancellationToken); if (process.ExitCode == 0) { @@ -138,7 +185,8 @@ public async Task MigrateRepo(Project project, CancellationToken cancellat var error = await process.StandardError.ReadToEndAsync(cancellationToken); var output = await process.StandardOutput.ReadToEndAsync(cancellationToken); - var description = $"rsync for project {project.Code} failed with exit code {process.ExitCode}. Error: {error}. Output: {output}"; + var description = + $"rsync for project {project.Code} failed with exit code {process.ExitCode}. Error: {error}. Output: {output}"; _logger.LogError(description); activity?.SetStatus(ActivityStatusCode.Error, description); return false; @@ -168,8 +216,11 @@ await Task.Run(() => }); } - private const UnixFileMode Permissions = UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.SetGroup | - UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.SetUser; + private const UnixFileMode Permissions = UnixFileMode.GroupRead | UnixFileMode.GroupWrite | + UnixFileMode.GroupExecute | UnixFileMode.SetGroup | + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.SetUser; + private static void CopyFilesRecursively(DirectoryInfo source, DirectoryInfo target) { foreach (DirectoryInfo dir in source.GetDirectories()) @@ -192,7 +243,8 @@ private static void CopyFilesRecursively(DirectoryInfo source, DirectoryInfo tar public async Task GetLastCommitTimeFromHg(string projectCode, ProjectMigrationStatus migrationStatus) { - var response = await GetClient(migrationStatus, projectCode).GetAsync($"{projectCode}/log?style=json-lex&rev=tip"); + var response = await GetClient(migrationStatus, projectCode) + .GetAsync($"{projectCode}/log?style=json-lex&rev=tip"); response.EnsureSuccessStatusCode(); var json = await response.Content.ReadFromJsonAsync(); //format is this: [1678687688, offset] offset is @@ -216,6 +268,7 @@ public async Task GetChangesets(string projectCode, ProjectMigratio } private static readonly string[] InvalidRepoNames = { DELETED_REPO_FOLDER, "api" }; + private void AssertIsSafeRepoName(string name) { if (InvalidRepoNames.Contains(name, StringComparer.OrdinalIgnoreCase)) @@ -240,7 +293,8 @@ public static string DetermineProjectUrlPrefix(HgType type, //all resumable redmine go to the same place (HgType.resumable, ProjectMigrationStatus.PublicRedmine) => hgConfig.RedmineHgResumableUrl, (HgType.resumable, ProjectMigrationStatus.PrivateRedmine) => hgConfig.RedmineHgResumableUrl, - _ => throw new ArgumentException($"Unknown request, HG request type: {type}, migration status: {migrationStatus}") + _ => throw new ArgumentException( + $"Unknown request, HG request type: {type}, migration status: {migrationStatus}") }; } } diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index e9391b9f4..4a2944980 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -44,6 +44,11 @@ public async Task CreateProject(CreateProjectInput input, Guid userId) return projectId; } + public async Task ProjectExists(string projectCode) + { + return await _dbContext.Projects.AnyAsync(p => p.Code == projectCode); + } + public async Task BackupProject(ResetProjectByAdminInput input) { var backupFile = await _hgService.BackupRepo(input.Code); @@ -52,9 +57,23 @@ public async Task CreateProject(CreateProjectInput input, Guid userId) public async Task ResetProject(ResetProjectByAdminInput input) { + var rowsAffected = await _dbContext.Projects.Where(p => p.Code == input.Code && p.ResetStatus == ResetStatus.None) + .ExecuteUpdateAsync(u => u.SetProperty(p => p.ResetStatus, ResetStatus.InProgress)); + if (rowsAffected == 0) throw new NotFoundException($"project {input.Code} not ready for reset, either already reset or not found"); await _hgService.ResetRepo(input.Code); } + public async Task FinishReset(string code, Stream zipFile) + { + var project = await _dbContext.Projects.Where(p => p.Code == code).SingleOrDefaultAsync(); + if (project is null) throw new NotFoundException($"project {code} not found"); + if (project.ResetStatus != ResetStatus.InProgress) throw ProjectResetException.NotReadyForUpload(code); + await _hgService.FinishReset(code, zipFile); + project.ResetStatus = ResetStatus.None; + project.LastCommit = await _hgService.GetLastCommitTimeFromHg(project.Code, project.MigrationStatus); + await _dbContext.SaveChangesAsync(); + } + public async Task UpdateLastCommit(string projectCode) { var project = await _dbContext.Projects.FirstOrDefaultAsync(p => p.Code == projectCode); diff --git a/backend/LexBoxApi/Services/TusService.cs b/backend/LexBoxApi/Services/TusService.cs new file mode 100644 index 000000000..9cc5d1596 --- /dev/null +++ b/backend/LexBoxApi/Services/TusService.cs @@ -0,0 +1,128 @@ +using System.Linq; +using System.Net; +using System.Text; +using LexBoxApi.Config; +using LexCore.Utils; +using LexSyncReverseProxy; +using Microsoft.Extensions.Options; +using tusdotnet.Interfaces; +using tusdotnet.Models; +using tusdotnet.Models.Configuration; +using tusdotnet.Stores; +using Path = System.IO.Path; + +namespace LexBoxApi.Services; + +public class TusService +{ + private readonly TusConfig _config; + private readonly ProjectService _projectService; + private static readonly string[] SupportZipTypes = new[] { "application/zip", "application/x-zip-compressed" }; + + public TusService(IOptions config, ProjectService projectService) + { + _projectService = projectService; + _config = config.Value; + Directory.CreateDirectory(Path.GetFullPath(_config.TestUploadPath)); + Directory.CreateDirectory(Path.GetFullPath(_config.ResetUploadPath)); + } + + public Task GetTestConfig(HttpContext context) + { + return Task.FromResult(new DefaultTusConfiguration + { + Store = new TusDiskStore(Path.GetFullPath(_config.TestUploadPath)), + Events = new Events + { + OnBeforeCreateAsync = createContext => + { + + var filetype = GetFiletype(createContext.Metadata); + if (string.IsNullOrEmpty(filetype)) + { + createContext.FailRequest(HttpStatusCode.BadRequest, "unknown file type"); + return Task.CompletedTask; + } + if (filetype != "application/png") + { + createContext.FailRequest(HttpStatusCode.BadRequest, $"file type {filetype} is not allowed"); + return Task.CompletedTask; + } + //validate the upload before it begins + return Task.CompletedTask; + }, + OnFileCompleteAsync = completeContext => + { + //do something with the uploaded file + return Task.CompletedTask; + } + } + }); + } + + private string? GetFiletype(Dictionary metadata) + { + if (!metadata.TryGetValue("filetype", out var filetypeMetadata)) return null; + return filetypeMetadata.GetString(Encoding.UTF8); + } + + public Task GetResetZipUploadConfig() + { + return Task.FromResult(new DefaultTusConfiguration + { + Store = new TusDiskStore(Path.GetFullPath(_config.ResetUploadPath)), + Events = new Events + { + OnBeforeCreateAsync = BeforeStartZipUpload, OnFileCompleteAsync = FinishZipUpload + } + }); + } + + private async Task BeforeStartZipUpload(BeforeCreateContext createContext) + { + var projectCode = createContext.HttpContext.Request.GetProjectCode(); + if (string.IsNullOrEmpty(projectCode)) + { + createContext.FailRequest(HttpStatusCode.BadRequest, "Missing project code"); + return; + } + + if (!await _projectService.ProjectExists(projectCode)) + { + createContext.FailRequest(HttpStatusCode.NotFound, $"Project {projectCode} not found"); + return; + } + + var filetype = GetFiletype(createContext.Metadata); + if (string.IsNullOrEmpty(filetype)) + { + createContext.FailRequest(HttpStatusCode.BadRequest, "unknown file type"); + return; + } + + if (!SupportZipTypes.Contains(filetype)) + { + createContext.FailRequest(HttpStatusCode.BadRequest, $"File type {filetype} is not allowed, only {string.Join(", ", SupportZipTypes)} are allowed"); + return; + } + } + + private async Task FinishZipUpload(FileCompleteContext completeContext) + { + try + { + var projectCode = completeContext.HttpContext.Request.GetProjectCode(); + ArgumentException.ThrowIfNullOrEmpty(projectCode); + var tusFile = await completeContext.GetFileAsync(); + await using var fileStream = await tusFile.GetContentAsync(completeContext.CancellationToken); + await _projectService.FinishReset(projectCode, fileStream); + } + finally + { + if (completeContext.Store is ITusTerminationStore terminationStore) + { + await terminationStore.DeleteFileAsync(completeContext.FileId, completeContext.CancellationToken); + } + } + } +} diff --git a/backend/LexBoxApi/appsettings.Development.json b/backend/LexBoxApi/appsettings.Development.json index a39e0920a..948cce9ff 100644 --- a/backend/LexBoxApi/appsettings.Development.json +++ b/backend/LexBoxApi/appsettings.Development.json @@ -24,13 +24,17 @@ "HgWebUrl": "http://localhost:8088/hg/", "HgResumableUrl": "http://localhost:8034/", "RedmineTrustToken": "redmine-dev-trust-token", - "LfMergeTrustToken": "lf-merge-dev-trust-token", + "LfMergeTrustToken": "lf-merge-dev-trust-token" }, "Authentication": { "Jwt": { "Secret": "d5cf1adc-16e6-4064-8041-4cfa00174210" } }, + "Tus" : { + "TestUploadPath": "testUploadPath", + "ResetUploadPath": "resetUploadPath", + }, "Email": { "SmtpHost": "localhost", "SmtpPort": 1025, diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs index 8b017f8ae..ffe3068a9 100644 --- a/backend/LexCore/Entities/Project.cs +++ b/backend/LexCore/Entities/Project.cs @@ -16,6 +16,7 @@ public class Project : EntityBase public required List Users { get; set; } public required DateTimeOffset? LastCommit { get; set; } public DateTimeOffset? DeletedDate { get; set; } + public ResetStatus ResetStatus { get; set; } = ResetStatus.None; public required ProjectMigrationStatus ProjectOrigin { get; set; } = ProjectMigrationStatus.Migrated; public required ProjectMigrationStatus MigrationStatus { get; set; } = ProjectMigrationStatus.Migrated; @@ -51,6 +52,12 @@ public enum ProjectMigrationStatus PublicRedmine, } +public enum ResetStatus +{ + None = 0, + InProgress = 1 +} + public enum ProjectType { Unknown = 0, diff --git a/backend/LexCore/Exceptions/ExceptionWithCode.cs b/backend/LexCore/Exceptions/ExceptionWithCode.cs new file mode 100644 index 000000000..653539424 --- /dev/null +++ b/backend/LexCore/Exceptions/ExceptionWithCode.cs @@ -0,0 +1,16 @@ +namespace LexCore.Exceptions; + +public interface IExceptionWithCode +{ + string Code { get; } +} + +public class ExceptionWithCode : Exception, IExceptionWithCode +{ + public ExceptionWithCode(string message, string code) : base(message) + { + Code = code; + } + + public string Code { get; } +} diff --git a/backend/LexCore/Exceptions/ProjectResetException.cs b/backend/LexCore/Exceptions/ProjectResetException.cs new file mode 100644 index 000000000..17711aa6f --- /dev/null +++ b/backend/LexCore/Exceptions/ProjectResetException.cs @@ -0,0 +1,14 @@ +namespace LexCore.Exceptions; + +public class ProjectResetException : ExceptionWithCode +{ + public static ProjectResetException ZipMissingHgFolder() => + new("Zip file does not contain a .hg folder", "ZIP_MISSING_HG_FOLDER"); + + public static ProjectResetException NotReadyForUpload(string projectCode) => + new($"project {projectCode} has not started the reset process", "RESET_NOT_STARTED"); + + private ProjectResetException(string message, string code) : base(message, code) + { + } +} diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs index b847b66c4..1018e4598 100644 --- a/backend/LexCore/ServiceInterfaces/IHgService.cs +++ b/backend/LexCore/ServiceInterfaces/IHgService.cs @@ -12,4 +12,5 @@ public interface IHgService Task BackupRepo(string code); Task ResetRepo(string code); Task MigrateRepo(Project project, CancellationToken cancellationToken); + Task FinishReset(string code, Stream zipFile); } diff --git a/backend/LexData/Migrations/20231018031108_AddProjectResetStatus.Designer.cs b/backend/LexData/Migrations/20231018031108_AddProjectResetStatus.Designer.cs new file mode 100644 index 000000000..b767b93da --- /dev/null +++ b/backend/LexData/Migrations/20231018031108_AddProjectResetStatus.Designer.cs @@ -0,0 +1,232 @@ +// +using System; +using LexData; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace LexData.Migrations +{ + [DbContext(typeof(LexBoxDbContext))] + [Migration("20231018031108_AddProjectResetStatus")] + partial class AddProjectResetStatus + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Npgsql:CollationDefinition:case_insensitive", "und-u-ks-level2,und-u-ks-level2,icu,False") + .HasAnnotation("ProductVersion", "7.0.5") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("DeletedDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("LastCommit") + .HasColumnType("timestamp with time zone"); + + b.Property("MigrationStatus") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("ProjectOrigin") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(1); + + b.Property("ResetStatus") + .HasColumnType("integer"); + + b.Property("RetentionPolicy") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.HasKey("Id"); + + b.HasIndex("Code") + .IsUnique(); + + b.HasIndex("ParentId"); + + b.ToTable("Projects"); + }); + + modelBuilder.Entity("LexCore.Entities.ProjectUsers", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("ProjectId") + .HasColumnType("uuid"); + + b.Property("Role") + .HasColumnType("integer"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("ProjectId"); + + b.HasIndex("UserId", "ProjectId") + .IsUnique(); + + b.ToTable("ProjectUsers"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CanCreateProjects") + .HasColumnType("boolean"); + + b.Property("CreatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text") + .UseCollation("case_insensitive"); + + b.Property("EmailVerified") + .HasColumnType("boolean"); + + b.Property("IsAdmin") + .HasColumnType("boolean"); + + b.Property("LastActive") + .HasColumnType("timestamp with time zone"); + + b.Property("LocalizationCode") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("text") + .HasDefaultValue("en"); + + b.Property("Locked") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("PasswordHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdatedDate") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now()"); + + b.Property("Username") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Email") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.HasOne("LexCore.Entities.Project", null) + .WithMany() + .HasForeignKey("ParentId"); + }); + + modelBuilder.Entity("LexCore.Entities.ProjectUsers", b => + { + b.HasOne("LexCore.Entities.Project", "Project") + .WithMany("Users") + .HasForeignKey("ProjectId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("LexCore.Entities.User", "User") + .WithMany("Projects") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Project"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("LexCore.Entities.Project", b => + { + b.Navigation("Users"); + }); + + modelBuilder.Entity("LexCore.Entities.User", b => + { + b.Navigation("Projects"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/LexData/Migrations/20231018031108_AddProjectResetStatus.cs b/backend/LexData/Migrations/20231018031108_AddProjectResetStatus.cs new file mode 100644 index 000000000..dfba7d87f --- /dev/null +++ b/backend/LexData/Migrations/20231018031108_AddProjectResetStatus.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace LexData.Migrations +{ + /// + public partial class AddProjectResetStatus : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "ResetStatus", + table: "Projects", + type: "integer", + nullable: false, + defaultValue: 0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "ResetStatus", + table: "Projects"); + } + } +} diff --git a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs index d0c6122e3..93c0c3eed 100644 --- a/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs +++ b/backend/LexData/Migrations/LexBoxDbContextModelSnapshot.cs @@ -64,6 +64,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasDefaultValue(1); + b.Property("ResetStatus") + .HasColumnType("integer"); + b.Property("RetentionPolicy") .HasColumnType("integer"); diff --git a/backend/Testing/Browser/Page/SandboxPage.cs b/backend/Testing/Browser/Page/SandboxPage.cs new file mode 100644 index 000000000..049ffd859 --- /dev/null +++ b/backend/Testing/Browser/Page/SandboxPage.cs @@ -0,0 +1,10 @@ +using Microsoft.Playwright; + +namespace Testing.Browser.Page; + +public class SandboxPage : BasePage +{ + public SandboxPage(IPage page) : base(page, "/sandbox", page.Locator(":text('Sandbox')")) + { + } +} diff --git a/backend/Testing/Browser/SandboxPageTests.cs b/backend/Testing/Browser/SandboxPageTests.cs new file mode 100644 index 000000000..d2b7bf8d2 --- /dev/null +++ b/backend/Testing/Browser/SandboxPageTests.cs @@ -0,0 +1,23 @@ +using Shouldly; +using Testing.Browser.Base; +using Testing.Browser.Page; + +namespace Testing.Browser; + +[Trait("Category", "Integration")] +public class SandboxPageTests : PageTest +{ + [Fact] + public async Task Goto500Works() + { + await new SandboxPage(Page).Goto(); + var request = await Page.RunAndWaitForRequestFinishedAsync(async () => + { + await Page.GetByText("goto 500 page").ClickAsync(); + }); + var response = await request.ResponseAsync(); + response.ShouldNotBeNull(); + response.Ok.ShouldBeFalse(); + response.Status.ShouldBe(500); + } +} diff --git a/backend/Testing/Browser/mask-playwright-traces.ps1 b/backend/Testing/Browser/mask-playwright-traces.ps1 deleted file mode 100644 index 446ea46e2..000000000 --- a/backend/Testing/Browser/mask-playwright-traces.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -param( - [Parameter(Mandatory)][string] $traceDir -) -Add-Type -Assembly System.IO.Compression.FileSystem - -# trace = Core trace info -# network = Network log -# json = Network log payloads -$filesToMask = "\.(json|trace|network)$" -$files = Get-ChildItem -path $traceDir -Filter *.zip -$secrets = Get-ChildItem env:PLAYWRIGHT_SECRET_* - -Write-Output "Masking $($secrets.Count) secrets in $($files.Count) traces."; - -foreach ($file in $files) { - try { - Write-Output "Masking $file" - $zip = [System.IO.Compression.ZipFile]::Open($file.FullName, "Update") - $entries = $zip.Entries.Where({ $_.Name -match $filesToMask }) - $traceMaskCount = 0; - foreach ($entry in $entries) { - $reader = [System.IO.StreamReader]::new($entry.Open()) - $content = $reader.ReadToEnd() - $entryMaskCount = 0; - foreach ($secret in $secrets) { - $pieces = $content -split "\b$($secret.Value)\b" - $entryMaskCount += $pieces.Count - 1; - $content = $pieces -join "*******" - } - $reader.Dispose() - $writer = [System.IO.StreamWriter]::new($entry.Open()) - $writer.BaseStream.SetLength(0) - $writer.Write($content) - $writer.Dispose() - Write-Output "- $entry ($entryMaskCount)" - } - Write-Output "Finished masking $file ($traceMaskCount)" - } - catch { - Write-Warning $_.Exception.Message - continue - } - finally { - if ($zip) { - $zip.Dispose() - } - } -} diff --git a/backend/Testing/LexCore/LexAuthUserTests.cs b/backend/Testing/LexCore/LexAuthUserTests.cs index 32a6b77ae..30d9d0b3d 100644 --- a/backend/Testing/LexCore/LexAuthUserTests.cs +++ b/backend/Testing/LexCore/LexAuthUserTests.cs @@ -7,6 +7,7 @@ using LexCore.Entities; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.Extensions.Options; using Microsoft.IdentityModel.Tokens; using Shouldly; @@ -14,16 +15,19 @@ namespace Testing.LexCore; public class LexAuthUserTests { + private readonly LexAuthService _lexAuthService = new LexAuthService( + new OptionsWrapper(JwtOptions.TestingOptions), + null, + null, + null); + private readonly LexAuthUser _user = new() { Id = Guid.NewGuid(), Email = "test@test.com", Role = UserRole.user, Name = "test", - Projects = new[] - { - new AuthUserProject("test-flex", ProjectRole.Manager, Guid.NewGuid()) - } + Projects = new[] { new AuthUserProject("test-flex", ProjectRole.Manager, Guid.NewGuid()) } }; [Fact] @@ -82,4 +86,15 @@ public void CanRoundTripClaimsWhenUsingSecurityTokenDescriptor() var newUser = JsonSerializer.Deserialize(Base64UrlEncoder.Decode(token.RawPayload)); _user.ShouldBeEquivalentTo(newUser); } + + [Fact] + public void CanRoundTripJwtFromUserThroughLexAuthService() + { + var (jwt, _) = _lexAuthService.GenerateJwt(_user); + var tokenHandler = new JwtSecurityTokenHandler(); + var outputJwt = tokenHandler.ReadJwtToken(jwt); + var principal = new ClaimsPrincipal(new ClaimsIdentity(outputJwt.Claims, "Testing")); + var newUser = LexAuthUser.FromClaimsPrincipal(principal); + _user.ShouldBeEquivalentTo(newUser); + } } diff --git a/backend/Testing/LexCore/Services/HgServiceTests.cs b/backend/Testing/LexCore/Services/HgServiceTests.cs index a1217109f..633552746 100644 --- a/backend/Testing/LexCore/Services/HgServiceTests.cs +++ b/backend/Testing/LexCore/Services/HgServiceTests.cs @@ -1,8 +1,12 @@ -using LexBoxApi.Services; +using System.IO.Compression; +using LexBoxApi.Services; using LexCore.Config; using LexCore.Entities; using LexCore.Exceptions; using LexSyncReverseProxy; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; using Shouldly; using Testing.Fixtures; @@ -15,17 +19,36 @@ public class HgServiceTests private const string RedmineResumable = "https://resumable.redmine"; private const string RedminePublic = "https://public.redmine"; private const string RedminePrivate = "https://private.redmine"; - private HgConfig _hgConfig = new HgConfig + private readonly string _basePath = Path.Join(Path.GetTempPath(), "HgServiceTests"); + private readonly HgConfig _hgConfig; + private readonly HgService _hgService; + + public HgServiceTests() { - RepoPath = "nothing", - HgWebUrl = LexboxHgWeb, - HgResumableUrl = LexboxResumable, - PublicRedmineHgWebUrl = RedminePublic, - PrivateRedmineHgWebUrl = RedminePrivate, - RedmineHgResumableUrl = RedmineResumable, - RedmineTrustToken = "tt", - LfMergeTrustToken = "tt" - }; + _hgConfig = new HgConfig + { + RepoPath = Path.Join(_basePath, "hg-repos"), + HgWebUrl = LexboxHgWeb, + HgResumableUrl = LexboxResumable, + PublicRedmineHgWebUrl = RedminePublic, + PrivateRedmineHgWebUrl = RedminePrivate, + RedmineHgResumableUrl = RedmineResumable, + RedmineTrustToken = "tt", + LfMergeTrustToken = "tt" + }; + _hgService = new HgService(new OptionsWrapper(_hgConfig), + Mock.Of(), + NullLogger.Instance); + CleanUpTempDir(); + } + + private void CleanUpTempDir() + { + if (Directory.Exists(_basePath)) + { + Directory.Delete(_basePath, true); + } + } [Theory] //lexbox @@ -49,4 +72,53 @@ public void ThrowsIfMigrating(HgType type) var act = () => HgService.DetermineProjectUrlPrefix(type, "test", ProjectMigrationStatus.Migrating, _hgConfig); act.ShouldThrow(); } + + [Theory] + [InlineData(".hg/important-file.bin")] + [InlineData("unzip-test/.hg/important-file.bin")] + public async Task CanFinishResetByUnZippingAnArchive(string filePath) + { + var code = "unzip-test"; + await _hgService.InitRepo(code); + var repoPath = Path.GetFullPath(Path.Join(_hgConfig.RepoPath, code)); + using var stream = new MemoryStream(); + using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) + { + CreateSimpleEntry(zipArchive, filePath); + CreateSimpleEntry(zipArchive, "random-subfolder/other-file.txt"); + } + + stream.Position = 0; + await _hgService.FinishReset(code, stream); + + Directory.EnumerateFiles(repoPath, "*", SearchOption.AllDirectories) + .Select(p => Path.GetRelativePath(repoPath, p)) + .ShouldHaveSingleItem().ShouldBe(Path.Join(".hg", "important-file.bin")); + } + + private void CreateSimpleEntry(ZipArchive zipArchive, string filePath) + { + var entry = zipArchive.CreateEntry(filePath); + using var fileStream = entry.Open(); + Span buff = stackalloc byte[100]; + Random.Shared.NextBytes(buff); + fileStream.Write(buff); + fileStream.Flush(); + } + + [Fact] + public async Task ThrowsIfNoHgFolderIsFound() + { + var code = "unzip-test-no-hg"; + await _hgService.InitRepo(code); + using var stream = new MemoryStream(); + using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) + { + CreateSimpleEntry(zipArchive, "random-subfolder/other-file.txt"); + } + + stream.Position = 0; + var act = () => _hgService.FinishReset(code, stream); + act.ShouldThrow(); + } } diff --git a/deployment/base/lexbox-deployment.yaml b/deployment/base/lexbox-deployment.yaml index b4f0bcaf6..988bc236c 100644 --- a/deployment/base/lexbox-deployment.yaml +++ b/deployment/base/lexbox-deployment.yaml @@ -83,7 +83,7 @@ spec: - name: ASPNETCORE_ENVIRONMENT value: Development - name: OTEL_METRIC_EXPORT_INTERVAL - value: '15000' # 15 seconds + value: '60000' # 60s - name: POSTGRES_DB valueFrom: secretKeyRef: @@ -142,9 +142,13 @@ spec: name: email - name: Email__BaseUrl value: http://localhost + - name: Tus__TestUploadPath + value: /tmp/tus-test-upload + - name: Tus__ResetUploadPath + value: /tmp/tus-reset-upload - name: otel-collector - image: otel/opentelemetry-collector:0.73.0 + image: otel/opentelemetry-collector-contrib:0.87.0 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers resources: requests: # TODO: need to establish resource limits, possibly after seeing it in action for some regular and/or load testing @@ -155,7 +159,7 @@ spec: - containerPort: 4318 - containerPort: 4317 volumeMounts: - - mountPath: /etc/otelcol + - mountPath: /etc/otelcol-contrib name: otel-config readOnly: true diff --git a/deployment/base/ui-deployment.yaml b/deployment/base/ui-deployment.yaml index 1398bd25e..f637de897 100644 --- a/deployment/base/ui-deployment.yaml +++ b/deployment/base/ui-deployment.yaml @@ -63,6 +63,8 @@ spec: name: ui - name: PUBLIC_ENV_NAME value: dev + - name: PUBLIC_TUS_CHUNK_SIZE_MB + value: "180" - name: BACKEND_HOST value: http://lexbox:5158 - name: OTEL_ENDPOINT diff --git a/deployment/production/lexbox-deployment.patch.yaml b/deployment/production/lexbox-deployment.patch.yaml index 6cac9666e..d4fe87d10 100644 --- a/deployment/production/lexbox-deployment.patch.yaml +++ b/deployment/production/lexbox-deployment.patch.yaml @@ -26,11 +26,11 @@ spec: - name: Email__BaseUrl value: "https://prod.languagedepot.org" - name: HgConfig__PublicRedmineHgWebUrl - value: "https://hg-public.languageforge.org" + value: "https://hg-public.languageforge.org/" - name: HgConfig__PrivateRedmineHgWebUrl - value: "https://hg-private.languageforge.org" + value: "https://hg-private.languageforge.org/" - name: HgConfig__RedmineHgResumableUrl - value: "https://hgresumable.languageforge.org" + value: "https://hg-resumable.languageforge.org" initContainers: - name: db-migrations env: diff --git a/frontend/.env b/frontend/.env index a8df5cf25..384a02a02 100644 --- a/frontend/.env +++ b/frontend/.env @@ -1,6 +1,7 @@ # https://developers.cloudflare.com/turnstile/reference/testing PUBLIC_TURNSTILE_SITE_KEY=1x00000000000000000000AA PUBLIC_ENV_NAME=dev +PUBLIC_TUS_CHUNK_SIZE_MB=180 # these are here so the app can be run outside of a docker if desired BACKEND_HOST=http://localhost:5158 # Avoids CORS issues in dev, see https://kit.svelte.dev/faq#how-do-i-use-a-different-backend-api-server diff --git a/frontend/package.json b/frontend/package.json index 45f512093..874233723 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -78,6 +78,7 @@ "mjml": "^4.14.1", "svelte-intl-precompile": "^0.12.3", "svelte-markdown": "^0.4.0", - "sveltekit-search-params": "^1.0.15" + "sveltekit-search-params": "^1.0.15", + "tus-js-client": "^3.1.1" } } diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 37825c083..a3827f3c8 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -53,6 +53,9 @@ dependencies: sveltekit-search-params: specifier: ^1.0.15 version: 1.0.15(@sveltejs/kit@1.24.1)(svelte@4.2.0) + tus-js-client: + specifier: ^3.1.1 + version: 3.1.1 devDependencies: '@egoist/tailwindcss-icons': @@ -4316,6 +4319,10 @@ packages: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} dev: true + /buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + dev: false + /buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} dependencies: @@ -4600,6 +4607,13 @@ packages: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} dev: true + /combine-errors@3.0.3: + resolution: {integrity: sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==} + dependencies: + custom-error-instance: 2.1.1 + lodash.uniqby: 4.5.0 + dev: false + /commander@2.20.3: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} dev: false @@ -4721,6 +4735,10 @@ packages: hasBin: true dev: true + /custom-error-instance@2.1.1: + resolution: {integrity: sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==} + dev: false + /daisyui@3.4.0: resolution: {integrity: sha512-s9EvNxnw/ubCIopKul+3ddASzZG+jMOmZgLJ0BFOVnxFQy6HJ5+EbMx5yY7Ef0cA6qjeUMl88SwhE94WLSLtHQ==} engines: {node: '>=16.9.0'} @@ -5409,7 +5427,6 @@ packages: /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} - dev: true /graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} @@ -5748,7 +5765,6 @@ packages: /is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} - dev: true /is-unc-path@1.0.0: resolution: {integrity: sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ==} @@ -5820,6 +5836,10 @@ packages: resolution: {integrity: sha512-j8GhLiKmUAh+dsFXlX1aJCbt5KMibuKb+d7j1JaOJG6s2UjX1PQlW+OKB/sD4a/5ZYF4RcmYmLSndOoU3Lt/3g==} dev: true + /js-base64@3.7.5: + resolution: {integrity: sha512-3MEt5DTINKqfScXKfJFrRbxkrnk2AxPWGBL/ycjz4dK8iqiSJ06UxD8jh8xuh6p10TX4t2+7FsBYVxxQbMg+qA==} + dev: false + /js-beautify@1.14.7: resolution: {integrity: sha512-5SOX1KXPFKx+5f6ZrPsIPEY7NwKeQz47n3jm2i+XeHx9MoRsfQenlOP13FQhWvg8JRS0+XLO6XYUQ2GX+q+T9A==} engines: {node: '>=10'} @@ -5972,6 +5992,37 @@ packages: p-locate: 5.0.0 dev: true + /lodash._baseiteratee@4.7.0: + resolution: {integrity: sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==} + dependencies: + lodash._stringtopath: 4.8.0 + dev: false + + /lodash._basetostring@4.12.0: + resolution: {integrity: sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==} + dev: false + + /lodash._baseuniq@4.6.0: + resolution: {integrity: sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==} + dependencies: + lodash._createset: 4.0.3 + lodash._root: 3.0.1 + dev: false + + /lodash._createset@4.0.3: + resolution: {integrity: sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==} + dev: false + + /lodash._root@3.0.1: + resolution: {integrity: sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==} + dev: false + + /lodash._stringtopath@4.8.0: + resolution: {integrity: sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==} + dependencies: + lodash._basetostring: 4.12.0 + dev: false + /lodash.camelcase@4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} dev: false @@ -5991,6 +6042,17 @@ packages: resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==} dev: true + /lodash.throttle@4.1.1: + resolution: {integrity: sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==} + dev: false + + /lodash.uniqby@4.5.0: + resolution: {integrity: sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==} + dependencies: + lodash._baseiteratee: 4.7.0 + lodash._baseuniq: 4.6.0 + dev: false + /lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} @@ -7170,6 +7232,14 @@ packages: asap: 2.0.6 dev: true + /proper-lockfile@4.1.2: + resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + dependencies: + graceful-fs: 4.2.11 + retry: 0.12.0 + signal-exit: 3.0.7 + dev: false + /proto-list@1.2.4: resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} dev: false @@ -7217,6 +7287,10 @@ packages: engines: {node: '>=6.0.0'} dev: true + /querystringify@2.2.0: + resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} + dev: false + /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} dev: true @@ -7317,6 +7391,10 @@ packages: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true + /requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + dev: false + /resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -7343,6 +7421,11 @@ packages: signal-exit: 3.0.7 dev: true + /retry@0.12.0: + resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} + engines: {node: '>= 4'} + dev: false + /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -7506,7 +7589,6 @@ packages: /signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} - dev: true /signedsource@1.0.0: resolution: {integrity: sha512-6+eerH9fEnNmi/hyM1DXcRK3pWdoMQtlkQ+ns0ntzunjKqp5i3sKCc80ym8Fib3iaYhdJUOPdhlJWj1tvge2Ww==} @@ -7999,6 +8081,18 @@ packages: resolution: {integrity: sha512-MJXbAeRPpORqPViZbHrpu8LWOHv1dp9i5GYSfoedu7GUmkUZpHWlAVLCFAhWlpRA5PQLBRworOGCEu4o6Ks7Zw==} dev: true + /tus-js-client@3.1.1: + resolution: {integrity: sha512-SZzWP62jEFLmROSRZx+uoGLKqsYWMGK/m+PiNehPVWbCm7/S9zRIMaDxiaOcKdMnFno4luaqP5E+Y1iXXPjP0A==} + dependencies: + buffer-from: 1.1.2 + combine-errors: 3.0.3 + is-stream: 2.0.1 + js-base64: 3.7.5 + lodash.throttle: 4.1.1 + proper-lockfile: 4.1.2 + url-parse: 1.5.10 + dev: false + /type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -8107,6 +8201,13 @@ packages: punycode: 2.3.0 dev: true + /url-parse@1.5.10: + resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} + dependencies: + querystringify: 2.2.0 + requires-port: 1.0.0 + dev: false + /urlpattern-polyfill@8.0.2: resolution: {integrity: sha512-Qp95D4TPJl1kC9SKigDcqgyM2VDVO4RiJc2d4qe5GrYm+zbIQCWWKAFaJNQ4BhdFeDGwBmAxqJBwWSJDb9T3BQ==} dev: true diff --git a/frontend/schema.graphql b/frontend/schema.graphql index 07db8d9f5..0a5f3bf91 100644 --- a/frontend/schema.graphql +++ b/frontend/schema.graphql @@ -134,6 +134,7 @@ type Project { users: [ProjectUsers!]! lastCommit: DateTime deletedDate: DateTime + resetStatus: ResetStatus! projectOrigin: ProjectMigrationStatus! userCount: Int! id: UUID! @@ -325,6 +326,7 @@ input ProjectFilterInput { users: ListFilterInputTypeOfProjectUsersFilterInput lastCommit: DateTimeOperationFilterInput deletedDate: DateTimeOperationFilterInput + resetStatus: ResetStatusOperationFilterInput projectOrigin: ProjectMigrationStatusOperationFilterInput migrationStatus: ProjectMigrationStatusOperationFilterInput userCount: IntOperationFilterInput @@ -356,6 +358,7 @@ input ProjectSortInput { type: SortEnumType lastCommit: SortEnumType deletedDate: SortEnumType + resetStatus: SortEnumType projectOrigin: SortEnumType migrationStatus: SortEnumType userCount: SortEnumType @@ -389,6 +392,13 @@ input RemoveProjectMemberInput { userId: UUID! } +input ResetStatusOperationFilterInput { + eq: ResetStatus + neq: ResetStatus + in: [ResetStatus!] + nin: [ResetStatus!] +} + input RetentionPolicyOperationFilterInput { eq: RetentionPolicy neq: RetentionPolicy @@ -513,6 +523,11 @@ enum ProjectType { OUR_WORD } +enum ResetStatus { + NONE + IN_PROGRESS +} + enum RetentionPolicy { UNKNOWN VERIFIED diff --git a/frontend/src/hooks.server.ts b/frontend/src/hooks.server.ts index d5cbaa0a7..7c020dbf2 100644 --- a/frontend/src/hooks.server.ts +++ b/frontend/src/hooks.server.ts @@ -1,5 +1,5 @@ import { getUser, isAuthn } from '$lib/user' -import { apiVersion } from '$lib/util/verstion'; +import { apiVersion } from '$lib/util/version'; import { redirect, type Handle, type HandleFetch, type HandleServerError, type ResolveOptions } from '@sveltejs/kit' import { loadI18n } from '$lib/i18n'; import { ensureErrorIsTraced, traceRequest, traceFetch } from '$lib/otel/otel.server' diff --git a/frontend/src/lib/app.postcss b/frontend/src/lib/app.postcss index aa051df45..3c3df868a 100644 --- a/frontend/src/lib/app.postcss +++ b/frontend/src/lib/app.postcss @@ -112,3 +112,15 @@ table tr:nth-last-child(-n + 2) .dropdown { @apply h-full; } } + +.file-input { + /* some make-up for what looks like a Chrome rendering bug */ + border-width: 2px; +} + +.steps .step-primary { + &:before, + &:after { + transition: background-color 0.5s; + } +} diff --git a/frontend/src/lib/components/Badges/Badge.svelte b/frontend/src/lib/components/Badges/Badge.svelte index acc0c6443..51758f742 100644 --- a/frontend/src/lib/components/Badges/Badge.svelte +++ b/frontend/src/lib/components/Badges/Badge.svelte @@ -1,6 +1,6 @@ {#if showing < total} -
+
{$t('table.refine_filter', { remainingRows: total - showing })}
diff --git a/frontend/src/lib/components/TusUpload.svelte b/frontend/src/lib/components/TusUpload.svelte new file mode 100644 index 000000000..b81387f46 --- /dev/null +++ b/frontend/src/lib/components/TusUpload.svelte @@ -0,0 +1,145 @@ + + +
+
+ + + + + +
+ +
+ +
+

{$t('tus.upload_progress')}

+ +
+
diff --git a/frontend/src/lib/components/modals/ConfirmDeleteModal.svelte b/frontend/src/lib/components/modals/ConfirmDeleteModal.svelte index 520ef9210..36b79f128 100644 --- a/frontend/src/lib/components/modals/ConfirmDeleteModal.svelte +++ b/frontend/src/lib/components/modals/ConfirmDeleteModal.svelte @@ -12,7 +12,7 @@ - diff --git a/frontend/src/lib/gql/gql-client.ts b/frontend/src/lib/gql/gql-client.ts index 7ea016943..f74b2ffe7 100644 --- a/frontend/src/lib/gql/gql-client.ts +++ b/frontend/src/lib/gql/gql-client.ts @@ -8,7 +8,8 @@ import { fetchExchange, type CombinedError, queryStore, - type OperationResultSource + type OperationResultSource, + type OperationResultStore } from '@urql/svelte'; import {createClient} from '@urql/svelte'; import {browser} from '$app/environment'; @@ -84,17 +85,30 @@ class GqlClient { ); } - async queryStore( + queryStore( fetch: Fetch, query: TypedDocumentNode, variables: Variables, - context: QueryOperationOptions = {}): Promise> { + context: QueryOperationOptions = {}): OperationResultStore { const resultStore = queryStore({ client: this.client, query, variables, context: {fetch, ...context} }); + + return derived(resultStore, (result) => { + this.throwAnyUnexpectedErrors(result); + return result; + }); + } + async awaitedQueryStore( + fetch: Fetch, + query: TypedDocumentNode, + variables: Variables, + context: QueryOperationOptions = {}): Promise> { + const resultStore = this.queryStore(fetch, query, variables, context); + const results = await new Promise>((resolve) => { let invalidate = undefined as Unsubscriber | undefined; invalidate = resultStore.subscribe(value => { @@ -104,7 +118,6 @@ class GqlClient { }); }); - this.throwAnyUnexpectedErrors(results); const keys = Object.keys(results.data ?? {}) as Array; const resultData = {} as Record>; for (const key of keys) { diff --git a/frontend/src/lib/i18n/index.ts b/frontend/src/lib/i18n/index.ts index 83d764924..fb638e328 100644 --- a/frontend/src/lib/i18n/index.ts +++ b/frontend/src/lib/i18n/index.ts @@ -8,6 +8,7 @@ import type I18n from '../i18n/locales/en.json'; import { derived, type Readable } from 'svelte/store'; // @ts-ignore there's an error here because this is a synthetic path import en from '$locales/en'; +import type { Get } from 'type-fest'; export async function loadI18n(): Promise { addMessages('en', en); @@ -32,10 +33,17 @@ export default t; export type Translater = StoreType; -export function tScoped(scope: I18nShapeKey): Readable<(key: DeepPathsToString, values?: InterpolationValues) => string> { +export function tScoped(scope: Scope): Readable<(key: DeepPathsToString>, values?: InterpolationValues) => string> { + // I can't quite figure out why this needs to be cast + return tTypeScoped>(scope as I18nShapeKey>); +} + +export function tTypeScoped(scope: I18nShapeKey): Readable<(key: DeepPathsToString, values?: InterpolationValues) => string> { return derived(t, tFunc => (key: DeepPathsToString, values?: InterpolationValues) => tFunc(`${String(scope)}.${String(key)}`, values)); } type I18nKey = DeepPaths; +type I18nScope = DeepPathsToType; +type I18nShape = Get; export type I18nShapeKey = DeepPathsToType; diff --git a/frontend/src/lib/i18n/locales/en.json b/frontend/src/lib/i18n/locales/en.json index fe4795545..92cdfb771 100644 --- a/frontend/src/lib/i18n/locales/en.json +++ b/frontend/src/lib/i18n/locales/en.json @@ -160,14 +160,30 @@ the [Linguistics Institute at Payap University](https://li.payap.ac.th/) in Chia "title": "Choose role for {name}" }, "reset_project_modal": { - "title": "Reset project", + "title": "Reset project {code}", "submit": "Reset project", - "download_button": "Download backup file", - "confirm_downloaded": "I confirm that I have downloaded the backup file and verified that it works. I am ready to completely reset the repository history.", - "confirm_downloaded_error": "Please check the box to confirm you have downloaded the backup file", + "close": "Close", + "i_have_working_backup": "I have a working backup", + "next": "Next", + "back": "Back", + "download_instruction": "First, download a backup of the project that you can use to restore it in step 3:", + "download_button": "Download project backup", + "confirm_downloaded": "I confirm that I have downloaded a backup of the project and verified that it works. I am ready to completely reset/delete the contents of the project repository.", + "confirm_downloaded_error": "Please confirm you have downloaded a backup", "confirm_project_code": "Enter project code to confirm reset", "confirm_project_code_error": "Please type the project code to confirm reset", - "reset_project_notification": "Successfully reset project {code}" + "reset_project_notification": "Successfully reset project {code}", + "upload_instruction": "The project repository was successfully reset. Now upload a zip file to restore it:", + "upload_project": "Upload Project", + "select_zip": "Project zip file", + "should_only_contain_hg": "Should only contain the .hg folder at the root", + "reset_success": "Project successfully reset.", + "backup_step": "Backup", + "reset_step": "Reset", + "restore_step": "Restore", + "finished_step": "Finished", + "reset_in_progress": "Project Reset in progress", + "click_to_continue": "Click to continue", }, "notifications": { "role_change": "Project role of {name} set to {role}.", @@ -193,6 +209,7 @@ the [Linguistics Institute at Payap University](https://li.payap.ac.th/) in Chia "hg": { "open_in_hgweb": "Open in HgWeb", "no_history": "No history", + "loading": "Loading history...", "date_header": "Date", "author_header": "Author", "log_header": "Message" @@ -332,4 +349,15 @@ the [Linguistics Institute at Payap University](https://li.payap.ac.th/) in Chia "turnstile": { "invalid": "Captcha Error, try again", }, + "tus": { + "upload": "Upload", + "upload_progress": "Upload progress", + "status": "Status: {status, select, NoFile {No file} Ready {Ready} Error {Error} Uploading {Uploading} Paused {Paused} Complete {Complete} other {Unknown}}", + "zip_only": "Only .zip files are allowed", + "no_file_selected": "Please choose a file", + "select_file": "Select file", + "server_error_codes": { + "ZIP_MISSING_HG_FOLDER": "The ZIP file does not contain a folder named .hg" + } + }, } diff --git a/frontend/src/lib/layout/AppMenu.svelte b/frontend/src/lib/layout/AppMenu.svelte index 69c79d80e..6c8dca39d 100644 --- a/frontend/src/lib/layout/AppMenu.svelte +++ b/frontend/src/lib/layout/AppMenu.svelte @@ -3,7 +3,7 @@ import {AdminIcon, AuthenticatedUserIcon, HomeIcon, LogoutIcon} from '$lib/icons'; import AdminContent from './AdminContent.svelte'; import Badge from '$lib/components/Badges/Badge.svelte'; - import {APP_VERSION} from '$lib/util/verstion'; + import {APP_VERSION} from '$lib/util/version'; import type {LexAuthUser} from '$lib/user'; export let serverVersion: string; diff --git a/frontend/src/lib/otel/otel.client.ts b/frontend/src/lib/otel/otel.client.ts index 43090ede6..98d6e6596 100644 --- a/frontend/src/lib/otel/otel.client.ts +++ b/frontend/src/lib/otel/otel.client.ts @@ -1,7 +1,6 @@ import { BatchSpanProcessor, WebTracerProvider } from '@opentelemetry/sdk-trace-web' import { SERVICE_NAME, ensureErrorIsTraced, tracer } from '.' - -import {APP_VERSION} from '$lib/util/verstion'; +import { APP_VERSION } from '$lib/util/version'; import { OTLPTraceExporterBrowserWithXhrRetry } from './trace-exporter-browser-with-xhr-retry'; import { Resource } from '@opentelemetry/resources' import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions' diff --git a/frontend/src/lib/otel/otel.server.ts b/frontend/src/lib/otel/otel.server.ts index eaec683a8..d5ae499ea 100644 --- a/frontend/src/lib/otel/otel.server.ts +++ b/frontend/src/lib/otel/otel.server.ts @@ -1,4 +1,4 @@ -import {APP_VERSION} from '$lib/util/verstion'; +import {APP_VERSION} from '$lib/util/version'; import { TraceFlags, context, diff --git a/frontend/src/lib/type.utils.ts b/frontend/src/lib/type.utils.ts index b1684cb2b..8914a7710 100644 --- a/frontend/src/lib/type.utils.ts +++ b/frontend/src/lib/type.utils.ts @@ -7,6 +7,8 @@ import type { Readable } from 'svelte/store'; */ type OmitNever = { [K in keyof T as IfNever]: T[K] }; +type OrIfNever = IfNever; + /** * Creates a union of all possible deep paths / nested keys (e.g. `obj.nestedObj.prop`) of an object */ @@ -20,9 +22,9 @@ export type DeepPaths = /** * Create a union of all possible deep paths of an object who's value fulfill `Condition` */ -export type DeepPathsToType = keyof OmitNever<{ +export type DeepPathsToType = OrIfNever extends Type ? Property : never; -}>; +}>, Readonly<`No paths match type:`> | Type>; /** * Create a union of all possible deep paths of an object who's value type is `string` diff --git a/frontend/src/lib/util/verstion.ts b/frontend/src/lib/util/version.ts similarity index 100% rename from frontend/src/lib/util/verstion.ts rename to frontend/src/lib/util/version.ts diff --git a/frontend/src/routes/(authenticated)/+page.ts b/frontend/src/routes/(authenticated)/+page.ts index 2caf1bb17..677fa8f0f 100644 --- a/frontend/src/routes/(authenticated)/+page.ts +++ b/frontend/src/routes/(authenticated)/+page.ts @@ -7,7 +7,7 @@ export async function load(event: PageLoadEvent) { // Currently Svelte-Kit is skipping re-running this load if you log out and back in, which results in stale project lists const client = getClient(); //language=GraphQL - const results = await client.queryStore(event.fetch, graphql(` + const results = await client.awaitedQueryStore(event.fetch, graphql(` query loadProjects { myProjects { code diff --git a/frontend/src/routes/(authenticated)/admin/+page.ts b/frontend/src/routes/(authenticated)/admin/+page.ts index b6d360800..1944f2933 100644 --- a/frontend/src/routes/(authenticated)/admin/+page.ts +++ b/frontend/src/routes/(authenticated)/admin/+page.ts @@ -35,7 +35,7 @@ export async function load(event: PageLoadEvent) { }; //language=GraphQL - const projectResultsPromise = client.queryStore(event.fetch, graphql(` + const projectResultsPromise = client.awaitedQueryStore(event.fetch, graphql(` query loadAdminDashboardProjects($withDeletedProjects: Boolean, $filter: ProjectFilterInput) { projects( where: $filter, @@ -54,7 +54,7 @@ export async function load(event: PageLoadEvent) { } `), { withDeletedProjects, filter: projectFilter }); - const userResultsPromise = client.queryStore(event.fetch, graphql(` + const userResultsPromise = client.awaitedQueryStore(event.fetch, graphql(` query loadAdminDashboardUsers($userSearch: String, $take: Int!) { users( where: {or: [ diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte b/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte index a980a441d..930e2b6c0 100644 --- a/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte +++ b/frontend/src/routes/(authenticated)/project/[project_code]/+page.svelte @@ -29,8 +29,8 @@ import MoreSettings from '$lib/components/MoreSettings.svelte'; import { AdminContent, Page } from '$lib/layout'; import SvelteMarkdown from 'svelte-markdown'; - import {ProjectMigrationStatus} from '$lib/gql/generated/graphql'; - import {onMount} from 'svelte'; + import { ProjectMigrationStatus, ResetStatus } from '$lib/gql/generated/graphql'; + import { onMount } from 'svelte'; import Button from '$lib/forms/Button.svelte'; import Icon from '$lib/icons/Icon.svelte'; @@ -39,6 +39,7 @@ let projectStore = data.project; $: project = $projectStore; $: _project = project as NonNullable; + $: changesetStore = data.changesets; $: projectHgUrl = import.meta.env.DEV ? `http://hg.${$page.url.host}/${data.code}` @@ -64,7 +65,7 @@ let resetProjectModal: ResetProjectModal; async function resetProject(): Promise { - await resetProjectModal.open(_project.code); + await resetProjectModal.open(_project.code, _project.resetStatus); } let removeUserModal: DeleteModal; @@ -139,13 +140,13 @@ [ProjectMigrationStatus.Migrating]: 'Migrating', [ProjectMigrationStatus.Unknown]: 'Unknown', [ProjectMigrationStatus.PrivateRedmine]: 'Not Migrated (private)', - [ProjectMigrationStatus.PublicRedmine]: 'Not Migrated (public)', + [ProjectMigrationStatus.PublicRedmine]: 'Not Migrated (public)', } satisfies Record; onMount(() => { - migrationStatus = project?.migrationStatus ?? ProjectMigrationStatus.Unknown; - if (migrationStatus === ProjectMigrationStatus.Migrating) { - void watchMigrationStatus(); - } + migrationStatus = project?.migrationStatus ?? ProjectMigrationStatus.Unknown; + if (migrationStatus === ProjectMigrationStatus.Migrating) { + void watchMigrationStatus(); + } }); async function watchMigrationStatus(): Promise { @@ -176,55 +177,59 @@
{#if migrationStatus !== ProjectMigrationStatus.Migrating} - - - -
-
-
- - -
- - -
- -
- {#if copiedToClipboard} - - {:else} - - {/if} -
-
- - + + + +
+
+
+
+ + +
+ +
+ {#if copiedToClipboard} + + {:else} + + {/if} +
+
+ +
- +
+ {/if} {#if migrationStatus === ProjectMigrationStatus.PublicRedmine || migrationStatus === ProjectMigrationStatus.PrivateRedmine} {/if} @@ -241,15 +246,28 @@
- + - + {#if migrationStatus === ProjectMigrationStatus.Migrating} - Migrating + {:else} {migrationStatusTable[migrationStatus]} {/if} + {#if project.resetStatus === ResetStatus.InProgress} + + {/if}
@@ -285,12 +303,11 @@ {#each project.users as member} - {@const canManageMember = canManage && (member.user.id !== userId || isAdmin(user))} + {@const canManageMember = canManage && (member.user.id !== userId || isAdmin(user))} + canManage={canManageMember} />
@@ -348,10 +364,10 @@ {$t('delete_project_modal.submit')} - - + + {/if} diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/+page.ts b/frontend/src/routes/(authenticated)/project/[project_code]/+page.ts index a2f6402d5..21395f5a4 100644 --- a/frontend/src/routes/(authenticated)/project/[project_code]/+page.ts +++ b/frontend/src/routes/(authenticated)/project/[project_code]/+page.ts @@ -14,6 +14,7 @@ import type { import { getClient, graphql } from '$lib/gql'; import type { PageLoadEvent } from './$types'; +import { derived } from 'svelte/store'; type Project = NonNullable; export type ProjectUser = Project['users'][number]; @@ -21,8 +22,8 @@ export type ProjectUser = Project['users'][number]; export async function load(event: PageLoadEvent) { const client = getClient(); const projectCode = event.params.project_code; - const result = await client - .queryStore(event.fetch, + const projectResult = await client + .awaitedQueryStore(event.fetch, graphql(` query projectPage($projectCode: String!) { projectByCode(code: $projectCode) { @@ -32,6 +33,7 @@ export async function load(event: PageLoadEvent) { description type migrationStatus + resetStatus lastCommit createdDate retentionPolicy @@ -43,22 +45,38 @@ export async function load(event: PageLoadEvent) { name } } - changesets { - node - parents - date - user - desc - } } } `), { projectCode } ); + const changesetResultStore = client + .queryStore(event.fetch, + graphql(` + query projectChangesets($projectCode: String!) { + projectByCode(code: $projectCode) { + id + code + changesets { + node + parents + date + user + desc + } + } + } + `), + { projectCode } + ); event.depends(`project:${projectCode}`); return { - project: result.projectByCode, + project: projectResult.projectByCode, + changesets: derived(changesetResultStore, result => ({ + fetching: result.fetching, + changesets: result.data?.projectByCode?.changesets ?? [], + })), code: projectCode, }; } @@ -88,7 +106,7 @@ export async function _addProjectMember(input: AddProjectMemberInput): $OpResult } `), { input: input } - ); + ); return result; } @@ -178,3 +196,28 @@ export async function _deleteProjectUser(projectId: string, userId: string): $Op ); return result; } + +export async function _refreshProjectMigrationStatusAndRepoInfo(projectCode: string): Promise { + const result = await getClient().query(graphql(` + query refreshProjectStatus($projectCode: String!) { + projectByCode(code: $projectCode) { + id + resetStatus + migrationStatus + lastCommit + changesets { + node + parents + date + user + desc + } + } + } + `), { projectCode }, { requestPolicy: 'network-only' }); + + if (result.error) { + // this should be meaningless, but just in case and it makes the linter happy + throw result.error; + } +} diff --git a/frontend/src/routes/(authenticated)/project/[project_code]/ResetProjectModal.svelte b/frontend/src/routes/(authenticated)/project/[project_code]/ResetProjectModal.svelte index 7139b1488..724ee0225 100644 --- a/frontend/src/routes/(authenticated)/project/[project_code]/ResetProjectModal.svelte +++ b/frontend/src/routes/(authenticated)/project/[project_code]/ResetProjectModal.svelte @@ -1,82 +1,173 @@ - - -
- - {$t('title')} -
- +
+ +

{$t('title', { code })}

+
    +
  • {$t('backup_step')}
    • +
  • = ResetSteps.Reset}>{$t('reset_step')}
    • +
  • = ResetSteps.Upload}>{$t('restore_step')}
    • +
  • = ResetSteps.Finished}>{$t('finished_step')}
    • +
+ +
+ + {#if currentStep === ResetSteps.Download} +

+ {$t('download_instruction')} +

+ {$t('download_button')} -
- - - - {$t('submit')} - + {:else if currentStep === ResetSteps.Reset} +
+ + + + {:else if currentStep === ResetSteps.Upload} +
+ {$t('upload_instruction')} +
+ + {:else if currentStep === ResetSteps.Finished} +
+

+ {$t('reset_success')} +

+ +
+ {:else} + Unknown step + {/if} + + {#if currentStep === ResetSteps.Reset} + + {/if} + + + {#if currentStep === ResetSteps.Download} + + {:else if currentStep === ResetSteps.Reset} + + {:else if currentStep === ResetSteps.Finished} + + {/if} - +
+ + diff --git a/frontend/src/routes/(unauthenticated)/sandbox/+page.svelte b/frontend/src/routes/(unauthenticated)/sandbox/+page.svelte new file mode 100644 index 000000000..0b84a412a --- /dev/null +++ b/frontend/src/routes/(unauthenticated)/sandbox/+page.svelte @@ -0,0 +1,20 @@ + +
+

Sandbox

+
+ Goto 500 page +
+
+
+ +
+
+
+ diff --git a/frontend/src/routes/(unauthenticated)/version/+server.ts b/frontend/src/routes/(unauthenticated)/version/+server.ts index 339eb0fcd..6aec1d428 100644 --- a/frontend/src/routes/(unauthenticated)/version/+server.ts +++ b/frontend/src/routes/(unauthenticated)/version/+server.ts @@ -1,5 +1,4 @@ -import { APP_VERSION } from '$lib/util/verstion'; - +import { APP_VERSION } from '$lib/util/version'; //used externally to get the app version export function GET(): Response { diff --git a/frontend/src/routes/+layout.server.ts b/frontend/src/routes/+layout.server.ts index d7faae6da..c5d3d72a8 100644 --- a/frontend/src/routes/+layout.server.ts +++ b/frontend/src/routes/+layout.server.ts @@ -1,4 +1,4 @@ -import { APP_VERSION, apiVersion } from '$lib/util/verstion'; +import { APP_VERSION, apiVersion } from '$lib/util/version'; import type { LayoutServerLoadEvent } from './$types' import { USER_LOAD_KEY } from '$lib/user'; diff --git a/frontend/src/routes/healthz/+server.ts b/frontend/src/routes/healthz/+server.ts index c8105d0c8..ab2714d98 100644 --- a/frontend/src/routes/healthz/+server.ts +++ b/frontend/src/routes/healthz/+server.ts @@ -1,4 +1,4 @@ -import { APP_VERSION } from '$lib/util/verstion'; +import { APP_VERSION } from '$lib/util/version'; import type { RequestEvent } from './$types' import { text } from '@sveltejs/kit' diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js index 037390aa9..7ec4a3305 100644 --- a/frontend/svelte.config.js +++ b/frontend/svelte.config.js @@ -8,7 +8,7 @@ const config = { }, kit: { version: { - pollInterval: 5000, + pollInterval: 60000 * 3, // 3m }, adapter: adapter({ precompress: true diff --git a/otel/collector-config.yaml b/otel/collector-config.yaml index 390762db0..969b3f9e2 100644 --- a/otel/collector-config.yaml +++ b/otel/collector-config.yaml @@ -21,6 +21,11 @@ processors: memory_limiter: check_interval: 1s limit_mib: 400 + redaction: # https://pkg.go.dev/github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor#section-readme + allow_all_keys: true + blocked_values: + - "[A-Za-z0-9-_]{10,}\\.[A-Za-z0-9-_]{20,}\\.[A-Za-z0-9-_]{10,}" # jwt + summary: debug exporters: logging: @@ -44,7 +49,7 @@ service: pipelines: traces: receivers: [otlp] - processors: [memory_limiter] + processors: [memory_limiter, redaction, batch] exporters: [otlp, logging] metrics: receivers: [otlp]