New multi-request publishing workflow.

This API allows publishing via multiple API requests. The flow is start -> file -> file -> finish. This means the files are directly transmitted through the HTTP POST body.

This should make faster publishes that don't rely on GitHub's artifacts or anything stupid like that. It does require configuring the web server to allow large client request bodies, but eh.

Robust.Cdn/Config/ManifestOptions.cs

@@ -12,6 +12,15 @@ public sealed class ManifestOptions
     public string FileDiskPath { get; set; } = "";
 
     public Dictionary<string, ManifestForkOptions> Forks { get; set; } = new();
+
+    /// <summary>
+    /// Time, in minutes, before an in-progress publish "times out" and can be deleted.
+    /// </summary>
+    /// <remarks>
+    /// Generally, publishes really should not take longer than more than a few minutes.
+    /// If a publish takes longer, it likely indicates an error caused it to be aborted.
+    /// </remarks>
+    public int InProgressPublishTimeoutMinutes { get; set; } = 60;
 }
 
 public sealed class ManifestForkOptions

Robust.Cdn/Controllers/ForkPublishController.Multi.cs

@@ -0,0 +1,189 @@
+using Dapper;
+using Microsoft.AspNetCore.Mvc;
+using Robust.Cdn.Helpers;
+
+namespace Robust.Cdn.Controllers;
+
+public sealed partial class ForkPublishController
+{
+    // Code for "multi-request" publishes.
+    // i.e. start, followed by files, followed by finish call.
+
+    [HttpPost("start")]
+    public async Task<IActionResult> MultiPublishStart(
+        string fork,
+        [FromBody] PublishMultiRequest request,
+        CancellationToken cancel)
+    {
+        if (!authHelper.IsAuthValid(fork, out _, out var failureResult))
+            return failureResult;
+
+        baseUrlManager.ValidateBaseUrl();
+
+        if (!ValidVersionRegex.IsMatch(request.Version))
+            return BadRequest("Invalid version name");
+
+        if (VersionAlreadyExists(fork, request.Version))
+            return Conflict("Version already exists");
+
+        var dbCon = manifestDatabase.Connection;
+
+        await using var tx = await dbCon.BeginTransactionAsync(cancel);
+
+        logger.LogInformation("Starting multi publish for fork {Fork} version {Version}", fork, request.Version);
+
+        var hasExistingPublish = dbCon.QuerySingleOrDefault<bool>(
+            "SELECT 1 FROM PublishInProgress WHERE Version = @Version ",
+            new { request.Version });
+        if (hasExistingPublish)
+        {
+            // If a publish with this name already exists we abort it and start again.
+            // We do this so you can "just" retry a mid-way-failed publish without an extra API call required.
+
+            logger.LogWarning("Already had an in-progress publish for this version, aborting it and restarting.");
+            publishManager.AbortMultiPublish(fork, request.Version, tx, commit: false);
+        }
+
+        var forkId = dbCon.QuerySingle<int>("SELECT Id FROM Fork WHERE Name = @Name", new { Name = fork });
+
+        await dbCon.ExecuteAsync("""
+            INSERT INTO PublishInProgress (Version, ForkId, StartTime, EngineVersion)
+            VALUES (@Version, @ForkId, @StartTime, @EngineVersion)
+            """,
+            new
+            {
+                request.Version,
+                request.EngineVersion,
+                ForkId = forkId,
+                StartTime = DateTime.UtcNow
+            });
+
+        var versionDir = buildDirectoryManager.GetBuildVersionPath(fork, request.Version);
+        Directory.CreateDirectory(versionDir);
+
+        await tx.CommitAsync(cancel);
+
+        logger.LogInformation("Multi publish initiated. Waiting for subsequent API requests...");
+
+        return NoContent();
+    }
+
+    [HttpPost("file")]
+    [RequestSizeLimit(2048L * 1024 * 1024)]
+    public async Task<IActionResult> MultiPublishFile(
+        string fork,
+        [FromHeader(Name = "Robust-Cdn-Publish-File")]
+        string fileName,
+        [FromHeader(Name = "Robust-Cdn-Publish-Version")]
+        string version,
+        CancellationToken cancel)
+    {
+        if (!authHelper.IsAuthValid(fork, out _, out var failureResult))
+            return failureResult;
+
+        if (!ValidFileRegex.IsMatch(fileName))
+            return BadRequest("Invalid artifact file name");
+
+        var dbCon = manifestDatabase.Connection;
+        await using var tx = await dbCon.BeginTransactionAsync(cancel);
+
+        var forkId = dbCon.QuerySingle<int>("SELECT Id FROM Fork WHERE Name = @Name", new { Name = fork });
+        var versionId = dbCon.QuerySingleOrDefault<int?>("""
+            SELECT Id
+            FROM PublishInProgress
+            WHERE Version = @Name AND ForkId = @Fork
+            """,
+            new { Name = version, Fork = forkId });
+
+        if (versionId == null)
+            return NotFound("Unknown in-progress version");
+
+        var versionDir = buildDirectoryManager.GetBuildVersionPath(fork, version);
+        var filePath = Path.Combine(versionDir, fileName);
+
+        if (System.IO.File.Exists(filePath))
+            return Conflict("File already published");
+
+        logger.LogDebug("Receiving file {FileName} for multi-publish version {Version}", fileName, version);
+
+        await using var file = System.IO.File.Create(filePath, 4096, FileOptions.Asynchronous);
+
+        await Request.Body.CopyToAsync(file, cancel);
+
+        logger.LogDebug("Successfully Received file {FileName}", fileName);
+
+        return NoContent();
+    }
+
+    [HttpPost("finish")]
+    public async Task<IActionResult> MultiPublishFinish(
+        string fork,
+        [FromBody] PublishFinishRequest request,
+        CancellationToken cancel)
+    {
+        if (!authHelper.IsAuthValid(fork, out var forkConfig, out var failureResult))
+            return failureResult;
+
+        var dbCon = manifestDatabase.Connection;
+        await using var tx = await dbCon.BeginTransactionAsync(cancel);
+
+        var forkId = dbCon.QuerySingle<int>("SELECT Id FROM Fork WHERE Name = @Name", new { Name = fork });
+        var versionMetadata = dbCon.QuerySingleOrDefault<VersionMetadata>("""
+            SELECT Version, EngineVersion
+            FROM PublishInProgress
+            WHERE Version = @Name AND ForkId = @Fork
+            """,
+            new { Name = request.Version, Fork = forkId });
+
+        if (versionMetadata == null)
+            return NotFound("Unknown in-progress version");
+
+        logger.LogInformation("Finishing multi publish {Version} for fork {Fork}", request.Version, fork);
+
+        var versionDir = buildDirectoryManager.GetBuildVersionPath(fork, request.Version);
+
+        logger.LogDebug("Classifying entries...");
+
+        var artifacts = ClassifyEntries(
+            forkConfig,
+            Directory.GetFiles(versionDir),
+            item => Path.GetRelativePath(versionDir, item));
+
+        var clientArtifact = artifacts.SingleOrNull(art => art.artifact.Type == ArtifactType.Client);
+        if (clientArtifact == null)
+        {
+            publishManager.AbortMultiPublish(fork, request.Version, tx, commit: true);
+            return UnprocessableEntity("Publish failed: no client zip was provided");
+        }
+
+        var diskFiles = artifacts.ToDictionary(i => i.artifact, i => i.key);
+
+        var buildJson = GenerateBuildJson(diskFiles, clientArtifact.Value.artifact, versionMetadata, fork);
+        InjectBuildJsonIntoServers(diskFiles, buildJson);
+
+        AddVersionToDatabase(clientArtifact.Value.artifact, diskFiles, fork, versionMetadata);
+
+        dbCon.Execute(
+            "DELETE FROM PublishInProgress WHERE Version = @Name AND ForkId = @Fork",
+            new { Name = request.Version, Fork = forkId });
+
+        tx.Commit();
+
+        await QueueIngestJobAsync(fork);
+
+        logger.LogInformation("Publish succeeded!");
+
+        return NoContent();
+    }
+
+    public sealed class PublishMultiRequest
+    {
+        public required string Version { get; set; }
+        public required string EngineVersion { get; set; }
+    }
+
+    public sealed class PublishFinishRequest
+    {
+        public required string Version { get; set; }
+    }
+}

Robust.Cdn/Controllers/ForkPublishController.OneShot.cs

@@ -0,0 +1,117 @@
+using System.IO.Compression;
+using Microsoft.AspNetCore.Mvc;
+using Robust.Cdn.Helpers;
+
+namespace Robust.Cdn.Controllers;
+
+public sealed partial class ForkPublishController
+{
+    // Code for the "one shot" publish workflow where everything is done in a single request.
+
+    [HttpPost]
+    public async Task<IActionResult> PostPublish(
+        string fork,
+        [FromBody] PublishRequest request,
+        CancellationToken cancel)
+    {
+        if (!authHelper.IsAuthValid(fork, out var forkConfig, out var failureResult))
+            return failureResult;
+
+        baseUrlManager.ValidateBaseUrl();
+
+        if (string.IsNullOrWhiteSpace(request.Archive))
+            return BadRequest("Archive is empty");
+
+        if (!ValidVersionRegex.IsMatch(request.Version))
+            return BadRequest("Invalid version name");
+
+        if (VersionAlreadyExists(fork, request.Version))
+            return Conflict("Version already exists");
+
+        logger.LogInformation("Starting one-shot publish for fork {Fork} version {Version}", fork, request.Version);
+
+        var httpClient = httpFactory.CreateClient();
+
+        await using var tmpFile = CreateTempFile();
+
+        logger.LogDebug("Downloading publish archive {Archive} to temp file", request.Archive);
+
+        await using var response = await httpClient.GetStreamAsync(request.Archive, cancel);
+        await response.CopyToAsync(tmpFile, cancel);
+        tmpFile.Seek(0, SeekOrigin.Begin);
+
+        using var archive = new ZipArchive(tmpFile, ZipArchiveMode.Read);
+
+        logger.LogDebug("Classifying archive entries...");
+
+        var artifacts = ClassifyEntries(forkConfig, archive.Entries, e => e.FullName);
+        var clientArtifact = artifacts.SingleOrNull(art => art.artifact.Type == ArtifactType.Client);
+        if (clientArtifact == null)
+            return BadRequest("Client zip is missing!");
+
+        var versionDir = buildDirectoryManager.GetBuildVersionPath(fork, request.Version);
+
+        var metadata = new VersionMetadata { Version = request.Version, EngineVersion = request.EngineVersion };
+
+        try
+        {
+            Directory.CreateDirectory(versionDir);
+
+            var diskFiles = ExtractZipToVersionDir(artifacts, versionDir);
+            var buildJson = GenerateBuildJson(diskFiles, clientArtifact.Value.artifact, metadata, fork);
+            InjectBuildJsonIntoServers(diskFiles, buildJson);
+
+            using var tx = manifestDatabase.Connection.BeginTransaction();
+
+            AddVersionToDatabase(
+                clientArtifact.Value.artifact,
+                diskFiles,
+                fork,
+                metadata);
+
+            tx.Commit();
+
+            await QueueIngestJobAsync(fork);
+
+            logger.LogInformation("Publish succeeded!");
+
+            return NoContent();
+        }
+        catch
+        {
+            // Clean up after ourselves if something goes wrong.
+            Directory.Delete(versionDir, true);
+
+            throw;
+        }
+    }
+
+    private Dictionary<Artifact, string> ExtractZipToVersionDir(
+        List<(ZipArchiveEntry entry, Artifact artifact)> artifacts,
+        string versionDir)
+    {
+        logger.LogDebug("Extracting artifacts to directory {Directory}", versionDir);
+
+        var dict = new Dictionary<Artifact, string>();
+
+        foreach (var (entry, artifact) in artifacts)
+        {
+            if (!ValidFileRegex.IsMatch(entry.Name))
+            {
+                logger.LogTrace("Skipping artifact {Name}: invalid name", entry.FullName);
+                continue;
+            }
+
+            var filePath = Path.Combine(versionDir, entry.Name);
+            logger.LogTrace("Extracting artifact {Name}", entry.FullName);
+
+            using var entryStream = entry.Open();
+            using var file = System.IO.File.Create(filePath);
+
+            entryStream.CopyTo(file);
+            dict.Add(artifact, filePath);
+        }
+
+        return dict;
+    }
+}