From 6e4c3f3b7e5d13f80e0aae6bf0d5335e9a86372f Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 2 Sep 2024 20:41:56 -0400 Subject: [PATCH] feat: handle potential race conditions during download Prevent a source file from being overwritten during duplication. --- DownloadTask.cs | 105 +++++++++++++++++++++++------------------------- ImportTask.cs | 2 +- 2 files changed, 51 insertions(+), 56 deletions(-) diff --git a/DownloadTask.cs b/DownloadTask.cs index 030aad9..188a9ef 100644 --- a/DownloadTask.cs +++ b/DownloadTask.cs @@ -46,6 +46,8 @@ internal class DownloadTask : IDisposable { internal required IActiveNotification? Notification { get; set; } private string? PenumbraModPath { get; set; } + private string? FilesPath { get; set; } + private string? HashesPath { get; set; } internal string? PackageName { get; private set; } internal string? VariantName { get; private set; } @@ -59,15 +61,8 @@ internal class DownloadTask : IDisposable { private bool SupportsHardLinks { get; set; } private SemaphoreSlim DuplicateMutex { get; } = new(1, 1); - /// - /// A mapping of existing file paths to their hashes. Paths are relative. - /// - private IReadOnlyDictionary ExistingPathHashes { get; set; } = new Dictionary(); - /// - /// A mapping of existing hashes to their file paths. Paths are relative. - /// - private IReadOnlyDictionary ExistingHashPaths { get; set; } = new Dictionary(); + private HashSet ExistingHashes { get; } = []; /// /// A list of files expected by the group jsons made by this task. These @@ -177,8 +172,9 @@ private async Task Run() { await this.HashExistingFiles(); await this.DownloadFiles(info); await this.ConstructModPack(info); - await this.AddMod(info); + this.RemoveWorkingDirectories(); this.RemoveOldFiles(); + await this.AddMod(info); // before setting state to finished, set the directory name @@ -300,6 +296,12 @@ private async Task GetPackageInfo() { private void GenerateModDirectoryPath(IDownloadTask_GetVersion info) { var dirName = HeliosphereMeta.ModDirectoryName(info.Variant.Package.Id, info.Variant.Package.Name, info.Version, info.Variant.Id); this.PenumbraModPath = Path.Join(this.ModDirectory, dirName); + this.FilesPath = Path.GetFullPath(Path.Join(this.PenumbraModPath, "files")); + this.HashesPath = Path.GetFullPath(Path.Join(this.PenumbraModPath, ".hs-hashes")); + + Plugin.Resilience.Execute(() => Directory.CreateDirectory(this.FilesPath)); + var di = Plugin.Resilience.Execute(() => Directory.CreateDirectory(this.HashesPath)); + di.Attributes |= FileAttributes.Hidden; } private async Task TestHardLinks() { @@ -364,14 +366,13 @@ private async Task HashExistingFiles() { this.State = State.CheckingExistingFiles; this.SetStateData(0, 0); - var filesPath = Path.Join(this.PenumbraModPath, "files"); - if (!Directory.Exists(filesPath)) { - return; + if (this.FilesPath == null) { + throw new Exception("files path was null"); } // path => hash var hashes = new ConcurrentDictionary(); - var allFiles = DirectoryHelper.GetFilesRecursive(filesPath).ToList(); + var allFiles = DirectoryHelper.GetFilesRecursive(this.FilesPath).ToList(); this.StateDataMax = (uint) allFiles.Count; @@ -391,37 +392,31 @@ await Parallel.ForEachAsync( await blake3.ComputeHashAsync(file, token); var hash = Base64.Url.Encode(blake3.Hash); - var relativePath = PathHelper.MakeRelativeSub(filesPath, path); - if (relativePath == null) { - throw new Exception($"path was not relative: {path}"); - } - - hashes.TryAdd(relativePath, hash); + hashes.TryAdd(path, hash); this.StateData += 1; } ); - this.ExistingPathHashes = hashes.ToImmutableDictionary(); - - var existingHashPaths = new Dictionary(); - foreach (var (path, hash) in this.ExistingPathHashes) { - existingHashPaths[hash] = path; + Action action = this.SupportsHardLinks + ? FileHelper.CreateHardLink + : File.Move; + foreach (var (path, hash) in hashes) { + // move/link each path to the hashes path + Plugin.Resilience.Execute(() => action( + path, + Path.Join(this.HashesPath, hash) + )); + + this.ExistingHashes.Add(hash); } - - this.ExistingHashPaths = existingHashPaths.ToImmutableDictionary(); } private async Task DownloadFiles(IDownloadTask_GetVersion info) { using var span = this.Transaction?.StartChild(nameof(this.DownloadFiles)); - var filesPath = Path.Join(this.PenumbraModPath, "files"); - if (!await PathHelper.CreateDirectory(filesPath)) { - throw new DirectoryNotFoundException($"Directory '{filesPath}' could not be found after waiting"); - } - var task = info.Batched - ? this.DownloadBatchedFiles(info.NeededFiles, info.Batches, filesPath) - : this.DownloadNormalFiles(info.NeededFiles, filesPath); + ? this.DownloadBatchedFiles(info.NeededFiles, info.Batches, this.FilesPath!) + : this.DownloadNormalFiles(info.NeededFiles, this.FilesPath!); await task; } @@ -483,17 +478,17 @@ private Task DownloadBatchedFiles(IDownloadTask_GetVersion_NeededFiles neededFil // find which files from this batch we already have a hash for var toDuplicate = new HashSet(); foreach (var hash in batchedFiles.Keys) { - if (!this.ExistingHashPaths.TryGetValue(hash, out var path)) { + if (!this.ExistingHashes.Contains(hash)) { continue; } - toDuplicate.Add(path); + toDuplicate.Add(hash); } // sort files in batch by offset, removing already-downloaded files var listOfFiles = batchedFiles .Select(pair => (Hash: pair.Key, Info: pair.Value)) - .Where(pair => !this.ExistingHashPaths.ContainsKey(pair.Hash)) + .Where(pair => !this.ExistingHashes.Contains(pair.Hash)) .OrderBy(pair => pair.Info.Offset).ToList(); if (listOfFiles.Count > 0) { @@ -585,18 +580,13 @@ await Plugin.Resilience.ExecuteAsync( } } - foreach (var path in toDuplicate) { - var joined = Path.Join(filesPath, path); - + foreach (var hash in toDuplicate) { + var joined = Path.Join(this.HashesPath, hash); if (!File.Exists(joined)) { Plugin.Log.Warning($"{joined} was supposed to be duplicated but no longer exists"); continue; } - if (!this.ExistingPathHashes.TryGetValue(path, out var hash)) { - throw new Exception("missing hash for file to duplicate"); - } - var gamePaths = neededFiles.Files.Files[hash]; var outputPaths = GetOutputPaths(gamePaths); @@ -838,10 +828,8 @@ private void RemoveOldFiles() { this.SetStateData(0, 0); // find old, normal files no longer being used to remove - var filesPath = Path.Join(this.PenumbraModPath, "files"); - - var presentFiles = DirectoryHelper.GetFilesRecursive(filesPath) - .Select(path => PathHelper.MakeRelativeSub(filesPath, path)) + var presentFiles = DirectoryHelper.GetFilesRecursive(this.FilesPath!) + .Select(path => PathHelper.MakeRelativeSub(this.FilesPath!, path)) .Where(path => !string.IsNullOrEmpty(path)) .Cast() .Select(path => path.ToLowerInvariant()) @@ -856,7 +844,7 @@ private void RemoveOldFiles() { var done = 0u; foreach (var extra in presentFiles) { - var extraPath = Path.Join(filesPath, extra); + var extraPath = Path.Join(this.FilesPath, extra); Plugin.Log.Info($"removing extra file {extraPath}"); Plugin.Resilience.Execute(() => FileHelper.Delete(extraPath)); @@ -865,7 +853,7 @@ private void RemoveOldFiles() { } // remove any empty directories - DirectoryHelper.RemoveEmptyDirectories(filesPath); + DirectoryHelper.RemoveEmptyDirectories(this.FilesPath!); } private async Task DownloadFile(Uri baseUri, string filesPath, string[] outputPaths, string hash) { @@ -889,8 +877,9 @@ private async Task DownloadFile(Uri baseUri, string filesPath, string[] outputPa } // find an existing path that has this hash - if (this.ExistingHashPaths.TryGetValue(hash, out var validPath)) { - validPath = Path.Join(filesPath, validPath); + string validPath; + if (this.ExistingHashes.Contains(hash)) { + validPath = Path.Join(this.HashesPath, hash); goto Duplicate; } @@ -1536,8 +1525,6 @@ private async Task DuplicateUiFiles(DefaultMod defaultMod, List modGro // each reference. const string uiPrefix = "ui/"; - var filesPath = Path.Join(this.PenumbraModPath, "files"); - // first record unique references var references = new Dictionary>)>(); UpdateReferences(defaultMod.Files); @@ -1568,11 +1555,11 @@ private async Task DuplicateUiFiles(DefaultMod defaultMod, List modGro ? FileHelper.CreateHardLink : File.Copy; - var src = Path.Join(filesPath, outputPath); + var src = Path.Join(this.FilesPath, outputPath); for (var i = 0; i < refs; i++) { var ext = $".{i + 1}" + Path.GetExtension(outputPath); var newRelative = Path.ChangeExtension(outputPath, ext); - var dst = Path.Join(filesPath, newRelative); + var dst = Path.Join(this.FilesPath, newRelative); FileHelper.DeleteIfExists(dst); @@ -1620,6 +1607,14 @@ void UpdateReferences(Dictionary files) { } } + private void RemoveWorkingDirectories() { + if (this.HashesPath == null) { + return; + } + + Plugin.Resilience.Execute(() => Directory.Delete(this.HashesPath, true)); + } + private async Task AddMod(IDownloadTask_GetVersion info) { using var span = this.Transaction?.StartChild(nameof(this.AddMod)); diff --git a/ImportTask.cs b/ImportTask.cs index f1d2078..63cd6b3 100644 --- a/ImportTask.cs +++ b/ImportTask.cs @@ -186,7 +186,7 @@ private void Rename() { this.StateMax = this.Data!.Files.Have; // first create the files directory - var filesPath = Path.Join(this._fullDirectory!, "files"); + var filesPath = Path.GetFullPath(Path.Join(this._fullDirectory!, "files")); Directory.CreateDirectory(filesPath); // rename all the files we have and need to their hashes