From c20869817e47d5a62d8c7da2e18000a6c4a952ed Mon Sep 17 00:00:00 2001 From: Anna Date: Sat, 17 Aug 2024 02:30:12 -0400 Subject: [PATCH] feat: add convert task --- ConvertTask.cs | 168 ++++++++++++++++++++++++++++++++++++ DownloadTask.cs | 4 +- Model/HeliosphereMeta.cs | 27 +++++- Ui/Tabs/Manager.cs | 33 +++++++ heliosphere-plugin.yaml | 1 + queries/ConvertTask.graphql | 7 ++ 6 files changed, 236 insertions(+), 4 deletions(-) create mode 100644 ConvertTask.cs create mode 100644 queries/ConvertTask.graphql diff --git a/ConvertTask.cs b/ConvertTask.cs new file mode 100644 index 0000000..02e76cf --- /dev/null +++ b/ConvertTask.cs @@ -0,0 +1,168 @@ +using System.Security; +using System.Text; +using Blake3; +using Dalamud.Interface.ImGuiNotification; +using gfoidl.Base64; +using Heliosphere.Model; +using Heliosphere.Util; +using Newtonsoft.Json; +using StrawberryShake; +using Windows.Win32; + +namespace Heliosphere; + +internal class ConvertTask { + private HeliosphereMeta Package { get; } + private IActiveNotification Notification { get; } + + internal ConvertTask(HeliosphereMeta pkg, IActiveNotification notification) { + this.Package = pkg; + this.Notification = notification; + } + + internal async Task Run() { + if (!Plugin.Instance.DownloadCodes.TryGetCode(this.Package.Id, out var code)) { + code = null; + } + + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Content = "Downloading package information"; + }); + + var resp = await Plugin.GraphQl.ConvertTask.ExecuteAsync( + this.Package.VersionId, + this.Package.SelectedOptions, + code, + this.Package.FullInstall, + Model.Generated.DownloadKind.Update + ); + resp.EnsureNoErrors(); + + var neededFiles = resp.Data?.GetVersion?.NeededFiles.Files.Files; + if (neededFiles == null) { + throw new Exception("TODO"); + } + + if (!Plugin.Instance.Penumbra.TryGetModDirectory(out var modDirectory)) { + throw new Exception("TODO"); + } + + var dirName = this.Package.ModDirectoryName(); + var penumbraModPath = Path.Join(modDirectory, dirName); + var filesPath = Path.Join(penumbraModPath, "files"); + + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Content = "Checking existing files"; + notif.Progress = 0; + }); + + // gather a list of all files in the files directory + var installedFiles = Directory.EnumerateFileSystemEntries(filesPath, "*", SearchOption.AllDirectories) + .Where(path => (File.GetAttributes(path) & FileAttributes.Directory) == 0) + .Select(Path.GetFileName) + .Where(path => path != null) + .Cast() + .ToHashSet(); + + // create a mapping of each file and its hash (hash => path) + var finished = 0; + var total = installedFiles.Count; + var hashPaths = new Dictionary(); + using var blake3 = new Blake3HashAlgorithm(); + foreach (var path in installedFiles) { + blake3.Initialize(); + var file = FileHelper.OpenSharedReadIfExists(Path.Join(filesPath, path)); + if (file == null) { + continue; + } + + var expected = PathHelper.GetBaseName(path); + if (hashPaths.ContainsKey(expected)) { + continue; + } + + var rawHash = await blake3.ComputeHashAsync(file); + var hash = Base64.Url.Encode(rawHash); + hashPaths[hash] = path; + + finished += 1; + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Progress = (float) finished / total; + }); + } + + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Content = "Converting file layout"; + notif.Progress = 0; + }); + + finished = 0; + total = neededFiles.Count; + + // loop through all needed files and find their corresponding hash, + // then link each output path to it + foreach (var (hash, files) in neededFiles) { + if (!hashPaths.TryGetValue(hash, out var existingPath)) { + Plugin.Log.Warning($"missing a file for {hash}"); + continue; + } + + existingPath = Path.Join(filesPath, existingPath); + var outputPaths = DownloadTask.GetOutputPaths(files); + foreach (var shortOutputPath in outputPaths) { + var outputPath = Path.Join(filesPath, shortOutputPath); + if (PathHelper.MakeRelativeSub(filesPath, Path.GetFullPath(outputPath)) == null) { + throw new SecurityException("path from mod was attempting to leave the files directory"); + } + + var parent = PathHelper.GetParent(outputPath); + Directory.CreateDirectory(parent); + + if (!PInvoke.CreateHardLink(@$"\\?\{outputPath}", @$"\\?\{existingPath}")) { + throw new IOException($"failed to create hard link: {existingPath} -> {outputPath}"); + } + } + + finished += 1; + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Progress = (float) finished / total; + }); + } + + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Content = "Removing old files"; + notif.Progress = 0; + }); + + finished = 0; + total = installedFiles.Count; + + // delete all previously-existing files + foreach (var path in installedFiles) { + File.Delete(path); + + finished += 1; + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Progress = (float) total / finished; + }); + } + + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Content = "Updating package metadata"; + notif.Progress = 0; + }); + + // update package + this.Package.FileStorageMethod = FileStorageMethod.Original; + var json = JsonConvert.SerializeObject(this.Package, Formatting.Indented); + var metaPath = Path.Join(penumbraModPath, "heliosphere.json"); + await using var metaFile = FileHelper.Create(metaPath); + await metaFile.WriteAsync(Encoding.UTF8.GetBytes(json)); + + this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => { + notif.Type = NotificationType.Success; + notif.Content = "Successfully converted file layout"; + notif.InitialDuration = TimeSpan.FromSeconds(3); + }); + } +} diff --git a/DownloadTask.cs b/DownloadTask.cs index f9411ed..9a08cbc 100644 --- a/DownloadTask.cs +++ b/DownloadTask.cs @@ -417,7 +417,7 @@ private async Task> DownloadBatchedFiles(IDownloadTask_GetVers var (batch, batchedFiles) = pair; // determine which pre-existing files to duplicate in this batch - var toDuplicate = new List(); + var toDuplicate = new HashSet(); foreach (var (hash, path) in installedHashes) { if (!batchedFiles.ContainsKey(hash)) { continue; @@ -683,7 +683,7 @@ private static string MakeFileNameSafe(string input) { return sb.ToString(); } - private static string[] GetOutputPaths(IReadOnlyCollection> files) { + internal static string[] GetOutputPaths(IReadOnlyCollection> files) { return files .Select(file => { var outputPath = file[3]; diff --git a/Model/HeliosphereMeta.cs b/Model/HeliosphereMeta.cs index 6c0312b..ae276c7 100644 --- a/Model/HeliosphereMeta.cs +++ b/Model/HeliosphereMeta.cs @@ -12,7 +12,7 @@ namespace Heliosphere.Model; [Serializable] internal class HeliosphereMeta { - internal const uint LatestVersion = 2; + internal const uint LatestVersion = 3; public uint MetaVersion { get; set; } = LatestVersion; @@ -40,6 +40,8 @@ internal class HeliosphereMeta { public string? ModHash { get; set; } + public FileStorageMethod FileStorageMethod { get; set; } + internal string ErrorName => $"{this.Name} v{this.Version} (P:{this.Id.ToCrockford()} Va:{this.VariantId.ToCrockford()} Ve:{this.VersionId.ToCrockford()})"; internal static async Task Load(string path) { @@ -60,6 +62,11 @@ internal class HeliosphereMeta { } private static async Task<(HeliosphereMeta, bool)> Convert(JObject config) { + var changed = await RunMigrations(config); + return (config.ToObject()!, changed); + } + + private static async Task RunMigrations(JObject config) { var version = GetVersion(); var changed = false; while (version < LatestVersion) { @@ -67,6 +74,9 @@ internal class HeliosphereMeta { case 1: await MigrateV1(config); break; + case 2: + MigrateV2(config); + break; default: throw new Exception("Invalid Heliosphere meta - unknown version"); } @@ -76,7 +86,7 @@ internal class HeliosphereMeta { } if (version == LatestVersion) { - return (config.ToObject()!, changed); + return changed; } throw new Exception("Could not migrate Heliosphere meta version"); @@ -138,6 +148,14 @@ private static async Task MigrateV1(JObject config) { config[nameof(MetaVersion)] = 2u; } + private static void MigrateV2(JObject config) { + if (!config.ContainsKey(nameof(FileStorageMethod))) { + config[nameof(FileStorageMethod)] = (int) FileStorageMethod.Hash; + } + + config[nameof(MetaVersion)] = 3u; + } + internal bool IsSimple() { return this.FullInstall && this.SelectedOptions.Count == 0; } @@ -260,3 +278,8 @@ internal readonly record struct HeliosphereDirectoryInfo( Guid VariantId, string Version ); + +internal enum FileStorageMethod { + Hash, + Original, +} \ No newline at end of file diff --git a/Ui/Tabs/Manager.cs b/Ui/Tabs/Manager.cs index e60527a..ba28ee0 100644 --- a/Ui/Tabs/Manager.cs +++ b/Ui/Tabs/Manager.cs @@ -29,6 +29,7 @@ internal class Manager : IDisposable { private bool _downloadingUpdates; private bool _checkingForUpdates; + private bool _converting; private string _filter = string.Empty; private bool _forced; @@ -384,6 +385,38 @@ await InstallerWindow.OpenAndAdd(new InstallerWindow.OpenOptions { this.Plugin.Penumbra.OpenMod(pkg.ModDirectoryName()); } + if (pkg.FileStorageMethod == FileStorageMethod.Hash) { + using var disabled = ImGuiHelper.DisabledIf(this._converting); + if (ImGuiHelper.CentredWideButton("Convert to original file layout") && !this._converting) { + this._converting = true; + Task.Run(async () => { + try { + var notif = this.Plugin.NotificationManager.AddNotification(new Notification { + Type = NotificationType.Info, + Title = pkg.Name, + Content = "Converting mod to original file layout...", + InitialDuration = TimeSpan.MaxValue, + Minimized = false, + }); + + try { + await new ConvertTask(pkg, notif).Run(); + } catch (Exception ex) { + Plugin.Log.Error(ex, "Failed to convert package"); + notif.AddOrUpdate(this.Plugin.NotificationManager, (notif, _) => { + notif.InitialDuration = TimeSpan.FromSeconds(5); + notif.Type = NotificationType.Error; + notif.Content = "An error occured while converting."; + }); + } + await this.Plugin.State.UpdatePackages(); + } finally { + this._converting = false; + } + }); + } + } + if (ImGuiHelper.CentredWideButton("Open on Heliosphere website")) { var url = $"https://heliosphere.app/mod/{pkg.Id.ToCrockford()}"; Process.Start(new ProcessStartInfo(url) { diff --git a/heliosphere-plugin.yaml b/heliosphere-plugin.yaml index 4a3b05b..9d16653 100644 --- a/heliosphere-plugin.yaml +++ b/heliosphere-plugin.yaml @@ -24,3 +24,4 @@ tags: - textools repo_url: https://heliosphere.app/ icon_url: https://repo.heliosphere.app/icon.png +accepts_feedback: false diff --git a/queries/ConvertTask.graphql b/queries/ConvertTask.graphql new file mode 100644 index 0000000..0a99514 --- /dev/null +++ b/queries/ConvertTask.graphql @@ -0,0 +1,7 @@ +query ConvertTask($versionId: UUID!, $options: Options, $key: String, $full: Boolean, $downloadKind: DownloadKind!) { + getVersion(id: $versionId) { + neededFiles(options: $options, full: $full, downloadKey: $key, downloadKind: $downloadKind) { + files + } + } +}