Skip to content

Commit

Permalink
feat: add convert task
Browse files Browse the repository at this point in the history
  • Loading branch information
anna-is-cute committed Aug 17, 2024
1 parent c849eea commit c208698
Show file tree
Hide file tree
Showing 6 changed files with 236 additions and 4 deletions.
168 changes: 168 additions & 0 deletions ConvertTask.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
using System.Security;
using System.Text;
using Blake3;
using Dalamud.Interface.ImGuiNotification;
using gfoidl.Base64;
using Heliosphere.Model;
using Heliosphere.Util;
using Newtonsoft.Json;
using StrawberryShake;
using Windows.Win32;

namespace Heliosphere;

internal class ConvertTask {
private HeliosphereMeta Package { get; }
private IActiveNotification Notification { get; }

internal ConvertTask(HeliosphereMeta pkg, IActiveNotification notification) {
this.Package = pkg;
this.Notification = notification;
}

internal async Task Run() {
if (!Plugin.Instance.DownloadCodes.TryGetCode(this.Package.Id, out var code)) {
code = null;
}

this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Content = "Downloading package information";
});

var resp = await Plugin.GraphQl.ConvertTask.ExecuteAsync(
this.Package.VersionId,
this.Package.SelectedOptions,
code,
this.Package.FullInstall,
Model.Generated.DownloadKind.Update
);
resp.EnsureNoErrors();

var neededFiles = resp.Data?.GetVersion?.NeededFiles.Files.Files;
if (neededFiles == null) {
throw new Exception("TODO");
}

if (!Plugin.Instance.Penumbra.TryGetModDirectory(out var modDirectory)) {
throw new Exception("TODO");
}

var dirName = this.Package.ModDirectoryName();
var penumbraModPath = Path.Join(modDirectory, dirName);
var filesPath = Path.Join(penumbraModPath, "files");

this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Content = "Checking existing files";
notif.Progress = 0;
});

// gather a list of all files in the files directory
var installedFiles = Directory.EnumerateFileSystemEntries(filesPath, "*", SearchOption.AllDirectories)
.Where(path => (File.GetAttributes(path) & FileAttributes.Directory) == 0)
.Select(Path.GetFileName)
.Where(path => path != null)
.Cast<string>()
.ToHashSet();

// create a mapping of each file and its hash (hash => path)
var finished = 0;
var total = installedFiles.Count;
var hashPaths = new Dictionary<string, string>();
using var blake3 = new Blake3HashAlgorithm();
foreach (var path in installedFiles) {
blake3.Initialize();
var file = FileHelper.OpenSharedReadIfExists(Path.Join(filesPath, path));
if (file == null) {
continue;
}

var expected = PathHelper.GetBaseName(path);
if (hashPaths.ContainsKey(expected)) {
continue;
}

var rawHash = await blake3.ComputeHashAsync(file);
var hash = Base64.Url.Encode(rawHash);
hashPaths[hash] = path;

finished += 1;
this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Progress = (float) finished / total;
});
}

this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Content = "Converting file layout";
notif.Progress = 0;
});

finished = 0;
total = neededFiles.Count;

// loop through all needed files and find their corresponding hash,
// then link each output path to it
foreach (var (hash, files) in neededFiles) {
if (!hashPaths.TryGetValue(hash, out var existingPath)) {
Plugin.Log.Warning($"missing a file for {hash}");
continue;
}

existingPath = Path.Join(filesPath, existingPath);
var outputPaths = DownloadTask.GetOutputPaths(files);
foreach (var shortOutputPath in outputPaths) {
var outputPath = Path.Join(filesPath, shortOutputPath);
if (PathHelper.MakeRelativeSub(filesPath, Path.GetFullPath(outputPath)) == null) {
throw new SecurityException("path from mod was attempting to leave the files directory");
}

var parent = PathHelper.GetParent(outputPath);
Directory.CreateDirectory(parent);

if (!PInvoke.CreateHardLink(@$"\\?\{outputPath}", @$"\\?\{existingPath}")) {
throw new IOException($"failed to create hard link: {existingPath} -> {outputPath}");
}
}

finished += 1;
this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Progress = (float) finished / total;
});
}

this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Content = "Removing old files";
notif.Progress = 0;
});

finished = 0;
total = installedFiles.Count;

// delete all previously-existing files
foreach (var path in installedFiles) {
File.Delete(path);

finished += 1;
this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Progress = (float) total / finished;
});
}

this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Content = "Updating package metadata";
notif.Progress = 0;
});

// update package
this.Package.FileStorageMethod = FileStorageMethod.Original;
var json = JsonConvert.SerializeObject(this.Package, Formatting.Indented);
var metaPath = Path.Join(penumbraModPath, "heliosphere.json");
await using var metaFile = FileHelper.Create(metaPath);
await metaFile.WriteAsync(Encoding.UTF8.GetBytes(json));

this.Notification.AddOrUpdate(Plugin.Instance.NotificationManager, (notif, _) => {
notif.Type = NotificationType.Success;
notif.Content = "Successfully converted file layout";
notif.InitialDuration = TimeSpan.FromSeconds(3);
});
}
}
4 changes: 2 additions & 2 deletions DownloadTask.cs
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,7 @@ private async Task<IEnumerable<Task>> DownloadBatchedFiles(IDownloadTask_GetVers
var (batch, batchedFiles) = pair;
// determine which pre-existing files to duplicate in this batch
var toDuplicate = new List<string>();
var toDuplicate = new HashSet<string>();
foreach (var (hash, path) in installedHashes) {
if (!batchedFiles.ContainsKey(hash)) {
continue;
Expand Down Expand Up @@ -683,7 +683,7 @@ private static string MakeFileNameSafe(string input) {
return sb.ToString();
}

private static string[] GetOutputPaths(IReadOnlyCollection<List<string?>> files) {
internal static string[] GetOutputPaths(IReadOnlyCollection<List<string?>> files) {
return files
.Select(file => {
var outputPath = file[3];
Expand Down
27 changes: 25 additions & 2 deletions Model/HeliosphereMeta.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ namespace Heliosphere.Model;

[Serializable]
internal class HeliosphereMeta {
internal const uint LatestVersion = 2;
internal const uint LatestVersion = 3;

public uint MetaVersion { get; set; } = LatestVersion;

Expand Down Expand Up @@ -40,6 +40,8 @@ internal class HeliosphereMeta {

public string? ModHash { get; set; }

public FileStorageMethod FileStorageMethod { get; set; }

internal string ErrorName => $"{this.Name} v{this.Version} (P:{this.Id.ToCrockford()} Va:{this.VariantId.ToCrockford()} Ve:{this.VersionId.ToCrockford()})";

internal static async Task<HeliosphereMeta?> Load(string path) {
Expand All @@ -60,13 +62,21 @@ internal class HeliosphereMeta {
}

private static async Task<(HeliosphereMeta, bool)> Convert(JObject config) {
var changed = await RunMigrations(config);
return (config.ToObject<HeliosphereMeta>()!, changed);
}

private static async Task<bool> RunMigrations(JObject config) {
var version = GetVersion();
var changed = false;
while (version < LatestVersion) {
switch (version) {
case 1:
await MigrateV1(config);
break;
case 2:
MigrateV2(config);
break;
default:
throw new Exception("Invalid Heliosphere meta - unknown version");
}
Expand All @@ -76,7 +86,7 @@ internal class HeliosphereMeta {
}

if (version == LatestVersion) {
return (config.ToObject<HeliosphereMeta>()!, changed);
return changed;
}

throw new Exception("Could not migrate Heliosphere meta version");
Expand Down Expand Up @@ -138,6 +148,14 @@ private static async Task MigrateV1(JObject config) {
config[nameof(MetaVersion)] = 2u;
}

private static void MigrateV2(JObject config) {
if (!config.ContainsKey(nameof(FileStorageMethod))) {
config[nameof(FileStorageMethod)] = (int) FileStorageMethod.Hash;
}

config[nameof(MetaVersion)] = 3u;
}

internal bool IsSimple() {
return this.FullInstall && this.SelectedOptions.Count == 0;
}
Expand Down Expand Up @@ -260,3 +278,8 @@ internal readonly record struct HeliosphereDirectoryInfo(
Guid VariantId,
string Version
);

internal enum FileStorageMethod {
Hash,
Original,
}
33 changes: 33 additions & 0 deletions Ui/Tabs/Manager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ internal class Manager : IDisposable {

private bool _downloadingUpdates;
private bool _checkingForUpdates;
private bool _converting;
private string _filter = string.Empty;
private bool _forced;

Expand Down Expand Up @@ -384,6 +385,38 @@ await InstallerWindow.OpenAndAdd(new InstallerWindow.OpenOptions {
this.Plugin.Penumbra.OpenMod(pkg.ModDirectoryName());
}

if (pkg.FileStorageMethod == FileStorageMethod.Hash) {
using var disabled = ImGuiHelper.DisabledIf(this._converting);
if (ImGuiHelper.CentredWideButton("Convert to original file layout") && !this._converting) {
this._converting = true;
Task.Run(async () => {
try {
var notif = this.Plugin.NotificationManager.AddNotification(new Notification {
Type = NotificationType.Info,
Title = pkg.Name,
Content = "Converting mod to original file layout...",
InitialDuration = TimeSpan.MaxValue,
Minimized = false,
});
try {
await new ConvertTask(pkg, notif).Run();
} catch (Exception ex) {
Plugin.Log.Error(ex, "Failed to convert package");
notif.AddOrUpdate(this.Plugin.NotificationManager, (notif, _) => {
notif.InitialDuration = TimeSpan.FromSeconds(5);
notif.Type = NotificationType.Error;
notif.Content = "An error occured while converting.";
});
}
await this.Plugin.State.UpdatePackages();
} finally {
this._converting = false;
}
});
}
}

if (ImGuiHelper.CentredWideButton("Open on Heliosphere website")) {
var url = $"https://heliosphere.app/mod/{pkg.Id.ToCrockford()}";
Process.Start(new ProcessStartInfo(url) {
Expand Down
1 change: 1 addition & 0 deletions heliosphere-plugin.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,4 @@ tags:
- textools
repo_url: https://heliosphere.app/
icon_url: https://repo.heliosphere.app/icon.png
accepts_feedback: false
7 changes: 7 additions & 0 deletions queries/ConvertTask.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
query ConvertTask($versionId: UUID!, $options: Options, $key: String, $full: Boolean, $downloadKind: DownloadKind!) {
getVersion(id: $versionId) {
neededFiles(options: $options, full: $full, downloadKey: $key, downloadKind: $downloadKind) {
files
}
}
}

0 comments on commit c208698

Please sign in to comment.