Skip to content

Commit

Permalink
Merge #4153 Refactor ZIP importing
Browse files Browse the repository at this point in the history
  • Loading branch information
@HebaruSan
HebaruSan committed Aug 9, 2024
2 parents be75f8e + 8fbd354 commit 9bbff6b
Show file tree
Hide file tree
Showing 18 changed files with 260 additions and 173 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ All notable changes to this project will be documented in this file.
- [Multiple] Update changeset on Replace checkbox change, other `replaced_by` fixes (#4128 by: HebaruSan)
- [Multiple] Stop checking multiple hashes (#4135 by: HebaruSan)
- [Core] Fix max version column for wildcard compat (#4142 by: HebaruSan)
- [Multiple] Refactor ZIP importing (#4153 by: HebaruSan)

### Internal

Expand Down
8 changes: 5 additions & 3 deletions Core/Extensions/CryptoExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@ public static class CryptoExtensions
/// <param name="progress">Callback to notify as we traverse the input, called with percentages from 0 to 100</param>
/// <param name="cancelToken">A cancellation token that can be used to abort the hash</param>
/// <returns>The requested hash of the input stream</returns>
public static byte[] ComputeHash(this HashAlgorithm hashAlgo, Stream stream,
IProgress<long> progress, CancellationToken cancelToken = default)
public static byte[] ComputeHash(this HashAlgorithm hashAlgo,
Stream stream,
IProgress<int> progress,
CancellationToken cancelToken = default)
{
const int bufSize = 1024 * 1024;
var buffer = new byte[bufSize];
Expand All @@ -44,7 +46,7 @@ public static byte[] ComputeHash(this HashAlgorithm hashAlgo, Stream stream,
}

totalBytesRead += bytesRead;
progress.Report(100 * totalBytesRead / stream.Length);
progress.Report((int)(100 * totalBytesRead / stream.Length));
}
return hashAlgo.Hash;
}
Expand Down
183 changes: 121 additions & 62 deletions Core/ModuleInstaller.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public static string Download(CkanModule module, string filename, NetModuleCache
ServiceLocator.Container.Resolve<IConfiguration>().PreferredHosts))
.First());

return cache.Store(module, tmp_file, new Progress<long>(bytes => {}), filename, true);
return cache.Store(module, tmp_file, new Progress<int>(percent => {}), filename, true);
}

/// <summary>
Expand Down Expand Up @@ -1501,6 +1501,37 @@ public bool CanInstall(List<CkanModule> toInstall,

#endregion

private bool TryGetFileHashMatches(HashSet<FileInfo> files,
Registry registry,
out Dictionary<FileInfo, List<CkanModule>> matched,
out List<FileInfo> notFound,
IProgress<int> percentProgress)
{
matched = new Dictionary<FileInfo, List<CkanModule>>();
notFound = new List<FileInfo>();
var index = registry.GetDownloadHashesIndex();
var progress = new ProgressScalePercentsByFileSizes(percentProgress,
files.Select(fi => fi.Length));
foreach (var fi in files.Distinct())
{
if (index.TryGetValue(Cache.GetFileHashSha256(fi.FullName, progress),
out List<CkanModule> modules)
// The progress bar will jump back and "redo" the same span
// for non-matched files, but that's... OK?
|| index.TryGetValue(Cache.GetFileHashSha1(fi.FullName, progress),
out modules))
{
matched.Add(fi, modules);
}
else
{
notFound.Add(fi);
}
progress.NextFile();
}
return matched.Count > 0;
}

/// <summary>
/// Import a list of files into the download cache, with progress bar and
/// interactive prompts for installation and deletion.
Expand All @@ -1509,69 +1540,106 @@ public bool CanInstall(List<CkanModule> toInstall,
/// <param name="user">Object for user interaction</param>
/// <param name="installMod">Function to call to mark a mod for installation</param>
/// <param name="allowDelete">True to ask user whether to delete imported files, false to leave the files as is</param>
public void ImportFiles(HashSet<FileInfo> files, IUser user, Action<CkanModule> installMod, Registry registry, bool allowDelete = true)
public bool ImportFiles(HashSet<FileInfo> files,
IUser user,
Action<CkanModule> installMod,
Registry registry,
bool allowDelete = true)
{
HashSet<CkanModule> installable = new HashSet<CkanModule>();
List<FileInfo> deletable = new List<FileInfo>();
// Get the mapping of known hashes to modules
var index = registry.GetDownloadHashesIndex();
int i = 0;
foreach (FileInfo f in files)
if (!TryGetFileHashMatches(files, registry,
out Dictionary<FileInfo, List<CkanModule>> matched,
out List<FileInfo> notFound,
new Progress<int>(p =>
user.RaiseProgress(Properties.Resources.ModuleInstallerImportScanningFiles,
p))))
{
int percent = i * 100 / files.Count;
user.RaiseProgress(string.Format(Properties.Resources.ModuleInstallerImporting, f.Name, percent), percent);
// Find SHA-256 or SHA-1 sum in registry (potentially multiple)
if (index.TryGetValue(Cache.GetFileHashSha256(f.FullName, new Progress<long>(bytes => {})),
out List<CkanModule> matches)
|| index.TryGetValue(Cache.GetFileHashSha1(f.FullName, new Progress<long>(bytes => {})),
out matches))
{
deletable.Add(f);
foreach (var mod in matches)
{
if (mod.IsCompatible(instance.VersionCriteria()))
{
installable.Add(mod);
}
if (Cache.IsMaybeCachedZip(mod))
{
user.RaiseMessage(Properties.Resources.ModuleInstallerImportAlreadyCached, f.Name);
}
else
{
user.RaiseMessage(Properties.Resources.ModuleInstallerImportingMod,
mod.identifier, StripEpoch(mod.version));
Cache.Store(mod, f.FullName, new Progress<long>(bytes => {}));
}
}
}
else
// We're not going to do anything, so let the user know they failed
user.RaiseError(Properties.Resources.ModuleInstallerImportNotFound,
string.Join(", ", notFound.Select(fi => fi.Name)));
return false;
}

if (notFound.Count > 0)
{
// At least one was found, so just warn about the rest
user.RaiseMessage(" ");
user.RaiseMessage(Properties.Resources.ModuleInstallerImportNotFound,
string.Join(", ", notFound.Select(fi => fi.Name)));
}
var installable = matched.Values.SelectMany(modules => modules)
.Where(m => registry.IdentifierCompatible(m.identifier,
instance.VersionCriteria()))
.ToHashSet();

var deletable = matched.Keys.ToList();
var delete = allowDelete
&& deletable.Count > 0
&& user.RaiseYesNoDialog(string.Format(Properties.Resources.ModuleInstallerImportDeletePrompt,
deletable.Count));

// Store once per "primary" URL since each has its own URL hash
var cachedGroups = matched.SelectMany(kvp => kvp.Value.DistinctBy(m => m.download.First())
.Select(m => (File: kvp.Key,
Module: m)))
.GroupBy(tuple => Cache.IsMaybeCachedZip(tuple.Module))
.ToDictionary(grp => grp.Key,
grp => grp.ToArray());
if (cachedGroups.TryGetValue(true, out (FileInfo File, CkanModule Module)[] alreadyStored))
{
// Notify about files that are already cached
user.RaiseMessage(" ");
user.RaiseMessage(Properties.Resources.ModuleInstallerImportAlreadyCached,
string.Join(", ", alreadyStored.Select(tuple => $"{tuple.Module} ({tuple.File.Name})")));
}
if (cachedGroups.TryGetValue(false, out (FileInfo File, CkanModule Module)[] toStore))
{
// Store any new files
user.RaiseMessage(" ");
var description = "";
var progress = new ProgressScalePercentsByFileSizes(
new Progress<int>(p =>
user.RaiseProgress(string.Format(Properties.Resources.ModuleInstallerImporting,
description),
p)),
toStore.Select(tuple => tuple.File.Length));
foreach ((FileInfo fi, CkanModule module) in toStore)
{
user.RaiseMessage(Properties.Resources.ModuleInstallerImportNotFound, f.Name);

// Update the progress string
description = $"{module} ({fi.Name})";
Cache.Store(module, fi.FullName, progress,
// Move if user said we could delete and we don't need to make any more copies
move: delete && toStore.Last(tuple => tuple.File == fi).Module == module,
// Skip revalidation because we had to check the hashes to get here!
validate: false);
progress.NextFile();
}
++i;
}
if (installable.Count > 0 && user.RaiseYesNoDialog(string.Format(
Properties.Resources.ModuleInstallerImportInstallPrompt,
installable.Count, instance.Name, instance.GameDir())))

// Here we have installable containing mods that can be installed, and the importable files have been stored in cache.
if (installable.Count > 0
&& user.RaiseYesNoDialog(string.Format(Properties.Resources.ModuleInstallerImportInstallPrompt,
installable.Count,
instance.Name,
instance.GameDir())))
{
// Install the imported mods
foreach (CkanModule mod in installable)
foreach (var mod in installable)
{
installMod(mod);
}
}
if (allowDelete && deletable.Count > 0 && user.RaiseYesNoDialog(string.Format(
Properties.Resources.ModuleInstallerImportDeletePrompt, deletable.Count)))
if (delete)
{
// Delete old files
foreach (FileInfo f in deletable)
// Delete old files that weren't already moved into cache
foreach (var f in deletable.Where(f => f.Exists))
{
f.Delete();
}
}

EnforceCacheSizeLimit(registry);
return true;
}

private void EnforceCacheSizeLimit(Registry registry)
Expand All @@ -1588,47 +1656,38 @@ private void EnforceCacheSizeLimit(Registry registry)
/// Remove prepending v V. Version_ etc
/// </summary>
public static string StripV(string version)
{
Match match = Regex.Match(version, @"^(?<num>\d\:)?[vV]+(ersion)?[_.]*(?<ver>\d.*)$");

return match.Success
? match.Groups["num"].Value + match.Groups["ver"].Value
: version;
}
=> Regex.Match(version, @"^(?<num>\d\:)?[vV]+(ersion)?[_.]*(?<ver>\d.*)$") is Match match
&& match.Success
? match.Groups["num"].Value + match.Groups["ver"].Value
: version;

/// <summary>
/// Returns a version string shorn of any leading epoch as delimited by a single colon
/// </summary>
/// <param name="version">A version that might contain an epoch</param>
public static string StripEpoch(ModuleVersion version)
{
return StripEpoch(version.ToString());
}
=> StripEpoch(version.ToString());

/// <summary>
/// Returns a version string shorn of any leading epoch as delimited by a single colon
/// </summary>
/// <param name="version">A version string that might contain an epoch</param>
public static string StripEpoch(string version)
{
// If our version number starts with a string of digits, followed by
// a colon, and then has no more colons, we're probably safe to assume
// the first string of digits is an epoch
return epochMatch.IsMatch(version)
=> epochMatch.IsMatch(version)
? epochReplace.Replace(version, @"$2")
: version;
}

/// <summary>
/// As above, but includes the original in parentheses
/// </summary>
/// <param name="version">A version string that might contain an epoch</param>
public static string WithAndWithoutEpoch(string version)
{
return epochMatch.IsMatch(version)
=> epochMatch.IsMatch(version)
? $"{epochReplace.Replace(version, @"$2")} ({version})"
: version;
}

private static readonly Regex epochMatch = new Regex(@"^[0-9][0-9]*:[^:]+$", RegexOptions.Compiled);
private static readonly Regex epochReplace = new Regex(@"^([^:]+):([^:]+)$", RegexOptions.Compiled);
Expand Down
2 changes: 1 addition & 1 deletion Core/Net/NetAsyncModulesDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ private void ModuleDownloadComplete(NetAsyncDownloader.DownloadTarget target,
|| m.InternetArchiveDownload == url);
User.RaiseMessage(Properties.Resources.NetAsyncDownloaderValidating, module);
cache.Store(module, filename,
new Progress<long>(percent => StoreProgress?.Invoke(module, 100 - percent, 100)),
new Progress<int>(percent => StoreProgress?.Invoke(module, 100 - percent, 100)),
module.StandardName(),
false,
cancelTokenSrc.Token);
Expand Down
6 changes: 3 additions & 3 deletions Core/Net/NetFileCache.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ public static string CreateURLHash(Uri url)
/// <returns>
/// SHA1 hash, in all-caps hexadecimal format
/// </returns>
public string GetFileHashSha1(string filePath, IProgress<long> progress, CancellationToken cancelToken = default)
public string GetFileHashSha1(string filePath, IProgress<int> progress, CancellationToken cancelToken = default)
=> GetFileHash(filePath, "sha1", sha1Cache, SHA1.Create, progress, cancelToken);

/// <summary>
Expand All @@ -580,7 +580,7 @@ public string GetFileHashSha1(string filePath, IProgress<long> progress, Cancell
/// <returns>
/// SHA256 hash, in all-caps hexadecimal format
/// </returns>
public string GetFileHashSha256(string filePath, IProgress<long> progress, CancellationToken cancelToken = default)
public string GetFileHashSha256(string filePath, IProgress<int> progress, CancellationToken cancelToken = default)
=> GetFileHash(filePath, "sha256", sha256Cache, SHA256.Create, progress, cancelToken);

/// <summary>
Expand All @@ -595,7 +595,7 @@ private string GetFileHash(string filePath,
string hashSuffix,
Dictionary<string, string> cache,
Func<HashAlgorithm> getHashAlgo,
IProgress<long> progress,
IProgress<int> progress,
CancellationToken cancelToken)
{
string hashFile = $"{filePath}.{hashSuffix}";
Expand Down
Loading

0 comments on commit 9bbff6b

Please sign in to comment.