Skip to content

Commit

Permalink
Skip temp files for repo updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@HebaruSan
HebaruSan committed Jun 6, 2024
1 parent 101b88d commit 298dbe7
Show file tree
Hide file tree
Showing 13 changed files with 226 additions and 101 deletions.
4 changes: 2 additions & 2 deletions Cmdline/Action/Install.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options)
try
{
var targets = options.ckan_files
.Select(arg => new NetAsyncDownloader.DownloadTarget(getUri(arg)))
.ToList();
.Select(arg => new NetAsyncDownloader.DownloadTargetFile(getUri(arg)))
.ToArray();
log.DebugFormat("Urls: {0}", targets.SelectMany(t => t.urls));
new NetAsyncDownloader(new NullUser()).DownloadAndWait(targets);
log.DebugFormat("Files: {0}", targets.Select(t => t.filename));
Expand Down
4 changes: 2 additions & 2 deletions Core/AutoUpdate/GithubReleaseCkanUpdate.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ public GitHubReleaseCkanUpdate(GitHubReleaseInfo releaseJson = null)

public override IList<NetAsyncDownloader.DownloadTarget> Targets => new[]
{
new NetAsyncDownloader.DownloadTarget(
new NetAsyncDownloader.DownloadTargetFile(
UpdaterDownload, updaterFilename, UpdaterSize),
new NetAsyncDownloader.DownloadTarget(
new NetAsyncDownloader.DownloadTargetFile(
ReleaseDownload, ckanFilename, ReleaseSize),
};

Expand Down
4 changes: 2 additions & 2 deletions Core/AutoUpdate/S3BuildCkanUpdate.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ public S3BuildCkanUpdate(S3BuildVersionInfo versionJson = null)

public override IList<NetAsyncDownloader.DownloadTarget> Targets => new[]
{
new NetAsyncDownloader.DownloadTarget(
new NetAsyncDownloader.DownloadTargetFile(
UpdaterDownload, updaterFilename),
new NetAsyncDownloader.DownloadTarget(
new NetAsyncDownloader.DownloadTargetFile(
ReleaseDownload, ckanFilename),
};

Expand Down
11 changes: 6 additions & 5 deletions Core/FileIdentifier.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System.IO;
using System.Linq;

using ICSharpCode.SharpZipLib.GZip;

using CKAN.Extensions;

namespace CKAN
Expand Down Expand Up @@ -173,11 +175,10 @@ public static FileType IdentifyFile(Stream stream)
if (CheckGZip(stream))
{
// This may contain a tar file inside, create a new stream and check.
stream.Seek (0, SeekOrigin.Begin);
using (ICSharpCode.SharpZipLib.GZip.GZipInputStream gz_stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream (stream))
{
type = CheckTar(gz_stream) ? FileType.TarGz : FileType.GZip;
}
stream.Seek(0, SeekOrigin.Begin);
type = CheckTar(new GZipInputStream(stream))
? FileType.TarGz
: FileType.GZip;
}
else if (CheckTar(stream))
{
Expand Down
8 changes: 3 additions & 5 deletions Core/Net/NetAsyncDownloader.DownloadPart.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System;
using System.IO;
using System.ComponentModel;

using Autofac;
Expand All @@ -14,7 +13,6 @@ public partial class NetAsyncDownloader
private class DownloadPart
{
public readonly DownloadTarget target;
public readonly string path;

public DateTime lastProgressUpdateTime;
public long lastProgressUpdateSize;
Expand All @@ -38,14 +36,14 @@ private class DownloadPart
public DownloadPart(DownloadTarget target)
{
this.target = target;
path = target.filename ?? Path.GetTempFileName();
size = bytesLeft = target.size;
lastProgressUpdateTime = DateTime.Now;
triedDownloads = 0;
}

public void Download(Uri url, string path)
public void Download()
{
var url = CurrentUri;
ResetAgent();
// Check whether to use an auth token for this host
if (url.IsAbsoluteUri
Expand All @@ -56,7 +54,7 @@ public void Download(Uri url, string path)
// Send our auth token to the GitHub API (or whoever else needs one)
agent.Headers.Add("Authorization", $"token {token}");
}
agent.DownloadFileAsyncWithResume(url, path);
target.DownloadWith(agent, url);
}

public Uri CurrentUri => target.urls[triedDownloads];
Expand Down
101 changes: 79 additions & 22 deletions Core/Net/NetAsyncDownloader.DownloadTarget.cs
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,99 @@
using System;
using System.IO;
using System.Collections.Generic;

using ChinhDo.Transactions.FileManager;

namespace CKAN
{
public partial class NetAsyncDownloader
{
public class DownloadTarget
public abstract class DownloadTarget
{
public List<Uri> urls { get; private set; }
public string filename { get; private set; }
public long size { get; set; }
public string mimeType { get; private set; }
public List<Uri> urls { get; protected set; }
public long size { get; protected set; }
public string mimeType { get; protected set; }

public DownloadTarget(List<Uri> urls,
string filename = null,
long size = 0,
string mimeType = "")
protected DownloadTarget(List<Uri> urls,
long size = 0,
string mimeType = "")
{
var FileTransaction = new TxFileManager();

this.urls = urls;
this.filename = string.IsNullOrEmpty(filename)
? FileTransaction.GetTempFileName()
: filename;
this.size = size;
this.mimeType = mimeType;
}

public DownloadTarget(Uri url,
string filename = null,
long size = 0,
string mimeType = "")
: this(new List<Uri> { url },
filename, size, mimeType)
public abstract long CalculateSize();
public abstract void DownloadWith(ResumingWebClient wc, Uri url);
}

public sealed class DownloadTargetFile : DownloadTarget
{
public string filename { get; private set; }

public DownloadTargetFile(List<Uri> urls,
string filename = null,
long size = 0,
string mimeType = "")
: base(urls, size, mimeType)
{
this.filename = filename ?? Path.GetTempFileName();
}

public DownloadTargetFile(Uri url,
string filename = null,
long size = 0,
string mimeType = "")
: this(new List<Uri> { url }, filename, size, mimeType)
{
}

public override long CalculateSize()
{
size = new FileInfo(filename).Length;
return size;
}

public override void DownloadWith(ResumingWebClient wc, Uri url)
{
wc.DownloadFileAsyncWithResume(url, filename);
}
}

public sealed class DownloadTargetStream : DownloadTarget, IDisposable
{
public Stream contents { get; private set; }

public DownloadTargetStream(List<Uri> urls,
long size = 0,
string mimeType = "")
: base(urls, size, mimeType)
{
contents = new MemoryStream();
}

public DownloadTargetStream(Uri url,
long size = 0,
string mimeType = "")
: this(new List<Uri> { url }, size, mimeType)
{
}

public override long CalculateSize()
{
size = contents.Length;
return size;
}

public override void DownloadWith(ResumingWebClient wc, Uri url)
{
wc.DownloadFileAsyncWithResume(url, contents);
}

public void Dispose()
{
// Close the stream
contents.Dispose();
}
}

}
}
25 changes: 13 additions & 12 deletions Core/Net/NetAsyncDownloader.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -36,7 +35,13 @@ public partial class NetAsyncDownloader
private volatile bool download_canceled;
private readonly ManualResetEvent complete_or_canceled;

public event Action<Uri, string, Exception, string> onOneCompleted;
/// <summary>
/// Invoked when a download completes or fails.
/// </summary>
/// <param>The download that is done</param>
/// <param>Exception thrown if failed</param>
/// <param>ETag of the URL</param>
public event Action<DownloadTarget, Exception, string> onOneCompleted;

/// <summary>
/// Returns a perfectly boring NetAsyncDownloader
Expand All @@ -54,7 +59,7 @@ public static string DownloadWithProgress(Uri url, string filename = null, IUser
{
var targets = new[]
{
new DownloadTarget(url, filename)
new DownloadTargetFile(url, filename)
};
DownloadWithProgress(targets, user);
return targets.First().filename;
Expand All @@ -63,16 +68,12 @@ public static string DownloadWithProgress(Uri url, string filename = null, IUser
public static void DownloadWithProgress(IList<DownloadTarget> downloadTargets, IUser user = null)
{
var downloader = new NetAsyncDownloader(user ?? new NullUser());
downloader.onOneCompleted += (url, filename, error, etag) =>
downloader.onOneCompleted += (target, error, etag) =>
{
if (error != null)
{
user?.RaiseError(error.ToString());
}
else
{
File.Move(filename, downloadTargets.First(p => p.urls.Contains(url)).filename);
}
};
downloader.DownloadAndWait(downloadTargets);
}
Expand All @@ -88,7 +89,7 @@ public void DownloadAndWait(IList<DownloadTarget> targets)
if (downloads.Count + queuedDownloads.Count > completed_downloads)
{
// Some downloads are still in progress, add to the current batch
foreach (DownloadTarget target in targets)
foreach (var target in targets)
{
DownloadModule(new DownloadPart(target));
}
Expand Down Expand Up @@ -259,7 +260,7 @@ private void DownloadModule(DownloadPart dl)
dl.CurrentUri.ToString().Replace(" ", "%20"));

// Start the download!
dl.Download(dl.CurrentUri, dl.path);
dl.Download();
}
}

Expand Down Expand Up @@ -403,15 +404,15 @@ private void FileDownloadComplete(int index, Exception error, bool canceled, str
log.InfoFormat("Finished downloading {0}", string.Join(", ", dl.target.urls));
dl.bytesLeft = 0;
// Let calling code find out how big this file is
dl.target.size = new FileInfo(dl.target.filename).Length;
dl.target.CalculateSize();
}

PopFromQueue(doneUri.Host);

try
{
// Tell calling code that this file is ready
onOneCompleted?.Invoke(dl.target.urls.First(), dl.path, dl.error, etag);
onOneCompleted?.Invoke(dl.target, dl.error, etag);
}
catch (Exception exc)
{
Expand Down
15 changes: 10 additions & 5 deletions Core/Net/NetAsyncModulesDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,16 @@ public NetAsyncModulesDownloader(IUser user, NetModuleCache cache)
this.cache = cache;
}

internal NetAsyncDownloader.DownloadTarget TargetFromModuleGroup(
internal NetAsyncDownloader.DownloadTargetFile TargetFromModuleGroup(
HashSet<CkanModule> group,
string[] preferredHosts)
=> TargetFromModuleGroup(group, group.OrderBy(m => m.identifier).First(), preferredHosts);

private NetAsyncDownloader.DownloadTarget TargetFromModuleGroup(
private NetAsyncDownloader.DownloadTargetFile TargetFromModuleGroup(
HashSet<CkanModule> group,
CkanModule first,
string[] preferredHosts)
=> new NetAsyncDownloader.DownloadTarget(
=> new NetAsyncDownloader.DownloadTargetFile(
group.SelectMany(mod => mod.download)
.Concat(group.Select(mod => mod.InternetArchiveDownload)
.Where(uri => uri != null)
Expand Down Expand Up @@ -90,7 +90,7 @@ public void DownloadModules(IEnumerable<CkanModule> modules)
.Where(grp => grp.All(mod => mod.download.All(dlUri => !activeURLs.Contains(dlUri))))
// Each group gets one target containing all the URLs
.Select(grp => TargetFromModuleGroup(grp, preferredHosts))
.ToList());
.ToArray());
this.modules.Clear();
AllComplete?.Invoke();
}
Expand Down Expand Up @@ -125,8 +125,13 @@ public void CancelDownload()
private readonly NetModuleCache cache;
private CancellationTokenSource cancelTokenSrc;

private void ModuleDownloadComplete(Uri url, string filename, Exception error, string etag)
private void ModuleDownloadComplete(NetAsyncDownloader.DownloadTarget target,
Exception error,
string etag)
{
var url = target.urls.First();
var filename = (target as NetAsyncDownloader.DownloadTargetFile)?.filename;

log.DebugFormat("Received download completion: {0}, {1}, {2}",
url, filename, error?.Message);
if (error != null)
Expand Down
Loading

0 comments on commit 298dbe7

Please sign in to comment.