Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Skip temp files for repo updates #4102

Merged
merged 1 commit into from
Jun 6, 2024
Merged

Skip temp files for repo updates #4102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cmdline/Action/Install.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options)
try
{
var targets = options.ckan_files
.Select(arg => new NetAsyncDownloader.DownloadTarget(getUri(arg)))
.ToList();
.Select(arg => new NetAsyncDownloader.DownloadTargetFile(getUri(arg)))
.ToArray();
log.DebugFormat("Urls: {0}", targets.SelectMany(t => t.urls));
new NetAsyncDownloader(new NullUser()).DownloadAndWait(targets);
log.DebugFormat("Files: {0}", targets.Select(t => t.filename));
Expand Down
4 changes: 2 additions & 2 deletions Core/AutoUpdate/GithubReleaseCkanUpdate.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ public GitHubReleaseCkanUpdate(GitHubReleaseInfo releaseJson = null)

public override IList<NetAsyncDownloader.DownloadTarget> Targets => new[]
{
new NetAsyncDownloader.DownloadTarget(
new NetAsyncDownloader.DownloadTargetFile(
UpdaterDownload, updaterFilename, UpdaterSize),
new NetAsyncDownloader.DownloadTarget(
new NetAsyncDownloader.DownloadTargetFile(
ReleaseDownload, ckanFilename, ReleaseSize),
};

Expand Down
4 changes: 2 additions & 2 deletions Core/AutoUpdate/S3BuildCkanUpdate.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ public S3BuildCkanUpdate(S3BuildVersionInfo versionJson = null)

public override IList<NetAsyncDownloader.DownloadTarget> Targets => new[]
{
new NetAsyncDownloader.DownloadTarget(
new NetAsyncDownloader.DownloadTargetFile(
UpdaterDownload, updaterFilename),
new NetAsyncDownloader.DownloadTarget(
new NetAsyncDownloader.DownloadTargetFile(
ReleaseDownload, ckanFilename),
};

Expand Down
11 changes: 6 additions & 5 deletions Core/FileIdentifier.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System.IO;
using System.Linq;

using ICSharpCode.SharpZipLib.GZip;

using CKAN.Extensions;

namespace CKAN
Expand Down Expand Up @@ -173,11 +175,10 @@ public static FileType IdentifyFile(Stream stream)
if (CheckGZip(stream))
{
// This may contain a tar file inside, create a new stream and check.
stream.Seek (0, SeekOrigin.Begin);
using (ICSharpCode.SharpZipLib.GZip.GZipInputStream gz_stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream (stream))
{
type = CheckTar(gz_stream) ? FileType.TarGz : FileType.GZip;
}
stream.Seek(0, SeekOrigin.Begin);
type = CheckTar(new GZipInputStream(stream))
? FileType.TarGz
: FileType.GZip;
}
else if (CheckTar(stream))
{
Expand Down
8 changes: 3 additions & 5 deletions Core/Net/NetAsyncDownloader.DownloadPart.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System;
using System.IO;
using System.ComponentModel;

using Autofac;
Expand All @@ -14,7 +13,6 @@ public partial class NetAsyncDownloader
private class DownloadPart
{
public readonly DownloadTarget target;
public readonly string path;

public DateTime lastProgressUpdateTime;
public long lastProgressUpdateSize;
Expand All @@ -38,14 +36,14 @@ private class DownloadPart
public DownloadPart(DownloadTarget target)
{
this.target = target;
path = target.filename ?? Path.GetTempFileName();
size = bytesLeft = target.size;
lastProgressUpdateTime = DateTime.Now;
triedDownloads = 0;
}

public void Download(Uri url, string path)
public void Download()
{
var url = CurrentUri;
ResetAgent();
// Check whether to use an auth token for this host
if (url.IsAbsoluteUri
Expand All @@ -56,7 +54,7 @@ public void Download(Uri url, string path)
// Send our auth token to the GitHub API (or whoever else needs one)
agent.Headers.Add("Authorization", $"token {token}");
}
agent.DownloadFileAsyncWithResume(url, path);
target.DownloadWith(agent, url);
}

public Uri CurrentUri => target.urls[triedDownloads];
Expand Down
101 changes: 79 additions & 22 deletions Core/Net/NetAsyncDownloader.DownloadTarget.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,42 +1,99 @@
using System;
using System.IO;
using System.Collections.Generic;

using ChinhDo.Transactions.FileManager;

namespace CKAN
{
public partial class NetAsyncDownloader
{
public class DownloadTarget
public abstract class DownloadTarget
{
public List<Uri> urls { get; private set; }
public string filename { get; private set; }
public long size { get; set; }
public string mimeType { get; private set; }
public List<Uri> urls { get; protected set; }
public long size { get; protected set; }
public string mimeType { get; protected set; }

public DownloadTarget(List<Uri> urls,
string filename = null,
long size = 0,
string mimeType = "")
protected DownloadTarget(List<Uri> urls,
long size = 0,
string mimeType = "")
{
var FileTransaction = new TxFileManager();

this.urls = urls;
this.filename = string.IsNullOrEmpty(filename)
? FileTransaction.GetTempFileName()
: filename;
this.size = size;
this.mimeType = mimeType;
}

public DownloadTarget(Uri url,
string filename = null,
long size = 0,
string mimeType = "")
: this(new List<Uri> { url },
filename, size, mimeType)
public abstract long CalculateSize();
public abstract void DownloadWith(ResumingWebClient wc, Uri url);
}

public sealed class DownloadTargetFile : DownloadTarget
{
public string filename { get; private set; }

public DownloadTargetFile(List<Uri> urls,
string filename = null,
long size = 0,
string mimeType = "")
: base(urls, size, mimeType)
{
this.filename = filename ?? Path.GetTempFileName();
}

public DownloadTargetFile(Uri url,
string filename = null,
long size = 0,
string mimeType = "")
: this(new List<Uri> { url }, filename, size, mimeType)
{
}

public override long CalculateSize()
{
size = new FileInfo(filename).Length;
return size;
}

public override void DownloadWith(ResumingWebClient wc, Uri url)
{
wc.DownloadFileAsyncWithResume(url, filename);
}
}

public sealed class DownloadTargetStream : DownloadTarget, IDisposable
{
public Stream contents { get; private set; }

public DownloadTargetStream(List<Uri> urls,
long size = 0,
string mimeType = "")
: base(urls, size, mimeType)
{
contents = new MemoryStream();
}

public DownloadTargetStream(Uri url,
long size = 0,
string mimeType = "")
: this(new List<Uri> { url }, size, mimeType)
{
}

public override long CalculateSize()
{
size = contents.Length;
return size;
}

public override void DownloadWith(ResumingWebClient wc, Uri url)
{
wc.DownloadFileAsyncWithResume(url, contents);
}

public void Dispose()
{
// Close the stream
contents.Dispose();
}
}

}
}
25 changes: 13 additions & 12 deletions Core/Net/NetAsyncDownloader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -36,7 +35,13 @@ public partial class NetAsyncDownloader
private volatile bool download_canceled;
private readonly ManualResetEvent complete_or_canceled;

public event Action<Uri, string, Exception, string> onOneCompleted;
/// <summary>
/// Invoked when a download completes or fails.
/// </summary>
/// <param>The download that is done</param>
/// <param>Exception thrown if failed</param>
/// <param>ETag of the URL</param>
public event Action<DownloadTarget, Exception, string> onOneCompleted;

/// <summary>
/// Returns a perfectly boring NetAsyncDownloader
Expand All @@ -54,7 +59,7 @@ public static string DownloadWithProgress(Uri url, string filename = null, IUser
{
var targets = new[]
{
new DownloadTarget(url, filename)
new DownloadTargetFile(url, filename)
};
DownloadWithProgress(targets, user);
return targets.First().filename;
Expand All @@ -63,16 +68,12 @@ public static string DownloadWithProgress(Uri url, string filename = null, IUser
public static void DownloadWithProgress(IList<DownloadTarget> downloadTargets, IUser user = null)
{
var downloader = new NetAsyncDownloader(user ?? new NullUser());
downloader.onOneCompleted += (url, filename, error, etag) =>
downloader.onOneCompleted += (target, error, etag) =>
{
if (error != null)
{
user?.RaiseError(error.ToString());
}
else
{
File.Move(filename, downloadTargets.First(p => p.urls.Contains(url)).filename);
}
};
downloader.DownloadAndWait(downloadTargets);
}
Expand All @@ -88,7 +89,7 @@ public void DownloadAndWait(IList<DownloadTarget> targets)
if (downloads.Count + queuedDownloads.Count > completed_downloads)
{
// Some downloads are still in progress, add to the current batch
foreach (DownloadTarget target in targets)
foreach (var target in targets)
{
DownloadModule(new DownloadPart(target));
}
Expand Down Expand Up @@ -259,7 +260,7 @@ private void DownloadModule(DownloadPart dl)
dl.CurrentUri.ToString().Replace(" ", "%20"));

// Start the download!
dl.Download(dl.CurrentUri, dl.path);
dl.Download();
}
}

Expand Down Expand Up @@ -403,15 +404,15 @@ private void FileDownloadComplete(int index, Exception error, bool canceled, str
log.InfoFormat("Finished downloading {0}", string.Join(", ", dl.target.urls));
dl.bytesLeft = 0;
// Let calling code find out how big this file is
dl.target.size = new FileInfo(dl.target.filename).Length;
dl.target.CalculateSize();
}

PopFromQueue(doneUri.Host);

try
{
// Tell calling code that this file is ready
onOneCompleted?.Invoke(dl.target.urls.First(), dl.path, dl.error, etag);
onOneCompleted?.Invoke(dl.target, dl.error, etag);
}
catch (Exception exc)
{
Expand Down
15 changes: 10 additions & 5 deletions Core/Net/NetAsyncModulesDownloader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,16 @@ public NetAsyncModulesDownloader(IUser user, NetModuleCache cache)
this.cache = cache;
}

internal NetAsyncDownloader.DownloadTarget TargetFromModuleGroup(
internal NetAsyncDownloader.DownloadTargetFile TargetFromModuleGroup(
HashSet<CkanModule> group,
string[] preferredHosts)
=> TargetFromModuleGroup(group, group.OrderBy(m => m.identifier).First(), preferredHosts);

private NetAsyncDownloader.DownloadTarget TargetFromModuleGroup(
private NetAsyncDownloader.DownloadTargetFile TargetFromModuleGroup(
HashSet<CkanModule> group,
CkanModule first,
string[] preferredHosts)
=> new NetAsyncDownloader.DownloadTarget(
=> new NetAsyncDownloader.DownloadTargetFile(
group.SelectMany(mod => mod.download)
.Concat(group.Select(mod => mod.InternetArchiveDownload)
.Where(uri => uri != null)
Expand Down Expand Up @@ -90,7 +90,7 @@ public void DownloadModules(IEnumerable<CkanModule> modules)
.Where(grp => grp.All(mod => mod.download.All(dlUri => !activeURLs.Contains(dlUri))))
// Each group gets one target containing all the URLs
.Select(grp => TargetFromModuleGroup(grp, preferredHosts))
.ToList());
.ToArray());
this.modules.Clear();
AllComplete?.Invoke();
}
Expand Down Expand Up @@ -125,8 +125,13 @@ public void CancelDownload()
private readonly NetModuleCache cache;
private CancellationTokenSource cancelTokenSrc;

private void ModuleDownloadComplete(Uri url, string filename, Exception error, string etag)
private void ModuleDownloadComplete(NetAsyncDownloader.DownloadTarget target,
Exception error,
string etag)
{
var url = target.urls.First();
var filename = (target as NetAsyncDownloader.DownloadTargetFile)?.filename;

log.DebugFormat("Received download completion: {0}, {1}, {2}",
url, filename, error?.Message);
if (error != null)
Expand Down
Loading
Loading