From 3e226b08a166e493f883588c9d0fb8f03e96a061 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Wed, 26 Jul 2023 14:54:41 -0500 Subject: [PATCH 1/4] Multi-document netkans, download as list --- CKAN.schema | 16 +- Netkan/CKAN-netkan.csproj | 1 + Netkan/Extensions/YamlExtensions.cs | 7 +- Netkan/Model/Metadata.cs | 60 +++++-- Netkan/Processors/Inflator.cs | 15 +- Netkan/Processors/QueueHandler.cs | 17 +- Netkan/Program.cs | 53 +++--- Netkan/Services/ModuleService.cs | 3 +- Netkan/Transformers/MetaNetkanTransformer.cs | 55 +++--- Netkan/Transformers/NetkanTransformer.cs | 5 +- Netkan/Validators/CkanValidator.cs | 1 + Netkan/Validators/DownloadArrayValidator.cs | 25 +++ Spec.md | 8 + .../NetKAN/Extensions/YamlExtensionsTests.cs | 2 +- Tests/NetKAN/Model/MetadataTests.cs | 158 ++++++++++++++++++ .../Validators/DownloadArrayValidatorTests.cs | 69 ++++++++ 16 files changed, 407 insertions(+), 88 deletions(-) create mode 100644 Netkan/Validators/DownloadArrayValidator.cs create mode 100644 Tests/NetKAN/Model/MetadataTests.cs create mode 100644 Tests/NetKAN/Validators/DownloadArrayValidatorTests.cs diff --git a/CKAN.schema b/CKAN.schema index cef4795c06..e2fd249e06 100644 --- a/CKAN.schema +++ b/CKAN.schema @@ -57,8 +57,20 @@ }, "download" : { "description" : "URL where mod can be downloaded by tools", - "type" : "string", - "format" : "uri" + "oneOf" : [ + { + "type" : "string", + "format" : "uri" + }, + { + "type" : "array", + "items" : { + "type" : "string", + "format" : "uri" + }, + "uniqueItems" : true + } + ] }, "download_size" : { "description" : "The size of the download in bytes", diff --git a/Netkan/CKAN-netkan.csproj b/Netkan/CKAN-netkan.csproj index b0e8c02e48..9c8a5530ef 100644 --- a/Netkan/CKAN-netkan.csproj +++ b/Netkan/CKAN-netkan.csproj @@ -138,6 +138,7 @@ + diff --git a/Netkan/Extensions/YamlExtensions.cs b/Netkan/Extensions/YamlExtensions.cs index aa489b2c9d..e59fb97f9c 100644 --- a/Netkan/Extensions/YamlExtensions.cs +++ b/Netkan/Extensions/YamlExtensions.cs @@ -9,16 +9,17 @@ namespace CKAN.NetKAN.Extensions { internal static class YamlExtensions { - public static YamlMappingNode Parse(string input) + public static YamlMappingNode[] Parse(string input) { return Parse(new StringReader(input)); } - public static YamlMappingNode Parse(TextReader input) + public static YamlMappingNode[] Parse(TextReader input) { var stream = new YamlStream(); stream.Load(input); - return stream.Documents.FirstOrDefault()?.RootNode as YamlMappingNode; + return stream.Documents.Select(doc => doc?.RootNode as YamlMappingNode) + .ToArray(); } /// diff --git a/Netkan/Model/Metadata.cs b/Netkan/Model/Metadata.cs index 7a0f289224..bd7c5aefde 100644 --- a/Netkan/Model/Metadata.cs +++ b/Netkan/Model/Metadata.cs @@ -1,8 +1,11 @@ using System; +using System.Collections.Generic; using System.Linq; -using CKAN.Versioning; + using Newtonsoft.Json.Linq; using YamlDotNet.RepresentationModel; + +using CKAN.Versioning; using CKAN.NetKAN.Extensions; namespace CKAN.NetKAN.Model @@ -20,7 +23,7 @@ internal sealed class Metadata private readonly JObject _json; - public string Identifier { get { return (string)_json["identifier"]; } } + public string Identifier => (string)_json["identifier"]; public RemoteRef Kref { get; private set; } public RemoteRef Vref { get; private set; } public ModuleVersion SpecVersion { get; private set; } @@ -96,7 +99,10 @@ public Metadata(JObject json) JToken downloadToken; if (json.TryGetValue(DownloadPropertyName, out downloadToken)) { - Download = new Uri((string)downloadToken); + Download = new Uri( + downloadToken.Type == JTokenType.String + ? (string)downloadToken + : (string)downloadToken.Children().First()); } JToken stagedToken; @@ -123,6 +129,41 @@ public Metadata(YamlMappingNode yaml) : this(yaml?.ToJObject()) { } + public static Metadata Merge(Metadata[] modules) + => modules.Length == 1 ? modules[0] + : new Metadata(MergeJson(modules.Select(m => m._json) + .ToArray())); + + private static JObject MergeJson(JObject[] jsons) + { + var mergeSettings = new JsonMergeSettings() + { + MergeArrayHandling = MergeArrayHandling.Replace, + MergeNullValueHandling = MergeNullValueHandling.Merge, + }; + var downloads = jsons.SelectMany(json => json[DownloadPropertyName] is JArray + ? json[DownloadPropertyName].Children() + : Enumerable.Repeat(json[DownloadPropertyName], 1)) + .Distinct() + .ToArray(); + var first = jsons.First(); + foreach (var other in jsons.Skip(1)) + { + if ((string)first["download_size"] != (string)other["download_size"] + || (string)first["download_hash"]["sha1"] != (string)other["download_hash"]["sha1"] + || (string)first["download_hash"]["sha256"] != (string)other["download_hash"]["sha256"]) + { + // Can't treat the URLs as equivalent if they're different files + throw new Kraken(string.Format( + "Download from {0} does not match download from {1}", + first["download"], other["download"])); + } + first.Merge(other, mergeSettings); + } + first[DownloadPropertyName] = JArray.FromObject(downloads); + return first; + } + public string[] Licenses { get @@ -142,13 +183,7 @@ public string[] Licenses } } - public bool Redistributable - { - get - { - return Licenses.Any(lic => new License(lic).Redistributable); - } - } + public bool Redistributable => Licenses.Any(lic => new License(lic).Redistributable); public Uri FallbackDownload { @@ -175,9 +210,6 @@ public Uri FallbackDownload } } - public JObject Json() - { - return (JObject)_json.DeepClone(); - } + public JObject Json() => (JObject)_json.DeepClone(); } } diff --git a/Netkan/Processors/Inflator.cs b/Netkan/Processors/Inflator.cs index f0345c3076..4373cf4363 100644 --- a/Netkan/Processors/Inflator.cs +++ b/Netkan/Processors/Inflator.cs @@ -33,7 +33,7 @@ public Inflator(string cacheDir, bool overwriteCache, string githubToken, string transformer = new NetkanTransformer(http, fileService, moduleService, githubToken, gitlabToken, prerelease, game, netkanValidator); } - internal IEnumerable Inflate(string filename, Metadata netkan, TransformOptions opts) + internal IEnumerable Inflate(string filename, Metadata[] netkans, TransformOptions opts) { log.DebugFormat("Inflating {0}", filename); try @@ -41,17 +41,22 @@ internal IEnumerable Inflate(string filename, Metadata netkan, Transfo // Tell the downloader that we're starting a new request http.ClearRequestedURLs(); - netkanValidator.ValidateNetkan(netkan, filename); + foreach (var netkan in netkans) + { + netkanValidator.ValidateNetkan(netkan, filename); + } log.Debug("Input successfully passed pre-validation"); - IEnumerable ckans = transformer - .Transform(netkan, opts) + var ckans = netkans + .SelectMany(netkan => transformer.Transform(netkan, opts)) + .GroupBy(module => module.Version) + .Select(grp => Metadata.Merge(grp.ToArray())) .ToList(); log.Debug("Finished transformation"); foreach (Metadata ckan in ckans) { - ckanValidator.ValidateCkan(ckan, netkan); + ckanValidator.ValidateCkan(ckan, netkans[0]); } log.Debug("Output successfully passed post-validation"); return ckans; diff --git a/Netkan/Processors/QueueHandler.cs b/Netkan/Processors/QueueHandler.cs index 335482d819..b43cb65bcd 100644 --- a/Netkan/Processors/QueueHandler.cs +++ b/Netkan/Processors/QueueHandler.cs @@ -131,7 +131,9 @@ private void handleMessages(string url, int howMany, int timeoutMinutes) private IEnumerable Inflate(Message msg) { log.DebugFormat("Metadata returned: {0}", msg.Body); - var netkan = new Metadata(YamlExtensions.Parse(msg.Body)); + var netkans = YamlExtensions.Parse(msg.Body) + .Select(ymap => new Metadata(ymap)) + .ToArray(); int releases = 1; MessageAttributeValue releasesAttr; @@ -147,14 +149,15 @@ private IEnumerable Inflate(Message msg) highVer = new ModuleVersion(highVerAttr.StringValue); } - log.InfoFormat("Inflating {0}", netkan.Identifier); + log.InfoFormat("Inflating {0}", netkans.First().Identifier); IEnumerable ckans = null; bool caught = false; string caughtMessage = null; - var opts = new TransformOptions(releases, null, highVer, netkan.Staged, netkan.StagingReason); + var opts = new TransformOptions(releases, null, highVer, netkans.First().Staged, netkans.First().StagingReason); try { - ckans = inflator.Inflate($"{netkan.Identifier}.netkan", netkan, opts); + ckans = inflator.Inflate($"{netkans[0].Identifier}.netkan", netkans, opts) + .ToArray(); } catch (Exception e) { @@ -167,14 +170,14 @@ private IEnumerable Inflate(Message msg) } if (caught) { - yield return inflationMessage(null, netkan, opts, false, caughtMessage); + yield return inflationMessage(null, netkans.FirstOrDefault(), opts, false, caughtMessage); } if (ckans != null) { foreach (Metadata ckan in ckans) { log.InfoFormat("Sending {0}-{1}", ckan.Identifier, ckan.Version); - yield return inflationMessage(ckan, netkan, opts, true); + yield return inflationMessage(ckan, netkans.FirstOrDefault(), opts, true); } } } @@ -231,7 +234,7 @@ private SendMessageBatchRequestEntry inflationMessage(Metadata ckan, Metadata ne new MessageAttributeValue() { DataType = "String", - StringValue = Program.CkanFileName(ckan) + StringValue = Program.CkanFileName(ckan.Json()) } ); } diff --git a/Netkan/Program.cs b/Netkan/Program.cs index 124bdd77c5..e810b9720f 100644 --- a/Netkan/Program.cs +++ b/Netkan/Program.cs @@ -89,7 +89,7 @@ public static int Main(string[] args) { Log.InfoFormat("Transforming {0}", Options.File); - var netkan = ReadNetkan(); + var netkans = ReadNetkans(); Log.Info("Finished reading input"); var inf = new Inflator( @@ -101,19 +101,18 @@ public static int Main(string[] args) game ); var ckans = inf.Inflate( - Options.File, - netkan, - new TransformOptions( - ParseReleases(Options.Releases), - ParseSkipReleases(Options.SkipReleases), - ParseHighestVersion(Options.HighestVersion), - netkan.Staged, - netkan.StagingReason - ) - ); + Options.File, + netkans, + new TransformOptions( + ParseReleases(Options.Releases), + ParseSkipReleases(Options.SkipReleases), + ParseHighestVersion(Options.HighestVersion), + netkans[0].Staged, + netkans[0].StagingReason)) + .ToArray(); foreach (Metadata ckan in ckans) { - WriteCkan(ckan); + WriteCkan(ckan.Json()); } } else @@ -179,31 +178,29 @@ private static void ProcessArgs(string[] args) } } - private static Metadata ReadNetkan() + private static Metadata[] ReadNetkans() { if (!Options.File.EndsWith(".netkan")) { Log.WarnFormat("Input is not a .netkan file"); } - return new Metadata(YamlExtensions.Parse(File.OpenText(Options.File))); + return YamlExtensions.Parse(File.OpenText(Options.File)) + .Select(ymap => new Metadata(ymap)) + .ToArray(); } - internal static string CkanFileName(Metadata metadata) - { - return Path.Combine( - Options.OutputDir, - string.Format( - "{0}-{1}.ckan", - metadata.Identifier, - metadata.Version.ToString().Replace(':', '-') - ) - ); - } + internal static string CkanFileName(JObject json) + => Path.Combine( + Options.OutputDir, + string.Format( + "{0}-{1}.ckan", + (string)json["identifier"], + ((string)json["version"]).Replace(':', '-'))); - private static void WriteCkan(Metadata metadata) + private static void WriteCkan(JObject json) { - var finalPath = CkanFileName(metadata); + var finalPath = CkanFileName(json); var sb = new StringBuilder(); var sw = new StringWriter(sb); @@ -215,7 +212,7 @@ private static void WriteCkan(Metadata metadata) writer.IndentChar = ' '; var serializer = new JsonSerializer(); - serializer.Serialize(writer, metadata.Json()); + serializer.Serialize(writer, json); } File.WriteAllText(finalPath, sw + Environment.NewLine); diff --git a/Netkan/Services/ModuleService.cs b/Netkan/Services/ModuleService.cs index a5ed529a30..71637017b2 100644 --- a/Netkan/Services/ModuleService.cs +++ b/Netkan/Services/ModuleService.cs @@ -213,7 +213,8 @@ private static JObject DeserializeFromStream(Stream stream) { using (var sr = new StreamReader(stream)) { - return YamlExtensions.Parse(sr).ToJObject(); + // Only one document per internal .ckan + return YamlExtensions.Parse(sr).FirstOrDefault()?.ToJObject(); } } diff --git a/Netkan/Transformers/MetaNetkanTransformer.cs b/Netkan/Transformers/MetaNetkanTransformer.cs index c90cde65b8..9b400e7d25 100644 --- a/Netkan/Transformers/MetaNetkanTransformer.cs +++ b/Netkan/Transformers/MetaNetkanTransformer.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Linq; + using log4net; using Newtonsoft.Json.Linq; @@ -52,37 +54,42 @@ public IEnumerable Transform(Metadata metadata, TransformOptions opts) Log.DebugFormat("Target netkan:{0}{1}", Environment.NewLine, targetFileText); - var targetJson = YamlExtensions.Parse(targetFileText).ToJObject(); - var targetMetadata = new Metadata(targetJson); + var targetJsons = YamlExtensions.Parse(targetFileText) + .Select(ymap => ymap.ToJObject()) + .ToArray(); - if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan") + foreach (var targetJson in targetJsons) { - json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, targetMetadata.SpecVersion) - .ToSpecVersionJson(); - - if (targetJson["$kref"] != null) - { - json["$kref"] = targetJson["$kref"]; - } - else + var targetMetadata = new Metadata(targetJson); + if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan") { - json.Remove("$kref"); - } + json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, targetMetadata.SpecVersion) + .ToSpecVersionJson(); - json.SafeMerge("resources", targetJson["resources"]); + if (targetJson["$kref"] != null) + { + json["$kref"] = targetJson["$kref"]; + } + else + { + json.Remove("$kref"); + } - foreach (var property in targetJson.Properties()) - { - json.SafeAdd(property.Name, property.Value); - } + json.SafeMerge("resources", targetJson["resources"]); - Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); + foreach (var property in targetJson.Properties()) + { + json.SafeAdd(property.Name, property.Value); + } - yield return new Metadata(json); - } - else - { - throw new Kraken("The target of a metanetkan may not also be a metanetkan."); + Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); + + yield return new Metadata(json); + } + else + { + throw new Kraken("The target of a metanetkan may not also be a metanetkan."); + } } } else diff --git a/Netkan/Transformers/NetkanTransformer.cs b/Netkan/Transformers/NetkanTransformer.cs index d1f4b95617..a460afd287 100644 --- a/Netkan/Transformers/NetkanTransformer.cs +++ b/Netkan/Transformers/NetkanTransformer.cs @@ -73,9 +73,8 @@ public IEnumerable Transform(Metadata metadata, TransformOptions opts) Metadata[] modules = new Metadata[] { metadata }; foreach (ITransformer tr in _transformers) { - modules = modules - .SelectMany(meta => tr.Transform(meta, opts)) - .ToArray(); + modules = modules.SelectMany(meta => tr.Transform(meta, opts)) + .ToArray(); // The metadata should be valid after each step foreach (Metadata meta in modules) { diff --git a/Netkan/Validators/CkanValidator.cs b/Netkan/Validators/CkanValidator.cs index 9b6fb9e19a..ccd790c646 100644 --- a/Netkan/Validators/CkanValidator.cs +++ b/Netkan/Validators/CkanValidator.cs @@ -17,6 +17,7 @@ public CkanValidator(IHttpService downloader, IModuleService moduleService, IGam _validators = new List { new IsCkanModuleValidator(), + new DownloadArrayValidator(), new TagsValidator(), new InstallsFilesValidator(downloader, moduleService, game), new MatchesKnownGameVersionsValidator(game), diff --git a/Netkan/Validators/DownloadArrayValidator.cs b/Netkan/Validators/DownloadArrayValidator.cs new file mode 100644 index 0000000000..95a2954cb7 --- /dev/null +++ b/Netkan/Validators/DownloadArrayValidator.cs @@ -0,0 +1,25 @@ +using Newtonsoft.Json.Linq; + +using CKAN.Versioning; +using CKAN.NetKAN.Model; + +namespace CKAN.NetKAN.Validators +{ + internal sealed class DownloadArrayValidator : IValidator + { + public void Validate(Metadata metadata) + { + var json = metadata.Json(); + if (json.ContainsKey("download")) + { + if (metadata.SpecVersion < v1p34 && json["download"] is JArray) + { + throw new Kraken(ErrorMessage); + } + } + } + + private static readonly ModuleVersion v1p34 = new ModuleVersion("v1.34"); + internal const string ErrorMessage = "spec_version v1.34+ required for download as array"; + } +} diff --git a/Spec.md b/Spec.md index 3e800e8586..c335f69118 100644 --- a/Spec.md +++ b/Spec.md @@ -170,6 +170,9 @@ A fully formed URL, indicating where a machine may download the described version of the mod. Note: This field is not required if the `kind` is `metapackage` or `dlc`. +May also be an array of URLs (**v1.34**), any of which may be used to +download the mod. + ##### license The license (**v1.0**), or list of licenses (**v1.8**), under which the mod is released. @@ -709,6 +712,11 @@ NetKAN is the name the tool which is used to automatically generate CKAN files f consumes `.netkan` files to produce `.ckan` files. `.netkan` files are a *strict superset* of `.ckan` files. Every `.ckan` file is a valid `.netkan` file but not vice versa. NetKAN uses the following fields to produce `.ckan` files. +If a mod is hosted on multiple servers, a `.netkan` file may contain multiple metadata documents +(separated by `---` in YAML or separate top-level `{}` object blocks in JSON), one per server. +These will all be checked for new releases and the results merged based on the value of their `version` properties, +resulting in a `download` property containing an array of multiple download URLs. + ##### YAML Option A `.netkan` file may be in either JSON or YAML format. All examples shown below assume YAML, but the JSON equivalents will work the same way. diff --git a/Tests/NetKAN/Extensions/YamlExtensionsTests.cs b/Tests/NetKAN/Extensions/YamlExtensionsTests.cs index 927a781eab..1c0f785c01 100644 --- a/Tests/NetKAN/Extensions/YamlExtensionsTests.cs +++ b/Tests/NetKAN/Extensions/YamlExtensionsTests.cs @@ -35,7 +35,7 @@ public void Parse_ValidInput_Works() }); // Act - YamlMappingNode yaml = YamlExtensions.Parse(input); + YamlMappingNode yaml = YamlExtensions.Parse(input).First(); // Assert Assert.AreEqual("v1.4", (string)yaml["spec_version"]); diff --git a/Tests/NetKAN/Model/MetadataTests.cs b/Tests/NetKAN/Model/MetadataTests.cs new file mode 100644 index 0000000000..2b6fd35d68 --- /dev/null +++ b/Tests/NetKAN/Model/MetadataTests.cs @@ -0,0 +1,158 @@ +using System.Linq; + +using NUnit.Framework; +using Newtonsoft.Json.Linq; + +using CKAN; +using CKAN.NetKAN.Model; + +namespace Tests.NetKAN.Model +{ + [TestFixture] + public class MetadataTests + { + [Test, + // https://stackoverflow.com/a/61403175/2422988 + TestCase((object)new string[] + { + @"{ + ""spec_version"": ""v1.34"", + ""download_size"": 1000 + }", + @"{ + ""spec_version"": ""v1.34"", + ""download_size"": 1001 + }", + }), + TestCase((object)new string[] + { + @"{ + ""spec_version"": ""v1.34"", + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }", + @"{ + ""spec_version"": ""v1.34"", + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""8008580085"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }", + }), + TestCase((object)new string[] + { + @"{ + ""spec_version"": ""v1.34"", + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }", + @"{ + ""spec_version"": ""v1.34"", + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""8008580085"" + } + }", + }), + ] + public void Merge_MismatchedSizeOrHash_Throws(string[] moduleJsons) + { + // Arrange + var modules = moduleJsons.Select(j => new Metadata(JObject.Parse(j))) + .ToArray(); + + // Act / Assert + var exception = Assert.Throws(() => Metadata.Merge(modules)); + StringAssert.Contains("does not match download from", exception.Message); + } + + [Test, + // Two modules with one URL each, merged to download list + TestCase(new string[] + { + @"{ + ""spec_version"": ""v1.32"", + ""download"": ""https://github.com/"", + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }", + @"{ + ""spec_version"": ""v1.34"", + ""download"": ""https://spacedock.info/"", + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }", + }, + @"{ + ""spec_version"": ""v1.34"", + ""download"": [ ""https://github.com/"", ""https://spacedock.info/"" ], + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }"), + // Two modules, one with download list, merged to list without duplicates + TestCase(new string[] + { + @"{ + ""spec_version"": ""v1.32"", + ""download"": ""https://github.com/"", + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }", + @"{ + ""spec_version"": ""v1.34"", + ""download"": [ ""https://github.com/"", ""https://spacedock.info/"" ], + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }", + }, + @"{ + ""spec_version"": ""v1.34"", + ""download"": [ ""https://github.com/"", ""https://spacedock.info/"" ], + ""download_size"": 1000, + ""download_hash"": { + ""sha1"": ""DEADBEEFDEADBEEF"", + ""sha256"": ""DEADBEEFDEADBEEF"" + } + }"), + ] + public void Merge_WithModules_ModuleWithMergedDownload(string[] moduleJsons, + string correctResult) + { + // Arrange + var modules = moduleJsons.Select(j => new Metadata(JObject.Parse(j))) + .ToArray(); + var correctJson = JObject.Parse(correctResult); + + // Act + var mergedJson = Metadata.Merge(modules).Json(); + + // Assert + Assert.IsTrue(JToken.DeepEquals(correctJson, mergedJson), + "Expected {0}, got {1}", + correctJson, mergedJson); + } + } +} diff --git a/Tests/NetKAN/Validators/DownloadArrayValidatorTests.cs b/Tests/NetKAN/Validators/DownloadArrayValidatorTests.cs new file mode 100644 index 0000000000..e7ccb8461e --- /dev/null +++ b/Tests/NetKAN/Validators/DownloadArrayValidatorTests.cs @@ -0,0 +1,69 @@ +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +using CKAN; +using CKAN.NetKAN.Model; +using CKAN.NetKAN.Validators; + +namespace Tests.NetKAN.Validators +{ + [TestFixture] + public sealed class DownloadArrayValidatorTests + { + private static readonly DownloadArrayValidator validator = new DownloadArrayValidator(); + + [Test] + public void Validate_OldSpecNoArray_DoesNotThrow() + { + // Arrange + var jobj = JObject.Parse(@"{ + ""spec_version"": ""v1.33"", + ""download"": ""https://github.com/"" + }"); + + // Act / Assert + validator.Validate(new Metadata(jobj)); + } + + [Test] + public void Validate_OldSpecArray_Throws() + { + // Arrange + var jobj = JObject.Parse(@"{ + ""spec_version"": ""v1.33"", + ""download"": [ ""https://github.com/"" ] + }"); + + // Act / Assert + var exception = Assert.Throws(() => + validator.Validate(new Metadata(jobj))); + Assert.AreEqual(DownloadArrayValidator.ErrorMessage, exception.Message); + } + + [Test] + public void Validate_NewSpecNoArray_DoesNotThrow() + { + // Arrange + var jobj = JObject.Parse(@"{ + ""spec_version"": ""v1.34"", + ""download"": ""https://github.com/"" + }"); + + // Act / Assert + validator.Validate(new Metadata(jobj)); + } + + [Test] + public void Validate_NewSpecArray_DoesNotThrow() + { + // Arrange + var jobj = JObject.Parse(@"{ + ""spec_version"": ""v1.34"", + ""download"": [ ""https://github.com/"" ] + }"); + + // Act / Assert + validator.Validate(new Metadata(jobj)); + } + } +} From be91398fb8828ec3ec9cd8f33ac7d7e3ff02782c Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Sun, 30 Jul 2023 10:22:40 -0500 Subject: [PATCH 2/4] Download array in data model and downloader --- Cmdline/Action/Show.cs | 2 +- ConsoleUI/ModInfoScreen.cs | 17 +- Core/Converters/JsonSingleOrArrayConverter.cs | 5 +- .../DelimeterSeparatedValueExporter.cs | 2 +- Core/GameInstance.cs | 1 - Core/ModuleInstaller.cs | 10 +- Core/Net/AutoUpdate.cs | 7 +- Core/Net/Net.cs | 18 +- Core/Net/NetAsyncDownloader.cs | 225 +++++++------ Core/Net/NetAsyncModulesDownloader.cs | 98 ++++-- Core/Net/NetFileCache.cs | 8 + Core/Net/NetModuleCache.cs | 66 ++-- Core/Net/Repo.cs | 2 +- Core/Registry/Registry.cs | 17 +- Core/Types/CkanModule.cs | 5 +- GUI/Main/MainRepo.cs | 2 +- Tests/Core/Net/NetAsyncModulesDownloader.cs | 139 -------- .../Net/NetAsyncModulesDownloaderTests.cs | 317 ++++++++++++++++++ Tests/Core/Types/CkanModuleTests.cs | 4 +- 19 files changed, 612 insertions(+), 333 deletions(-) delete mode 100644 Tests/Core/Net/NetAsyncModulesDownloader.cs create mode 100644 Tests/Core/Net/NetAsyncModulesDownloaderTests.cs diff --git a/Cmdline/Action/Show.cs b/Cmdline/Action/Show.cs index a5c9bf138e..7a6d220f5b 100644 --- a/Cmdline/Action/Show.cs +++ b/Cmdline/Action/Show.cs @@ -299,7 +299,7 @@ private int ShowMod(CkanModule module, ShowOptions opts) if (!opts.without_files && !module.IsDLC) { // Compute the CKAN filename. - string file_uri_hash = NetFileCache.CreateURLHash(module.download); + string file_uri_hash = NetFileCache.CreateURLHash(module.download[0]); string file_name = CkanModule.StandardName(module.identifier, module.version); user.RaiseMessage(""); diff --git a/ConsoleUI/ModInfoScreen.cs b/ConsoleUI/ModInfoScreen.cs index 464e53121e..d4ff7ddfbc 100644 --- a/ConsoleUI/ModInfoScreen.cs +++ b/ConsoleUI/ModInfoScreen.cs @@ -512,11 +512,16 @@ private void addVersionBox(int l, int t, int r, int b, Func title, Func< private string HostedOn() { - string dl = mod.download?.ToString() ?? ""; - foreach (var kvp in hostDomains) { - if (dl.IndexOf(kvp.Key, StringComparison.CurrentCultureIgnoreCase) >= 0) { - return string.Format(Properties.Resources.ModInfoHostedOn, kvp.Value); - } + if (mod.download != null && mod.download.Count > 0) + { + var downloadHosts = mod.download + .Select(dlUri => dlUri.Host) + .Select(host => + hostDomains.TryGetValue(host, out string name) + ? name + : host); + return string.Format(Properties.Resources.ModInfoHostedOn, + string.Join(", ", downloadHosts)); } if (mod.resources != null) { if (mod.resources.bugtracker != null) { @@ -550,7 +555,7 @@ private string HostedOn() : Properties.Resources.ModInfoBuyFromKSPStoreOrSteamStore; } } - return mod.download?.Host ?? ""; + return ""; } private void Download(ConsoleTheme theme) diff --git a/Core/Converters/JsonSingleOrArrayConverter.cs b/Core/Converters/JsonSingleOrArrayConverter.cs index 9f5fa883ff..5db5b29db2 100644 --- a/Core/Converters/JsonSingleOrArrayConverter.cs +++ b/Core/Converters/JsonSingleOrArrayConverter.cs @@ -24,11 +24,12 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist return token.ToObject() == null ? null : new List { token.ToObject() }; } - public override bool CanWrite => false; + public override bool CanWrite => true; public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { - throw new NotImplementedException(); + var list = value as List; + serializer.Serialize(writer, list?.Count == 1 ? list[0] : value); } /// diff --git a/Core/Exporters/DelimeterSeparatedValueExporter.cs b/Core/Exporters/DelimeterSeparatedValueExporter.cs index b869a8a02b..3671d0b3da 100644 --- a/Core/Exporters/DelimeterSeparatedValueExporter.cs +++ b/Core/Exporters/DelimeterSeparatedValueExporter.cs @@ -65,7 +65,7 @@ public void Export(IRegistryQuerier registry, Stream stream) QuoteIfNecessary(mod.Module.description), QuoteIfNecessary(string.Join(";", mod.Module.author)), QuoteIfNecessary(mod.Module.kind), - WriteUri(mod.Module.download), + WriteUri(mod.Module.download[0]), mod.Module.download_size, mod.Module.ksp_version, mod.Module.ksp_version_min, diff --git a/Core/GameInstance.cs b/Core/GameInstance.cs index 42a6e4a062..82b77bd20d 100644 --- a/Core/GameInstance.cs +++ b/Core/GameInstance.cs @@ -393,7 +393,6 @@ public bool Scan() return dllChanged || dlcChanged; } - return false; } #endregion diff --git a/Core/ModuleInstaller.cs b/Core/ModuleInstaller.cs index e568c9bcf4..5335e9f245 100644 --- a/Core/ModuleInstaller.cs +++ b/Core/ModuleInstaller.cs @@ -1051,14 +1051,14 @@ public void Upgrade(IEnumerable modules, IDownloader netAsyncDownloa { User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingResuming, module.name, module.version, - module.download.Host, + string.Join(", ", module.download.Select(dl => dl.Host).Distinct()), CkanModule.FmtSize(module.download_size - inProgressFile.Length)); } else { User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingUncached, module.name, module.version, - module.download.Host, + string.Join(", ", module.download.Select(dl => dl.Host).Distinct()), CkanModule.FmtSize(module.download_size)); } } @@ -1093,13 +1093,15 @@ public void Upgrade(IEnumerable modules, IDownloader netAsyncDownloa { User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingResuming, module.name, installed.version, module.version, - module.download.Host, CkanModule.FmtSize(module.download_size - inProgressFile.Length)); + string.Join(", ", module.download.Select(dl => dl.Host).Distinct()), + CkanModule.FmtSize(module.download_size - inProgressFile.Length)); } else { User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingUncached, module.name, installed.version, module.version, - module.download.Host, CkanModule.FmtSize(module.download_size)); + string.Join(", ", module.download.Select(dl => dl.Host).Distinct()), + CkanModule.FmtSize(module.download_size)); } } else diff --git a/Core/Net/AutoUpdate.cs b/Core/Net/AutoUpdate.cs index 94be06f7d3..518d08b6f9 100644 --- a/Core/Net/AutoUpdate.cs +++ b/Core/Net/AutoUpdate.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Diagnostics; using System.Net; @@ -151,13 +152,11 @@ public void StartUpdateProcess(bool launchCKANAfterUpdate, IUser user = null) new[] { new Net.DownloadTarget( - latestUpdate.UpdaterDownload, - null, + new List { latestUpdate.UpdaterDownload }, updaterFilename, latestUpdate.UpdaterSize), new Net.DownloadTarget( - latestUpdate.ReleaseDownload, - null, + new List { latestUpdate.ReleaseDownload }, ckanFilename, latestUpdate.ReleaseSize), }, diff --git a/Core/Net/Net.cs b/Core/Net/Net.cs index 037eaf18a6..7bdd498722 100644 --- a/Core/Net/Net.cs +++ b/Core/Net/Net.cs @@ -146,18 +146,16 @@ public static string Download(string url, out string etag, string filename = nul public class DownloadTarget { - public Uri url { get; private set; } - public Uri fallbackUrl { get; private set; } - public string filename { get; private set; } - public long size { get; private set; } - public string mimeType { get; private set; } + public List urls { get; private set; } + public string filename { get; private set; } + public long size { get; private set; } + public string mimeType { get; private set; } - public DownloadTarget(Uri url, Uri fallback = null, string filename = null, long size = 0, string mimeType = "") + public DownloadTarget(List urls, string filename = null, long size = 0, string mimeType = "") { TxFileManager FileTransaction = new TxFileManager(); - this.url = url; - this.fallbackUrl = fallback; + this.urls = urls; this.filename = string.IsNullOrEmpty(filename) ? FileTransaction.GetTempFileName() : filename; @@ -174,7 +172,7 @@ public static string DownloadWithProgress(string url, string filename = null, IU public static string DownloadWithProgress(Uri url, string filename = null, IUser user = null) { var targets = new[] { - new DownloadTarget(url, null, filename) + new DownloadTarget(new List { url }, filename) }; DownloadWithProgress(targets, user); return targets.First().filename; @@ -191,7 +189,7 @@ public static void DownloadWithProgress(ICollection downloadTarg } else { - File.Move(filename, downloadTargets.First(p => p.url == url).filename); + File.Move(filename, downloadTargets.First(p => p.urls.Contains(url)).filename); } }; downloader.DownloadAndWait(downloadTargets); diff --git a/Core/Net/NetAsyncDownloader.cs b/Core/Net/NetAsyncDownloader.cs index e540a8801c..41b2d3fddc 100644 --- a/Core/Net/NetAsyncDownloader.cs +++ b/Core/Net/NetAsyncDownloader.cs @@ -24,13 +24,15 @@ private class NetAsyncDownloaderDownloadPart { public readonly Net.DownloadTarget target; public DateTime lastProgressUpdateTime; - public string path; + public long lastProgressUpdateSize; + public readonly string path; public long bytesLeft; public long size; public long bytesPerSecond; - public bool triedFallback; public Exception error; - public long lastProgressUpdateSize; + + // Number of target URLs already tried and failed + private int triedDownloads; /// /// Percentage, bytes received, total bytes to receive @@ -38,25 +40,46 @@ private class NetAsyncDownloaderDownloadPart public event Action Progress; public event Action Done; - private string mimeType; + private string mimeType => target.mimeType; private ResumingWebClient agent; public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target) { this.target = target; - this.mimeType = target.mimeType; - this.triedFallback = false; - this.path = target.filename ?? Path.GetTempFileName(); - this.size = bytesLeft = target.size; - this.lastProgressUpdateTime = DateTime.Now; + path = target.filename ?? Path.GetTempFileName(); + size = bytesLeft = target.size; + lastProgressUpdateTime = DateTime.Now; + triedDownloads = 0; } public void Download(Uri url, string path) { ResetAgent(); + // Check whether to use an auth token for this host + string token; + if (url.IsAbsoluteUri + && ServiceLocator.Container.Resolve().TryGetAuthToken(url.Host, out token) + && !string.IsNullOrEmpty(token)) + { + log.InfoFormat("Using auth token for {0}", url.Host); + // Send our auth token to the GitHub API (or whoever else needs one) + agent.Headers.Add("Authorization", $"token {token}"); + } agent.DownloadFileAsyncWithResume(url, path); } + public Uri CurrentUri => target.urls[triedDownloads]; + + public bool HaveMoreUris => triedDownloads + 1 < target.urls.Count; + + public void NextUri() + { + if (HaveMoreUris) + { + ++triedDownloads; + } + } + public void Abort() { agent?.CancelAsyncOverridden(); @@ -75,17 +98,6 @@ private void ResetAgent() agent.Headers.Add("Accept", mimeType); } - // Check whether to use an auth token for this host - string token; - if (this.target.url.IsAbsoluteUri - && ServiceLocator.Container.Resolve().TryGetAuthToken(this.target.url.Host, out token) - && !string.IsNullOrEmpty(token)) - { - log.InfoFormat("Using auth token for {0}", this.target.url.Host); - // Send our auth token to the GitHub API (or whoever else needs one) - agent.Headers.Add("Authorization", $"token {token}"); - } - // Forward progress and completion events to our listeners agent.DownloadProgressChanged += (sender, args) => { @@ -115,18 +127,20 @@ private void ResetAgent() public event Action Progress; private readonly object dlMutex = new object(); - private List downloads = new List(); - private List queuedDownloads = new List(); + // NOTE: Never remove anything from this, because closures have indexes into it! + // (Clearing completely after completion is OK) + private List downloads = new List(); + private List queuedDownloads = new List(); private int completed_downloads; - //Used for inter-thread communication. + // For inter-thread communication private volatile bool download_canceled; private readonly ManualResetEvent complete_or_canceled; public event Action onOneCompleted; /// - /// Returns a perfectly boring NetAsyncDownloader. + /// Returns a perfectly boring NetAsyncDownloader /// public NetAsyncDownloader(IUser user) { @@ -145,7 +159,7 @@ public void DownloadAndWait(ICollection targets) // Some downloads are still in progress, add to the current batch foreach (Net.DownloadTarget target in targets) { - DownloadModule(target); + DownloadModule(new NetAsyncDownloaderDownloadPart(target)); } // Wait for completion along with original caller // so we can handle completion tasks for the added mods @@ -174,17 +188,17 @@ public void DownloadAndWait(ICollection targets) // If the user cancelled our progress, then signal that. if (old_download_canceled) { - log.DebugFormat("User clicked cancel, discarding {0} queued downloads: {1}", queuedDownloads.Count, string.Join(", ", queuedDownloads.Select(dl => dl.url))); + log.DebugFormat("User clicked cancel, discarding {0} queued downloads: {1}", queuedDownloads.Count, string.Join(", ", queuedDownloads.SelectMany(dl => dl.target.urls))); // Ditch anything we haven't started queuedDownloads.Clear(); // Abort all our traditional downloads, if there are any. var inProgress = downloads.Where(dl => dl.bytesLeft > 0 && dl.error == null).ToList(); - log.DebugFormat("Telling {0} in progress downloads to abort: {1}", inProgress.Count, string.Join(", ", inProgress.Select(dl => dl.target.url))); + log.DebugFormat("Telling {0} in progress downloads to abort: {1}", inProgress.Count, string.Join(", ", inProgress.SelectMany(dl => dl.target.urls))); foreach (var download in inProgress) { - log.DebugFormat("Telling download of {0} to abort", download.target.url); + log.DebugFormat("Telling download of {0} to abort", string.Join(", ", download.target.urls)); download.Abort(); - log.DebugFormat("Done requesting abort of {0}", download.target.url); + log.DebugFormat("Done requesting abort of {0}", string.Join(", ", download.target.urls)); } log.Debug("Throwing cancellation kraken"); @@ -193,7 +207,7 @@ public void DownloadAndWait(ICollection targets) } // Check to see if we've had any errors. If so, then release the kraken! - List> exceptions = new List>(); + var exceptions = new List>(); for (int i = 0; i < downloads.Count; ++i) { if (downloads[i].error != null) @@ -211,11 +225,13 @@ public void DownloadAndWait(ICollection targets) { // Handle HTTP 403 used for throttling case HttpStatusCode.Forbidden: - Uri infoUrl; - if (downloads[i].target.url.IsAbsoluteUri - && Net.ThrottledHosts.TryGetValue(downloads[i].target.url.Host, out infoUrl)) + Uri infoUrl = null; + var throttledUri = downloads[i].target.urls.FirstOrDefault(uri => + uri.IsAbsoluteUri + && Net.ThrottledHosts.TryGetValue(uri.Host, out infoUrl)); + if (throttledUri != null) { - throw new DownloadThrottledKraken(downloads[i].target.url, infoUrl); + throw new DownloadThrottledKraken(throttledUri, infoUrl); } break; } @@ -257,48 +273,50 @@ private void Download(ICollection targets) { downloads.Clear(); queuedDownloads.Clear(); - queuedDownloads.AddRange(targets); - foreach (Net.DownloadTarget target in targets) + foreach (var t in targets) { - DownloadModule(target); + DownloadModule(new NetAsyncDownloaderDownloadPart(t)); } } - private void DownloadModule(Net.DownloadTarget target) + private void DownloadModule(NetAsyncDownloaderDownloadPart dl) { - if (shouldQueue(target)) + if (shouldQueue(dl.CurrentUri)) { - if (!queuedDownloads.Contains(target)) + if (!queuedDownloads.Contains(dl)) { - log.DebugFormat("Enqueuing download of {0}", target.url); + log.DebugFormat("Enqueuing download of {0}", string.Join(", ", dl.target.urls)); // Throttled host already downloading, we will get back to this later - queuedDownloads.Add(target); + queuedDownloads.Add(dl); } } else { - log.DebugFormat("Beginning download of {0}", target.url); - // We need a new variable for our closure/lambda, hence index = 1+prev max - int index = downloads.Count; + log.DebugFormat("Beginning download of {0}", string.Join(", ", dl.target.urls)); - var dl = new NetAsyncDownloaderDownloadPart(target); - downloads.Add(dl); - queuedDownloads.Remove(target); + if (!downloads.Contains(dl)) + { + // We need a new variable for our closure/lambda, hence index = 1+prev max + int index = downloads.Count; - // Encode spaces to avoid confusing URL parsers - User.RaiseMessage(Properties.Resources.NetAsyncDownloaderDownloading, - dl.target.url.ToString().Replace(" ", "%20")); + downloads.Add(dl); + queuedDownloads.Remove(dl); - // Schedule for us to get back progress reports. - dl.Progress += (ProgressPercentage, BytesReceived, TotalBytesToReceive) => - FileProgressReport(index, ProgressPercentage, BytesReceived, TotalBytesToReceive); + // Schedule for us to get back progress reports. + dl.Progress += (ProgressPercentage, BytesReceived, TotalBytesToReceive) => + FileProgressReport(index, ProgressPercentage, BytesReceived, TotalBytesToReceive); - // And schedule a notification if we're done (or if something goes wrong) - dl.Done += (sender, args, etag) => - FileDownloadComplete(index, args.Error, args.Cancelled, etag); + // And schedule a notification if we're done (or if something goes wrong) + dl.Done += (sender, args, etag) => + FileDownloadComplete(index, args.Error, args.Cancelled, etag); + } + + // Encode spaces to avoid confusing URL parsers + User.RaiseMessage(Properties.Resources.NetAsyncDownloaderDownloading, + dl.CurrentUri.ToString().Replace(" ", "%20")); // Start the download! - dl.Download(dl.target.url, dl.path); + dl.Download(dl.CurrentUri, dl.path); } } @@ -307,16 +325,16 @@ private void DownloadModule(Net.DownloadTarget target) /// Decision is made based on whether we're already downloading something /// else from the same host. /// - /// Info about a requested download + /// A URL we want to download /// /// true to queue, false to start immediately /// - private bool shouldQueue(Net.DownloadTarget target) - => downloads.Any(dl => - (!dl.target.url.IsAbsoluteUri || dl.target.url.Host == target.url.Host) - && dl.bytesLeft > 0 - // Consider done if already tried and failed - && dl.error == null); + private bool shouldQueue(Uri url) + // Ignore inactive downloads + => downloads.Except(queuedDownloads) + .Any(dl => (!dl.CurrentUri.IsAbsoluteUri || dl.CurrentUri.Host == url.Host) + // Consider done if already tried and failed + && dl.error == null); private void triggerCompleted() { @@ -366,13 +384,13 @@ private void FileProgressReport(int index, int percent, long bytesDownloaded, lo totalBytesLeft += t.bytesLeft; totalSize += t.size; } - foreach (Net.DownloadTarget t in queuedDownloads.ToList()) + foreach (var dl in queuedDownloads.ToList()) { // Somehow managed to get a NullRef for t here - if (t == null) + if (dl == null) continue; - totalBytesLeft += t.size; - totalSize += t.size; + totalBytesLeft += dl.target.size; + totalSize += dl.target.size; } int totalPercentage = (int)(((totalSize - totalBytesLeft) * 100) / (totalSize)); @@ -383,71 +401,70 @@ private void FileProgressReport(int index, int percent, long bytesDownloaded, lo totalPercentage); } + private void PopFromQueue(string host) + { + // Make sure the threads don't trip on one another + lock (dlMutex) + { + var next = queuedDownloads.FirstOrDefault(qDl => + !qDl.CurrentUri.IsAbsoluteUri || qDl.CurrentUri.Host == host); + if (next != null) + { + log.DebugFormat("Attempting to start queued download {0}", string.Join(", ", next.target.urls)); + // Start this host's next queued download + queuedDownloads.Remove(next); + DownloadModule(next); + } + } + } + /// /// This method gets called back by `WebClient` when a download is completed. /// It in turncalls the onCompleted hook when *all* downloads are finished. /// private void FileDownloadComplete(int index, Exception error, bool canceled, string etag) { + var dl = downloads[index]; + var doneUri = dl.CurrentUri; if (error != null) { - log.InfoFormat("Error downloading {0}: {1}", downloads[index].target.url, error.Message); + log.InfoFormat("Error downloading {0}: {1}", doneUri, error.Message); - // Check whether we were already downloading the fallback url - if (!canceled && !downloads[index].triedFallback && downloads[index].target.fallbackUrl != null) + // Check whether there are any alternate download URLs remaining + if (!canceled && dl.HaveMoreUris) { - log.InfoFormat("Trying fallback URL: {0}", downloads[index].target.fallbackUrl); - // Encode spaces to avoid confusing URL parsers - User.RaiseMessage(Properties.Resources.NetAsyncDownloaderTryingFallback, - downloads[index].target.url.ToString().Replace(" ", "%20"), - downloads[index].target.fallbackUrl.ToString().Replace(" ", "%20") - ); - // Try the fallbackUrl - downloads[index].triedFallback = true; - downloads[index].Download(downloads[index].target.fallbackUrl, downloads[index].path); + dl.NextUri(); + // Either re-queue this or start the next one, depending on active downloads + DownloadModule(dl); + // Check the host that just failed for queued downloads + PopFromQueue(doneUri.Host); // Short circuit the completion process so the fallback can run return; } else { - downloads[index].error = error; + dl.error = error; } } else { - log.InfoFormat("Finished downloading {0}", downloads[index].target.url); - downloads[index].bytesLeft = 0; + log.InfoFormat("Finished downloading {0}", string.Join(", ", dl.target.urls)); + dl.bytesLeft = 0; } - // Make sure the threads don't trip on one another - lock (dlMutex) - { - // Start next download, if any - if (!canceled) - { - var next = queuedDownloads.FirstOrDefault(dl => - !dl.url.IsAbsoluteUri || dl.url.Host == downloads[index].target.url.Host); - if (next != null) - { - log.DebugFormat("Attempting to start queued download {0}", next.url); - // Start this host's next queued download - queuedDownloads.Remove(next); - DownloadModule(next); - } - } - } + PopFromQueue(doneUri.Host); try { // Tell calling code that this file is ready - onOneCompleted?.Invoke(downloads[index].target.url, downloads[index].path, downloads[index].error, etag); + onOneCompleted?.Invoke(dl.target.urls.First(), dl.path, dl.error, etag); } catch (Exception exc) { - if (downloads[index].error == null) + if (dl.error == null) { // Capture anything that goes wrong with the post-download process as well - downloads[index].error = exc; + dl.error = exc; } } diff --git a/Core/Net/NetAsyncModulesDownloader.cs b/Core/Net/NetAsyncModulesDownloader.cs index bd5fb05f7c..aaa87d834b 100644 --- a/Core/Net/NetAsyncModulesDownloader.cs +++ b/Core/Net/NetAsyncModulesDownloader.cs @@ -6,6 +6,8 @@ using log4net; +using CKAN.Extensions; + namespace CKAN { /// @@ -28,7 +30,8 @@ public NetAsyncModulesDownloader(IUser user, NetModuleCache cache) downloader.onOneCompleted += ModuleDownloadComplete; downloader.Progress += (target, remaining, total) => { - var mod = modules.FirstOrDefault(m => m.download == target.url); + var mod = modules.FirstOrDefault(m => m.download?.Any(dlUri => target.urls.Contains(dlUri)) + ?? false); if (mod != null && Progress != null) { Progress(mod, remaining, total); @@ -37,41 +40,83 @@ public NetAsyncModulesDownloader(IUser user, NetModuleCache cache) this.cache = cache; } + internal static List> GroupByDownloads(IEnumerable modules) + { + // Each module is a vertex, each download URL is an edge + // We want to group the vertices by transitive connectedness + // We can go breadth first or depth first + // Once we encounter a mod, we never have to look at it again + var unsearched = modules.ToHashSet(); + var groups = new List>(); + while (unsearched.Count > 0) + { + // Find one group, remove it from unsearched, add it to groups + var searching = new List { unsearched.First() }; + unsearched.ExceptWith(searching); + var found = searching.ToHashSet(); + // Breadth first search to find all modules any URLs in common, transitively + while (searching.Count > 0) + { + var origin = searching.First(); + searching.Remove(origin); + var neighbors = origin.download + .SelectMany(dlUri => unsearched.Where(other => other.download.Contains(dlUri))) + .ToHashSet(); + unsearched.ExceptWith(neighbors); + searching.AddRange(neighbors); + found.UnionWith(neighbors); + } + groups.Add(found); + } + return groups; + } + + internal Net.DownloadTarget TargetFromModuleGroup(HashSet group, + string[] preferredHosts) + => TargetFromModuleGroup(group, group.OrderBy(m => m.identifier).First(), preferredHosts); + + private Net.DownloadTarget TargetFromModuleGroup(HashSet group, + CkanModule first, + string[] preferredHosts) + => new Net.DownloadTarget( + group.SelectMany(mod => mod.download) + .Concat(group.Select(mod => mod.InternetArchiveDownload) + .Where(uri => uri != null) + .OrderBy(uri => uri.ToString())) + .Distinct() + .OrderBy(u => u, + new PreferredHostUriComparer(preferredHosts)) + .ToList(), + cache.GetInProgressFileName(first), + first.download_size, + string.IsNullOrEmpty(first.download_content_type) + ? defaultMimeType + : $"{first.download_content_type};q=1.0,{defaultMimeType};q=0.9"); + /// /// /// public void DownloadModules(IEnumerable modules) { - // Walk through all our modules, but only keep the first of each - // one that has a unique download path (including active downloads). - var currentlyActive = new HashSet(this.modules.Select(m => m.download)); - Dictionary unique_downloads = modules - .GroupBy(module => module.download) - .Where(group => !currentlyActive.Contains(group.Key)) - .ToDictionary(group => group.Key, group => group.First()); - + var activeURLs = this.modules.SelectMany(m => m.download) + .ToHashSet(); + var moduleGroups = GroupByDownloads(modules); // Make sure we have enough space to download and cache - cache.CheckFreeSpace(unique_downloads.Values - .Select(m => m.download_size) - .Sum()); - - this.modules.AddRange(unique_downloads.Values); + cache.CheckFreeSpace(moduleGroups.Select(grp => grp.First().download_size) + .Sum()); + // Add all the requested modules + this.modules.AddRange(moduleGroups.SelectMany(grp => grp)); try { cancelTokenSrc = new CancellationTokenSource(); + var preferredHosts = ServiceLocator.Container.Resolve().PreferredHosts; // Start the downloads! - downloader.DownloadAndWait(unique_downloads - .Select(item => new Net.DownloadTarget( - item.Key, - item.Value.InternetArchiveDownload, - cache.GetInProgressFileName(item.Value), - item.Value.download_size, - // Send the MIME type to use for the Accept header - // The GitHub API requires this to include application/octet-stream - string.IsNullOrEmpty(item.Value.download_content_type) - ? defaultMimeType - : $"{item.Value.download_content_type};q=1.0,{defaultMimeType};q=0.9")) + downloader.DownloadAndWait(moduleGroups + // Skip any group that already has a URL in progress + .Where(grp => grp.All(mod => mod.download.All(dlUri => !activeURLs.Contains(dlUri)))) + // Each group gets one target containing all the URLs + .Select(grp => TargetFromModuleGroup(grp, preferredHosts)) .ToList()); this.modules.Clear(); AllComplete?.Invoke(); @@ -122,7 +167,8 @@ private void ModuleDownloadComplete(Uri url, string filename, Exception error, s CkanModule module = null; try { - module = modules.First(m => m.download == url); + module = modules.First(m => m.download?.Any(dlUri => dlUri == url) + ?? false); User.RaiseMessage(Properties.Resources.NetAsyncDownloaderValidating, module); cache.Store(module, filename, new Progress(percent => StoreProgress?.Invoke(module, 100 - percent, 100)), diff --git a/Core/Net/NetFileCache.cs b/Core/Net/NetFileCache.cs index 49bbdf738b..fba3c0a16e 100644 --- a/Core/Net/NetFileCache.cs +++ b/Core/Net/NetFileCache.cs @@ -124,6 +124,14 @@ public string GetInProgressFileName(Uri url, string description) => GetInProgressFileName(NetFileCache.CreateURLHash(url), description); + public string GetInProgressFileName(List urls, string description) + { + var filenames = urls.Select(url => GetInProgressFileName(NetFileCache.CreateURLHash(url), description)) + .ToArray(); + return filenames.FirstOrDefault(filename => File.Exists(filename)) + ?? filenames.FirstOrDefault(); + } + /// /// Called from our FileSystemWatcher. Use OnCacheChanged() /// without arguments to signal manually. diff --git a/Core/Net/NetModuleCache.cs b/Core/Net/NetModuleCache.cs index 09b6837f9d..3dbce8a8e1 100644 --- a/Core/Net/NetModuleCache.cs +++ b/Core/Net/NetModuleCache.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.IO; using System.Threading; using System.Security.Cryptography; @@ -48,29 +49,37 @@ public void MoveFrom(string fromDir) cache.MoveFrom(fromDir); } public bool IsCached(CkanModule m) - { - return cache.IsCached(m.download); - } + => m.download?.Any(dlUri => cache.IsCached(dlUri)) + ?? false; public bool IsCached(CkanModule m, out string outFilename) { - return cache.IsCached(m.download, out outFilename); + if (m.download != null) + { + foreach (var dlUri in m.download) + { + if (cache.IsCached(dlUri, out outFilename)) + { + return true; + } + } + } + outFilename = null; + return false; } public bool IsCachedZip(CkanModule m) - { - return cache.IsCachedZip(m.download); - } + => m.download?.Any(dlUri => cache.IsCachedZip(dlUri)) + ?? false; public bool IsMaybeCachedZip(CkanModule m) - { - return cache.IsMaybeCachedZip(m.download, m.release_date); - } + => m.download?.Any(dlUri => cache.IsMaybeCachedZip(dlUri, m.release_date)) + ?? false; public string GetCachedFilename(CkanModule m) - { - return cache.GetCachedFilename(m.download, m.release_date); - } + => m.download?.Select(dlUri => cache.GetCachedFilename(dlUri, m.release_date)) + .Where(filename => filename != null) + .FirstOrDefault(); public string GetCachedZip(CkanModule m) - { - return cache.GetCachedZip(m.download); - } + => m.download?.Select(dlUri => cache.GetCachedZip(dlUri)) + .Where(filename => filename != null) + .FirstOrDefault(); public void GetSizeInfo(out int numFiles, out long numBytes, out long bytesFree) { cache.GetSizeInfo(out numFiles, out numBytes, out bytesFree); @@ -90,10 +99,13 @@ public string GetInProgressFileName(CkanModule m) private static string DescribeUncachedAvailability(CkanModule m, FileInfo fi) => fi.Exists ? string.Format(Properties.Resources.NetModuleCacheModuleResuming, - m.name, m.version, m.download.Host ?? "", + m.name, m.version, + string.Join(", ", m.download.Select(dl => dl.Host).Distinct()), CkanModule.FmtSize(m.download_size - fi.Length)) : string.Format(Properties.Resources.NetModuleCacheModuleHostSize, - m.name, m.version, m.download.Host ?? "", CkanModule.FmtSize(m.download_size)); + m.name, m.version, + string.Join(", ", m.download.Select(dl => dl.Host).Distinct()), + CkanModule.FmtSize(m.download_size)); public string DescribeAvailability(CkanModule m) => m.IsMetapackage @@ -200,22 +212,32 @@ public string DescribeAvailability(CkanModule m) cancelToken.ThrowIfCancellationRequested(); // If no exceptions, then everything is fine - var success = cache.Store(module.download, path, description ?? module.StandardName(), move); + var success = cache.Store(module.download[0], path, description ?? module.StandardName(), move); // Make sure completion is signalled so progress bars go away progress?.Report(100); return success; } /// - /// Remove a module's download file from the cache + /// Remove a module's download files from the cache /// /// Module to purge /// - /// True if purged, false otherwise + /// True if all purged, false otherwise /// public bool Purge(CkanModule module) { - return cache.Remove(module.download); + if (module.download != null) + { + foreach (var dlUri in module.download) + { + if (!cache.Remove(dlUri)) + { + return false; + } + } + } + return true; } private NetFileCache cache; diff --git a/Core/Net/Repo.cs b/Core/Net/Repo.cs index 9e1f5f947f..cc69229235 100644 --- a/Core/Net/Repo.cs +++ b/Core/Net/Repo.cs @@ -62,7 +62,7 @@ public static RepoUpdateResult UpdateAllRepositories(RegistryManager registry_ma downloader.onOneCompleted += (url, filename, error, etag) => savedEtags.Add(url, etag); // Download metadata from all repos - var targets = repos.Select(r => new Net.DownloadTarget(r.uri)).ToArray(); + var targets = repos.Select(r => new Net.DownloadTarget(new List() { r.uri })).ToArray(); downloader.DownloadAndWait(targets); // If we get to this point, the downloads were successful diff --git a/Core/Registry/Registry.cs b/Core/Registry/Registry.cs index 25837531e4..6968a64d1c 100644 --- a/Core/Registry/Registry.cs +++ b/Core/Registry/Registry.cs @@ -1266,14 +1266,17 @@ public Dictionary> GetDownloadHashIndex() CkanModule mod = kvp2.Value; if (mod.download != null) { - string hash = NetFileCache.CreateURLHash(mod.download); - if (index.ContainsKey(hash)) + foreach (var dlUri in mod.download) { - index[hash].Add(mod); - } - else - { - index.Add(hash, new List() {mod}); + string hash = NetFileCache.CreateURLHash(dlUri); + if (index.ContainsKey(hash)) + { + index[hash].Add(mod); + } + else + { + index.Add(hash, new List() {mod}); + } } } } diff --git a/Core/Types/CkanModule.cs b/Core/Types/CkanModule.cs index c7761da147..10da7f12c4 100644 --- a/Core/Types/CkanModule.cs +++ b/Core/Types/CkanModule.cs @@ -128,7 +128,8 @@ public class CkanModule : IEquatable public ModuleRelationshipDescriptor replaced_by; [JsonProperty("download", Order = 25, NullValueHandling = NullValueHandling.Ignore)] - public Uri download; + [JsonConverter(typeof(JsonSingleOrArrayConverter))] + public List download; [JsonProperty("download_size", Order = 26, DefaultValueHandling = DefaultValueHandling.Ignore)] [DefaultValue(0)] @@ -319,7 +320,7 @@ public CkanModule( this.author = author; this.license = license; this.version = version; - this.download = download; + this.download = new List { download }; this.kind = kind; this._comparator = comparator ?? ServiceLocator.Container.Resolve(); CheckHealth(); diff --git a/GUI/Main/MainRepo.cs b/GUI/Main/MainRepo.cs index f09906028b..04052b988c 100644 --- a/GUI/Main/MainRepo.cs +++ b/GUI/Main/MainRepo.cs @@ -80,7 +80,7 @@ private void UpdateRepo(object sender, DoWorkEventArgs e) downloader.Progress += (target, remaining, total) => { var repo = repos - .Where(r => r.uri == target.url) + .Where(r => target.urls.Contains(r.uri)) .FirstOrDefault(); if (repo != null) { diff --git a/Tests/Core/Net/NetAsyncModulesDownloader.cs b/Tests/Core/Net/NetAsyncModulesDownloader.cs deleted file mode 100644 index 7a64056719..0000000000 --- a/Tests/Core/Net/NetAsyncModulesDownloader.cs +++ /dev/null @@ -1,139 +0,0 @@ -using System.Collections.Generic; - -using log4net; -using NUnit.Framework; - -using Tests.Data; -using CKAN; - -namespace Tests.Core.Net -{ - /// - /// Test the async downloader. - /// - - [TestFixture] - public class NetAsyncModulesDownloader - { - private CKAN.GameInstanceManager manager; - private CKAN.RegistryManager registry_manager; - private CKAN.Registry registry; - private DisposableKSP ksp; - private CKAN.IDownloader async; - private NetModuleCache cache; - private NetAsyncDownloader downloader; - - private static readonly ILog log = LogManager.GetLogger(typeof (NetAsyncModulesDownloader)); - - [SetUp] - public void Setup() - { - manager = new GameInstanceManager(new NullUser()); - // Give us a registry to play with. - ksp = new DisposableKSP(); - registry_manager = CKAN.RegistryManager.Instance(ksp.KSP); - registry = registry_manager.registry; - registry.ClearDlls(); - registry.Installed().Clear(); - // Make sure we have a registry we can use. - - registry.Repositories = new SortedDictionary() - { - { - "testRepo", - new Repository("testRepo", TestData.TestKANZip()) - } - }; - - downloader = new NetAsyncDownloader(new NullUser()); - - CKAN.Repo.UpdateAllRepositories(registry_manager, ksp.KSP, downloader, null, new NullUser()); - - // Ready our downloader. - async = new CKAN.NetAsyncModulesDownloader(new NullUser(), manager.Cache); - - // General shortcuts - cache = manager.Cache; - } - - [TearDown] - public void TearDown() - { - manager.Dispose(); - ksp.Dispose(); - } - - [Test] - [Category("Online")] - [Category("NetAsyncModulesDownloader")] - [Explicit] - public void SingleDownload() - { - log.Info("Performing single download test."); - - // We know kOS is in the TestKAN data, and hosted in KS. Let's get it. - - var modules = new List(); - - CkanModule kOS = registry.LatestAvailable("kOS", null); - Assert.IsNotNull(kOS); - - modules.Add(kOS); - - // Make sure we don't alread have kOS somehow. - Assert.IsFalse(cache.IsCached(kOS)); - - // - log.InfoFormat("Downloading kOS from {0}", kOS.download); - - // Download our module. - async.DownloadModules(modules); - - // Assert that we have it, and it passes zip validation. - Assert.IsTrue(cache.IsCachedZip(kOS)); - } - - [Test] - [Category("Online")] - [Category("NetAsyncModulesDownloader")] - [Explicit] - public void MultiDownload() - { - var modules = new List(); - - CkanModule kOS = registry.LatestAvailable("kOS", null); - CkanModule quick_revert = registry.LatestAvailable("QuickRevert", null); - - modules.Add(kOS); - modules.Add(quick_revert); - - Assert.IsFalse(cache.IsCachedZip(kOS)); - Assert.IsFalse(cache.IsCachedZip(quick_revert)); - - async.DownloadModules(modules); - - Assert.IsTrue(cache.IsCachedZip(kOS)); - Assert.IsTrue(cache.IsCachedZip(quick_revert)); - } - - [Test] - [Category("Online")] - [Category("NetAsyncModulesDownloader")] - [Explicit] - public void RandSdownload() - { - var modules = new List(); - - var rAndS = TestData.RandSCapsuleDyneModule(); - - modules.Add(rAndS); - - Assert.IsFalse(cache.IsCachedZip(rAndS), "Module not yet downloaded"); - - async.DownloadModules(modules); - - Assert.IsTrue(cache.IsCachedZip(rAndS),"Module download successful"); - } - - } -} diff --git a/Tests/Core/Net/NetAsyncModulesDownloaderTests.cs b/Tests/Core/Net/NetAsyncModulesDownloaderTests.cs new file mode 100644 index 0000000000..1586802e65 --- /dev/null +++ b/Tests/Core/Net/NetAsyncModulesDownloaderTests.cs @@ -0,0 +1,317 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using log4net; +using NUnit.Framework; + +using Tests.Data; +using CKAN; +using CKAN.Extensions; + +namespace Tests.Core.Net +{ + /// + /// Test the async downloader. + /// + + [TestFixture] + public class NetAsyncModulesDownloaderTests + { + private CKAN.GameInstanceManager manager; + private CKAN.RegistryManager registry_manager; + private CKAN.Registry registry; + private DisposableKSP ksp; + private CKAN.IDownloader async; + private NetModuleCache cache; + private NetAsyncDownloader downloader; + + private static readonly ILog log = LogManager.GetLogger(typeof(NetAsyncModulesDownloaderTests)); + + [SetUp] + public void Setup() + { + manager = new GameInstanceManager(new NullUser()); + // Give us a registry to play with. + ksp = new DisposableKSP(); + registry_manager = CKAN.RegistryManager.Instance(ksp.KSP); + registry = registry_manager.registry; + registry.ClearDlls(); + registry.Installed().Clear(); + // Make sure we have a registry we can use. + + registry.Repositories = new SortedDictionary() + { + { + "testRepo", + new Repository("testRepo", TestData.TestKANZip()) + } + }; + + downloader = new NetAsyncDownloader(new NullUser()); + + CKAN.Repo.UpdateAllRepositories(registry_manager, ksp.KSP, downloader, null, new NullUser()); + + // Ready our downloader. + async = new CKAN.NetAsyncModulesDownloader(new NullUser(), manager.Cache); + + // General shortcuts + cache = manager.Cache; + } + + [TearDown] + public void TearDown() + { + manager.Dispose(); + ksp.Dispose(); + } + + [Test, + // No mods, nothing to group + TestCase(new string[] { }, + // null is removed as a workaround for limitations of params keyword + null), + // One mod, one group + TestCase(new string[] + { + @"{ + ""identifier"": ""ModA"", + ""version"": ""1.0"", + ""download"": ""https://github.com/"" + }", + }, + new string[] { "ModA" }), + // Two unrelated, two groups + TestCase(new string[] + { + @"{ + ""identifier"": ""ModA"", + ""version"": ""1.0"", + ""download"": ""https://github.com/"" + }", + @"{ + ""identifier"": ""ModB"", + ""version"": ""1.0"", + ""download"": ""https://spacedock.info/"" + }", + }, + new string[] { "ModA" }, + new string[] { "ModB" }), + // Same URL, one group + TestCase(new string[] + { + @"{ + ""identifier"": ""ModA"", + ""version"": ""1.0"", + ""download"": ""https://github.com/"" + }", + @"{ + ""identifier"": ""ModB"", + ""version"": ""1.0"", + ""download"": ""https://github.com/"" + }", + }, + new string[] { "ModA", "ModB" }), + // Transitively shared URLs in one group, unrelated separate + TestCase(new string[] + { + @"{ + ""identifier"": ""ModA"", + ""version"": ""1.0"", + ""download"": [ ""https://github.com/"", ""https://spacedock.info/"" ] + }", + @"{ + ""identifier"": ""ModB"", + ""version"": ""1.0"", + ""download"": [ ""https://curseforge.com/"" ] + }", + @"{ + ""identifier"": ""ModC"", + ""version"": ""1.0"", + ""download"": [ ""https://spacedock.info/"", ""https://archive.org/"" ] + }", + @"{ + ""identifier"": ""ModD"", + ""version"": ""1.0"", + ""download"": [ ""https://drive.google.com/"" ] + }", + @"{ + ""identifier"": ""ModE"", + ""version"": ""1.0"", + ""download"": [ ""https://archive.org/"", ""https://taniwha.org/"" ] + }", + }, + new string[] { "ModA", "ModC", "ModE" }, + new string[] { "ModB" }, + new string[] { "ModD" }), + ] + public void GroupByDownloads_WithModules_GroupsBySharedURLs(string[] moduleJsons, params string[][] correctGroups) + { + // Arrange + var modules = moduleJsons.Select(j => CkanModule.FromJson(j)) + .ToArray(); + // Turn [null] into [] as Workaround for params argument not allowing no values + // (params argument itself is a workaround for TestCase not allowing string[][]) + correctGroups = correctGroups.Where(g => g != null).ToArray(); + + // Act + var result = NetAsyncModulesDownloader.GroupByDownloads(modules); + var groupIdentifiers = result.Select(grp => grp.OrderBy(m => m.identifier) + .Select(m => m.identifier) + .ToArray()) + .OrderBy(grp => grp.First()) + .ToArray(); + + // Assert + Assert.AreEqual(correctGroups, groupIdentifiers); + } + + [Test, + // No modules, not valid + TestCase(new string[] { }, + new string[] { }, + null), + // One module, no settings, preserve order + TestCase(new string[] + { + @"{ + ""identifier"": ""ModA"", + ""version"": ""1.0"", + ""download"": [ ""https://spacedock.info/"", ""https://github.com/"" ] + }", + }, + new string[] { }, + new string[] { "https://spacedock.info/", "https://github.com/" }), + // Multiple mods, redistributable license w/ hash, sort by priority w/ implicit archive.org fallback + TestCase(new string[] + { + @"{ + ""identifier"": ""ModA"", + ""version"": ""1.0"", + ""license"": ""GPL-3.0"", + ""download"": [ ""https://spacedock.info/"", ""https://github.com/"" ], + ""download_hash"": { ""sha1"": ""DEADBEEFDEADBEEF""} + }", + @"{ + ""identifier"": ""ModB"", + ""version"": ""1.0"", + ""license"": ""GPL-3.0"", + ""download"": [ ""https://spacedock.info/"", ""https://github.com/"" ], + ""download_hash"": { ""sha1"": ""DEADBEEFDEADBEEF""} + }", + @"{ + ""identifier"": ""ModC"", + ""version"": ""1.0"", + ""license"": ""GPL-3.0"", + ""download"": [ ""https://spacedock.info/"", ""https://github.com/"" ], + ""download_hash"": { ""sha1"": ""DEADBEEFDEADBEEF""} + }", + }, + new string[] { "github.com", null }, + new string[] + { + "https://github.com/", + "https://spacedock.info/", + "https://archive.org/download/ModA-1.0/DEADBEEF-ModA-1.0.zip", + "https://archive.org/download/ModB-1.0/DEADBEEF-ModB-1.0.zip", + "https://archive.org/download/ModC-1.0/DEADBEEF-ModC-1.0.zip" + }), + ] + public void TargetFromModuleGroup_WithModules_ExpectedTarget(string[] moduleJsons, string[] preferredHosts, string[] correctURLs) + { + // Arrange + var group = moduleJsons.Select(j => CkanModule.FromJson(j)) + .ToHashSet(); + var downloader = new NetAsyncModulesDownloader(new NullUser(), cache); + + if (correctURLs == null) + { + // Act / Assert + Assert.Throws(() => + downloader.TargetFromModuleGroup(group, preferredHosts)); + } + else + { + // Act + var result = downloader.TargetFromModuleGroup(group, preferredHosts); + var urls = result.urls.Select(u => u.ToString()).ToArray(); + + // Assert + Assert.AreEqual(correctURLs, urls); + } + } + + [Test] + [Category("Online")] + [Category("NetAsyncModulesDownloader")] + [Explicit] + public void SingleDownload() + { + log.Info("Performing single download test."); + + // We know kOS is in the TestKAN data, and hosted in KS. Let's get it. + + var modules = new List(); + + CkanModule kOS = registry.LatestAvailable("kOS", null); + Assert.IsNotNull(kOS); + + modules.Add(kOS); + + // Make sure we don't alread have kOS somehow. + Assert.IsFalse(cache.IsCached(kOS)); + + // + log.InfoFormat("Downloading kOS from {0}", kOS.download); + + // Download our module. + async.DownloadModules(modules); + + // Assert that we have it, and it passes zip validation. + Assert.IsTrue(cache.IsCachedZip(kOS)); + } + + [Test] + [Category("Online")] + [Category("NetAsyncModulesDownloader")] + [Explicit] + public void MultiDownload() + { + var modules = new List(); + + CkanModule kOS = registry.LatestAvailable("kOS", null); + CkanModule quick_revert = registry.LatestAvailable("QuickRevert", null); + + modules.Add(kOS); + modules.Add(quick_revert); + + Assert.IsFalse(cache.IsCachedZip(kOS)); + Assert.IsFalse(cache.IsCachedZip(quick_revert)); + + async.DownloadModules(modules); + + Assert.IsTrue(cache.IsCachedZip(kOS)); + Assert.IsTrue(cache.IsCachedZip(quick_revert)); + } + + [Test] + [Category("Online")] + [Category("NetAsyncModulesDownloader")] + [Explicit] + public void RandSdownload() + { + var modules = new List(); + + var rAndS = TestData.RandSCapsuleDyneModule(); + + modules.Add(rAndS); + + Assert.IsFalse(cache.IsCachedZip(rAndS), "Module not yet downloaded"); + + async.DownloadModules(modules); + + Assert.IsTrue(cache.IsCachedZip(rAndS),"Module download successful"); + } + + } +} diff --git a/Tests/Core/Types/CkanModuleTests.cs b/Tests/Core/Types/CkanModuleTests.cs index 989c2cd315..f4dcc369db 100644 --- a/Tests/Core/Types/CkanModuleTests.cs +++ b/Tests/Core/Types/CkanModuleTests.cs @@ -38,7 +38,7 @@ public void MetaData() Assert.AreEqual("kOS - Kerbal OS", module.name); Assert.AreEqual("kOS", module.identifier); Assert.AreEqual("A programming and automation environment for KSP craft.", module.@abstract); - Assert.AreEqual("https://github.com/KSP-KOS/KOS/releases/download/v0.14/kOS.v14.zip", module.download.ToString()); + Assert.AreEqual("https://github.com/KSP-KOS/KOS/releases/download/v0.14/kOS.v14.zip", module.download[0].ToString()); Assert.AreEqual("GPL-3.0", module.license.First().ToString()); Assert.AreEqual("0.14", module.version.ToString()); Assert.AreEqual("stable", module.release_status.ToString()); @@ -65,7 +65,7 @@ public void MetaData() public void SpacesPreservedInDownload() { CkanModule module = CkanModule.FromJson(TestData.DogeCoinFlag_101()); - Assert.AreEqual("https://kerbalstuff.com/mod/269/Dogecoin%20Flag/download/1.01", module.download.OriginalString); + Assert.AreEqual("https://kerbalstuff.com/mod/269/Dogecoin%20Flag/download/1.01", module.download[0].OriginalString); } [Test] From 2d405a84bbd95b6566528e697863d50fc4c73e33 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Sat, 5 Aug 2023 14:03:56 -0500 Subject: [PATCH 3/4] Ability to prioritize downloads by host --- Core/Configuration/IConfiguration.cs | 7 ++ Core/Configuration/JsonConfiguration.cs | 34 +++++-- .../Win32RegistryConfiguration.cs | 5 + Core/ModuleInstaller.cs | 19 +++- Core/Net/NetAsyncModulesDownloader.cs | 2 + Core/Net/NetModuleCache.cs | 4 +- Core/Net/PreferredHostUriComparer.cs | 29 ++++++ Core/Registry/AvailableModule.cs | 2 +- Tests/Core/Configuration/FakeConfiguration.cs | 2 + .../Core/Net/PreferredHostUriComparerTests.cs | 94 +++++++++++++++++++ 10 files changed, 182 insertions(+), 16 deletions(-) create mode 100644 Core/Net/PreferredHostUriComparer.cs create mode 100644 Tests/Core/Net/PreferredHostUriComparerTests.cs diff --git a/Core/Configuration/IConfiguration.cs b/Core/Configuration/IConfiguration.cs index 549ebc4374..97a89065c6 100644 --- a/Core/Configuration/IConfiguration.cs +++ b/Core/Configuration/IConfiguration.cs @@ -59,5 +59,12 @@ public interface IConfiguration /// Paths that should be excluded from all installations /// string[] GlobalInstallFilters { get; set; } + + /// + /// List of hosts in order of priority when there are multiple URLs to choose from. + /// The first null value represents where all other hosts should go. + /// + /// + string[] PreferredHosts { get; set; } } } diff --git a/Core/Configuration/JsonConfiguration.cs b/Core/Configuration/JsonConfiguration.cs index 10033d21d0..7a78debf98 100644 --- a/Core/Configuration/JsonConfiguration.cs +++ b/Core/Configuration/JsonConfiguration.cs @@ -2,7 +2,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; + using Newtonsoft.Json; + using CKAN.Games; namespace CKAN.Configuration @@ -23,20 +25,16 @@ private class Config public IList GameInstances { get; set; } = new List(); public IDictionary AuthTokens { get; set; } = new Dictionary(); public string[] GlobalInstallFilters { get; set; } = new string[] { }; + public string[] PreferredHosts { get; set; } = new string[] { }; } public class ConfigConverter : JsonPropertyNamesChangedConverter { protected override Dictionary mapping - { - get + => new Dictionary { - return new Dictionary - { - { "KspInstances", "GameInstances" } - }; - } - } + { "KspInstances", "GameInstances" } + }; } private class GameInstanceEntry @@ -326,6 +324,26 @@ public string[] GlobalInstallFilters } } + public string[] PreferredHosts + { + get + { + lock (_lock) + { + return config.PreferredHosts; + } + } + + set + { + lock (_lock) + { + config.PreferredHosts = value; + SaveConfig(); + } + } + } + // // Save the JSON configuration file. // diff --git a/Core/Configuration/Win32RegistryConfiguration.cs b/Core/Configuration/Win32RegistryConfiguration.cs index 9f3f8f96f9..2978020f59 100644 --- a/Core/Configuration/Win32RegistryConfiguration.cs +++ b/Core/Configuration/Win32RegistryConfiguration.cs @@ -178,6 +178,11 @@ public void SetAuthToken(string host, string token) /// public string[] GlobalInstallFilters { get; set; } + /// + /// Not implemented because the Windows registry is deprecated + /// + public string[] PreferredHosts { get; set; } + public static bool DoesRegistryConfigurationExist() { RegistryKey key = Microsoft.Win32.Registry.CurrentUser.OpenSubKey(CKAN_KEY_NO_PREFIX); diff --git a/Core/ModuleInstaller.cs b/Core/ModuleInstaller.cs index 5335e9f245..173d538415 100644 --- a/Core/ModuleInstaller.cs +++ b/Core/ModuleInstaller.cs @@ -66,7 +66,11 @@ public static string Download(CkanModule module, string filename, NetModuleCache { log.Info("Downloading " + filename); - string tmp_file = Net.Download(module.download); + string tmp_file = Net.Download(module.download + .OrderBy(u => u, + new PreferredHostUriComparer( + ServiceLocator.Container.Resolve().PreferredHosts)) + .First()); return cache.Store(module, tmp_file, new Progress(bytes => {}), filename, true); } @@ -1051,14 +1055,14 @@ public void Upgrade(IEnumerable modules, IDownloader netAsyncDownloa { User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingResuming, module.name, module.version, - string.Join(", ", module.download.Select(dl => dl.Host).Distinct()), + string.Join(", ", PrioritizedHosts(module.download)), CkanModule.FmtSize(module.download_size - inProgressFile.Length)); } else { User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingUncached, module.name, module.version, - string.Join(", ", module.download.Select(dl => dl.Host).Distinct()), + string.Join(", ", PrioritizedHosts(module.download)), CkanModule.FmtSize(module.download_size)); } } @@ -1093,14 +1097,14 @@ public void Upgrade(IEnumerable modules, IDownloader netAsyncDownloa { User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingResuming, module.name, installed.version, module.version, - string.Join(", ", module.download.Select(dl => dl.Host).Distinct()), + string.Join(", ", PrioritizedHosts(module.download)), CkanModule.FmtSize(module.download_size - inProgressFile.Length)); } else { User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingUncached, module.name, installed.version, module.version, - string.Join(", ", module.download.Select(dl => dl.Host).Distinct()), + string.Join(", ", PrioritizedHosts(module.download)), CkanModule.FmtSize(module.download_size)); } } @@ -1229,6 +1233,11 @@ public void Replace(IEnumerable replacements, RelationshipRes #endregion + public static IEnumerable PrioritizedHosts(IEnumerable urls) + => urls.OrderBy(u => u, new PreferredHostUriComparer(ServiceLocator.Container.Resolve().PreferredHosts)) + .Select(dl => dl.Host) + .Distinct(); + /// /// Makes sure all the specified mods are downloaded. /// diff --git a/Core/Net/NetAsyncModulesDownloader.cs b/Core/Net/NetAsyncModulesDownloader.cs index aaa87d834b..93a6f7b635 100644 --- a/Core/Net/NetAsyncModulesDownloader.cs +++ b/Core/Net/NetAsyncModulesDownloader.cs @@ -5,7 +5,9 @@ using System.Threading; using log4net; +using Autofac; +using CKAN.Configuration; using CKAN.Extensions; namespace CKAN diff --git a/Core/Net/NetModuleCache.cs b/Core/Net/NetModuleCache.cs index 3dbce8a8e1..381e9e90ba 100644 --- a/Core/Net/NetModuleCache.cs +++ b/Core/Net/NetModuleCache.cs @@ -100,11 +100,11 @@ private static string DescribeUncachedAvailability(CkanModule m, FileInfo fi) => fi.Exists ? string.Format(Properties.Resources.NetModuleCacheModuleResuming, m.name, m.version, - string.Join(", ", m.download.Select(dl => dl.Host).Distinct()), + string.Join(", ", ModuleInstaller.PrioritizedHosts(m.download)), CkanModule.FmtSize(m.download_size - fi.Length)) : string.Format(Properties.Resources.NetModuleCacheModuleHostSize, m.name, m.version, - string.Join(", ", m.download.Select(dl => dl.Host).Distinct()), + string.Join(", ", ModuleInstaller.PrioritizedHosts(m.download)), CkanModule.FmtSize(m.download_size)); public string DescribeAvailability(CkanModule m) diff --git a/Core/Net/PreferredHostUriComparer.cs b/Core/Net/PreferredHostUriComparer.cs new file mode 100644 index 0000000000..3801b08028 --- /dev/null +++ b/Core/Net/PreferredHostUriComparer.cs @@ -0,0 +1,29 @@ +using System; +using System.Linq; +using System.Collections.Generic; + +namespace CKAN +{ + public class PreferredHostUriComparer : IComparer + { + public PreferredHostUriComparer(IEnumerable hosts) + { + this.hosts = (hosts ?? Enumerable.Empty()).ToList(); + // null represents the position in the list for all other hosts + defaultPriority = this.hosts.IndexOf(null); + } + + public int Compare(Uri a, Uri b) + => GetPriority(a).CompareTo(GetPriority(b)); + + private int GetPriority(Uri u) + { + var index = hosts.IndexOf(u.Host); + return index == -1 ? defaultPriority + : index; + } + + private readonly List hosts; + private readonly int defaultPriority; + } +} diff --git a/Core/Registry/AvailableModule.cs b/Core/Registry/AvailableModule.cs index 1c30416cfe..d313749405 100644 --- a/Core/Registry/AvailableModule.cs +++ b/Core/Registry/AvailableModule.cs @@ -91,7 +91,7 @@ public void Remove(ModuleVersion version) /// Modules that are planned to be installed /// public CkanModule Latest( - GameVersionCriteria ksp_version = null, + GameVersionCriteria ksp_version = null, RelationshipDescriptor relationship = null, IEnumerable installed = null, IEnumerable toInstall = null diff --git a/Tests/Core/Configuration/FakeConfiguration.cs b/Tests/Core/Configuration/FakeConfiguration.cs index 008fc63588..2b3454af0b 100644 --- a/Tests/Core/Configuration/FakeConfiguration.cs +++ b/Tests/Core/Configuration/FakeConfiguration.cs @@ -161,6 +161,8 @@ public string Language public string[] GlobalInstallFilters { get; set; } = new string[] { }; + public string[] PreferredHosts { get; set; } = new string[] { }; + public void Dispose() { Directory.Delete(DownloadCacheDir, true); diff --git a/Tests/Core/Net/PreferredHostUriComparerTests.cs b/Tests/Core/Net/PreferredHostUriComparerTests.cs new file mode 100644 index 0000000000..22b03b16f6 --- /dev/null +++ b/Tests/Core/Net/PreferredHostUriComparerTests.cs @@ -0,0 +1,94 @@ +using System; +using System.Linq; + +using NUnit.Framework; + +using CKAN; + +namespace Tests.Core.Net +{ + [TestFixture] + public sealed class PreferredHostUriComparerTests + { + private static readonly Uri[] uris = new Uri[] + { + new Uri("https://taniwha.org/"), + new Uri("https://spacedock.info/"), + new Uri("https://github.com/"), + new Uri("https://archive.org/"), + }; + + // Reminder: null means "all other hosts" + + [Test, + // Null settings + TestCase(null, + new string[] + { + "https://taniwha.org/", + "https://spacedock.info/", + "https://github.com/", + "https://archive.org/", + }), + // Empty settings + TestCase(new string[] { }, + new string[] + { + "https://taniwha.org/", + "https://spacedock.info/", + "https://github.com/", + "https://archive.org/", + }), + // Irrelevant settings + TestCase(new string[] { "api.github.com", "curseforge.com", null, "www.dropbox.com", "drive.google.com" }, + new string[] + { + "https://taniwha.org/", + "https://spacedock.info/", + "https://github.com/", + "https://archive.org/", + }), + // Prioritize one + TestCase(new string[] { "github.com", null }, + new string[] + { + "https://github.com/", + "https://taniwha.org/", + "https://spacedock.info/", + "https://archive.org/", + }), + // De-prioritize one + TestCase(new string[] { null, "spacedock.info" }, + new string[] + { + "https://taniwha.org/", + "https://github.com/", + "https://archive.org/", + "https://spacedock.info/", + }), + // Prioritize one, de-prioritize another + TestCase(new string[] { "github.com", null, "spacedock.info" }, + new string[] + { + "https://github.com/", + "https://taniwha.org/", + "https://archive.org/", + "https://spacedock.info/", + }), + ] + public void OrderBy_WithPreferences_SortsCorrectly(string[] preferredHosts, + string[] correctAnswer) + { + // Arrange + var comparer = new PreferredHostUriComparer(preferredHosts); + + // Act + var result = uris.OrderBy(u => u, comparer) + .Select(u => u.ToString()) + .ToArray(); + + // Assert + Assert.AreEqual(correctAnswer, result); + } + } +} From 985292b577ff618b04aa02d27c90608566d38275 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Sat, 5 Aug 2023 15:42:14 -0500 Subject: [PATCH 4/4] UI for editing preferred hosts setting --- Core/HelpURLs.cs | 1 + Core/Registry/Registry.cs | 22 ++ GUI/CKAN-GUI.csproj | 12 + GUI/Controls/EditModpack.cs | 4 +- GUI/Dialogs/PreferredHostsDialog.Designer.cs | 234 ++++++++++++++++++ GUI/Dialogs/PreferredHostsDialog.cs | 176 +++++++++++++ GUI/Dialogs/PreferredHostsDialog.resx | 124 ++++++++++ .../en-US/PreferredHostsDialog.en-US.resx | 121 +++++++++ GUI/Main/Main.Designer.cs | 14 +- GUI/Main/Main.cs | 10 + GUI/Main/Main.resx | 1 + GUI/Properties/Resources.Designer.cs | 16 ++ GUI/Properties/Resources.resx | 5 + Tests/Core/Registry/Registry.cs | 70 ++++++ Tests/GUI/ResourcesTests.cs | 6 + 15 files changed, 812 insertions(+), 4 deletions(-) create mode 100644 GUI/Dialogs/PreferredHostsDialog.Designer.cs create mode 100644 GUI/Dialogs/PreferredHostsDialog.cs create mode 100644 GUI/Dialogs/PreferredHostsDialog.resx create mode 100644 GUI/Localization/en-US/PreferredHostsDialog.en-US.resx diff --git a/Core/HelpURLs.cs b/Core/HelpURLs.cs index 787bc95034..2f8019cd9f 100644 --- a/Core/HelpURLs.cs +++ b/Core/HelpURLs.cs @@ -14,6 +14,7 @@ public static class HelpURLs public const string CloneFakeInstances = "https://github.com/KSP-CKAN/CKAN/pull/2627"; public const string DeleteDirectories = "https://github.com/KSP-CKAN/CKAN/pull/2962"; + public const string PreferredHosts = "https://github.com/KSP-CKAN/CKAN/pull/3877"; public const string Filters = "https://github.com/KSP-CKAN/CKAN/pull/3458"; public const string Labels = "https://github.com/KSP-CKAN/CKAN/pull/2936"; public const string PlayTime = "https://github.com/KSP-CKAN/CKAN/pull/3543"; diff --git a/Core/Registry/Registry.cs b/Core/Registry/Registry.cs index 6968a64d1c..390d8626bf 100644 --- a/Core/Registry/Registry.cs +++ b/Core/Registry/Registry.cs @@ -1284,6 +1284,28 @@ public Dictionary> GetDownloadHashIndex() return index; } + /// + /// Return all hosts from latest versions of all available modules, + /// sorted by number of occurrences, most common first + /// + /// Host strings without duplicates + public IEnumerable GetAllHosts() + => available_modules.Values + // Pick all modules where download is not null + .Where(availMod => availMod?.Latest()?.download != null) + // Merge all the URLs into one sequence + .SelectMany(availMod => availMod.Latest().download) + // Skip relative URLs because they don't have hosts + .Where(dlUri => dlUri.IsAbsoluteUri) + // Group the URLs by host + .GroupBy(dlUri => dlUri.Host) + // Put most commonly used hosts first + .OrderByDescending(grp => grp.Count()) + // Alphanumeric sort if same number of usages + .ThenBy(grp => grp.Key) + // Return the host from each group + .Select(grp => grp.Key); + /// /// Partition all CkanModules in available_modules into /// compatible and incompatible groups. diff --git a/GUI/CKAN-GUI.csproj b/GUI/CKAN-GUI.csproj index 2d9673272e..681a127d2d 100644 --- a/GUI/CKAN-GUI.csproj +++ b/GUI/CKAN-GUI.csproj @@ -112,6 +112,12 @@ InstallFiltersDialog.cs + + Form + + + PreferredHostsDialog.cs + Form @@ -816,6 +822,12 @@ ..\..\Dialogs\InstallFiltersDialog.cs + + PreferredHostsDialog.cs + + + ..\..\Dialogs\PreferredHostsDialog.cs + Main.cs Designer diff --git a/GUI/Controls/EditModpack.cs b/GUI/Controls/EditModpack.cs index e82997bb81..33dfc0b3fb 100644 --- a/GUI/Controls/EditModpack.cs +++ b/GUI/Controls/EditModpack.cs @@ -20,8 +20,8 @@ public EditModpack() this.ToolTip.SetToolTip(NameTextBox, Properties.Resources.EditModpackTooltipName); this.ToolTip.SetToolTip(AbstractTextBox, Properties.Resources.EditModpackTooltipAbstract); this.ToolTip.SetToolTip(VersionTextBox, Properties.Resources.EditModpackTooltipVersion); - this.ToolTip.SetToolTip(GameVersionMinComboBox, Properties.Resources.EditModpackTooltipGameVersionMin); - this.ToolTip.SetToolTip(GameVersionMaxComboBox, Properties.Resources.EditModpackTooltipGameVersionMax); + this.ToolTip.SetToolTip(GameVersionMinComboBox, Properties.Resources.EditModpackTooltipGameVersionMin); + this.ToolTip.SetToolTip(GameVersionMaxComboBox, Properties.Resources.EditModpackTooltipGameVersionMax); this.ToolTip.SetToolTip(LicenseComboBox, Properties.Resources.EditModpackTooltipLicense); this.ToolTip.SetToolTip(IncludeVersionsCheckbox, Properties.Resources.EditModpackTooltipIncludeVersions); this.ToolTip.SetToolTip(DependsRadioButton, Properties.Resources.EditModpackTooltipDepends); diff --git a/GUI/Dialogs/PreferredHostsDialog.Designer.cs b/GUI/Dialogs/PreferredHostsDialog.Designer.cs new file mode 100644 index 0000000000..287cc78890 --- /dev/null +++ b/GUI/Dialogs/PreferredHostsDialog.Designer.cs @@ -0,0 +1,234 @@ +namespace CKAN.GUI +{ + partial class PreferredHostsDialog + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + this.components = new System.ComponentModel.Container(); + System.ComponentModel.ComponentResourceManager resources = new SingleAssemblyComponentResourceManager(typeof(PreferredHostsDialog)); + this.ToolTip = new System.Windows.Forms.ToolTip(); + this.ExplanationLabel = new System.Windows.Forms.Label(); + this.Splitter = new System.Windows.Forms.SplitContainer(); + this.AvailableHostsLabel = new System.Windows.Forms.Label(); + this.AvailableHostsListBox = new System.Windows.Forms.ListBox(); + this.MoveRightButton = new System.Windows.Forms.Button(); + this.MoveLeftButton = new System.Windows.Forms.Button(); + this.PreferredHostsLabel = new System.Windows.Forms.Label(); + this.PreferredHostsListBox = new System.Windows.Forms.ListBox(); + this.MoveUpButton = new System.Windows.Forms.Button(); + this.MoveDownButton = new System.Windows.Forms.Button(); + ((System.ComponentModel.ISupportInitialize)(this.Splitter)).BeginInit(); + this.Splitter.Panel1.SuspendLayout(); + this.Splitter.Panel2.SuspendLayout(); + this.Splitter.SuspendLayout(); + this.SuspendLayout(); + // + // ToolTip + // + this.ToolTip.AutoPopDelay = 10000; + this.ToolTip.InitialDelay = 250; + this.ToolTip.ReshowDelay = 250; + this.ToolTip.ShowAlways = true; + // + // ExplanationLabel + // + this.ExplanationLabel.Dock = System.Windows.Forms.DockStyle.Top; + this.ExplanationLabel.Font = new System.Drawing.Font(System.Drawing.SystemFonts.DefaultFont.Name, 12, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Pixel); + this.ExplanationLabel.Location = new System.Drawing.Point(5, 0); + this.ExplanationLabel.Name = "ExplanationLabel"; + this.ExplanationLabel.Padding = new System.Windows.Forms.Padding(10, 10, 10, 10); + this.ExplanationLabel.Size = new System.Drawing.Size(490, 60); + resources.ApplyResources(this.ExplanationLabel, "ExplanationLabel"); + // + // Splitter + // + this.Splitter.Dock = System.Windows.Forms.DockStyle.Fill; + this.Splitter.Location = new System.Drawing.Point(0, 0); + this.Splitter.Name = "Splitter"; + this.Splitter.Size = new System.Drawing.Size(534, 300); + this.Splitter.SplitterDistance = 262; + this.Splitter.SplitterWidth = 10; + this.Splitter.TabIndex = 0; + // + // Splitter.Panel1 + // + this.Splitter.Panel1.Controls.Add(this.AvailableHostsLabel); + this.Splitter.Panel1.Controls.Add(this.AvailableHostsListBox); + this.Splitter.Panel1.Controls.Add(this.MoveRightButton); + this.Splitter.Panel1.Controls.Add(this.MoveLeftButton); + this.Splitter.Panel1MinSize = 200; + // + // Splitter.Panel2 + // + this.Splitter.Panel2.Controls.Add(this.PreferredHostsLabel); + this.Splitter.Panel2.Controls.Add(this.PreferredHostsListBox); + this.Splitter.Panel2.Controls.Add(this.MoveUpButton); + this.Splitter.Panel2.Controls.Add(this.MoveDownButton); + this.Splitter.Panel2MinSize = 200; + // + // AvailableHostsLabel + // + this.AvailableHostsLabel.AutoSize = true; + this.AvailableHostsLabel.Anchor = ((System.Windows.Forms.AnchorStyles)(System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)); + this.AvailableHostsLabel.Font = new System.Drawing.Font(System.Drawing.SystemFonts.DefaultFont.Name, 12, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Pixel); + this.AvailableHostsLabel.Location = new System.Drawing.Point(10, 10); + this.AvailableHostsLabel.Name = "AvailableHostsLabel"; + this.AvailableHostsLabel.Size = new System.Drawing.Size(75, 23); + this.AvailableHostsLabel.TabIndex = 1; + resources.ApplyResources(this.AvailableHostsLabel, "AvailableHostsLabel"); + // + // AvailableHostsListBox + // + this.AvailableHostsListBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left | System.Windows.Forms.AnchorStyles.Right)))); + this.AvailableHostsListBox.FormattingEnabled = true; + this.AvailableHostsListBox.Location = new System.Drawing.Point(10, 35); + this.AvailableHostsListBox.Name = "AvailableHostsListBox"; + this.AvailableHostsListBox.Size = new System.Drawing.Size(210, 255); + this.AvailableHostsListBox.TabIndex = 2; + this.AvailableHostsListBox.SelectedIndexChanged += new System.EventHandler(this.AvailableHostsListBox_SelectedIndexChanged); + this.AvailableHostsListBox.DoubleClick += new System.EventHandler(this.AvailableHostsListBox_DoubleClick); + // + // MoveRightButton + // + this.MoveRightButton.Anchor = ((System.Windows.Forms.AnchorStyles)(System.Windows.Forms.AnchorStyles.Top + | System.Windows.Forms.AnchorStyles.Right)); + this.MoveRightButton.FlatStyle = System.Windows.Forms.FlatStyle.Flat; + this.MoveRightButton.Location = new System.Drawing.Point(230, 35); + this.MoveRightButton.Name = "MoveRightButton"; + this.MoveRightButton.Size = new System.Drawing.Size(32, 32); + this.MoveRightButton.TabIndex = 3; + this.MoveRightButton.Text = "▸"; + this.MoveRightButton.UseVisualStyleBackColor = true; + this.MoveRightButton.Click += new System.EventHandler(this.MoveRightButton_Click); + resources.ApplyResources(this.MoveRightButton, "MoveRightButton"); + // + // MoveLeftButton + // + this.MoveLeftButton.Anchor = ((System.Windows.Forms.AnchorStyles)(System.Windows.Forms.AnchorStyles.Top + | System.Windows.Forms.AnchorStyles.Right)); + this.MoveLeftButton.FlatStyle = System.Windows.Forms.FlatStyle.Flat; + this.MoveLeftButton.Location = new System.Drawing.Point(230, 72); + this.MoveLeftButton.Name = "MoveLeftButton"; + this.MoveLeftButton.Size = new System.Drawing.Size(32, 32); + this.MoveLeftButton.TabIndex = 4; + this.MoveLeftButton.Text = "◂"; + this.MoveLeftButton.UseVisualStyleBackColor = true; + this.MoveLeftButton.Click += new System.EventHandler(this.MoveLeftButton_Click); + resources.ApplyResources(this.MoveLeftButton, "MoveLeftButton"); + // + // PreferredHostsLabel + // + this.PreferredHostsLabel.AutoSize = true; + this.PreferredHostsLabel.Anchor = ((System.Windows.Forms.AnchorStyles)(System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)); + this.PreferredHostsLabel.Font = new System.Drawing.Font(System.Drawing.SystemFonts.DefaultFont.Name, 12F, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Pixel); + this.PreferredHostsLabel.Location = new System.Drawing.Point(0, 10); + this.PreferredHostsLabel.Name = "PreferredHostsLabel"; + this.PreferredHostsLabel.Size = new System.Drawing.Size(75, 23); + this.PreferredHostsLabel.TabIndex = 5; + resources.ApplyResources(this.PreferredHostsLabel, "PreferredHostsLabel"); + // + // PreferredHostsListBox + // + this.PreferredHostsListBox.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left) + | System.Windows.Forms.AnchorStyles.Right))); + this.PreferredHostsListBox.FormattingEnabled = true; + this.PreferredHostsListBox.Location = new System.Drawing.Point(0, 35); + this.PreferredHostsListBox.Name = "PreferredHostsListBox"; + this.PreferredHostsListBox.Size = new System.Drawing.Size(216, 255); + this.PreferredHostsListBox.TabIndex = 6; + this.PreferredHostsListBox.SelectedIndexChanged += new System.EventHandler(this.PreferredHostsListBox_SelectedIndexChanged); + this.PreferredHostsListBox.DoubleClick += new System.EventHandler(this.PreferredHostsListBox_DoubleClick); + // + // MoveUpButton + // + this.MoveUpButton.Anchor = ((System.Windows.Forms.AnchorStyles)(System.Windows.Forms.AnchorStyles.Top + | System.Windows.Forms.AnchorStyles.Right)); + this.MoveUpButton.FlatStyle = System.Windows.Forms.FlatStyle.Flat; + this.MoveUpButton.Location = new System.Drawing.Point(226, 35); + this.MoveUpButton.Name = "MoveUpButton"; + this.MoveUpButton.Size = new System.Drawing.Size(32, 32); + this.MoveUpButton.TabIndex = 7; + this.MoveUpButton.Text = "▴"; + this.MoveUpButton.UseVisualStyleBackColor = true; + this.MoveUpButton.Click += new System.EventHandler(this.MoveUpButton_Click); + resources.ApplyResources(this.MoveUpButton, "MoveUpButton"); + // + // MoveDownButton + // + this.MoveDownButton.Anchor = ((System.Windows.Forms.AnchorStyles)(System.Windows.Forms.AnchorStyles.Top + | System.Windows.Forms.AnchorStyles.Right)); + this.MoveDownButton.FlatStyle = System.Windows.Forms.FlatStyle.Flat; + this.MoveDownButton.Location = new System.Drawing.Point(226, 72); + this.MoveDownButton.Name = "MoveDownButton"; + this.MoveDownButton.Size = new System.Drawing.Size(32, 32); + this.MoveDownButton.TabIndex = 8; + this.MoveDownButton.Text = "▾"; + this.MoveDownButton.UseVisualStyleBackColor = true; + this.MoveDownButton.Click += new System.EventHandler(this.MoveDownButton_Click); + resources.ApplyResources(this.MoveDownButton, "MoveDownButton"); + // + // PreferredHostsDialog + // + this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.None; + this.ClientSize = new System.Drawing.Size(534, 360); + this.Controls.Add(this.Splitter); + this.Controls.Add(this.ExplanationLabel); + this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.Sizable; + this.Icon = Properties.Resources.AppIcon; + this.MaximizeBox = false; + this.MinimizeBox = false; + this.MinimumSize = new System.Drawing.Size(480, 300); + this.HelpButton = true; + this.Name = "PreferredHostsDialog"; + this.StartPosition = System.Windows.Forms.FormStartPosition.CenterParent; + this.Load += new System.EventHandler(this.PreferredHostsDialog_Load); + this.Closing += new System.ComponentModel.CancelEventHandler(this.PreferredHostsDialog_Closing); + resources.ApplyResources(this, "$this"); + this.Splitter.Panel1.ResumeLayout(false); + this.Splitter.Panel2.ResumeLayout(false); + ((System.ComponentModel.ISupportInitialize)(this.Splitter)).EndInit(); + this.Splitter.ResumeLayout(false); + this.ResumeLayout(false); + this.PerformLayout(); + + } + + #endregion + + private System.Windows.Forms.ToolTip ToolTip; + private System.Windows.Forms.Label ExplanationLabel; + private System.Windows.Forms.SplitContainer Splitter; + private System.Windows.Forms.Label AvailableHostsLabel; + private System.Windows.Forms.ListBox AvailableHostsListBox; + private System.Windows.Forms.Button MoveRightButton; + private System.Windows.Forms.Button MoveLeftButton; + private System.Windows.Forms.Label PreferredHostsLabel; + private System.Windows.Forms.ListBox PreferredHostsListBox; + private System.Windows.Forms.Button MoveUpButton; + private System.Windows.Forms.Button MoveDownButton; + } +} diff --git a/GUI/Dialogs/PreferredHostsDialog.cs b/GUI/Dialogs/PreferredHostsDialog.cs new file mode 100644 index 0000000000..9c1f2e5896 --- /dev/null +++ b/GUI/Dialogs/PreferredHostsDialog.cs @@ -0,0 +1,176 @@ +using System; +using System.Linq; +using System.ComponentModel; +using System.Windows.Forms; + +using CKAN.Configuration; + +namespace CKAN.GUI +{ + public partial class PreferredHostsDialog : Form + { + public PreferredHostsDialog(IConfiguration config, Registry registry) + { + InitializeComponent(); + this.config = config; + allHosts = registry.GetAllHosts().ToArray(); + placeholder = Properties.Resources.PreferredHostsPlaceholder; + + ToolTip.SetToolTip(MoveRightButton, Properties.Resources.PreferredHostsTooltipMoveRight); + ToolTip.SetToolTip(MoveLeftButton, Properties.Resources.PreferredHostsTooltipMoveLeft); + ToolTip.SetToolTip(MoveUpButton, Properties.Resources.PreferredHostsTooltipMoveUp); + ToolTip.SetToolTip(MoveDownButton, Properties.Resources.PreferredHostsTooltipMoveDown); + } + + /// + /// Open the user guide when the user presses F1 + /// + protected override void OnHelpRequested(HelpEventArgs evt) + { + evt.Handled = Util.TryOpenWebPage(HelpURLs.PreferredHosts); + } + + /// + /// Open the user guide when the user clicks the help button + /// + protected override void OnHelpButtonClicked(CancelEventArgs evt) + { + evt.Cancel = Util.TryOpenWebPage(HelpURLs.PreferredHosts); + } + + private void PreferredHostsDialog_Load(object sender, EventArgs e) + { + AvailableHostsListBox.Items.AddRange(allHosts + .Except(config.PreferredHosts) + .ToArray()); + PreferredHostsListBox.Items.AddRange(config.PreferredHosts + .Select(host => host ?? placeholder) + .ToArray()); + AvailableHostsListBox_SelectedIndexChanged(null, null); + PreferredHostsListBox_SelectedIndexChanged(null, null); + } + + private void PreferredHostsDialog_Closing(object sender, CancelEventArgs e) + { + config.PreferredHosts = PreferredHostsListBox.Items.Cast() + .Select(h => h == placeholder ? null : h) + .ToArray(); + } + + private void AvailableHostsListBox_SelectedIndexChanged(object sender, EventArgs e) + { + MoveRightButton.Enabled = AvailableHostsListBox.SelectedIndex > -1; + } + + private void PreferredHostsListBox_SelectedIndexChanged(object sender, EventArgs e) + { + var haveSelection = PreferredHostsListBox.SelectedIndex > -1; + MoveLeftButton.Enabled = haveSelection + && PreferredHostsListBox.SelectedItem != placeholder; + MoveUpButton.Enabled = PreferredHostsListBox.SelectedIndex > 0; + MoveDownButton.Enabled = haveSelection + && PreferredHostsListBox.SelectedIndex < PreferredHostsListBox.Items.Count - 1; + } + + private void AvailableHostsListBox_DoubleClick(object sender, EventArgs r) + { + MoveRightButton_Click(null, null); + } + + private void PreferredHostsListBox_DoubleClick(object sender, EventArgs r) + { + MoveLeftButton_Click(null, null); + } + + private void MoveRightButton_Click(object sender, EventArgs e) + { + if (AvailableHostsListBox.SelectedIndex > -1) + { + if (PreferredHostsListBox.Items.Count == 0) + { + PreferredHostsListBox.Items.Add(placeholder); + } + var fromWhere = AvailableHostsListBox.SelectedIndex; + var selected = AvailableHostsListBox.SelectedItem; + var toWhere = PreferredHostsListBox.Items.IndexOf(placeholder); + AvailableHostsListBox.Items.Remove(selected); + PreferredHostsListBox.Items.Insert(toWhere, selected); + // Preserve selection on same line + if (AvailableHostsListBox.Items.Count > 0) + { + AvailableHostsListBox.SetSelected(Math.Min(fromWhere, + AvailableHostsListBox.Items.Count - 1), + true); + } + else + { + // ListBox doesn't notify of selection changes that happen via removal + AvailableHostsListBox_SelectedIndexChanged(null, null); + } + } + } + + private void MoveLeftButton_Click(object sender, EventArgs e) + { + if (PreferredHostsListBox.SelectedIndex > -1) + { + var fromWhere = PreferredHostsListBox.SelectedIndex; + var selected = PreferredHostsListBox.SelectedItem; + if (selected != placeholder) + { + PreferredHostsListBox.Items.Remove(selected); + // Regenerate the list to put the item back in the original order + AvailableHostsListBox.Items.Clear(); + AvailableHostsListBox.Items.AddRange(allHosts + .Except(PreferredHostsListBox.Items.Cast()) + .ToArray()); + if (PreferredHostsListBox.Items.Count == 1) + { + PreferredHostsListBox.Items.Remove(placeholder); + } + // Preserve selection on same line + if (PreferredHostsListBox.Items.Count > 0) + { + PreferredHostsListBox.SetSelected(Math.Min(fromWhere, + PreferredHostsListBox.Items.Count - 1), + true); + } + } + } + } + + private void MoveUpButton_Click(object sender, EventArgs e) + { + if (PreferredHostsListBox.SelectedIndex > 0) + { + MoveItem(PreferredHostsListBox.SelectedIndex - 1, + PreferredHostsListBox.SelectedIndex); + // ListBox doesn't notify of selection changes that happen via removal + PreferredHostsListBox_SelectedIndexChanged(null, null); + } + } + + private void MoveDownButton_Click(object sender, EventArgs e) + { + if (PreferredHostsListBox.SelectedIndex > -1 + && PreferredHostsListBox.SelectedIndex < PreferredHostsListBox.Items.Count - 1) + { + MoveItem(PreferredHostsListBox.SelectedIndex + 1, + PreferredHostsListBox.SelectedIndex); + // ListBox doesn't notify of selection changes that happen via insertion + PreferredHostsListBox_SelectedIndexChanged(null, null); + } + } + + private void MoveItem(int from, int to) + { + var item = PreferredHostsListBox.Items[from]; + PreferredHostsListBox.Items.RemoveAt(from); + PreferredHostsListBox.Items.Insert(to, item); + } + + private readonly IConfiguration config; + private readonly string[] allHosts; + private readonly string placeholder; + } +} diff --git a/GUI/Dialogs/PreferredHostsDialog.resx b/GUI/Dialogs/PreferredHostsDialog.resx new file mode 100644 index 0000000000..334b026bd6 --- /dev/null +++ b/GUI/Dialogs/PreferredHostsDialog.resx @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + If a module has multiple download URLs, they will be prioritised by host according to the list on the right. + Available hosts (by popularity): + Preferred hosts (by priority): + Preferred Hosts + diff --git a/GUI/Localization/en-US/PreferredHostsDialog.en-US.resx b/GUI/Localization/en-US/PreferredHostsDialog.en-US.resx new file mode 100644 index 0000000000..48918f45af --- /dev/null +++ b/GUI/Localization/en-US/PreferredHostsDialog.en-US.resx @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + If a module has multiple download URLs, they will be prioritized by host according to the list on the right. + diff --git a/GUI/Main/Main.Designer.cs b/GUI/Main/Main.Designer.cs index ad9784fe62..b1df4e1eca 100644 --- a/GUI/Main/Main.Designer.cs +++ b/GUI/Main/Main.Designer.cs @@ -46,6 +46,7 @@ private void InitializeComponent() this.settingsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.cKANSettingsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.pluginsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.preferredHostsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.installFiltersToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.GameCommandlineToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.compatibleGameVersionsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); @@ -252,10 +253,11 @@ private void InitializeComponent() // this.settingsToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { this.cKANSettingsToolStripMenuItem, - this.pluginsToolStripMenuItem, this.GameCommandlineToolStripMenuItem, this.compatibleGameVersionsToolStripMenuItem, - this.installFiltersToolStripMenuItem}); + this.preferredHostsToolStripMenuItem, + this.installFiltersToolStripMenuItem, + this.pluginsToolStripMenuItem}); this.settingsToolStripMenuItem.Name = "settingsToolStripMenuItem"; this.settingsToolStripMenuItem.Size = new System.Drawing.Size(88, 29); resources.ApplyResources(this.settingsToolStripMenuItem, "settingsToolStripMenuItem"); @@ -274,6 +276,13 @@ private void InitializeComponent() this.pluginsToolStripMenuItem.Click += new System.EventHandler(this.pluginsToolStripMenuItem_Click); resources.ApplyResources(this.pluginsToolStripMenuItem, "pluginsToolStripMenuItem"); // + // preferredHostsToolStripMenuItem + // + this.preferredHostsToolStripMenuItem.Name = "preferredHostsToolStripMenuItem"; + this.preferredHostsToolStripMenuItem.Size = new System.Drawing.Size(247, 30); + this.preferredHostsToolStripMenuItem.Click += new System.EventHandler(this.preferredHostsToolStripMenuItem_Click); + resources.ApplyResources(this.preferredHostsToolStripMenuItem, "preferredHostsToolStripMenuItem"); + // // installFiltersToolStripMenuItem // this.installFiltersToolStripMenuItem.Name = "installFiltersToolStripMenuItem"; @@ -878,6 +887,7 @@ private void InitializeComponent() public System.Windows.Forms.ToolStripMenuItem settingsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem cKANSettingsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem pluginsToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem preferredHostsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem installFiltersToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem GameCommandlineToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem compatibleGameVersionsToolStripMenuItem; diff --git a/GUI/Main/Main.cs b/GUI/Main/Main.cs index dadcfc535f..8f97edafba 100644 --- a/GUI/Main/Main.cs +++ b/GUI/Main/Main.cs @@ -580,6 +580,16 @@ private void pluginsToolStripMenuItem_Click(object sender, EventArgs e) Enabled = true; } + private void preferredHostsToolStripMenuItem_Click(object sender, EventArgs e) + { + Enabled = false; + var dlg = new PreferredHostsDialog( + ServiceLocator.Container.Resolve(), + RegistryManager.Instance(CurrentInstance).registry); + dlg.ShowDialog(this); + Enabled = true; + } + private void installFiltersToolStripMenuItem_Click(object sender, EventArgs e) { Enabled = false; diff --git a/GUI/Main/Main.resx b/GUI/Main/Main.resx index 9a8d9c15f4..99d60b5d46 100644 --- a/GUI/Main/Main.resx +++ b/GUI/Main/Main.resx @@ -135,6 +135,7 @@ &Settings CKAN &settings CKAN &plugins + Preferred &hosts Installation &filters &Game command-line &Compatible game versions diff --git a/GUI/Properties/Resources.Designer.cs b/GUI/Properties/Resources.Designer.cs index c1f9ffa1e8..04b82c95a4 100644 --- a/GUI/Properties/Resources.Designer.cs +++ b/GUI/Properties/Resources.Designer.cs @@ -1221,5 +1221,21 @@ internal static string DeleteUnmanagedFileDelete { internal static string DeleteUnmanagedFileCancel { get { return (string)(ResourceManager.GetObject("DeleteUnmanagedFileCancel", resourceCulture)); } } + + internal static string PreferredHostsPlaceholder { + get { return (string)(ResourceManager.GetObject("PreferredHostsPlaceholder", resourceCulture)); } + } + internal static string PreferredHostsTooltipMoveRight { + get { return (string)(ResourceManager.GetObject("PreferredHostsTooltipMoveRight", resourceCulture)); } + } + internal static string PreferredHostsTooltipMoveLeft { + get { return (string)(ResourceManager.GetObject("PreferredHostsTooltipMoveLeft", resourceCulture)); } + } + internal static string PreferredHostsTooltipMoveUp { + get { return (string)(ResourceManager.GetObject("PreferredHostsTooltipMoveUp", resourceCulture)); } + } + internal static string PreferredHostsTooltipMoveDown { + get { return (string)(ResourceManager.GetObject("PreferredHostsTooltipMoveDown", resourceCulture)); } + } } } diff --git a/GUI/Properties/Resources.resx b/GUI/Properties/Resources.resx index 4db456ba9c..9888513f71 100644 --- a/GUI/Properties/Resources.resx +++ b/GUI/Properties/Resources.resx @@ -479,4 +479,9 @@ If you wish to remove this folder, uninstall these mods normally. This action cannot be undone! Delete Cancel + <ALL OTHER HOSTS>This string should not be valid as a hostname + Add selected host to preferences list + Remove selected host from preferences list + Make selected host higher priority + Make selected host lower priority diff --git a/Tests/Core/Registry/Registry.cs b/Tests/Core/Registry/Registry.cs index cf7730a29f..6d28b70c1c 100644 --- a/Tests/Core/Registry/Registry.cs +++ b/Tests/Core/Registry/Registry.cs @@ -297,6 +297,76 @@ public void HasUpdate_OtherModDependsOnCurrent_ReturnsFalse() } } + [Test, + // Empty registry, return nothing + TestCase(new string[] { }, + new string[] { }), + // One per host, sort by alphanumeric + TestCase(new string[] + { + @"{ + ""identifier"": ""ModA"", + ""version"": ""1.0"", + ""download"": [ + ""https://archive.org/"", + ""https://spacedock.info/"", + ""https://github.com/"" + ] + }", + }, + new string[] + { + "archive.org", "github.com", "spacedock.info" + }), + // Multiple per host, sort by frequency + TestCase(new string[] + { + @"{ + ""identifier"": ""ModA"", + ""version"": ""1.0"", + ""download"": [ + ""https://archive.org/"", + ""https://spacedock.info/"", + ""https://github.com/"" + ] + }", + @"{ + ""identifier"": ""ModB"", + ""version"": ""1.0"", + ""download"": [ + ""https://spacedock.info/"", + ""https://github.com/"" + ] + }", + @"{ + ""identifier"": ""ModC"", + ""version"": ""1.0"", + ""download"": [ + ""https://github.com/"" + ] + }", + }, + new string[] + { + "github.com", "spacedock.info", "archive.org" + }), + ] + public void GetAllHosts_WithModules_ReturnsCorrectList(string[] modules, + string[] correctAnswer) + { + // Arrange + foreach (var module in modules) + { + registry.AddAvailable(CkanModule.FromJson(module)); + } + + // Act + var allHosts = registry.GetAllHosts().ToArray(); + + // Assert + Assert.AreEqual(correctAnswer, allHosts); + } + [Test] public void TxEmbeddedCommit() { diff --git a/Tests/GUI/ResourcesTests.cs b/Tests/GUI/ResourcesTests.cs index b37c476484..12a5ab4db3 100644 --- a/Tests/GUI/ResourcesTests.cs +++ b/Tests/GUI/ResourcesTests.cs @@ -48,8 +48,11 @@ public void PropertiesResources_LanguageResource_NotSet() TestCase(typeof(CKAN.GUI.DeleteDirectories)), TestCase(typeof(CKAN.GUI.EditModpack)), TestCase(typeof(CKAN.GUI.EditModSearch)), + TestCase(typeof(CKAN.GUI.InstallationHistory)), TestCase(typeof(CKAN.GUI.ManageMods)), TestCase(typeof(CKAN.GUI.ModInfo)), + TestCase(typeof(CKAN.GUI.PlayTime)), + TestCase(typeof(CKAN.GUI.UnmanagedFiles)), TestCase(typeof(CKAN.GUI.Wait)), // Dialogs @@ -58,13 +61,16 @@ public void PropertiesResources_LanguageResource_NotSet() TestCase(typeof(CKAN.GUI.AskUserForAutoUpdatesDialog)), TestCase(typeof(CKAN.GUI.CloneGameInstanceDialog)), TestCase(typeof(CKAN.GUI.CompatibleGameVersionsDialog)), + TestCase(typeof(CKAN.GUI.DownloadsFailedDialog)), TestCase(typeof(CKAN.GUI.EditLabelsDialog)), TestCase(typeof(CKAN.GUI.ErrorDialog)), TestCase(typeof(CKAN.GUI.GameCommandLineOptionsDialog)), + TestCase(typeof(CKAN.GUI.InstallFiltersDialog)), TestCase(typeof(CKAN.GUI.ManageGameInstancesDialog)), TestCase(typeof(CKAN.GUI.NewRepoDialog)), TestCase(typeof(CKAN.GUI.NewUpdateDialog)), TestCase(typeof(CKAN.GUI.PluginsDialog)), + TestCase(typeof(CKAN.GUI.PreferredHostsDialog)), TestCase(typeof(CKAN.GUI.RenameInstanceDialog)), TestCase(typeof(CKAN.GUI.SelectionDialog)), TestCase(typeof(CKAN.GUI.YesNoDialog)),