Support multiple download URLs per module

CKAN.schema

@@ -57,8 +57,20 @@
         },
         "download" : {
             "description" : "URL where mod can be downloaded by tools",
-            "type"        : "string",
-            "format"      : "uri"
+            "oneOf" : [
+                {
+                    "type"   : "string",
+                    "format" : "uri"
+                },
+                {
+                    "type"        : "array",
+                    "items"       : {
+                        "type"    : "string",
+                        "format"  : "uri"
+                    },
+                    "uniqueItems" : true
+                }
+            ]
         },
         "download_size" : {
             "description" : "The size of the download in bytes",

Cmdline/Action/Show.cs

@@ -299,7 +299,7 @@ private int ShowMod(CkanModule module, ShowOptions opts)
             if (!opts.without_files && !module.IsDLC)
             {
                 // Compute the CKAN filename.
-                string file_uri_hash = NetFileCache.CreateURLHash(module.download);
+                string file_uri_hash = NetFileCache.CreateURLHash(module.download[0]);
                 string file_name = CkanModule.StandardName(module.identifier, module.version);
 
                 user.RaiseMessage("");

ConsoleUI/ModInfoScreen.cs

@@ -512,11 +512,16 @@ private void addVersionBox(int l, int t, int r, int b, Func<string> title, Func<
 
         private string HostedOn()
         {
-            string dl = mod.download?.ToString() ?? "";
-            foreach (var kvp in hostDomains) {
-                if (dl.IndexOf(kvp.Key, StringComparison.CurrentCultureIgnoreCase) >= 0) {
-                    return string.Format(Properties.Resources.ModInfoHostedOn, kvp.Value);
-                }
+            if (mod.download != null && mod.download.Count > 0)
+            {
+                var downloadHosts = mod.download
+                    .Select(dlUri => dlUri.Host)
+                    .Select(host =>
+                        hostDomains.TryGetValue(host, out string name)
+                            ? name
+                            : host);
+                return string.Format(Properties.Resources.ModInfoHostedOn,
+                                     string.Join(", ", downloadHosts));
             }
             if (mod.resources != null) {
                 if (mod.resources.bugtracker != null) {
@@ -550,7 +555,7 @@ private string HostedOn()
                         :                                     Properties.Resources.ModInfoBuyFromKSPStoreOrSteamStore;
                 }
             }
-            return mod.download?.Host ?? "";
+            return "";
         }
 
         private void Download(ConsoleTheme theme)

Core/Configuration/IConfiguration.cs

@@ -59,5 +59,12 @@ public interface IConfiguration
         /// Paths that should be excluded from all installations
         /// </summary>
         string[] GlobalInstallFilters { get; set; }
+
+        /// <summary>
+        /// List of hosts in order of priority when there are multiple URLs to choose from.
+        /// The first null value represents where all other hosts should go.
+        /// </summary>
+        /// <value></value>
+        string[] PreferredHosts { get; set; }
     }
 }

Core/Configuration/JsonConfiguration.cs

@@ -2,7 +2,9 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+
 using Newtonsoft.Json;
+
 using CKAN.Games;
 
 namespace CKAN.Configuration
@@ -23,20 +25,16 @@ private class Config
             public IList<GameInstanceEntry> GameInstances { get; set; } = new List<GameInstanceEntry>();
             public IDictionary<string, string> AuthTokens { get; set; } = new Dictionary<string, string>();
             public string[] GlobalInstallFilters { get; set; } = new string[] { };
+            public string[] PreferredHosts { get; set; } = new string[] { };
         }
 
         public class ConfigConverter : JsonPropertyNamesChangedConverter
         {
             protected override Dictionary<string, string> mapping
-            {
-                get
+                => new Dictionary<string, string>
                 {
-                    return new Dictionary<string, string>
-                    {
-                        { "KspInstances", "GameInstances" }
-                    };
-                }
-            }
+                    { "KspInstances", "GameInstances" }
+                };
         }
 
         private class GameInstanceEntry
@@ -326,6 +324,26 @@ public string[] GlobalInstallFilters
             }
         }
 
+        public string[] PreferredHosts
+        {
+            get
+            {
+                lock (_lock)
+                {
+                    return config.PreferredHosts;
+                }
+            }
+
+            set
+            {
+                lock (_lock)
+                {
+                    config.PreferredHosts = value;
+                    SaveConfig();
+                }
+            }
+        }
+
         // <summary>
         // Save the JSON configuration file.
         // </summary>

Core/Configuration/Win32RegistryConfiguration.cs

@@ -178,6 +178,11 @@ public void SetAuthToken(string host, string token)
         /// </summary>
         public string[] GlobalInstallFilters { get; set; }
 
+        /// <summary>
+        /// Not implemented because the Windows registry is deprecated
+        /// </summary>
+        public string[] PreferredHosts { get; set; }
+
         public static bool DoesRegistryConfigurationExist()
         {
             RegistryKey key = Microsoft.Win32.Registry.CurrentUser.OpenSubKey(CKAN_KEY_NO_PREFIX);

Core/Converters/JsonSingleOrArrayConverter.cs

@@ -24,11 +24,12 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
             return token.ToObject<T>() == null ? null : new List<T> { token.ToObject<T>() };
         }
 
-        public override bool CanWrite => false;
+        public override bool CanWrite => true;
 
         public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
         {
-            throw new NotImplementedException();
+            var list = value as List<T>;
+            serializer.Serialize(writer, list?.Count == 1 ? list[0] : value);
         }
 
         /// <summary>

Core/Exporters/DelimeterSeparatedValueExporter.cs

@@ -65,7 +65,7 @@ public void Export(IRegistryQuerier registry, Stream stream)
                         QuoteIfNecessary(mod.Module.description),
                         QuoteIfNecessary(string.Join(";", mod.Module.author)),
                         QuoteIfNecessary(mod.Module.kind),
-                        WriteUri(mod.Module.download),
+                        WriteUri(mod.Module.download[0]),
                         mod.Module.download_size,
                         mod.Module.ksp_version,
                         mod.Module.ksp_version_min,

Core/GameInstance.cs

@@ -393,7 +393,6 @@ public bool Scan()
 
                 return dllChanged || dlcChanged;
             }
-            return false;
         }
 
         #endregion

Core/HelpURLs.cs

@@ -14,6 +14,7 @@ public static class HelpURLs
 
         public const string CloneFakeInstances = "https://github.com/KSP-CKAN/CKAN/pull/2627";
         public const string DeleteDirectories = "https://github.com/KSP-CKAN/CKAN/pull/2962";
+        public const string PreferredHosts = "https://github.com/KSP-CKAN/CKAN/pull/3877";
         public const string Filters = "https://github.com/KSP-CKAN/CKAN/pull/3458";
         public const string Labels = "https://github.com/KSP-CKAN/CKAN/pull/2936";
         public const string PlayTime = "https://github.com/KSP-CKAN/CKAN/pull/3543";

Core/ModuleInstaller.cs

@@ -66,7 +66,11 @@ public static string Download(CkanModule module, string filename, NetModuleCache
         {
             log.Info("Downloading " + filename);
 
-            string tmp_file = Net.Download(module.download);
+            string tmp_file = Net.Download(module.download
+                .OrderBy(u => u,
+                         new PreferredHostUriComparer(
+                             ServiceLocator.Container.Resolve<IConfiguration>().PreferredHosts))
+                .First());
 
             return cache.Store(module, tmp_file, new Progress<long>(bytes => {}), filename, true);
         }
@@ -1051,14 +1055,14 @@ public void Upgrade(IEnumerable<CkanModule> modules, IDownloader netAsyncDownloa
                         {
                             User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingResuming,
                                 module.name, module.version,
-                                module.download.Host,
+                                string.Join(", ", PrioritizedHosts(module.download)),
                                 CkanModule.FmtSize(module.download_size - inProgressFile.Length));
                         }
                         else
                         {
                             User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingUncached,
                                 module.name, module.version,
-                                module.download.Host,
+                                string.Join(", ", PrioritizedHosts(module.download)),
                                 CkanModule.FmtSize(module.download_size));
                         }
                     }
@@ -1093,13 +1097,15 @@ public void Upgrade(IEnumerable<CkanModule> modules, IDownloader netAsyncDownloa
                             {
                                 User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingResuming,
                                     module.name, installed.version, module.version,
-                                    module.download.Host, CkanModule.FmtSize(module.download_size - inProgressFile.Length));
+                                    string.Join(", ", PrioritizedHosts(module.download)),
+                                    CkanModule.FmtSize(module.download_size - inProgressFile.Length));
                             }
                             else
                             {
                                 User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingUncached,
                                     module.name, installed.version, module.version,
-                                    module.download.Host, CkanModule.FmtSize(module.download_size));
+                                    string.Join(", ", PrioritizedHosts(module.download)),
+                                    CkanModule.FmtSize(module.download_size));
                             }
                         }
                         else
@@ -1227,6 +1233,11 @@ public void Replace(IEnumerable<ModuleReplacement> replacements, RelationshipRes
 
         #endregion
 
+        public static IEnumerable<string> PrioritizedHosts(IEnumerable<Uri> urls)
+            => urls.OrderBy(u => u, new PreferredHostUriComparer(ServiceLocator.Container.Resolve<IConfiguration>().PreferredHosts))
+                   .Select(dl => dl.Host)
+                   .Distinct();
+
         /// <summary>
         /// Makes sure all the specified mods are downloaded.
         /// </summary>

Core/Net/AutoUpdate.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Diagnostics;
 using System.Net;
@@ -151,13 +152,11 @@ public void StartUpdateProcess(bool launchCKANAfterUpdate, IUser user = null)
                 new[]
                 {
                     new Net.DownloadTarget(
-                        latestUpdate.UpdaterDownload,
-                        null,
+                        new List<Uri> { latestUpdate.UpdaterDownload },
                         updaterFilename,
                         latestUpdate.UpdaterSize),
                     new Net.DownloadTarget(
-                        latestUpdate.ReleaseDownload,
-                        null,
+                        new List<Uri> {  latestUpdate.ReleaseDownload },
                         ckanFilename,
                         latestUpdate.ReleaseSize),
                 },

Core/Net/Net.cs

@@ -146,18 +146,16 @@ public static string Download(string url, out string etag, string filename = nul
 
         public class DownloadTarget
         {
-            public Uri    url         { get; private set; }
-            public Uri    fallbackUrl { get; private set; }
-            public string filename    { get; private set; }
-            public long   size        { get; private set; }
-            public string mimeType    { get; private set; }
+            public List<Uri> urls     { get; private set; }
+            public string    filename { get; private set; }
+            public long      size     { get; private set; }
+            public string    mimeType { get; private set; }
 
-            public DownloadTarget(Uri url, Uri fallback = null, string filename = null, long size = 0, string mimeType = "")
+            public DownloadTarget(List<Uri> urls, string filename = null, long size = 0, string mimeType = "")
             {
                 TxFileManager FileTransaction = new TxFileManager();
 
-                this.url         = url;
-                this.fallbackUrl = fallback;
+                this.urls        = urls;
                 this.filename    = string.IsNullOrEmpty(filename)
                     ? FileTransaction.GetTempFileName()
                     : filename;
@@ -174,7 +172,7 @@ public static string DownloadWithProgress(string url, string filename = null, IU
         public static string DownloadWithProgress(Uri url, string filename = null, IUser user = null)
         {
             var targets = new[] {
-                new DownloadTarget(url, null, filename)
+                new DownloadTarget(new List<Uri> { url }, filename)
             };
             DownloadWithProgress(targets, user);
             return targets.First().filename;
@@ -191,7 +189,7 @@ public static void DownloadWithProgress(ICollection<DownloadTarget> downloadTarg
                 }
                 else
                 {
-                    File.Move(filename, downloadTargets.First(p => p.url == url).filename);
+                    File.Move(filename, downloadTargets.First(p => p.urls.Contains(url)).filename);
                 }
             };
             downloader.DownloadAndWait(downloadTargets);