Merge #4155 Autogenerate spec_version and make it optional in netkans

CHANGELOG.md

@@ -71,6 +71,7 @@ All notable changes to this project will be documented in this file.
 - [Multiple] Translation updates from Crowdin (#4105, #4149 by: vinix38, frankieorabona, ambition, Francesco Ricina, S.O.2; reviewed: HebaruSan)
 - [Netkan] Allow string "Harmony" in DLL parent folder names (#4123 by: HebaruSan)
 - [Netkan] Allow licenses to be absent from netkans (#4137 by: HebaruSan)
+- [Netkan] Autogenerate `spec_version` and make it optional in netkans (#4155 by: HebaruSan)
 
 ## v1.34.4 (Niven)
 

Netkan/Model/Metadata.cs

@@ -15,7 +15,7 @@ internal sealed class Metadata
     {
         private const string KrefPropertyName          = "$kref";
         private const string VrefPropertyName          = "$vref";
-        private const string SpecVersionPropertyName   = "spec_version";
+        public  const string SpecVersionPropertyName   = "spec_version";
         private const string VersionPropertyName       = "version";
         private const string DownloadPropertyName      = "download";
         public  const string UpdatedPropertyName       = "release_date";
@@ -85,10 +85,6 @@ public Metadata(JObject json)
                     ));
                 }
             }
-            else
-            {
-                throw new Kraken(string.Format("{0} must be specified.", SpecVersionPropertyName));
-            }
 
             if (json.TryGetValue(VersionPropertyName, out JToken versionToken))
             {

Netkan/Processors/Inflator.cs

@@ -48,6 +48,7 @@ internal IEnumerable<Metadata> Inflate(string filename, Metadata[] netkans, Tran
                     .SelectMany(netkan => transformer.Transform(netkan, opts))
                     .GroupBy(module => module.Version)
                     .Select(grp => Metadata.Merge(grp.ToArray()))
+                    .SelectMany(merged => specVersionTransformer.Transform(merged, opts))
                     .ToList();
                 log.Debug("Finished transformation");
 
@@ -120,6 +121,7 @@ private static void PurgeDownloads(IHttpService http, NetFileCache cache)
         private readonly IHttpService http;
 
         private readonly NetkanTransformer transformer;
+        private readonly SpecVersionTransformer specVersionTransformer = new SpecVersionTransformer();
 
         private readonly NetkanValidator netkanValidator = new NetkanValidator();
         private readonly CkanValidator   ckanValidator;

Netkan/Program.cs

@@ -4,20 +4,20 @@
 using System.Linq;
 using System.Net;
 using System.Text;
-using CommandLine;
 
+using CommandLine;
 using log4net;
 using log4net.Core;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using YamlDotNet.RepresentationModel;
 
 using CKAN.Games;
 using CKAN.Versioning;
 using CKAN.NetKAN.Model;
 using CKAN.NetKAN.Processors;
 using CKAN.NetKAN.Transformers;
 using CKAN.NetKAN.Extensions;
-using YamlDotNet.RepresentationModel;
 
 namespace CKAN.NetKAN
 {

Netkan/Transformers/GitlabTransformer.cs

@@ -12,7 +12,7 @@
 namespace CKAN.NetKAN.Transformers
 {
     /// <summary>
-    /// An <see cref="ITransformer"/> that looks up data from GitHub.
+    /// An <see cref="ITransformer"/> that looks up data from GitLab.
     /// </summary>
     internal sealed class GitlabTransformer : ITransformer
     {

Netkan/Transformers/InternalCkanTransformer.cs

@@ -61,9 +61,6 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                         json.SafeAdd(property.Name, property.Value);
                     }
 
-                    json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, new Metadata(internalJson).SpecVersion)
-                        .ToSpecVersionJson();
-
                     json.SafeMerge("resources", internalJson["resources"]);
 
                     Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);

Netkan/Transformers/MetaNetkanTransformer.cs

@@ -66,9 +66,6 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                     var targetMetadata =  new Metadata(targetJson);
                     if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan")
                     {
-                        json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, targetMetadata.SpecVersion)
-                            .ToSpecVersionJson();
-
                         if (targetJson["$kref"] != null)
                         {
                             json["$kref"] = targetJson["$kref"];

Netkan/Transformers/SpecVersionTransformer.cs

@@ -0,0 +1,153 @@
+using System.Collections.Generic;
+using System.Linq;
+
+using Newtonsoft.Json.Linq;
+using log4net;
+
+using CKAN.Versioning;
+using CKAN.NetKAN.Model;
+using CKAN.NetKAN.Extensions;
+
+namespace CKAN.NetKAN.Transformers
+{
+    /// <summary>
+    /// An <see cref="ITransformer"/> that sets the spec version to match the metadata.
+    /// </summary>
+    internal sealed class SpecVersionTransformer : ITransformer
+    {
+        /// <summary>
+        /// Defines the name of this transformer
+        /// </summary>
+        public string Name => "spec_version";
+
+        public IEnumerable<Metadata> Transform(Metadata         metadata,
+                                               TransformOptions opts)
+        {
+            var json       = metadata.Json();
+            var minVersion = MinimumSpecVersion(json);
+            if (metadata.SpecVersion == null || metadata.SpecVersion != minVersion)
+            {
+                log.InfoFormat("Setting spec version {0}", minVersion);
+                json[Metadata.SpecVersionPropertyName] = minVersion.ToSpecVersionJson();
+                yield return new Metadata(json);
+            }
+            else
+            {
+                yield return metadata;
+            }
+        }
+
+        private static ModuleVersion MinimumSpecVersion(JObject json)
+            // Add new stuff at the top, versions in this function should be in descending order
+            => json["download_hash"] is JObject hashes
+               && (!hashes.ContainsKey("sha256") || !hashes.ContainsKey("sha1")) ? v1p35
+
+             : json["download"] is JArray ? v1p34
+
+             : AllRelationships(json).Any(rel => rel.ContainsKey("any_of")
+                                              && rel.ContainsKey("choice_help_text")) ? v1p31
+
+             : HasLicense(json, "MPL-2.0") ? v1p30
+
+             : (json["install"] as JArray)?.OfType<JObject>().Any(stanza =>
+                 (string)stanza["install_to"] == "Ships" && (
+                     // find: .../Script, install_to: Ships
+                     ((string)stanza["find"])?.Split(new char[] {'/'})?.LastOrDefault() == "Script"
+                     // file: .../Script, install_to: Ships
+                     || ((string)stanza["file"])?.Split(new char[] {'/'})?.LastOrDefault() == "Script"
+                     // install_to: Ships, as: Script
+                     || (((string)stanza["as"])?.EndsWith("Script") ?? false))) ?? false ? v1p29
+
+             : (string)json["kind"] == "dlc" ? v1p28
+
+             : json.ContainsKey("replaced_by") ? v1p26
+
+             : AllRelationships(json).Any(rel => rel.ContainsKey("any_of")) ? v1p26
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => (string)stanza["install_to"] == "Missions") ?? false ? v1p25
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => stanza.ContainsKey("include_only")
+                                                      || stanza.ContainsKey("include_only_regexp")) ?? false ? v1p24
+
+             : HasLicense(json, "Unlicense") ? v1p18
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => stanza.ContainsKey("as")) ?? false ? v1p18
+
+             : json.ContainsKey("ksp_version_strict") ? v1p16
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => ((string)stanza["install_to"]).StartsWith("Ships/@thumbs")) ?? false ? v1p16
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => stanza.ContainsKey("find_matches_files")) ?? false ? v1p16
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => (string)stanza["install_to"] == "Scenarios") ?? false ? v1p14
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => ((string)stanza["install_to"]).StartsWith("Ships/")) ?? false ? v1p12
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => stanza.ContainsKey("find_regexp")
+                                                       || stanza.ContainsKey("filter_regexp")) ?? false ? v1p10
+
+             : json["license"] is JArray ? v1p8
+
+             : (string)json["kind"] == "metapackage" ? v1p6
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => stanza.ContainsKey("find")) ?? false ? v1p4
+
+             : HasLicense(json, "WTFPL") ? v1p2
+
+             : json.ContainsKey("supports") ? v1p2
+
+             : (json["install"] as JArray)?.OfType<JObject>()
+                                           .Any(stanza => ((string)stanza["install_to"]).StartsWith("GameData/")) ?? false ? v1p2
+
+             : v1p0;
+
+        private static bool HasLicense(JObject json,
+                                       string  name)
+            => json["license"] is JArray array ?  array.Contains(name)
+             : json["license"] is JToken token && ((string)token) == name;
+
+        private static IEnumerable<JObject> AllRelationships(JObject json)
+            => relProps.SelectMany(p => json[p] is JArray array ? array.OfType<JObject>()
+                                                                : Enumerable.Empty<JObject>());
+
+        private static readonly string[] relProps = new string[]
+        {
+            "depends",
+            "recommends",
+            "suggests",
+            "conflicts",
+            "supports"
+        };
+
+        private static readonly ModuleVersion v1p0  = new ModuleVersion("v1.0");
+        private static readonly ModuleVersion v1p2  = new ModuleVersion("v1.2");
+        private static readonly ModuleVersion v1p4  = new ModuleVersion("v1.4");
+        private static readonly ModuleVersion v1p6  = new ModuleVersion("v1.6");
+        private static readonly ModuleVersion v1p8  = new ModuleVersion("v1.8");
+        private static readonly ModuleVersion v1p10 = new ModuleVersion("v1.10");
+        private static readonly ModuleVersion v1p12 = new ModuleVersion("v1.12");
+        private static readonly ModuleVersion v1p14 = new ModuleVersion("v1.14");
+        private static readonly ModuleVersion v1p16 = new ModuleVersion("v1.16");
+        private static readonly ModuleVersion v1p18 = new ModuleVersion("v1.18");
+        private static readonly ModuleVersion v1p24 = new ModuleVersion("v1.24");
+        private static readonly ModuleVersion v1p25 = new ModuleVersion("v1.25");
+        private static readonly ModuleVersion v1p26 = new ModuleVersion("v1.26");
+        private static readonly ModuleVersion v1p28 = new ModuleVersion("v1.28");
+        private static readonly ModuleVersion v1p29 = new ModuleVersion("v1.29");
+        private static readonly ModuleVersion v1p30 = new ModuleVersion("v1.30");
+        private static readonly ModuleVersion v1p31 = new ModuleVersion("v1.31");
+        private static readonly ModuleVersion v1p34 = new ModuleVersion("v1.34");
+        private static readonly ModuleVersion v1p35 = new ModuleVersion("v1.35");
+
+        private static readonly ILog log = LogManager.GetLogger(typeof(SpecVersionTransformer));
+    }
+}

Netkan/Validators/CkanValidator.cs

@@ -18,6 +18,10 @@ public CkanValidator(IHttpService downloader, IModuleService moduleService, IGam
                 new DownloadArrayValidator(),
                 new TagsValidator(),
                 new LicensesValidator(),
+                new RelationshipsValidator(),
+                new VersionStrictValidator(),
+                new ReplacedByValidator(),
+                new InstallValidator(),
                 new InstallsFilesValidator(downloader, moduleService, game),
                 new MatchesKnownGameVersionsValidator(game),
                 new ObeysCKANSchemaValidator(),

Netkan/Validators/NetkanValidator.cs

@@ -12,17 +12,12 @@ public NetkanValidator()
         {
             _validators = new List<IValidator>()
             {
-                new SpecVersionFormatValidator(),
                 new HasIdentifierValidator(),
                 new KrefValidator(),
                 new AlphaNumericIdentifierValidator(),
-                new RelationshipsValidator(),
                 new KrefDownloadMutexValidator(),
                 new DownloadVersionValidator(),
                 new OverrideValidator(),
-                new VersionStrictValidator(),
-                new ReplacedByValidator(),
-                new InstallValidator(),
             };
         }
 

Spec.md

@@ -127,8 +127,8 @@ reference CKAN client that will read this file.
 For compatibility with pre-release clients, and the v1.0 client, the special
 *integer* `1` should be used.
 
-This document describes the CKAN specification 'v1.26'. Changes since spec `1`
-are marked with **v1.2** through to **v1.26** respectively. For maximum
+This document describes the CKAN specification 'v1.34'. Changes since spec `1`
+are marked with **v1.2** through to **v1.34** respectively. For maximum
 compatibility, using older spec versions is preferred when newer features are
 not required.
 

Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs

@@ -95,46 +95,5 @@ public void DoesNotOverrideExistingProperties()
                 "InternalCkanTransformer should not override existing properties."
             );
         }
-
-        [TestCase("v1.2", "v1.4", "v1.4")]
-        [TestCase("v1.4", "v1.2", "v1.4")]
-        public void HigherOfTwoSpecVersionsIsChosen(
-            string specVersion, string internalSpecVersion, string expectedSpecVersion
-        )
-        {
-            // Arrange
-            const string filePath = "/DoesNotExist.zip";
-
-            var internalCkan = new JObject();
-            internalCkan["spec_version"] = internalSpecVersion;
-
-            var mHttp = new Mock<IHttpService>();
-            var mModuleService = new Mock<IModuleService>();
-
-            mHttp.Setup(i => i.DownloadModule(It.IsAny<Metadata>()))
-                .Returns(filePath);
-
-            mModuleService.Setup(i => i.GetInternalCkan(
-                    It.IsAny<CkanModule>(), It.IsAny<string>(),
-                    It.IsAny<GameInstance>()))
-                .Returns(internalCkan);
-
-            var sut = new InternalCkanTransformer(mHttp.Object, mModuleService.Object, new KerbalSpaceProgram());
-
-            var json = new JObject();
-            json["spec_version"] = specVersion;
-            json["identifier"] = "DoesNotExist";
-            json["version"] = "1.0";
-            json["download"] = "https://awesomemod.example/AwesomeMod.zip";
-
-            // Act
-            var result = sut.Transform(new Metadata(json), opts).First();
-            var transformedJson = result.Json();
-
-            // Assert
-            Assert.That((string)transformedJson["spec_version"], Is.EqualTo(expectedSpecVersion),
-                "InternalCkanTransformer should use the higher of the two spec_versions."
-            );
-        }
     }
 }

Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs

@@ -123,34 +123,5 @@ public void TargetMetadataDoesNotOverrideExistingProperty()
                 "MetaNetkanTransformer should not override existing properties."
             );
         }
-
-        [TestCase("v1.2", "v1.4", "v1.4")]
-        [TestCase("v1.4", "v1.2", "v1.4")]
-        public void SelectsTheHigherSpecVresion(string specVersion, string targetSpecVersion, string expected)
-        {
-            // Arrange
-            var targetJson = new JObject();
-            targetJson["spec_version"] = targetSpecVersion;
-
-            var mHttp = new Mock<IHttpService>();
-
-            mHttp.Setup(i => i.DownloadText(It.IsAny<Uri>()))
-                .Returns(targetJson.ToString());
-
-            var sut = new MetaNetkanTransformer(mHttp.Object, null);
-
-            var json = new JObject();
-            json["spec_version"] = specVersion;
-            json["$kref"] = "#/ckan/netkan/http://awesomemod.example/AwesomeMod.netkan";
-
-            // Act
-            var result = sut.Transform(new Metadata(json), opts).First();
-            var transformedJson = result.Json();
-
-            // Assert
-            Assert.That((string)transformedJson["spec_version"], Is.EqualTo(expected),
-                "MetaNetkanTransformer should select the higher of the two spec_versions."
-            );
-        }
     }
 }

Tests/NetKAN/Validators/IsCkanModuleValidatorTests.cs

@@ -35,7 +35,6 @@ public void DoesNotThrowOnValidCkan()
             );
         }
 
-        [TestCase("spec_version")]
         [TestCase("identifier")]
         [TestCase("version")]
         [TestCase("download")]