From 0d5c7e26431d4fcd125b0e130aee315dbe836615 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Sat, 10 Aug 2024 00:20:07 -0500 Subject: [PATCH] Autogenerate spec_version and make it optional in netkans --- Netkan/Model/Metadata.cs | 6 +- Netkan/Processors/Inflator.cs | 2 + Netkan/Program.cs | 4 +- Netkan/Transformers/GitlabTransformer.cs | 2 +- .../Transformers/InternalCkanTransformer.cs | 3 - Netkan/Transformers/MetaNetkanTransformer.cs | 3 - Netkan/Transformers/SpecVersionTransformer.cs | 153 ++++++++++++++++++ Netkan/Validators/CkanValidator.cs | 4 + Netkan/Validators/NetkanValidator.cs | 5 - Spec.md | 4 +- .../InternalCkanTransformerTests.cs | 41 ----- .../MetaNetkanTransformerTests.cs | 29 ---- .../Validators/IsCkanModuleValidatorTests.cs | 1 - 13 files changed, 165 insertions(+), 92 deletions(-) create mode 100644 Netkan/Transformers/SpecVersionTransformer.cs diff --git a/Netkan/Model/Metadata.cs b/Netkan/Model/Metadata.cs index 42d2f2ccde..648adb4f39 100644 --- a/Netkan/Model/Metadata.cs +++ b/Netkan/Model/Metadata.cs @@ -15,7 +15,7 @@ internal sealed class Metadata { private const string KrefPropertyName = "$kref"; private const string VrefPropertyName = "$vref"; - private const string SpecVersionPropertyName = "spec_version"; + public const string SpecVersionPropertyName = "spec_version"; private const string VersionPropertyName = "version"; private const string DownloadPropertyName = "download"; public const string UpdatedPropertyName = "release_date"; @@ -85,10 +85,6 @@ public Metadata(JObject json) )); } } - else - { - throw new Kraken(string.Format("{0} must be specified.", SpecVersionPropertyName)); - } if (json.TryGetValue(VersionPropertyName, out JToken versionToken)) { diff --git a/Netkan/Processors/Inflator.cs b/Netkan/Processors/Inflator.cs index 87aa6288a4..97a89178f7 100644 --- a/Netkan/Processors/Inflator.cs +++ b/Netkan/Processors/Inflator.cs @@ -48,6 +48,7 @@ internal IEnumerable Inflate(string filename, Metadata[] netkans, Tran .SelectMany(netkan => transformer.Transform(netkan, opts)) .GroupBy(module => module.Version) .Select(grp => Metadata.Merge(grp.ToArray())) + .SelectMany(merged => specVersionTransformer.Transform(merged, opts)) .ToList(); log.Debug("Finished transformation"); @@ -120,6 +121,7 @@ private static void PurgeDownloads(IHttpService http, NetFileCache cache) private readonly IHttpService http; private readonly NetkanTransformer transformer; + private readonly SpecVersionTransformer specVersionTransformer = new SpecVersionTransformer(); private readonly NetkanValidator netkanValidator = new NetkanValidator(); private readonly CkanValidator ckanValidator; diff --git a/Netkan/Program.cs b/Netkan/Program.cs index c63b32434c..8efcda7b8b 100644 --- a/Netkan/Program.cs +++ b/Netkan/Program.cs @@ -4,12 +4,13 @@ using System.Linq; using System.Net; using System.Text; -using CommandLine; +using CommandLine; using log4net; using log4net.Core; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using YamlDotNet.RepresentationModel; using CKAN.Games; using CKAN.Versioning; @@ -17,7 +18,6 @@ using CKAN.NetKAN.Processors; using CKAN.NetKAN.Transformers; using CKAN.NetKAN.Extensions; -using YamlDotNet.RepresentationModel; namespace CKAN.NetKAN { diff --git a/Netkan/Transformers/GitlabTransformer.cs b/Netkan/Transformers/GitlabTransformer.cs index 7b6c24b7fc..01554f238b 100644 --- a/Netkan/Transformers/GitlabTransformer.cs +++ b/Netkan/Transformers/GitlabTransformer.cs @@ -12,7 +12,7 @@ namespace CKAN.NetKAN.Transformers { /// - /// An that looks up data from GitHub. + /// An that looks up data from GitLab. /// internal sealed class GitlabTransformer : ITransformer { diff --git a/Netkan/Transformers/InternalCkanTransformer.cs b/Netkan/Transformers/InternalCkanTransformer.cs index a8369da937..044b3db2f1 100644 --- a/Netkan/Transformers/InternalCkanTransformer.cs +++ b/Netkan/Transformers/InternalCkanTransformer.cs @@ -61,9 +61,6 @@ public IEnumerable Transform(Metadata metadata, TransformOptions opts) json.SafeAdd(property.Name, property.Value); } - json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, new Metadata(internalJson).SpecVersion) - .ToSpecVersionJson(); - json.SafeMerge("resources", internalJson["resources"]); Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); diff --git a/Netkan/Transformers/MetaNetkanTransformer.cs b/Netkan/Transformers/MetaNetkanTransformer.cs index 1ca2642f75..f20ca66fe0 100644 --- a/Netkan/Transformers/MetaNetkanTransformer.cs +++ b/Netkan/Transformers/MetaNetkanTransformer.cs @@ -66,9 +66,6 @@ public IEnumerable Transform(Metadata metadata, TransformOptions opts) var targetMetadata = new Metadata(targetJson); if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan") { - json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, targetMetadata.SpecVersion) - .ToSpecVersionJson(); - if (targetJson["$kref"] != null) { json["$kref"] = targetJson["$kref"]; diff --git a/Netkan/Transformers/SpecVersionTransformer.cs b/Netkan/Transformers/SpecVersionTransformer.cs new file mode 100644 index 0000000000..a262aef753 --- /dev/null +++ b/Netkan/Transformers/SpecVersionTransformer.cs @@ -0,0 +1,153 @@ +using System.Collections.Generic; +using System.Linq; + +using Newtonsoft.Json.Linq; +using log4net; + +using CKAN.Versioning; +using CKAN.NetKAN.Model; +using CKAN.NetKAN.Extensions; + +namespace CKAN.NetKAN.Transformers +{ + /// + /// An that sets the spec version to match the metadata. + /// + internal sealed class SpecVersionTransformer : ITransformer + { + /// + /// Defines the name of this transformer + /// + public string Name => "spec_version"; + + public IEnumerable Transform(Metadata metadata, + TransformOptions opts) + { + var json = metadata.Json(); + var minVersion = MinimumSpecVersion(json); + if (metadata.SpecVersion == null || metadata.SpecVersion != minVersion) + { + log.InfoFormat("Setting spec version {0}", minVersion); + json[Metadata.SpecVersionPropertyName] = minVersion.ToSpecVersionJson(); + yield return new Metadata(json); + } + else + { + yield return metadata; + } + } + + private static ModuleVersion MinimumSpecVersion(JObject json) + // Add new stuff at the top, versions in this function should be in descending order + => json["download_hash"] is JObject hashes + && (!hashes.ContainsKey("sha256") || !hashes.ContainsKey("sha1")) ? v1p35 + + : json["download"] is JArray ? v1p34 + + : AllRelationships(json).Any(rel => rel.ContainsKey("any_of") + && rel.ContainsKey("choice_help_text")) ? v1p31 + + : HasLicense(json, "MPL-2.0") ? v1p30 + + : (json["install"] as JArray)?.OfType().Any(stanza => + (string)stanza["install_to"] == "Ships" && ( + // find: .../Script, install_to: Ships + ((string)stanza["find"])?.Split(new char[] {'/'})?.LastOrDefault() == "Script" + // file: .../Script, install_to: Ships + || ((string)stanza["file"])?.Split(new char[] {'/'})?.LastOrDefault() == "Script" + // install_to: Ships, as: Script + || (((string)stanza["as"])?.EndsWith("Script") ?? false))) ?? false ? v1p29 + + : (string)json["kind"] == "dlc" ? v1p28 + + : json.ContainsKey("replaced_by") ? v1p26 + + : AllRelationships(json).Any(rel => rel.ContainsKey("any_of")) ? v1p26 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => (string)stanza["install_to"] == "Missions") ?? false ? v1p25 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => stanza.ContainsKey("include_only") + || stanza.ContainsKey("include_only_regexp")) ?? false ? v1p24 + + : HasLicense(json, "Unlicense") ? v1p18 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => stanza.ContainsKey("as")) ?? false ? v1p18 + + : json.ContainsKey("ksp_version_strict") ? v1p16 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => ((string)stanza["install_to"]).StartsWith("Ships/@thumbs")) ?? false ? v1p16 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => stanza.ContainsKey("find_matches_files")) ?? false ? v1p16 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => (string)stanza["install_to"] == "Scenarios") ?? false ? v1p14 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => ((string)stanza["install_to"]).StartsWith("Ships/")) ?? false ? v1p12 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => stanza.ContainsKey("find_regexp") + || stanza.ContainsKey("filter_regexp")) ?? false ? v1p10 + + : json["license"] is JArray ? v1p8 + + : (string)json["kind"] == "metapackage" ? v1p6 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => stanza.ContainsKey("find")) ?? false ? v1p4 + + : HasLicense(json, "WTFPL") ? v1p2 + + : json.ContainsKey("supports") ? v1p2 + + : (json["install"] as JArray)?.OfType() + .Any(stanza => ((string)stanza["install_to"]).StartsWith("GameData/")) ?? false ? v1p2 + + : v1p0; + + private static bool HasLicense(JObject json, + string name) + => json["license"] is JArray array ? array.Contains(name) + : json["license"] is JToken token && ((string)token) == name; + + private static IEnumerable AllRelationships(JObject json) + => relProps.SelectMany(p => json[p] is JArray array ? array.OfType() + : Enumerable.Empty()); + + private static readonly string[] relProps = new string[] + { + "depends", + "recommends", + "suggests", + "conflicts", + "supports" + }; + + private static readonly ModuleVersion v1p0 = new ModuleVersion("v1.0"); + private static readonly ModuleVersion v1p2 = new ModuleVersion("v1.2"); + private static readonly ModuleVersion v1p4 = new ModuleVersion("v1.4"); + private static readonly ModuleVersion v1p6 = new ModuleVersion("v1.6"); + private static readonly ModuleVersion v1p8 = new ModuleVersion("v1.8"); + private static readonly ModuleVersion v1p10 = new ModuleVersion("v1.10"); + private static readonly ModuleVersion v1p12 = new ModuleVersion("v1.12"); + private static readonly ModuleVersion v1p14 = new ModuleVersion("v1.14"); + private static readonly ModuleVersion v1p16 = new ModuleVersion("v1.16"); + private static readonly ModuleVersion v1p18 = new ModuleVersion("v1.18"); + private static readonly ModuleVersion v1p24 = new ModuleVersion("v1.24"); + private static readonly ModuleVersion v1p25 = new ModuleVersion("v1.25"); + private static readonly ModuleVersion v1p26 = new ModuleVersion("v1.26"); + private static readonly ModuleVersion v1p28 = new ModuleVersion("v1.28"); + private static readonly ModuleVersion v1p29 = new ModuleVersion("v1.29"); + private static readonly ModuleVersion v1p30 = new ModuleVersion("v1.30"); + private static readonly ModuleVersion v1p31 = new ModuleVersion("v1.31"); + private static readonly ModuleVersion v1p34 = new ModuleVersion("v1.34"); + private static readonly ModuleVersion v1p35 = new ModuleVersion("v1.35"); + + private static readonly ILog log = LogManager.GetLogger(typeof(SpecVersionTransformer)); + } +} diff --git a/Netkan/Validators/CkanValidator.cs b/Netkan/Validators/CkanValidator.cs index 2b26bc3042..9b0f64e8eb 100644 --- a/Netkan/Validators/CkanValidator.cs +++ b/Netkan/Validators/CkanValidator.cs @@ -18,6 +18,10 @@ public CkanValidator(IHttpService downloader, IModuleService moduleService, IGam new DownloadArrayValidator(), new TagsValidator(), new LicensesValidator(), + new RelationshipsValidator(), + new VersionStrictValidator(), + new ReplacedByValidator(), + new InstallValidator(), new InstallsFilesValidator(downloader, moduleService, game), new MatchesKnownGameVersionsValidator(game), new ObeysCKANSchemaValidator(), diff --git a/Netkan/Validators/NetkanValidator.cs b/Netkan/Validators/NetkanValidator.cs index 2660852958..5d8d7a78af 100644 --- a/Netkan/Validators/NetkanValidator.cs +++ b/Netkan/Validators/NetkanValidator.cs @@ -12,17 +12,12 @@ public NetkanValidator() { _validators = new List() { - new SpecVersionFormatValidator(), new HasIdentifierValidator(), new KrefValidator(), new AlphaNumericIdentifierValidator(), - new RelationshipsValidator(), new KrefDownloadMutexValidator(), new DownloadVersionValidator(), new OverrideValidator(), - new VersionStrictValidator(), - new ReplacedByValidator(), - new InstallValidator(), }; } diff --git a/Spec.md b/Spec.md index 887d6a209e..f0b5174b35 100644 --- a/Spec.md +++ b/Spec.md @@ -127,8 +127,8 @@ reference CKAN client that will read this file. For compatibility with pre-release clients, and the v1.0 client, the special *integer* `1` should be used. -This document describes the CKAN specification 'v1.26'. Changes since spec `1` -are marked with **v1.2** through to **v1.26** respectively. For maximum +This document describes the CKAN specification 'v1.34'. Changes since spec `1` +are marked with **v1.2** through to **v1.34** respectively. For maximum compatibility, using older spec versions is preferred when newer features are not required. diff --git a/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs b/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs index 25d3cf934b..8abef80c7b 100644 --- a/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs +++ b/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs @@ -95,46 +95,5 @@ public void DoesNotOverrideExistingProperties() "InternalCkanTransformer should not override existing properties." ); } - - [TestCase("v1.2", "v1.4", "v1.4")] - [TestCase("v1.4", "v1.2", "v1.4")] - public void HigherOfTwoSpecVersionsIsChosen( - string specVersion, string internalSpecVersion, string expectedSpecVersion - ) - { - // Arrange - const string filePath = "/DoesNotExist.zip"; - - var internalCkan = new JObject(); - internalCkan["spec_version"] = internalSpecVersion; - - var mHttp = new Mock(); - var mModuleService = new Mock(); - - mHttp.Setup(i => i.DownloadModule(It.IsAny())) - .Returns(filePath); - - mModuleService.Setup(i => i.GetInternalCkan( - It.IsAny(), It.IsAny(), - It.IsAny())) - .Returns(internalCkan); - - var sut = new InternalCkanTransformer(mHttp.Object, mModuleService.Object, new KerbalSpaceProgram()); - - var json = new JObject(); - json["spec_version"] = specVersion; - json["identifier"] = "DoesNotExist"; - json["version"] = "1.0"; - json["download"] = "https://awesomemod.example/AwesomeMod.zip"; - - // Act - var result = sut.Transform(new Metadata(json), opts).First(); - var transformedJson = result.Json(); - - // Assert - Assert.That((string)transformedJson["spec_version"], Is.EqualTo(expectedSpecVersion), - "InternalCkanTransformer should use the higher of the two spec_versions." - ); - } } } diff --git a/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs b/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs index d27f0c9eae..65641d6797 100644 --- a/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs +++ b/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs @@ -123,34 +123,5 @@ public void TargetMetadataDoesNotOverrideExistingProperty() "MetaNetkanTransformer should not override existing properties." ); } - - [TestCase("v1.2", "v1.4", "v1.4")] - [TestCase("v1.4", "v1.2", "v1.4")] - public void SelectsTheHigherSpecVresion(string specVersion, string targetSpecVersion, string expected) - { - // Arrange - var targetJson = new JObject(); - targetJson["spec_version"] = targetSpecVersion; - - var mHttp = new Mock(); - - mHttp.Setup(i => i.DownloadText(It.IsAny())) - .Returns(targetJson.ToString()); - - var sut = new MetaNetkanTransformer(mHttp.Object, null); - - var json = new JObject(); - json["spec_version"] = specVersion; - json["$kref"] = "#/ckan/netkan/http://awesomemod.example/AwesomeMod.netkan"; - - // Act - var result = sut.Transform(new Metadata(json), opts).First(); - var transformedJson = result.Json(); - - // Assert - Assert.That((string)transformedJson["spec_version"], Is.EqualTo(expected), - "MetaNetkanTransformer should select the higher of the two spec_versions." - ); - } } } diff --git a/Tests/NetKAN/Validators/IsCkanModuleValidatorTests.cs b/Tests/NetKAN/Validators/IsCkanModuleValidatorTests.cs index dcfd74d1fd..1c32d5c586 100644 --- a/Tests/NetKAN/Validators/IsCkanModuleValidatorTests.cs +++ b/Tests/NetKAN/Validators/IsCkanModuleValidatorTests.cs @@ -35,7 +35,6 @@ public void DoesNotThrowOnValidCkan() ); } - [TestCase("spec_version")] [TestCase("identifier")] [TestCase("version")] [TestCase("download")]