Merge #4172 SourceForge kref

CHANGELOG.md

@@ -79,6 +79,7 @@ All notable changes to this project will be documented in this file.
 - [Infra] Trigger mod installer deploy after APT repo update (#4158 by: HebaruSan)
 - [CLI] Ability to update repos without a game instance (#4161 by: HebaruSan)
 - [Multiple] Nullable references, net8.0, blend registry alert dot, netkan fixes (#4171 by: HebaruSan)
+- [Netkan] SourceForge kref (#4172 by: HebaruSan)
 
 ## v1.34.4 (Niven)
 

Core/Extensions/RegexExtensions.cs

@@ -12,7 +12,7 @@ public static class RegexExtensions
         /// <param name="value">The string to check</param>
         /// <param name="match">Object representing the match, if any</param>
         /// <returns>True if the regex matched the value, false otherwise</returns>
-        public static bool TryMatch(this Regex regex, string value,
+        public static bool TryMatch(this Regex regex, string? value,
                                     [NotNullWhen(returnValue: true)] out Match? match)
         {
             if (value == null)

Core/Net/Net.cs

@@ -197,12 +197,13 @@ public static string Download(string url, out string? etag, string? filename = n
             return null;
         }
 
-        public static Uri? ResolveRedirect(Uri url)
+        public static Uri? ResolveRedirect(Uri     url,
+                                           string? userAgent = "")
         {
             const int maxRedirects = 6;
             for (int redirects = 0; redirects <= maxRedirects; ++redirects)
             {
-                var rwClient = new RedirectWebClient();
+                var rwClient = new RedirectWebClient(userAgent);
                 using (rwClient.OpenRead(url)) { }
                 var location = rwClient.ResponseHeaders?["Location"];
                 if (location == null)
@@ -239,7 +240,7 @@ public static string Download(string url, out string? etag, string? filename = n
             // Is it supposed to turn a "&" into part of the content of a form field,
             // or is it supposed to assume that it separates different form fields?
             // https://github.com/dotnet/runtime/issues/31387
-            // So now we have to just substitude certain characters ourselves one by one.
+            // So now we have to just substitute certain characters ourselves one by one.
 
             // Square brackets are "reserved characters" that should not appear
             // in strings to begin with, so C# doesn't try to escape them in case

Core/Net/RedirectWebClient.cs

@@ -8,9 +8,9 @@ namespace CKAN
     // HttpClient doesn't handle redirects well on Mono, but net7.0 considers WebClient obsolete
     internal sealed class RedirectWebClient : WebClient
     {
-        public RedirectWebClient()
+        public RedirectWebClient(string? userAgent = null)
         {
-            Headers.Add("User-Agent", Net.UserAgentString);
+            Headers.Add("User-Agent", userAgent ?? Net.UserAgentString);
         }
 
         protected override WebRequest GetWebRequest(Uri address)

Netkan/CKAN-netkan.csproj

@@ -37,11 +37,13 @@
     <PackageReference Include="Namotion.Reflection" Version="2.1.2" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="YamlDotNet" Version="9.1.0" />
+    <PackageReference Include="System.ServiceModel.Syndication" Version="8.0.0" />
     <PackageReference Include="Nullable" Version="1.3.1" PrivateAssets="all" />
     <PackageReference Include="IndexRange" Version="1.0.3" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'net48' ">
     <Reference Include="System" />
+    <Reference Include="System.ServiceModel" />
   </ItemGroup>
   <ItemGroup>
     <None Include="app.config" />

Netkan/Sources/SourceForge/ISourceForgeApi.cs

@@ -0,0 +1,7 @@
+namespace CKAN.NetKAN.Sources.SourceForge
+{
+    internal interface ISourceForgeApi
+    {
+        SourceForgeMod GetMod(SourceForgeRef sfRef);
+    }
+}

Netkan/Sources/SourceForge/SourceForgeApi.cs

@@ -0,0 +1,26 @@
+using System;
+using System.IO;
+using System.Xml;
+using System.ServiceModel.Syndication;
+
+using CKAN.NetKAN.Services;
+
+namespace CKAN.NetKAN.Sources.SourceForge
+{
+    internal sealed class SourceForgeApi : ISourceForgeApi
+    {
+        public SourceForgeApi(IHttpService httpSvc)
+        {
+            this.httpSvc = httpSvc;
+        }
+
+        public SourceForgeMod GetMod(SourceForgeRef sfRef)
+            => new SourceForgeMod(sfRef,
+                                  SyndicationFeed.Load(XmlReader.Create(new StringReader(
+                                      httpSvc.DownloadText(new Uri(
+                                          $"https://sourceforge.net/projects/{sfRef.Name}/rss"))
+                                      ?? ""))));
+
+        private readonly IHttpService httpSvc;
+    }
+}

Netkan/Sources/SourceForge/SourceForgeMod.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Linq;
+using System.ServiceModel.Syndication;
+
+namespace CKAN.NetKAN.Sources.SourceForge
+{
+    internal class SourceForgeMod
+    {
+        public SourceForgeMod(SourceForgeRef  sfRef,
+                              SyndicationFeed feed)
+        {
+            Title          = feed.Title.Text;
+            Description    = feed.Description.Text;
+            HomepageLink   = $"https://sourceforge.net/projects/{sfRef.Name}/";
+            RepositoryLink = $"https://sourceforge.net/p/{sfRef.Name}/code/";
+            BugTrackerLink = $"https://sourceforge.net/p/{sfRef.Name}/bugs/";
+            Versions       = feed.Items.Where(item => item.Title.Text.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
+                                       .Select(item => new SourceForgeVersion(item))
+                                       .ToArray();
+        }
+
+        public readonly string               Title;
+        public readonly string               Description;
+        public readonly string               HomepageLink;
+        public readonly string               RepositoryLink;
+        public readonly string               BugTrackerLink;
+        public readonly SourceForgeVersion[] Versions;
+    }
+}

Netkan/Sources/SourceForge/SourceForgeRef.cs

@@ -0,0 +1,40 @@
+using System.Text.RegularExpressions;
+
+using CKAN.Extensions;
+using CKAN.NetKAN.Model;
+
+namespace CKAN.NetKAN.Sources.SourceForge
+{
+    /// <summary>
+    /// Represents a SourceForge $kref
+    /// </summary>
+    internal sealed class SourceForgeRef : RemoteRef
+    {
+        /// <summary>
+        /// Initialize the SourceForge reference
+        /// </summary>
+        /// <param name="reference">The base $kref object from a netkan</param>
+        public SourceForgeRef(RemoteRef reference)
+            : base(reference)
+        {
+            if (Pattern.TryMatch(reference.Id, out Match? match))
+            {
+                Name = match.Groups["name"].Value;
+            }
+            else
+            {
+                throw new Kraken(string.Format(@"Could not parse reference: ""{0}""",
+                                 reference));
+            }
+        }
+
+        /// <summary>
+        /// The name of the project on SourceForge
+        /// </summary>
+        public readonly string Name;
+
+        private static readonly Regex Pattern =
+            new Regex(@"^(?<name>[^/]+)$",
+                      RegexOptions.Compiled);
+    }
+}

Netkan/Sources/SourceForge/SourceForgeVersion.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Linq;
+using System.ServiceModel.Syndication;
+
+namespace CKAN.NetKAN.Sources.SourceForge
+{
+    internal class SourceForgeVersion
+    {
+        public SourceForgeVersion(SyndicationItem item)
+        {
+            Title     = item.Title.Text.TrimStart('/');
+            // Throw an exception on missing or multiple <link/>s
+            Link      = item.Links.Single().Uri;
+            Timestamp = item.PublishDate;
+        }
+
+        public readonly string         Title;
+        public readonly Uri            Link;
+        public readonly DateTimeOffset Timestamp;
+    }
+}

Netkan/Transformers/NetkanTransformer.cs

@@ -10,6 +10,7 @@
 using CKAN.NetKAN.Sources.Jenkins;
 using CKAN.NetKAN.Sources.Spacedock;
 using CKAN.Games;
+using CKAN.NetKAN.Sources.SourceForge;
 
 namespace CKAN.NetKAN.Transformers
 {
@@ -35,6 +36,7 @@ public NetkanTransformer(IHttpService   http,
             _validator = validator;
             var ghApi = new GithubApi(http, githubToken);
             var glApi = new GitlabApi(http, gitlabToken);
+            var sfApi = new SourceForgeApi(http);
             _transformers = InjectVersionedOverrideTransformers(new List<ITransformer>
             {
                 new StagingTransformer(game),
@@ -43,6 +45,7 @@ public NetkanTransformer(IHttpService   http,
                 new CurseTransformer(new CurseApi(http)),
                 new GithubTransformer(ghApi, prerelease),
                 new GitlabTransformer(glApi),
+                new SourceForgeTransformer(sfApi),
                 new HttpTransformer(),
                 new JenkinsTransformer(new JenkinsApi(http)),
                 new AvcKrefTransformer(http, ghApi),

Netkan/Transformers/SourceForgeTransformer.cs

@@ -0,0 +1,90 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+
+using Newtonsoft.Json.Linq;
+using log4net;
+
+using CKAN.NetKAN.Model;
+using CKAN.NetKAN.Extensions;
+using CKAN.NetKAN.Sources.SourceForge;
+
+namespace CKAN.NetKAN.Transformers
+{
+    /// <summary>
+    /// An <see cref="ITransformer"/> that looks up data from GitLab.
+    /// </summary>
+    internal sealed class SourceForgeTransformer : ITransformer
+    {
+        /// <summary>
+        /// Initialize the transformer
+        /// </summary>
+        /// <param name="api">Object to use for accessing the SourceForge API</param>
+        public SourceForgeTransformer(ISourceForgeApi api)
+        {
+            this.api = api;
+        }
+
+        /// <summary>
+        /// Defines the name of this transformer
+        /// </summary>
+        public string Name => "sourceforge";
+
+        /// <summary>
+        /// If input metadata has a GitLab kref, inflate it with whatever info we can get,
+        /// otherwise return it unchanged
+        /// </summary>
+        /// <param name="metadata">Input netkan</param>
+        /// <param name="opts">Inflation options from command line</param>
+        /// <returns></returns>
+        public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions? opts)
+        {
+            if (metadata.Kref?.Source == Name)
+            {
+                log.InfoFormat("Executing SourceForge transformation with {0}", metadata.Kref);
+                var reference = new SourceForgeRef(metadata.Kref);
+                var mod       = api.GetMod(reference);
+                var releases  = mod.Versions
+                                   .Skip(opts?.SkipReleases ?? 0)
+                                   .Take(opts?.Releases     ?? 1)
+                                   .ToArray();
+                if (releases.Length < 1)
+                {
+                    log.WarnFormat("No releases found for {0}", reference);
+                    return Enumerable.Repeat(metadata, 1);
+                }
+                return releases.Select(ver => TransformOne(metadata.Json(), mod, ver));
+            }
+            else
+            {
+                // Passthrough for non-GitLab mods
+                return Enumerable.Repeat(metadata, 1);
+            }
+        }
+
+        private static Metadata TransformOne(JObject            json,
+                                             SourceForgeMod     mod,
+                                             SourceForgeVersion version)
+        {
+            json.SafeAdd("name",     mod.Title);
+            json.SafeMerge("resources", JObject.FromObject(new Dictionary<string, string?>()
+            {
+                { "homepage",   mod.HomepageLink   },
+                { "repository", mod.RepositoryLink },
+                { "bugtracker", mod.BugTrackerLink },
+            }));
+            // SourceForge doesn't send redirects to user agents it considers browser-like
+            json.SafeAdd("download", Net.ResolveRedirect(version.Link, "Wget")
+                                        ?.OriginalString);
+            json.SafeAdd(Metadata.UpdatedPropertyName, version.Timestamp);
+
+            json.Remove("$kref");
+
+            log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
+            return new Metadata(json);
+        }
+
+        private        readonly ISourceForgeApi api;
+        private static readonly ILog            log = LogManager.GetLogger(typeof(GitlabTransformer));
+    }
+}

Netkan/Validators/KrefValidator.cs

@@ -22,6 +22,7 @@ public void Validate(Metadata metadata)
                 case "jenkins":
                 case "netkan":
                 case "spacedock":
+                case "sourceforge":
                     // We know this $kref, looks good
                     break;
 

Spec.md

@@ -882,6 +882,29 @@ An `x_netkan_gitlab` field must be provided to customize how the metadata is fet
   Specifies that the source ZIP of the release will be used instead of any discrete assets.<br/>
   Note that this must be `true`! GitLab only offers source ZIP assets, so we can only index mods that use them. If at some point in the future GitLab adds support for non-source assets, we will be able to add support for setting this property to `false` or omitting it.
 
+###### `#/ckan/sourceforge/:repo`
+
+Indicates that data should be fetched from SourceForge using the `:repo` provided.
+For example: `'#/ckan/sourceforge/ksre`
+
+When used, the following fields will be auto-filled if not already present:
+
+- `name`
+- `resources.homepage`
+- `resources.repository`
+- `resources.bugtracker`
+- `download`
+- `download_size`
+- `download_hash`
+- `download_content_type`
+- `release_date`
+
+An example `.netkan` excerpt:
+
+```yaml
+$kref: '#/ckan/sourceforge/ksre'
+```
+
 ###### `#/ckan/jenkins/:joburl`
 
 Indicates data should be fetched from a [Jenkins CI server](https://jenkins-ci.org/) using the `:joburl` provided. For