Fix broken license downloads from GitHub

src/NuGetUtility/LicenseValidator/LicenseValidator.cs

@@ -217,12 +217,25 @@ private async Task ValidateLicenseByUrl(IPackageMetadata info,
             }
         }
 
+        private Uri FixupLicenseUrl(Uri licenseUrl)
+        {
+            if ((licenseUrl.Host == "github.com" || licenseUrl.Host == "www.github.com")
+                && licenseUrl.Query == ""
+                && licenseUrl.Segments.Length >= 5
+                && licenseUrl.Segments[3] == "blob/")
+            {
+                return new Uri(licenseUrl.ToString() + "?raw=true");
+            }
+            return licenseUrl;
+        }
+
         private async Task DownloadLicenseAsync(Uri licenseUrl, PackageIdentity identity, string context, CancellationToken token)
         {
+            licenseUrl = FixupLicenseUrl(licenseUrl);
             try
             {
                 await _fileDownloader.DownloadFile(licenseUrl,
-                    $"{identity.Id}__{identity.Version}.html",
+                    $"{identity.Id}__{identity.Version}",
                     token);
             }
             catch (OperationCanceledException)

src/NuGetUtility/Wrapper/HttpClientWrapper/FileDownloader.cs

@@ -19,14 +19,14 @@ public FileDownloader(HttpClient client, string downloadDirectory)
             _downloadDirectory = downloadDirectory;
         }
 
-        public async Task DownloadFile(Uri url, string fileName, CancellationToken token)
+        public async Task DownloadFile(Uri url, string fileNameStem, CancellationToken token)
         {
             await _parallelDownloadLimiter.WaitAsync(token);
             try
             {
                 for (int i = 0; i < MAX_RETRIES; i++)
                 {
-                    if (await TryDownload(fileName, url, token))
+                    if (await TryDownload(fileNameStem, url, token))
                     {
                         return;
                     }
@@ -40,9 +40,8 @@ public async Task DownloadFile(Uri url, string fileName, CancellationToken token
         }
 
 #if NETFRAMEWORK
-        private async Task<bool> TryDownload(string fileName, Uri url, CancellationToken _)
+        private async Task<bool> TryDownload(string fileNameStem, Uri url, CancellationToken _)
         {
-            using FileStream file = File.OpenWrite(Path.Combine(_downloadDirectory, fileName));
             var request = new HttpRequestMessage(HttpMethod.Get, url);
 
             HttpResponseMessage response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
@@ -52,16 +51,22 @@ private async Task<bool> TryDownload(string fileName, Uri url, CancellationToken
                 return false;
             }
             response.EnsureSuccessStatusCode();
+
+            string extension = "html";
+            if (response.Content.Headers.ContentType.MediaType == "text/plain")
+            {
+                extension = "txt";
+            }
+            string fileName = fileNameStem + "." + extension;
+            using FileStream file = File.OpenWrite(Path.Combine(_downloadDirectory, fileName));
             using Stream downloadStream = await response.Content.ReadAsStreamAsync();
 
             await downloadStream.CopyToAsync(file);
             return true;
         }
 #else
-        private async Task<bool> TryDownload(string fileName, Uri url, CancellationToken token)
+        private async Task<bool> TryDownload(string fileNameStem, Uri url, CancellationToken token)
         {
-
-            await using FileStream file = File.OpenWrite(Path.Combine(_downloadDirectory, fileName));
             var request = new HttpRequestMessage(HttpMethod.Get, url);
 
             HttpResponseMessage response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
@@ -70,7 +75,15 @@ private async Task<bool> TryDownload(string fileName, Uri url, CancellationToken
                 return false;
             }
             response.EnsureSuccessStatusCode();
-            using Stream downloadStream = await response.Content.ReadAsStreamAsync(token);
+
+            string extension = "html";
+            if (response.Content.Headers.ContentType?.MediaType == "text/plain")
+            {
+                extension = "txt";
+            }
+            string fileName = fileNameStem + "." + extension;
+            using FileStream file = File.OpenWrite(Path.Combine(_downloadDirectory, fileName));
+            using Stream downloadStream = await response.Content.ReadAsStreamAsync();
 
             await downloadStream.CopyToAsync(file, token);
             return true;

tests/NuGetUtility.Test/LicenseValidator/LicenseValidatorTest.cs

@@ -423,7 +423,7 @@ public async Task ValidatingLicensesWithExpression_Should_StartDownloadingSaidLi
             _ = await _uut.Validate(CreateInput(package, _context), _token.Token);
 
             await _fileDownloader.Received(1).DownloadFile(new Uri($"https://licenses.nuget.org/({expression})"),
-                    $"{package.Identity.Id}__{package.Identity.Version}.html",
+                    $"{package.Identity.Id}__{package.Identity.Version}",
                     _token.Token);
         }
 
@@ -893,7 +893,7 @@ public async Task ValidatingLicensesWithUrlInformation_Should_StartDownloadingSa
             _ = await _uut.Validate(CreateInput(package, _context), _token.Token);
 
             await _fileDownloader.Received(1).DownloadFile(package.LicenseUrl!,
-                    $"{package.Identity.Id}__{package.Identity.Version}.html",
+                    $"{package.Identity.Id}__{package.Identity.Version}",
                     _token.Token);
         }