From f62a4c09954abf05486d7cbb28de835797cee8a8 Mon Sep 17 00:00:00 2001 From: cocoa-dev Date: Tue, 9 Nov 2021 16:36:22 +0900 Subject: [PATCH 1/4] Fixed update process of list.json --- .../ITemporaryExposureKeyBlobService.cs | 2 +- .../TemporaryExposureKeyBlobService.cs | 29 ++++++++++--------- .../TemporaryExposureKeyExportBatchService.cs | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/Covid19Radar.Background/Services/ITemporaryExposureKeyBlobService.cs b/src/Covid19Radar.Background/Services/ITemporaryExposureKeyBlobService.cs index 62e4ae742..f20ed5e87 100644 --- a/src/Covid19Radar.Background/Services/ITemporaryExposureKeyBlobService.cs +++ b/src/Covid19Radar.Background/Services/ITemporaryExposureKeyBlobService.cs @@ -16,6 +16,6 @@ public interface ITemporaryExposureKeyBlobService Task DeleteAsync(TemporaryExposureKeyExportModel model); - Task WriteFilesJsonAsync(IEnumerable models); + Task WriteFilesJsonAsync(IEnumerable models, string[] supportRegions); } } diff --git a/src/Covid19Radar.Background/Services/TemporaryExposureKeyBlobService.cs b/src/Covid19Radar.Background/Services/TemporaryExposureKeyBlobService.cs index effc6bedd..c312256ed 100644 --- a/src/Covid19Radar.Background/Services/TemporaryExposureKeyBlobService.cs +++ b/src/Covid19Radar.Background/Services/TemporaryExposureKeyBlobService.cs @@ -86,34 +86,37 @@ public async Task DeleteAsync(TemporaryExposureKeyExportModel model) await blockBlob.DeleteIfExistsAsync(); } - public async Task WriteFilesJsonAsync(IEnumerable models) + public async Task WriteFilesJsonAsync(IEnumerable models, string[] supportRegions) { Logger.LogInformation($"start {nameof(WriteFilesJsonAsync)}"); var blobContainerName = $"{TekExportBlobStorageContainerPrefix}".ToLower(); var cloudBlobContainer = BlobClient.GetContainerReference(blobContainerName); await cloudBlobContainer.CreateIfNotExistsAsync(BlobContainerPublicAccessType.Blob, new BlobRequestOptions(), new OperationContext()); - foreach (var grp in models.GroupBy(_ => _.Region)) + var postedGroupedRegions = models.GroupBy(_ => _.Region); + foreach (var region in supportRegions) { - // per region - var region = grp.Key; var blobDirectory = cloudBlobContainer.GetDirectoryReference($"{region}".ToLower()); - // Filename is inferable as batch number var exportFileName = "list.json"; - //var blockBlob = cloudBlobContainer.GetBlockBlobReference(exportFileName); var blockBlob = blobDirectory.GetBlockBlobReference(exportFileName); - var files = grp.Select(_ => new TemporaryExposureKeyExportFileModel() - { - Region = region, - Created = _.TimestampSecondsSinceEpoch, - Url = $"{TekExportKeyUrl}/{TekExportBlobStorageContainerPrefix}/{region}/{_.BatchNum}.zip" + var grp = postedGroupedRegions?.FirstOrDefault(_ => _.Key == region); - }).ToArray(); + var filesJson = "[]"; + if (grp != null) + { + var files = grp.Select(_ => new TemporaryExposureKeyExportFileModel() + { + Region = region, + Created = _.TimestampSecondsSinceEpoch, + Url = $"{TekExportKeyUrl}/{TekExportBlobStorageContainerPrefix}/{region}/{_.BatchNum}.zip" + }).ToArray(); + + filesJson = JsonConvert.SerializeObject(files); + } using (var stream = new MemoryStream()) using (var writer = new StreamWriter(stream)) { - var filesJson = JsonConvert.SerializeObject(files); await writer.WriteAsync(filesJson); await writer.FlushAsync(); await stream.FlushAsync(); diff --git a/src/Covid19Radar.Background/Services/TemporaryExposureKeyExportBatchService.cs b/src/Covid19Radar.Background/Services/TemporaryExposureKeyExportBatchService.cs index e1a774b2f..855ed3af3 100644 --- a/src/Covid19Radar.Background/Services/TemporaryExposureKeyExportBatchService.cs +++ b/src/Covid19Radar.Background/Services/TemporaryExposureKeyExportBatchService.cs @@ -98,7 +98,7 @@ await CreateAsync((ulong)kv.Key.RollingStartUnixTimeSeconds, // Write Export Files json var models = await TekExportRepository.GetKeysAsync(0); - await BlobService.WriteFilesJsonAsync(models); + await BlobService.WriteFilesJsonAsync(models, Regions); } catch (Exception ex) { From ce72c6f778702c8ab6761da20bc1a1669161ce6a Mon Sep 17 00:00:00 2001 From: cocoa-dev Date: Tue, 9 Nov 2021 16:36:57 +0900 Subject: [PATCH 2/4] Add log of iOS device check. --- .../Services/DeviceValidationAppleService.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Covid19Radar.Api/Services/DeviceValidationAppleService.cs b/src/Covid19Radar.Api/Services/DeviceValidationAppleService.cs index 9e2e5cc38..3bb96c555 100644 --- a/src/Covid19Radar.Api/Services/DeviceValidationAppleService.cs +++ b/src/Covid19Radar.Api/Services/DeviceValidationAppleService.cs @@ -101,7 +101,7 @@ public async Task Validation(DiagnosisSubmissionParameter param, DateTimeO if (!response.IsSuccessStatusCode) { var responseBody = await response.Content.ReadAsStringAsync(); - Logger.LogWarning($"POST {UrlApple} {response.StatusCode} {response.ReasonPhrase} {responseBody}"); + Logger.LogWarning($"iOS device check failed.\r\n{nameof(HttpRequestMessage)} : {request}\r\n{nameof(HttpResponseMessage)} : {response}"); } //switch (response.StatusCode) @@ -117,7 +117,8 @@ public async Task Validation(DiagnosisSubmissionParameter param, DateTimeO // break; //} if (response.StatusCode != System.Net.HttpStatusCode.OK) - return false; + // FIXME: When call iOS Device check, return error sometimes, Until the cause is known, ignored device check + return true; return true; } From f7e4cfd2ff5d8302303ba982422f0394d77df714 Mon Sep 17 00:00:00 2001 From: cocoa-dev Date: Tue, 9 Nov 2021 16:38:06 +0900 Subject: [PATCH 3/4] Accept chaff request Add debug log --- .../V2DiagnosisApiTest.cs | 48 +++++++++++++++---- src/Covid19Radar.Api/DiagnosisApi.cs | 7 ++- src/Covid19Radar.Api/V2DiagnosisApi.cs | 19 +++++++- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/src/Covid19Radar.Api.Tests/V2DiagnosisApiTest.cs/V2DiagnosisApiTest.cs b/src/Covid19Radar.Api.Tests/V2DiagnosisApiTest.cs/V2DiagnosisApiTest.cs index e613eac19..23003ba27 100644 --- a/src/Covid19Radar.Api.Tests/V2DiagnosisApiTest.cs/V2DiagnosisApiTest.cs +++ b/src/Covid19Radar.Api.Tests/V2DiagnosisApiTest.cs/V2DiagnosisApiTest.cs @@ -6,15 +6,16 @@ using Covid19Radar.Api.Models; using Covid19Radar.Api.Services; using Microsoft.AspNetCore.Http; -using Microsoft.DotNet.PlatformAbstractions; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; -using System.Collections.Generic; +using System.Diagnostics; +using System.Net; using System.Security.Cryptography; -using System.Text; using System.Threading.Tasks; +using System.Web.Http; namespace Covid19Radar.Api.Tests { @@ -44,16 +45,24 @@ public void CreateMethod() } [DataTestMethod] - [DataRow(true, true, "RegionX", "xxxxx", "ios")] - [DataRow(true, true, "RegionX", "xxxxx", "")] - [DataRow(false, false, "Region1", "xxxxx", "ios")] - [DataRow(true, false, "Region1", "xxxxx", "ios")] - [DataRow(true, true, "Region1", "xxxxx", "ios")] + [DataRow(true, true, "NotSupportRegion", "xxxxx", "ios", false, HttpStatusCode.BadRequest)] + [DataRow(true, true, "NotSupportRegion", "xxxxx", "", false, HttpStatusCode.BadRequest)] + [DataRow(false, false, "Region1", "xxxxx", "ios", false, HttpStatusCode.BadRequest)] + [DataRow(true, false, "Region1", "xxxxx", "ios", false, HttpStatusCode.BadRequest)] + [DataRow(true, true, "Region1", "xxxxx", "ios", false, HttpStatusCode.NoContent)] + [DataRow(true, true, "NotSupportRegion", "xxxxx", "ios", true, HttpStatusCode.BadRequest)] + [DataRow(true, true, "NotSupportRegion", "xxxxx", "", true, HttpStatusCode.BadRequest)] + [DataRow(false, false, "Region1", "xxxxx", "ios", true, HttpStatusCode.BadRequest)] + [DataRow(true, false, "Region1", "xxxxx", "ios", true, HttpStatusCode.BadRequest)] + [DataRow(true, true, "Region1", "xxxxx", "ios", true, HttpStatusCode.NoContent)] public async Task RunAsyncMethod(bool isValid, bool isValidDevice, string region, string verificationPayload, - string platform) + string platform, + bool isChaffRequest, + HttpStatusCode expectedStatusCode + ) { // preparation var config = new Mock(); @@ -104,10 +113,29 @@ public async Task RunAsyncMethod(bool isValid, await writer.FlushAsync(); } stream.Seek(0, System.IO.SeekOrigin.Begin); + context.Setup(_ => _.Request.Body).Returns(stream); + + if (isChaffRequest) + { + IHeaderDictionary headers = new HeaderDictionary() { + { "X-Chaff", "Foo" /* Server will check X-Chaff header existence, content no matter. */ } + }; + context.Setup(_ => _.Request.Headers).Returns(headers); + } + // action - await diagnosisApi.RunAsync(context.Object.Request); + var result = await diagnosisApi.RunAsync(context.Object.Request); + // assert + if (result is StatusCodeResult statusCodeResult) + { + Assert.AreEqual(((int)expectedStatusCode), statusCodeResult.StatusCode); + } + else if (result is BadRequestErrorMessageResult) + { + Assert.AreEqual(expectedStatusCode, HttpStatusCode.BadRequest); + } } } } diff --git a/src/Covid19Radar.Api/DiagnosisApi.cs b/src/Covid19Radar.Api/DiagnosisApi.cs index d714f4d19..ce60ea721 100644 --- a/src/Covid19Radar.Api/DiagnosisApi.cs +++ b/src/Covid19Radar.Api/DiagnosisApi.cs @@ -82,7 +82,12 @@ public async Task RunAsync( // TODO: Security Consider, additional validation for user uuid. - // validation device + // validation device + Logger.LogInformation("regions: " + (diagnosis?.Regions != null && diagnosis.Regions.Count() != 0 ? string.Join(", ", diagnosis.Regions) : "Empty") + ", " + + $"platform: {diagnosis?.Platform}, " + + $"deviceVerificationPayload: {diagnosis?.DeviceVerificationPayload}, " + + $"appPackageName: {diagnosis?.AppPackageName}, " + + $"padding: {diagnosis?.Padding}"); if (false == await DeviceCheck.Validation(diagnosis, reqTime)) { Logger.LogInformation($"Invalid Device"); diff --git a/src/Covid19Radar.Api/V2DiagnosisApi.cs b/src/Covid19Radar.Api/V2DiagnosisApi.cs index a0ce0357e..dbb6527f0 100644 --- a/src/Covid19Radar.Api/V2DiagnosisApi.cs +++ b/src/Covid19Radar.Api/V2DiagnosisApi.cs @@ -1,4 +1,4 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public +/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ @@ -22,6 +22,8 @@ namespace Covid19Radar.Api { public class V2DiagnosisApi { + private const string CHAFF_HEADER = "X-Chaff"; + private readonly IDiagnosisRepository DiagnosisRepository; private readonly ITemporaryExposureKeyRepository TekRepository; private readonly IDeviceValidationService DeviceCheck; @@ -79,13 +81,26 @@ public async Task RunAsync( return new BadRequestErrorMessageResult("Regions not supported."); } - // validation device + // validation device + Logger.LogInformation("regions: " + (diagnosis?.Regions != null && diagnosis.Regions.Count() != 0 ? string.Join(", ", diagnosis.Regions) : "Empty") + ", " + + $"platform: {diagnosis?.Platform}, " + + $"deviceVerificationPayload: {diagnosis?.DeviceVerificationPayload}, " + + $"appPackageName: {diagnosis?.AppPackageName}, " + + $"padding: {diagnosis?.Padding}"); if (false == await DeviceCheck.Validation(diagnosis, reqTime)) { Logger.LogInformation($"Invalid Device"); return new BadRequestErrorMessageResult("Invalid Device"); } + // Check Chaff request for production + // https://google.github.io/exposure-notifications-server/server_functional_requirements.html + if (req.Headers?.ContainsKey(CHAFF_HEADER) ?? false) + { + return new NoContentResult(); + } + + // validatetion VerificationPayload var verificationResult = await VerificationService.VerificationAsync(diagnosis.VerificationPayload); if (verificationResult != 200) From 0b3f7ede8bca1706e057f86292c1b03ba15b8558 Mon Sep 17 00:00:00 2001 From: ARIYAMA Keiji Date: Thu, 25 Nov 2021 18:31:11 +0900 Subject: [PATCH 4/4] Hot-fix --- .../Services/Migration/Migrator_1_3_0.cs | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/Covid19Radar/Covid19Radar/Services/Migration/Migrator_1_3_0.cs b/Covid19Radar/Covid19Radar/Services/Migration/Migrator_1_3_0.cs index a571f00c7..ee481792a 100644 --- a/Covid19Radar/Covid19Radar/Services/Migration/Migrator_1_3_0.cs +++ b/Covid19Radar/Covid19Radar/Services/Migration/Migrator_1_3_0.cs @@ -15,6 +15,7 @@ internal class Migrator_1_3_0 private const string TERMS_OF_SERVICE_LAST_UPDATE_DATETIME = "TermsOfServiceLastUpdateDateTime"; private const string PRIVACY_POLICY_LAST_UPDATE_DATETIME = "PrivacyPolicyLastUpdateDateTime"; + private readonly DateTime FALLBACK_DATETIME = new DateTime(2020, 6, 19); private readonly TimeSpan TIME_DIFFERENCIAL_JST_UTC = TimeSpan.FromHours(+9); private readonly IPreferencesService _preferencesService; @@ -33,7 +34,7 @@ public Task ExecuteAsync() { if (_preferencesService.ContainsKey(START_DATETIME)) { - MigrateDateTimeToEpoch(START_DATETIME, PreferenceKey.StartDateTimeEpoch, TimeSpan.Zero); + MigrateDateTimeToEpoch(START_DATETIME, PreferenceKey.StartDateTimeEpoch, TimeSpan.Zero, DateTime.UtcNow); } if (_preferencesService.ContainsKey(TERMS_OF_SERVICE_LAST_UPDATE_DATETIME)) @@ -41,7 +42,8 @@ public Task ExecuteAsync() MigrateDateTimeToEpoch( TERMS_OF_SERVICE_LAST_UPDATE_DATETIME, PreferenceKey.TermsOfServiceLastUpdateDateTimeEpoch, - -TIME_DIFFERENCIAL_JST_UTC + -TIME_DIFFERENCIAL_JST_UTC, + FALLBACK_DATETIME ); } @@ -50,27 +52,38 @@ public Task ExecuteAsync() MigrateDateTimeToEpoch( PRIVACY_POLICY_LAST_UPDATE_DATETIME, PreferenceKey.PrivacyPolicyLastUpdateDateTimeEpoch, - -TIME_DIFFERENCIAL_JST_UTC + -TIME_DIFFERENCIAL_JST_UTC, + FALLBACK_DATETIME ); } return Task.CompletedTask; } - private void MigrateDateTimeToEpoch(string dateTimeKey, string epochKey, TimeSpan differential) + private void MigrateDateTimeToEpoch(string dateTimeKey, string epochKey, TimeSpan differential, DateTime fallbackDateTime) { - string dateTimeStr = _preferencesService.GetValue(dateTimeKey, DateTime.UtcNow.ToString()); + string dateTimeStr = _preferencesService.GetValue(dateTimeKey, fallbackDateTime.ToString()); DateTime dateTime; try { - dateTime = DateTime.SpecifyKind(DateTime.Parse(dateTimeStr) + differential, DateTimeKind.Utc); + dateTime = DateTime.SpecifyKind(DateTime.Parse(dateTimeStr), DateTimeKind.Utc); } catch (FormatException exception) { _loggerService.Exception($"Parse dateTime FormatException occurred. {dateTimeStr}", exception); - dateTime = DateTime.UtcNow; + dateTime = fallbackDateTime; } + + try + { + dateTime += differential; + } + catch (ArgumentOutOfRangeException exception) + { + _loggerService.Exception($"{dateTimeStr} {differential} The added or subtracted value results in an un-representable DateTime.", exception); + } + _preferencesService.SetValue(epochKey, dateTime.ToUnixEpoch()); _preferencesService.RemoveValue(dateTimeKey); }