From e9855a38287b4b4c96cc1f2f7a039700dd90b224 Mon Sep 17 00:00:00 2001 From: Neil South Date: Fri, 26 May 2023 16:34:41 +0100 Subject: [PATCH 1/8] remove the need for a double copy to minio Signed-off-by: Neil South --- src/Api/Storage/FileStorageMetadata.cs | 2 ++ .../Connectors/DataRetrievalService.cs | 2 ++ .../Services/Connectors/IPayloadAssembler.cs | 5 ++-- .../Services/Connectors/PayloadAssembler.cs | 5 ++-- .../Connectors/PayloadMoveActionHandler.cs | 16 +++++----- .../Services/Scp/ApplicationEntityHandler.cs | 6 ++-- .../Services/Storage/ObjectUploadService.cs | 29 +++++++++++++------ .../Storage/ObjectUploadServiceTest.cs | 1 + .../ExportServicesStepDefinitions.cs | 2 +- .../StepDefinitions/SharedDefinitions.cs | 2 +- 10 files changed, 45 insertions(+), 25 deletions(-) diff --git a/src/Api/Storage/FileStorageMetadata.cs b/src/Api/Storage/FileStorageMetadata.cs index 06b66bff8..8a045ed52 100644 --- a/src/Api/Storage/FileStorageMetadata.cs +++ b/src/Api/Storage/FileStorageMetadata.cs @@ -116,5 +116,7 @@ public virtual void SetFailed() { File.SetFailed(); } + + public string PayloadId { get; set; } } } diff --git a/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs b/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs index 88cf26757..054e318b1 100644 --- a/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs +++ b/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs @@ -198,6 +198,8 @@ private async Task NotifyNewInstance(InferenceRequest inferenceRequest, Dictiona { retrievedFiles[key].SetWorkflows(inferenceRequest.Application.Id); } + var FileMeta = retrievedFiles[key]; + FileMeta.PayloadId = inferenceRequest.TransactionId; _uploadQueue.Queue(retrievedFiles[key]); await _payloadAssembler.Queue(inferenceRequest.TransactionId, retrievedFiles[key]).ConfigureAwait(false); } diff --git a/src/InformaticsGateway/Services/Connectors/IPayloadAssembler.cs b/src/InformaticsGateway/Services/Connectors/IPayloadAssembler.cs index 2a5ebde02..5206e90d7 100644 --- a/src/InformaticsGateway/Services/Connectors/IPayloadAssembler.cs +++ b/src/InformaticsGateway/Services/Connectors/IPayloadAssembler.cs @@ -14,6 +14,7 @@ * limitations under the License. */ +using System; using System.Threading; using System.Threading.Tasks; using Monai.Deploy.InformaticsGateway.Api.Storage; @@ -30,7 +31,7 @@ internal interface IPayloadAssembler /// /// The bucket group the file belongs to. /// Path to the file to be added to the payload bucket. - Task Queue(string bucket, FileStorageMetadata file); + Task Queue(string bucket, FileStorageMetadata file); /// /// Queue a new file for the spcified payload bucket. @@ -38,7 +39,7 @@ internal interface IPayloadAssembler /// The bucket group the file belongs to. /// Path to the file to be added to the payload bucket. /// Number of seconds to wait for additional files. - Task Queue(string bucket, FileStorageMetadata file, uint timeout); + Task Queue(string bucket, FileStorageMetadata file, uint timeout); /// /// Dequeue a payload from the queue for the message broker to notify subscribers. diff --git a/src/InformaticsGateway/Services/Connectors/PayloadAssembler.cs b/src/InformaticsGateway/Services/Connectors/PayloadAssembler.cs index 265e4dbd0..83bf3973e 100644 --- a/src/InformaticsGateway/Services/Connectors/PayloadAssembler.cs +++ b/src/InformaticsGateway/Services/Connectors/PayloadAssembler.cs @@ -87,7 +87,7 @@ private async Task RemovePendingPayloads() /// /// Name of the bucket where the file would be added to /// Instance to be queued - public async Task Queue(string bucket, FileStorageMetadata file) => await Queue(bucket, file, DEFAULT_TIMEOUT).ConfigureAwait(false); + public async Task Queue(string bucket, FileStorageMetadata file) => await Queue(bucket, file, DEFAULT_TIMEOUT).ConfigureAwait(false); /// /// Queues a new instance of . @@ -95,7 +95,7 @@ private async Task RemovePendingPayloads() /// Name of the bucket where the file would be added to /// Instance to be queued /// Number of seconds the bucket shall wait before sending the payload to be processed. Note: timeout cannot be modified once the bucket is created. - public async Task Queue(string bucket, FileStorageMetadata file, uint timeout) + public async Task Queue(string bucket, FileStorageMetadata file, uint timeout) { Guard.Against.Null(file); @@ -106,6 +106,7 @@ public async Task Queue(string bucket, FileStorageMetadata file, uint timeout) var payload = await CreateOrGetPayload(bucket, file.CorrelationId, timeout).ConfigureAwait(false); payload.Add(file); _logger.FileAddedToBucket(payload.Key, payload.Count); + return payload.PayloadId; } /// diff --git a/src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs b/src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs index bfc65d558..c59a5d046 100644 --- a/src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs +++ b/src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs @@ -183,12 +183,12 @@ private async Task MoveFile(Guid payloadId, string identity, StorageObjectMetada try { - await _storageService.CopyObjectAsync( - file.TemporaryBucketName, - file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), - _options.Value.Storage.StorageServiceBucketName, - file.GetPayloadPath(payloadId), - cancellationToken).ConfigureAwait(false); + //await _storageService.CopyObjectAsync( + // file.TemporaryBucketName, + // file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), + // _options.Value.Storage.StorageServiceBucketName, + // file.GetPayloadPath(payloadId), + // cancellationToken).ConfigureAwait(false); await VerifyFileExists(payloadId, file, cancellationToken).ConfigureAwait(false); } @@ -212,8 +212,8 @@ await _storageService.CopyObjectAsync( try { - _logger.DeletingFileFromTemporaryBbucket(file.TemporaryBucketName, identity, file.TemporaryPath); - await _storageService.RemoveObjectAsync(file.TemporaryBucketName, file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), cancellationToken).ConfigureAwait(false); + //_logger.DeletingFileFromTemporaryBbucket(file.TemporaryBucketName, identity, file.TemporaryPath); + //await _storageService.RemoveObjectAsync(file.TemporaryBucketName, file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), cancellationToken).ConfigureAwait(false); } catch (Exception) { diff --git a/src/InformaticsGateway/Services/Scp/ApplicationEntityHandler.cs b/src/InformaticsGateway/Services/Scp/ApplicationEntityHandler.cs index 34d7337ff..b2d9ca5af 100644 --- a/src/InformaticsGateway/Services/Scp/ApplicationEntityHandler.cs +++ b/src/InformaticsGateway/Services/Scp/ApplicationEntityHandler.cs @@ -113,12 +113,14 @@ public async Task HandleInstanceAsync(DicomCStoreRequest request, string calledA } await dicomInfo.SetDataStreams(request.File, request.File.ToJson(_dicomJsonOptions, _validateDicomValueOnJsonSerialization), _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.LocalTemporaryStoragePath).ConfigureAwait(false); - _uploadQueue.Queue(dicomInfo); var dicomTag = FellowOakDicom.DicomTag.Parse(_configuration.Grouping); _logger.QueueInstanceUsingDicomTag(dicomTag); var key = request.Dataset.GetSingleValue(dicomTag); - await _payloadAssembler.Queue(key, dicomInfo, _configuration.Timeout).ConfigureAwait(false); + + var payloadid = await _payloadAssembler.Queue(key, dicomInfo, _configuration.Timeout).ConfigureAwait(false); + dicomInfo.PayloadId = payloadid.ToString(); + _uploadQueue.Queue(dicomInfo); } private bool AcceptsSopClass(string sopClassUid) diff --git a/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs b/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs index 89159e3a2..99c815812 100644 --- a/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs +++ b/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs @@ -158,12 +158,12 @@ private async Task ProcessObject(int thread, FileStorageMetadata blob) case DicomFileStorageMetadata dicom: if (!string.IsNullOrWhiteSpace(dicom.JsonFile.TemporaryPath)) { - await UploadFileAndConfirm(dicom.Id, dicom.JsonFile, dicom.Source, dicom.Workflows, _cancellationTokenSource.Token).ConfigureAwait(false); + await UploadFileAndConfirm(dicom.Id, dicom.JsonFile, dicom.Source, dicom.Workflows, blob.PayloadId, _cancellationTokenSource.Token).ConfigureAwait(false); } break; } - await UploadFileAndConfirm(blob.Id, blob.File, blob.Source, blob.Workflows, _cancellationTokenSource.Token).ConfigureAwait(false); + await UploadFileAndConfirm(blob.Id, blob.File, blob.Source, blob.Workflows, blob.PayloadId, _cancellationTokenSource.Token).ConfigureAwait(false); } catch (Exception ex) { @@ -177,7 +177,7 @@ private async Task ProcessObject(int thread, FileStorageMetadata blob) } } - private async Task UploadFileAndConfirm(string identifier, StorageObjectMetadata storageObjectMetadata, string source, List workflows, CancellationToken cancellationToken) + private async Task UploadFileAndConfirm(string identifier, StorageObjectMetadata storageObjectMetadata, string source, List workflows, string payloadId, CancellationToken cancellationToken) { Guard.Against.NullOrWhiteSpace(identifier); Guard.Against.Null(storageObjectMetadata); @@ -192,12 +192,15 @@ private async Task UploadFileAndConfirm(string identifier, StorageObjectMetadata var count = 3; do { - await UploadFile(storageObjectMetadata, source, workflows, cancellationToken).ConfigureAwait(false); + await UploadFile(storageObjectMetadata, source, workflows, payloadId, cancellationToken).ConfigureAwait(false); if (count-- <= 0) { throw new FileUploadException($"Failed to upload file after retries {identifier}."); } - } while (!(await VerifyExists(storageObjectMetadata.GetTempStoragPath(_configuration.Value.Storage.RemoteTemporaryStoragePath), cancellationToken).ConfigureAwait(false))); + } while (!( + //await VerifyExists(storageObjectMetadata.GetTempStoragPath(_configuration.Value.Storage.RemoteTemporaryStoragePath), cancellationToken).ConfigureAwait(false) + await VerifyExists(storageObjectMetadata.GetPayloadPath(Guid.Parse(payloadId)), cancellationToken).ConfigureAwait(false) + )); } private async Task VerifyExists(string path, CancellationToken cancellationToken) @@ -214,14 +217,15 @@ private async Task VerifyExists(string path, CancellationToken cancellatio { var internalCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); internalCancellationTokenSource.CancelAfter(_configuration.Value.Storage.StorageServiceListTimeout); - var exists = await _storageService.VerifyObjectExistsAsync(_configuration.Value.Storage.TemporaryStorageBucket, path).ConfigureAwait(false); + //var exists = await _storageService.VerifyObjectExistsAsync(_configuration.Value.Storage.TemporaryStorageBucket, path).ConfigureAwait(false); + var exists = await _storageService.VerifyObjectExistsAsync(_configuration.Value.Storage.StorageServiceBucketName, path).ConfigureAwait(false); _logger.VerifyFileExists(path, exists); return exists; }) .ConfigureAwait(false); } - private async Task UploadFile(StorageObjectMetadata storageObjectMetadata, string source, List workflows, CancellationToken cancellationToken) + private async Task UploadFile(StorageObjectMetadata storageObjectMetadata, string source, List workflows, string payloadId, CancellationToken cancellationToken) { _logger.UploadingFileToTemporaryStore(storageObjectMetadata.TemporaryPath); var metadata = new Dictionary @@ -242,10 +246,17 @@ await Policy { if (storageObjectMetadata.IsUploaded) { return; } + //////////////////// + var bucket = _configuration.Value.Storage.StorageServiceBucketName; + var path = storageObjectMetadata.GetPayloadPath(Guid.Parse(payloadId)); + ////////////////// + storageObjectMetadata.Data.Seek(0, System.IO.SeekOrigin.Begin); await _storageService.PutObjectAsync( - _configuration.Value.Storage.TemporaryStorageBucket, - storageObjectMetadata.GetTempStoragPath(_configuration.Value.Storage.RemoteTemporaryStoragePath), + //_configuration.Value.Storage.TemporaryStorageBucket, + bucket, + //storageObjectMetadata.GetTempStoragPath(_configuration.Value.Storage.RemoteTemporaryStoragePath), + path, storageObjectMetadata.Data, storageObjectMetadata.Data.Length, storageObjectMetadata.ContentType, diff --git a/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs b/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs index 7aa593aab..dd4fe8cbf 100644 --- a/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs +++ b/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs @@ -166,6 +166,7 @@ private async Task GenerateDicomFileStorageMetadata() }; var dicomFile = new DicomFile(dataset); await file.SetDataStreams(dicomFile, "[]", TemporaryDataStorageLocation.Memory); + file.PayloadId = Guid.NewGuid().ToString(); return file; } } diff --git a/tests/Integration.Test/StepDefinitions/ExportServicesStepDefinitions.cs b/tests/Integration.Test/StepDefinitions/ExportServicesStepDefinitions.cs index 6d62a6723..ecdce7b2e 100644 --- a/tests/Integration.Test/StepDefinitions/ExportServicesStepDefinitions.cs +++ b/tests/Integration.Test/StepDefinitions/ExportServicesStepDefinitions.cs @@ -36,7 +36,7 @@ namespace Monai.Deploy.InformaticsGateway.Integration.Test.StepDefinitions [CollectionDefinition("SpecFlowNonParallelizableFeatures", DisableParallelization = true)] public class DicomDimseScuServicesStepDefinitions { - internal static readonly TimeSpan DicomScpWaitTimeSpan = TimeSpan.FromMinutes(3); + internal static readonly TimeSpan DicomScpWaitTimeSpan = TimeSpan.FromMinutes(20); private readonly InformaticsGatewayConfiguration _informaticsGatewayConfiguration; private readonly Configurations _configuration; private readonly DicomScp _dicomServer; diff --git a/tests/Integration.Test/StepDefinitions/SharedDefinitions.cs b/tests/Integration.Test/StepDefinitions/SharedDefinitions.cs index 896923162..da3f396d2 100644 --- a/tests/Integration.Test/StepDefinitions/SharedDefinitions.cs +++ b/tests/Integration.Test/StepDefinitions/SharedDefinitions.cs @@ -26,7 +26,7 @@ namespace Monai.Deploy.InformaticsGateway.Integration.Test.StepDefinitions [CollectionDefinition("SpecFlowNonParallelizableFeatures", DisableParallelization = true)] public class SharedDefinitions { - internal static readonly TimeSpan MessageWaitTimeSpan = TimeSpan.FromMinutes(10); + internal static readonly TimeSpan MessageWaitTimeSpan = TimeSpan.FromMinutes(3); private readonly InformaticsGatewayConfiguration _informaticsGatewayConfiguration; private readonly RabbitMqConsumer _receivedMessages; private readonly Assertions _assertions; From 1aa4468f0e9e15d399a2183bb89df456c8edc302 Mon Sep 17 00:00:00 2001 From: Neil South Date: Fri, 26 May 2023 17:44:21 +0100 Subject: [PATCH 2/8] moved call to SetMoved to earlier in process Signed-off-by: Neil South --- src/InformaticsGateway/Services/Storage/ObjectUploadService.cs | 3 ++- tests/Integration.Test/Common/DicomCStoreDataClient.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs b/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs index 99c815812..ed089efc5 100644 --- a/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs +++ b/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs @@ -262,8 +262,9 @@ await _storageService.PutObjectAsync( storageObjectMetadata.ContentType, metadata, cancellationToken).ConfigureAwait(false); - storageObjectMetadata.SetUploaded(_configuration.Value.Storage.TemporaryStorageBucket); + storageObjectMetadata.SetUploaded(_configuration.Value.Storage.TemporaryStorageBucket); // deletes local file _logger.UploadedFileToTemporaryStore(storageObjectMetadata.TemporaryPath); + storageObjectMetadata.SetMoved(_configuration.Value.Storage.StorageServiceBucketName); }) .ConfigureAwait(false); } diff --git a/tests/Integration.Test/Common/DicomCStoreDataClient.cs b/tests/Integration.Test/Common/DicomCStoreDataClient.cs index 848590694..5b4400860 100644 --- a/tests/Integration.Test/Common/DicomCStoreDataClient.cs +++ b/tests/Integration.Test/Common/DicomCStoreDataClient.cs @@ -62,7 +62,7 @@ public async Task SendAsync(DataProvider dataProvider, params object[] args) var failureStatus = new List(); for (int i = 0; i < associations; i++) { - var files = dataProvider.DicomSpecs.Files.Skip(i * filesPerAssociations).Take(filesPerAssociations).ToList(); + var files = dataProvider.DicomSpecs.Files.Skip(i * filesPerAssociations).Take(filesPerAssociations).ToList(); // .Take(20) if (i + 1 == associations && dataProvider.DicomSpecs.Files.Count > (i + 1) * filesPerAssociations) { files.AddRange(dataProvider.DicomSpecs.Files.Skip(i * filesPerAssociations)); From 86d17c210cb18b27ffc7f50d6de3deb95f418225 Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 30 May 2023 12:29:10 +0100 Subject: [PATCH 3/8] small clean ups Signed-off-by: Neil South --- .../Connectors/PayloadMoveActionHandler.cs | 144 ------------------ .../Services/Storage/ObjectUploadService.cs | 10 +- 2 files changed, 2 insertions(+), 152 deletions(-) diff --git a/src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs b/src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs index c59a5d046..0906f8f4f 100644 --- a/src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs +++ b/src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs @@ -77,7 +77,6 @@ public async Task MoveFilesAsync(Payload payload, ActionBlock moveQueue var stopwatch = Stopwatch.StartNew(); try { - await Move(payload, cancellationToken).ConfigureAwait(false); await NotifyIfCompleted(payload, notificationQueue, cancellationToken).ConfigureAwait(false); } catch (Exception ex) @@ -127,149 +126,6 @@ private async Task NotifyIfCompleted(Payload payload, ActionBlock notif } } - private async Task Move(Payload payload, CancellationToken cancellationToken) - { - Guard.Against.Null(payload); - - _logger.MovingFIlesInPayload(payload.PayloadId, _options.Value.Storage.StorageServiceBucketName); - - var options = new ParallelOptions - { - CancellationToken = cancellationToken, - MaxDegreeOfParallelism = _options.Value.Storage.ConcurrentUploads - }; - - var exceptions = new List(); - await Parallel.ForEachAsync(payload.Files, options, async (file, cancellationToke) => - { - try - { - switch (file) - { - case DicomFileStorageMetadata dicom: - if (!string.IsNullOrWhiteSpace(dicom.JsonFile.TemporaryPath)) - { - await MoveFile(payload.PayloadId, dicom.Id, dicom.JsonFile, cancellationToken).ConfigureAwait(false); - } - break; - } - - await MoveFile(payload.PayloadId, file.Id, file.File, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - exceptions.Add(ex); - } - }).ConfigureAwait(false); - - if (exceptions.Any()) - { - throw new AggregateException(exceptions); - } - } - - private async Task MoveFile(Guid payloadId, string identity, StorageObjectMetadata file, CancellationToken cancellationToken) - { - Guard.Against.NullOrWhiteSpace(identity); - Guard.Against.Null(file); - - if (file.IsMoveCompleted) - { - _logger.AlreadyMoved(payloadId, file.UploadPath); - return; - } - - _logger.MovingFileToPayloadDirectory(payloadId, file.UploadPath); - - try - { - //await _storageService.CopyObjectAsync( - // file.TemporaryBucketName, - // file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), - // _options.Value.Storage.StorageServiceBucketName, - // file.GetPayloadPath(payloadId), - // cancellationToken).ConfigureAwait(false); - - await VerifyFileExists(payloadId, file, cancellationToken).ConfigureAwait(false); - } - catch (StorageObjectNotFoundException ex) when (ex.Message.Contains("Not found", StringComparison.OrdinalIgnoreCase)) // TODO: StorageLib shall not throw any errors from MINIO - { - // when file cannot be found on the Storage Service, we assume file has been moved previously by verifying the file exists on destination. - _logger.FileMissingInPayload(payloadId, file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), ex); - await VerifyFileExists(payloadId, file, cancellationToken).ConfigureAwait(false); - } - catch (StorageConnectionException ex) - { - _logger.StorageServiceConnectionError(ex); - throw new PayloadNotifyException(PayloadNotifyException.FailureReason.ServiceUnavailable); - } - catch (Exception ex) - { - _logger.PayloadMoveException(ex); - await LogFilesInMinIo(file.TemporaryBucketName, cancellationToken).ConfigureAwait(false); - throw new FileMoveException(file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), file.UploadPath, ex); - } - - try - { - //_logger.DeletingFileFromTemporaryBbucket(file.TemporaryBucketName, identity, file.TemporaryPath); - //await _storageService.RemoveObjectAsync(file.TemporaryBucketName, file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), cancellationToken).ConfigureAwait(false); - } - catch (Exception) - { - _logger.ErrorDeletingFileAfterMoveComplete(file.TemporaryBucketName, identity, file.TemporaryPath); - } - finally - { - file.SetMoved(_options.Value.Storage.StorageServiceBucketName); - } - } - - private async Task VerifyFileExists(Guid payloadId, StorageObjectMetadata file, CancellationToken cancellationToken) - { - await Policy - .Handle() - .WaitAndRetryAsync( - _options.Value.Storage.Retries.RetryDelays, - (exception, timeSpan, retryCount, context) => - { - _logger.ErrorUploadingFileToTemporaryStore(timeSpan, retryCount, exception); - }) - .ExecuteAsync(async () => - { - var internalCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - internalCancellationTokenSource.CancelAfter(_options.Value.Storage.StorageServiceListTimeout); - var exists = await _storageService.VerifyObjectExistsAsync(_options.Value.Storage.StorageServiceBucketName, file.GetPayloadPath(payloadId), cancellationToken).ConfigureAwait(false); - if (!exists) - { - _logger.FileMovedVerificationFailure(payloadId, file.UploadPath); - throw new PayloadNotifyException(PayloadNotifyException.FailureReason.MoveFailure, false); - } - }) - .ConfigureAwait(false); - } - - private async Task LogFilesInMinIo(string bucketName, CancellationToken cancellationToken) - { - try - { - var internalCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - internalCancellationTokenSource.CancelAfter(_options.Value.Storage.StorageServiceListTimeout); - var listingResults = await _storageService.ListObjectsAsync(bucketName, recursive: true, cancellationToken: internalCancellationTokenSource.Token).ConfigureAwait(false); - _logger.FilesFounddOnStorageService(bucketName, listingResults.Count); - var files = new List(); - foreach (var item in listingResults) - { - files.Add(item.FilePath); - } - _logger.FileFounddOnStorageService(bucketName, string.Join(Environment.NewLine, files)); - } - catch (Exception ex) - { - _logger.ErrorListingFilesOnStorageService(ex); - } - } - private async Task UpdatePayloadState(Payload payload, Exception ex, CancellationToken cancellationToken = default) { Guard.Against.Null(payload); diff --git a/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs b/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs index ed089efc5..11ca607a8 100644 --- a/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs +++ b/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs @@ -198,7 +198,6 @@ private async Task UploadFileAndConfirm(string identifier, StorageObjectMetadata throw new FileUploadException($"Failed to upload file after retries {identifier}."); } } while (!( - //await VerifyExists(storageObjectMetadata.GetTempStoragPath(_configuration.Value.Storage.RemoteTemporaryStoragePath), cancellationToken).ConfigureAwait(false) await VerifyExists(storageObjectMetadata.GetPayloadPath(Guid.Parse(payloadId)), cancellationToken).ConfigureAwait(false) )); } @@ -217,7 +216,6 @@ private async Task VerifyExists(string path, CancellationToken cancellatio { var internalCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); internalCancellationTokenSource.CancelAfter(_configuration.Value.Storage.StorageServiceListTimeout); - //var exists = await _storageService.VerifyObjectExistsAsync(_configuration.Value.Storage.TemporaryStorageBucket, path).ConfigureAwait(false); var exists = await _storageService.VerifyObjectExistsAsync(_configuration.Value.Storage.StorageServiceBucketName, path).ConfigureAwait(false); _logger.VerifyFileExists(path, exists); return exists; @@ -246,25 +244,21 @@ await Policy { if (storageObjectMetadata.IsUploaded) { return; } - //////////////////// var bucket = _configuration.Value.Storage.StorageServiceBucketName; var path = storageObjectMetadata.GetPayloadPath(Guid.Parse(payloadId)); - ////////////////// storageObjectMetadata.Data.Seek(0, System.IO.SeekOrigin.Begin); await _storageService.PutObjectAsync( - //_configuration.Value.Storage.TemporaryStorageBucket, bucket, - //storageObjectMetadata.GetTempStoragPath(_configuration.Value.Storage.RemoteTemporaryStoragePath), path, storageObjectMetadata.Data, storageObjectMetadata.Data.Length, storageObjectMetadata.ContentType, metadata, cancellationToken).ConfigureAwait(false); - storageObjectMetadata.SetUploaded(_configuration.Value.Storage.TemporaryStorageBucket); // deletes local file + storageObjectMetadata.SetUploaded(_configuration.Value.Storage.TemporaryStorageBucket); // deletes local file and sets uploaded to true _logger.UploadedFileToTemporaryStore(storageObjectMetadata.TemporaryPath); - storageObjectMetadata.SetMoved(_configuration.Value.Storage.StorageServiceBucketName); + storageObjectMetadata.SetMoved(_configuration.Value.Storage.StorageServiceBucketName); // set bucket, date moved, and move complete }) .ConfigureAwait(false); } From d748ae2c19351b8f8560bce2757eabc0b63ec74d Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 30 May 2023 14:58:11 +0100 Subject: [PATCH 4/8] fix for other incomming types Signed-off-by: Neil South --- .../Services/Connectors/DataRetrievalService.cs | 4 +++- src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs | 6 ++++-- src/InformaticsGateway/Services/Fhir/FhirService.cs | 3 ++- src/InformaticsGateway/Services/HealthLevel7/MllpService.cs | 3 ++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs b/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs index 054e318b1..7bd0f950f 100644 --- a/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs +++ b/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs @@ -201,7 +201,9 @@ private async Task NotifyNewInstance(InferenceRequest inferenceRequest, Dictiona var FileMeta = retrievedFiles[key]; FileMeta.PayloadId = inferenceRequest.TransactionId; _uploadQueue.Queue(retrievedFiles[key]); - await _payloadAssembler.Queue(inferenceRequest.TransactionId, retrievedFiles[key]).ConfigureAwait(false); + + var payloadId = await _payloadAssembler.Queue(inferenceRequest.TransactionId, retrievedFiles[key]).ConfigureAwait(false); + //retrievedFiles[key].PayloadId = payloadId.ToString(); } } diff --git a/src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs b/src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs index 7f9cbb2df..1c3cfad60 100644 --- a/src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs +++ b/src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs @@ -168,12 +168,14 @@ private async Task SaveInstance(Stream stream, string studyInstanceUid, string w { dicomInfo.SetWorkflows(workflowName); } + // for DICOMweb, use correlation ID as the grouping key + var payloadId = await _payloadAssembler.Queue(correlationId, dicomInfo, _configuration.Value.DicomWeb.Timeout).ConfigureAwait(false); + dicomInfo.PayloadId = payloadId.ToString(); await dicomInfo.SetDataStreams(dicomFile, dicomFile.ToJson(_configuration.Value.Dicom.WriteDicomJson, _configuration.Value.Dicom.ValidateDicomOnSerialization), _configuration.Value.Storage.TemporaryDataStorage, _fileSystem, _configuration.Value.Storage.LocalTemporaryStoragePath).ConfigureAwait(false); _uploadQueue.Queue(dicomInfo); - // for DICOMweb, use correlation ID as the grouping key - await _payloadAssembler.Queue(correlationId, dicomInfo, _configuration.Value.DicomWeb.Timeout).ConfigureAwait(false); + _logger.QueuedStowInstance(); AddSuccess(null, uids); diff --git a/src/InformaticsGateway/Services/Fhir/FhirService.cs b/src/InformaticsGateway/Services/Fhir/FhirService.cs index f1cc9eab6..45b706ce9 100644 --- a/src/InformaticsGateway/Services/Fhir/FhirService.cs +++ b/src/InformaticsGateway/Services/Fhir/FhirService.cs @@ -87,8 +87,9 @@ public async Task StoreAsync(HttpRequest request, string correl throw new FhirStoreException(correlationId, $"Provided resource is of type '{content.InternalResourceType}' but request targeted type '{resourceType}'.", IssueType.Invalid); } + var payloadId = await _payloadAssembler.Queue(correlationId, content.Metadata, Resources.PayloadAssemblerTimeout).ConfigureAwait(false); + content.Metadata.PayloadId = payloadId.ToString(); _uploadQueue.Queue(content.Metadata); - await _payloadAssembler.Queue(correlationId, content.Metadata, Resources.PayloadAssemblerTimeout).ConfigureAwait(false); _logger.QueuedStowInstance(); content.StatusCode = StatusCodes.Status201Created; diff --git a/src/InformaticsGateway/Services/HealthLevel7/MllpService.cs b/src/InformaticsGateway/Services/HealthLevel7/MllpService.cs index 8e2ae602d..5746a9ba6 100644 --- a/src/InformaticsGateway/Services/HealthLevel7/MllpService.cs +++ b/src/InformaticsGateway/Services/HealthLevel7/MllpService.cs @@ -170,8 +170,9 @@ private async Task OnDisconnect(IMllpClient client, MllpClientResult result) { var hl7Fileetadata = new Hl7FileStorageMetadata(client.ClientId.ToString()); await hl7Fileetadata.SetDataStream(message.HL7Message, _configuration.Value.Storage.TemporaryDataStorage, _fileSystem, _configuration.Value.Storage.LocalTemporaryStoragePath).ConfigureAwait(false); + var payloadId = await _payloadAssembler.Queue(client.ClientId.ToString(), hl7Fileetadata).ConfigureAwait(false); + hl7Fileetadata.PayloadId = payloadId.ToString(); _uploadQueue.Queue(hl7Fileetadata); - await _payloadAssembler.Queue(client.ClientId.ToString(), hl7Fileetadata).ConfigureAwait(false); } } catch (Exception ex) From 2ff65924ac49140e7ba7f2fbddae0074ce80b863 Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 30 May 2023 16:24:23 +0100 Subject: [PATCH 5/8] fix up for tests Signed-off-by: Neil South --- .../Connectors/PayloadMoveActionHandlerTest.cs | 14 ++++++++++---- .../Services/Storage/ObjectUploadServiceTest.cs | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/InformaticsGateway/Test/Services/Connectors/PayloadMoveActionHandlerTest.cs b/src/InformaticsGateway/Test/Services/Connectors/PayloadMoveActionHandlerTest.cs index 748229209..2bf054a68 100644 --- a/src/InformaticsGateway/Test/Services/Connectors/PayloadMoveActionHandlerTest.cs +++ b/src/InformaticsGateway/Test/Services/Connectors/PayloadMoveActionHandlerTest.cs @@ -190,25 +190,31 @@ public async Task GivenAPayload_WhenAllFilesAreMove_ExpectPayloadToBeAddedToNoti }); var correlationId = Guid.NewGuid(); + var file1 = new DicomFileStorageMetadata(correlationId.ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + file1.File.SetMoved("test"); + file1.JsonFile.SetMoved("test"); + var file2 = new FhirFileStorageMetadata(correlationId.ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Api.Rest.FhirStorageFormat.Json); + file2.File.SetMoved("test"); var payload = new Payload("key", correlationId.ToString(), 0) { RetryCount = 3, State = Payload.PayloadState.Move, Files = new List { - new DicomFileStorageMetadata(correlationId.ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()), - new FhirFileStorageMetadata(correlationId.ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Api.Rest.FhirStorageFormat.Json), + file1, + file2, }, }; + var handler = new PayloadMoveActionHandler(_serviceScopeFactory.Object, _logger.Object, _options); await handler.MoveFilesAsync(payload, moveAction, notifyAction, _cancellationTokenSource.Token); Assert.True(notifyEvent.Wait(TimeSpan.FromSeconds(5))); - _storageService.Verify(p => p.CopyObjectAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); - _storageService.Verify(p => p.RemoveObjectAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); + //_storageService.Verify(p => p.CopyObjectAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); + //_storageService.Verify(p => p.RemoveObjectAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); } } } diff --git a/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs b/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs index dd4fe8cbf..b601a5876 100644 --- a/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs +++ b/src/InformaticsGateway/Test/Services/Storage/ObjectUploadServiceTest.cs @@ -119,7 +119,7 @@ public async Task GivenADicomFileStorageMetadata_WhenQueuedForUpload_ExpectTwoFi _storageService.Verify(p => p.PutObjectAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny()), Times.Exactly(2)); } - [RetryFact(10, 250)] + [RetryFact(10, 25000)] public async Task GivenAFhirFileStorageMetadata_WhenQueuedForUpload_ExpectSingleFileToBeUploaded() { var countdownEvent = new CountdownEvent(1); @@ -146,6 +146,7 @@ private async Task GenerateFhirFileStorageMetadata() var file = new FhirFileStorageMetadata(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), FhirStorageFormat.Json); await file.SetDataStream("[]", TemporaryDataStorageLocation.Memory); + file.PayloadId = Guid.NewGuid().ToString(); return file; } From 0385d73967e26ed1d3318eb0f595dff3a0f210f1 Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 30 May 2023 17:08:59 +0100 Subject: [PATCH 6/8] fix for diacomWeb Signed-off-by: Neil South --- .../Services/Connectors/DataRetrievalService.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs b/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs index 7bd0f950f..c664e9aeb 100644 --- a/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs +++ b/src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs @@ -199,11 +199,10 @@ private async Task NotifyNewInstance(InferenceRequest inferenceRequest, Dictiona retrievedFiles[key].SetWorkflows(inferenceRequest.Application.Id); } var FileMeta = retrievedFiles[key]; - FileMeta.PayloadId = inferenceRequest.TransactionId; - _uploadQueue.Queue(retrievedFiles[key]); var payloadId = await _payloadAssembler.Queue(inferenceRequest.TransactionId, retrievedFiles[key]).ConfigureAwait(false); - //retrievedFiles[key].PayloadId = payloadId.ToString(); + retrievedFiles[key].PayloadId = payloadId.ToString(); + _uploadQueue.Queue(retrievedFiles[key]); } } From dab25f2815bec510bacfceeff7043571d71ca9e3 Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 30 May 2023 17:25:44 +0100 Subject: [PATCH 7/8] fixing warnings Signed-off-by: Neil South --- src/Api/Storage/FileStorageMetadata.cs | 2 +- .../Test/SourceApplicationEntityRepositoryTest.cs | 1 + .../Integration.Test/SourceApplicationEntityRepositoryTest.cs | 1 + tests/Integration.Test/Drivers/RabbitMqConsumer.cs | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Api/Storage/FileStorageMetadata.cs b/src/Api/Storage/FileStorageMetadata.cs index 8a045ed52..ec3e3b310 100644 --- a/src/Api/Storage/FileStorageMetadata.cs +++ b/src/Api/Storage/FileStorageMetadata.cs @@ -117,6 +117,6 @@ public virtual void SetFailed() File.SetFailed(); } - public string PayloadId { get; set; } + public string? PayloadId { get; set; } } } diff --git a/src/Database/EntityFramework/Test/SourceApplicationEntityRepositoryTest.cs b/src/Database/EntityFramework/Test/SourceApplicationEntityRepositoryTest.cs index 536e165df..ad6487ef8 100644 --- a/src/Database/EntityFramework/Test/SourceApplicationEntityRepositoryTest.cs +++ b/src/Database/EntityFramework/Test/SourceApplicationEntityRepositoryTest.cs @@ -119,6 +119,7 @@ public async Task GivenAAETitleName_WhenFindByAETAsyncIsCalled_ExpectItToReturnM Assert.Equal("AET1", actual.FirstOrDefault()!.Name); actual = await store.FindByAETAsync("AET6").ConfigureAwait(false); + Assert.NotNull(actual); Assert.Empty(actual); } diff --git a/src/Database/MongoDB/Integration.Test/SourceApplicationEntityRepositoryTest.cs b/src/Database/MongoDB/Integration.Test/SourceApplicationEntityRepositoryTest.cs index 3760f2d7b..83ec01f0f 100644 --- a/src/Database/MongoDB/Integration.Test/SourceApplicationEntityRepositoryTest.cs +++ b/src/Database/MongoDB/Integration.Test/SourceApplicationEntityRepositoryTest.cs @@ -124,6 +124,7 @@ public async Task GivenAETitle_WhenFindByAETitleAsyncIsCalled_ExpectItToReturnMa Assert.Equal("AET1", actual.FirstOrDefault()!.Name); actual = await store.FindByAETAsync("AET6").ConfigureAwait(false); + Assert.NotNull(actual); Assert.Empty(actual); } diff --git a/tests/Integration.Test/Drivers/RabbitMqConsumer.cs b/tests/Integration.Test/Drivers/RabbitMqConsumer.cs index 38f07d716..e82f53d1a 100644 --- a/tests/Integration.Test/Drivers/RabbitMqConsumer.cs +++ b/tests/Integration.Test/Drivers/RabbitMqConsumer.cs @@ -45,10 +45,10 @@ public RabbitMqConsumer(RabbitMQMessageSubscriberService subscriberService, stri _outputHelper = outputHelper ?? throw new ArgumentNullException(nameof(outputHelper)); _messages = new ConcurrentBag(); - subscriberService.Subscribe( + subscriberService.SubscribeAsync( queueName, queueName, - (eventArgs) => + async (eventArgs) => { _outputHelper.WriteLine($"Message received from queue {queueName} for {queueName}."); _messages.Add(eventArgs.Message); From 556d2c3bddf6eb56e69e9a5246e9d8eb7fbfeabf Mon Sep 17 00:00:00 2001 From: Neil South Date: Tue, 6 Jun 2023 12:03:47 +0100 Subject: [PATCH 8/8] clean up of log messages Signed-off-by: Neil South --- src/Configuration/ConfigurationValidator.cs | 2 +- .../Logging/Log.4000.ObjectUploadService.cs | 10 +++++----- .../Services/Storage/ObjectUploadService.cs | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Configuration/ConfigurationValidator.cs b/src/Configuration/ConfigurationValidator.cs index c19d175d4..f54381be2 100644 --- a/src/Configuration/ConfigurationValidator.cs +++ b/src/Configuration/ConfigurationValidator.cs @@ -120,7 +120,7 @@ private bool IsValidDirectory(string source, string directory) private bool IsValidBucketName(string source, string bucketName) { var valid = IsNotNullOrWhiteSpace(source, bucketName); - var regex = new Regex("(?=^.{3,63}$)(^[a-z0-9]+[a-z0-9\\-]+[a-z0-9]+$)"); + var regex = new Regex("(?=^.{3,63}$)(^[a-z0-9]+[a-z0-9\\-]+[a-z0-9]+$)", new RegexOptions(), TimeSpan.FromSeconds(5)); if (!regex.IsMatch(bucketName)) { valid = false; diff --git a/src/InformaticsGateway/Logging/Log.4000.ObjectUploadService.cs b/src/InformaticsGateway/Logging/Log.4000.ObjectUploadService.cs index e6e9957c4..afdad8c78 100644 --- a/src/InformaticsGateway/Logging/Log.4000.ObjectUploadService.cs +++ b/src/InformaticsGateway/Logging/Log.4000.ObjectUploadService.cs @@ -27,8 +27,8 @@ public static partial class Log [LoggerMessage(EventId = 4001, Level = LogLevel.Debug, Message = "Upload statistics: {threads} threads, {seconds} seconds.")] public static partial void UploadStats(this ILogger logger, int threads, double seconds); - [LoggerMessage(EventId = 4002, Level = LogLevel.Debug, Message = "Uploading file to temporary store at {filePath}.")] - public static partial void UploadingFileToTemporaryStore(this ILogger logger, string filePath); + [LoggerMessage(EventId = 4002, Level = LogLevel.Debug, Message = "Uploading file to storeage at {filePath}.")] + public static partial void UploadingFileToStoreage(this ILogger logger, string filePath); [LoggerMessage(EventId = 4003, Level = LogLevel.Information, Message = "Instance queued for upload {identifier}. Items in queue {count} using memory {memoryUsageKb}KB.")] public static partial void InstanceAddedToUploadQueue(this ILogger logger, string identifier, int count, double memoryUsageKb); @@ -36,11 +36,11 @@ public static partial class Log [LoggerMessage(EventId = 4004, Level = LogLevel.Debug, Message = "Error removing objects that are pending upload during startup.")] public static partial void ErrorRemovingPendingUploadObjects(this ILogger logger, Exception ex); - [LoggerMessage(EventId = 4005, Level = LogLevel.Error, Message = "Error uploading temporary store. Waiting {timeSpan} before next retry. Retry attempt {retryCount}.")] + [LoggerMessage(EventId = 4005, Level = LogLevel.Error, Message = "Error uploading storeage. Waiting {timeSpan} before next retry. Retry attempt {retryCount}.")] public static partial void ErrorUploadingFileToTemporaryStore(this ILogger logger, TimeSpan timespan, int retryCount, Exception ex); - [LoggerMessage(EventId = 4006, Level = LogLevel.Information, Message = "File uploaded to temporary store at {filePath}.")] - public static partial void UploadedFileToTemporaryStore(this ILogger logger, string filePath); + [LoggerMessage(EventId = 4006, Level = LogLevel.Information, Message = "File uploaded to storeage at {filePath}.")] + public static partial void UploadedFileToStoreage(this ILogger logger, string filePath); [LoggerMessage(EventId = 4007, Level = LogLevel.Debug, Message = "Items in queue {count}.")] public static partial void InstanceInUploadQueue(this ILogger logger, int count); diff --git a/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs b/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs index 11ca607a8..fb30a5d30 100644 --- a/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs +++ b/src/InformaticsGateway/Services/Storage/ObjectUploadService.cs @@ -225,7 +225,7 @@ private async Task VerifyExists(string path, CancellationToken cancellatio private async Task UploadFile(StorageObjectMetadata storageObjectMetadata, string source, List workflows, string payloadId, CancellationToken cancellationToken) { - _logger.UploadingFileToTemporaryStore(storageObjectMetadata.TemporaryPath); + _logger.UploadingFileToStoreage(storageObjectMetadata.TemporaryPath); var metadata = new Dictionary { { FileMetadataKeys.Source, source }, @@ -257,7 +257,7 @@ await _storageService.PutObjectAsync( metadata, cancellationToken).ConfigureAwait(false); storageObjectMetadata.SetUploaded(_configuration.Value.Storage.TemporaryStorageBucket); // deletes local file and sets uploaded to true - _logger.UploadedFileToTemporaryStore(storageObjectMetadata.TemporaryPath); + _logger.UploadedFileToStoreage(storageObjectMetadata.TemporaryPath); storageObjectMetadata.SetMoved(_configuration.Value.Storage.StorageServiceBucketName); // set bucket, date moved, and move complete }) .ConfigureAwait(false);