Merge pull request #952 from Project-MONAI/nds-logging

Nds improve logging

src/TaskManager/TaskManager/Program.cs

@@ -154,6 +154,7 @@ private static Logger ConfigureNLog(string assemblyVersionNumber)
                 ext.RegisterLayoutRenderer("servicename", logEvent => typeof(Program).Namespace);
                 ext.RegisterLayoutRenderer("serviceversion", logEvent => assemblyVersionNumber);
                 ext.RegisterLayoutRenderer("machinename", logEvent => Environment.MachineName);
+                ext.RegisterLayoutRenderer("appname", logEvent => "TaskManager");
             })
             .LoadConfigurationFromAppSettings()
             .GetCurrentClassLogger();

src/TaskManager/TaskManager/nlog.config

@@ -52,6 +52,7 @@ limitations under the License.
         <attribute name="Tag" layout="${environment:ENVIRONMENT_NAME}" />
         <attribute name="ServiceName" layout="${servicename}" />
         <attribute name="ServiceVersion" layout="${serviceversion}" />
+        <attribute name="ApplicationName" layout="${appname}" />
         <attribute name="MachineName" layout="${machinename}" />
         <attribute name="CorrelationId" layout="${mdlc:item=correlationId}" />
         <attribute name="LoggerName" layout="${logger}" />

src/WorkflowManager/Contracts/Migrations/M004_WorkflowRevision_addDataRetension.cs

@@ -26,9 +26,8 @@ public M004_WorkflowRevision_AddDataRetension() : base("1.0.1") { }
 
         public override void Up(BsonDocument document)
         {
-//            document.Add("Workflow.DataRetentionDays", BsonNull.Create(null).ToJson(), true);
             var workflow = document["Workflow"].AsBsonDocument;
-            workflow.Add("DataRetentionDays", BsonNull.Create(null).ToJson(), true);
+            workflow.Add("DataRetentionDays", -1, true);
         }
 
         public override void Down(BsonDocument document)

src/WorkflowManager/WorkflowExecuter/Services/WorkflowExecuterService.cs

@@ -253,7 +253,7 @@ await _artifactsRepository
 
         private async Task ProcessArtifactReceivedOutputs(ArtifactsReceivedEvent message, WorkflowInstance workflowInstance, TaskObject taskTemplate, string taskId)
         {
-            var artifactList = message.Artifacts.Select(a => $"{a.Path}").ToList();
+            var artifactList = message.Artifacts.Select(a => a.Path).ToList();
             var artifactsInStorage = (await _storageService.VerifyObjectsExistAsync(workflowInstance.BucketId, artifactList, default)) ?? new Dictionary<string, bool>();
             if (artifactsInStorage.Any(a => a.Value) is false)
             {

src/WorkflowManager/WorkflowManager/Program.cs

@@ -189,6 +189,7 @@ private static Logger ConfigureNLog(string assemblyVersionNumber)
                 ext.RegisterLayoutRenderer("servicename", logEvent => typeof(Program).Namespace);
                 ext.RegisterLayoutRenderer("serviceversion", logEvent => assemblyVersionNumber);
                 ext.RegisterLayoutRenderer("machinename", logEvent => Environment.MachineName);
+                ext.RegisterLayoutRenderer("appname", logEvent => "WorkflowManager");
             })
             .LoadConfigurationFromAppSettings()
             .GetCurrentClassLogger();

src/WorkflowManager/WorkflowManager/nlog.config

@@ -52,6 +52,7 @@ limitations under the License.
         <attribute name="Tag" layout="${environment:ENVIRONMENT_NAME}" />
         <attribute name="ServiceName" layout="${servicename}" />
         <attribute name="ServiceVersion" layout="${serviceversion}" />
+        <attribute name="ApplicationName" layout="${appname}" />
         <attribute name="MachineName" layout="${machinename}" />
         <attribute name="CorrelationId" layout="${mdlc:item=correlationId}" />
         <attribute name="LoggerName" layout="${logger}" />

tests/UnitTests/WorkflowExecuter.Tests/Services/WorkflowExecuterServiceTests.cs

@@ -3703,6 +3703,182 @@ public async Task ProcessTaskUpdate_ValidTaskUpdateEventWithExportHl7TaskDestina
             response.Should().BeTrue();
         }
 
+        //[Fact]
+        public async Task ProcessPayload_With_Multiple_Taskdestinations_One_Has_Inputs()
+        {
+            var workflowInstanceId = Guid.NewGuid().ToString();
+            var workflowId1 = Guid.NewGuid().ToString();
+            var workflowId2 = Guid.NewGuid().ToString();
+            var workflowRequest = new WorkflowRequestEvent
+            {
+                Bucket = "testbucket",
+                DataTrigger = new DataOrigin { Source = "aetitle", Destination = "aetitle" },
+                CorrelationId = Guid.NewGuid().ToString(),
+                Timestamp = DateTime.UtcNow,
+                Workflows = new List<string>
+                {
+                    workflowId1.ToString()
+                }
+            };
+
+            var workflows = new List<WorkflowRevision>
+            {
+                new WorkflowRevision
+                {
+                    Id = Guid.NewGuid().ToString(),
+                    WorkflowId = workflowId1,
+                    Revision = 1,
+                    Workflow = new Workflow
+                    {
+                        Name = "Workflowname1",
+                        Description = "Workflowdesc1",
+                        Version = "1",
+                        InformaticsGateway = new InformaticsGateway
+                        {
+                            AeTitle = "aetitle",
+                            ExportDestinations = new string[] { "PROD_PACS" }
+                        },
+                        Tasks = new TaskObject[]
+                        {
+                            new TaskObject {
+                                Id = "router",
+                                Type = "router",
+                                Description = "router",
+                                Artifacts = new ArtifactMap
+                                {
+                                    Input = new Artifact[] { new Artifact { Name = "dicomexport", Value = "{{ context.input }}" } },
+                                    Output = new OutputArtifact[]
+                                {
+                                    new OutputArtifact
+                                    {
+                                        Name = "Artifact1",
+                                        Value = "Artifact1Value",
+                                        Mandatory = true,
+                                        Type = ArtifactType.DOC
+                                    },
+                                    new OutputArtifact
+                                    {
+                                        Name = "Artifact2",
+                                        Value = "Artifact2Value",
+                                        Mandatory = true,
+                                        Type = ArtifactType.CT
+                                    }
+                                }
+                                },
+                                TaskDestinations = new TaskDestination[]
+                                {
+                                    new TaskDestination
+                                    {
+                                        Name = "export1"
+                                    },
+                                    new TaskDestination
+                                    {
+                                        Name = "export2"
+                                    }
+                                }
+                            },
+                            new TaskObject
+                            {
+                                Id ="export1",
+                                Type = "export",
+                                Artifacts = new ArtifactMap
+                                {
+                                    Input = new Artifact[] { new Artifact { Name = "artifact", Value = "{{ context.executions.router.artifacts.output.Artifact1 }}" } }
+                                },
+                                ExportDestinations = new ExportDestination[]
+                                {
+                                }
+                            },
+                            new TaskObject
+                            {
+                                Id ="export2",
+                                Type = "export",
+                                Artifacts = new ArtifactMap
+                                {
+                                    Input = new Artifact[] { new Artifact { Name = "artifact2", Value = "{{ context.executions.router.artifacts.output.Artifact2 }}" } }
+                                },
+                                ExportDestinations = new ExportDestination[]
+                                {
+                                }
+                            },
+                        }
+                    }
+                }
+            };
+            var workflowInstance = new WorkflowInstance
+            {
+                Id = workflowInstanceId,
+                WorkflowId = workflowId1,
+                WorkflowName = workflows.First()!.Workflow!.Name,
+                PayloadId = Guid.NewGuid().ToString(),
+                Status = Status.Created,
+                BucketId = "bucket",
+                Tasks = new List<TaskExecution>
+                {
+                        new TaskExecution
+                        {
+                            TaskId = "router",
+                            Status = TaskExecutionStatus.Created
+                        },
+                        //new TaskExecution
+                        //{
+                        //    TaskId = "export1",
+                        //    Status = TaskExecutionStatus.Created
+                        //},
+                        //new TaskExecution
+                        //{
+                        //    TaskId = "export2",
+                        //    Status = TaskExecutionStatus.Created
+                        //}
+                    }
+            };
+
+            var artifactDict = new List<Messaging.Common.Storage>
+                {
+                    new Messaging.Common.Storage
+                    {
+                        Name = "artifactname",
+                        RelativeRootPath = "path/to/artifact"
+                    }
+                };
+
+            _workflowInstanceRepository.Setup(w => w.GetByWorkflowInstanceIdAsync(workflowInstance.Id)).ReturnsAsync(workflowInstance);
+
+            _workflowRepository.Setup(w => w.GetByWorkflowsIdsAsync(new List<string> { workflowId1.ToString() })).ReturnsAsync(workflows);
+            _workflowRepository.Setup(w => w.GetByWorkflowIdAsync(workflowId1.ToString())).ReturnsAsync(workflows[0]);
+            _workflowInstanceRepository.Setup(w => w.CreateAsync(It.IsAny<List<WorkflowInstance>>())).ReturnsAsync(true);
+            _workflowInstanceRepository.Setup(w => w.UpdateTasksAsync(It.IsAny<string>(), It.IsAny<List<TaskExecution>>())).ReturnsAsync(true);
+            _workflowInstanceRepository.Setup(w => w.GetByWorkflowsIdsAsync(It.IsAny<List<string>>())).ReturnsAsync(new List<WorkflowInstance>());
+            _workflowInstanceRepository.Setup(w => w.UpdateTaskStatusAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<TaskExecutionStatus>())).ReturnsAsync(true);
+            var dcmInfo = new Dictionary<string, string>() { { "dicomexport", "/dcm" } };
+            _artifactMapper.Setup(a => a.TryConvertArtifactVariablesToPath(It.IsAny<Artifact[]>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), out dcmInfo)).Returns(true);
+
+            _messageBrokerPublisherService.Setup(m => m.Publish(It.IsAny<string>(), It.IsAny<Message>()));
+
+            var pathList = artifactDict.Select(a => a.RelativeRootPath).ToList();
+
+            _storageService.Setup(w => w.VerifyObjectsExistAsync(
+                workflowInstance.BucketId, It.Is<IReadOnlyList<string>>(l => l.Any(a => pathList.Any(p => p == a))), It.IsAny<CancellationToken>()))
+                .ReturnsAsync(new Dictionary<string, bool>() { { pathList.First(), true } });
+
+            var mess = new ArtifactsReceivedEvent
+            {
+                WorkflowInstanceId = workflowInstance.Id,
+                TaskId = "router",
+                Artifacts = [new Messaging.Common.Artifact { Type = ArtifactType.DOC, Path = "path/to/artifact" }]
+            };
+
+
+            var response = await WorkflowExecuterService.ProcessArtifactReceivedAsync(mess);
+
+            Assert.True(response);
+            //_workflowInstanceRepository.Verify(w => w.UpdateTaskStatusAsync(workflowInstanceId, "router", TaskExecutionStatus.Succeeded));
+            _workflowInstanceRepository.Verify(w => w.UpdateTaskStatusAsync(workflowInstanceId, "export1", TaskExecutionStatus.Succeeded));
+
+
+
+#pragma warning restore CS8604 // Possible null reference argument.
+        }
     }
 
 #pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type.