Merge pull request #446 from atlas-bi/dev

runner/scripts/em_python.py

@@ -70,7 +70,9 @@ def run(self) -> Optional[List[Path]]:
             # debug statements
             output = ast.literal_eval(self.output.splitlines()[-1])
             if isinstance(output, List):
-                return output
+                # check if they are existing
+                valid_paths = [x for x in output if Path(x).is_file()]
+                return valid_paths or None
         except BaseException:
             # will raise an exception on some output.
             pass

runner/scripts/task_runner.py

@@ -729,24 +729,33 @@ def __process(self) -> None:
                 f"Processing script generated files:\n{output}",
             )
 
-            # convert the files into a set of named temp files and remove them.
-            self.source_files = []
-            for file in output:
-                original = Path(file)
-                original_name = str(original.absolute())
-                with tempfile.NamedTemporaryFile(
-                    mode="wb+", delete=False, dir=self.temp_path
-                ) as data_file:
-                    # write contents
-                    data_file.write(original.read_bytes())
-
-                    # set name and remove original
-                    original.unlink()
-
-                    os.link(data_file.name, original_name)
-
-                data_file.name = original_name  # type: ignore[misc]
-                self.source_files.append(data_file)
+            try:
+                # convert the files into a set of named temp files and remove them.
+                self.source_files = []
+                for file in output:
+                    original = Path(file)
+                    original_name = str(original.absolute())
+                    with tempfile.NamedTemporaryFile(
+                        mode="wb+", delete=False, dir=self.temp_path
+                    ) as data_file:
+                        # write contents
+                        data_file.write(original.read_bytes())
+
+                        # set name and remove original
+                        original.unlink()
+
+                        os.link(data_file.name, original_name)
+
+                    data_file.name = original_name  # type: ignore[misc]
+                    self.source_files.append(data_file)
+
+            except BaseException as e:
+                raise RunnerException(
+                    self.task,
+                    self.run_id,
+                    8,
+                    f"Failed to load files from processing script output:\n{e}",
+                )
 
     def __store_files(self) -> None:
         if not self.source_files or len(self.source_files) == 0: