Fix FileNotFoundError for log file (#609)

* Fix `FileNotFoundError` for log file
- Add module level instantiation of `logger` in `logger.py` to avoid repeating logger outputs in `move_log_to_prov_dir()`

e3sm_diags/logger.py

@@ -1,6 +1,7 @@
 """Logger module for setting up a custom logger."""
 import logging
 import logging.handlers
+import os
 import shutil
 
 LOG_FILENAME = "e3sm_diags_run.log"
@@ -9,6 +10,13 @@
 def custom_logger(name: str, propagate: bool = True) -> logging.Logger:
     """Sets up a custom logger.
 
+    `force` is set to `True` to automatically remove root handlers whenever
+    `basicConfig` called. This is required for cases where multiple e3sm_diags
+    runs are executed. Otherwise, the logger objects attempt to share the same
+    root file reference (which gets deleted between runs), resulting in
+    `FileNotFoundError: [Errno 2] No such file or directory: 'e3sm_diags_run.log'`.
+    More info here: https://stackoverflow.com/a/49202811
+
     Parameters
     ----------
     name : str
@@ -59,6 +67,7 @@ def custom_logger(name: str, propagate: bool = True) -> logging.Logger:
         filename=LOG_FILENAME,
         filemode=log_filemode,
         level=logging.INFO,
+        force=True,
     )
     logging.captureWarnings(True)
 
@@ -74,6 +83,9 @@ def custom_logger(name: str, propagate: bool = True) -> logging.Logger:
     return logger
 
 
+logger = custom_logger(__name__)
+
+
 def move_log_to_prov_dir(results_dir: str):
     """Moves the e3sm diags log file to the provenance directory.
 
@@ -87,7 +99,8 @@ def move_log_to_prov_dir(results_dir: str):
     """
     provenance_dir = f"{results_dir}/prov/{LOG_FILENAME}"
 
-    logger = custom_logger(__name__)
+    # Must copy and then delete because shutil.move does not work if different
+    # filesystems are used for the source and destination directories.
+    shutil.copy(LOG_FILENAME, provenance_dir)
+    os.remove(LOG_FILENAME)
     logger.info(f"Log file saved in {provenance_dir}")
-
-    shutil.move(LOG_FILENAME, provenance_dir)