From 8f0541cd61755e74e1f3116dedcc3afc8fa9cda1 Mon Sep 17 00:00:00 2001 From: Innocent Souopgui <162634017+InnocentSouopgui-NOAA@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:41:46 -0500 Subject: [PATCH] Move to contrib installation of spack-stack on Jet (#2878) Migrates Global Workflow to use contrib installation of spack-stack on Jet. Following the failure of the storage /lfs4 on Jet, the installation of spack spack moved to /contrib. All softwares relying on spack-stack on Jet needs update. Resolves #2841 Refs NOAA-EMC/gfs-utils#78 Refs NOAA-EMC/GSI#786 Refs NOAA-EMC/GSI-Monitor#143 Refs NOAA-EMC/GSI-utils#51 Refs ufs-community/UFS_UTILS#977 --- docs/note_fixfield.txt | 2 +- docs/source/components.rst | 2 +- docs/source/init.rst | 4 ++-- modulefiles/module_base.jet.lua | 2 +- modulefiles/module_gwsetup.jet.lua | 2 +- parm/config/gfs/config.aero | 2 +- sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/gsi_monitor.fd | 2 +- sorc/link_workflow.sh | 2 +- sorc/ufs_model.fd | 2 +- sorc/ufs_utils.fd | 2 +- ush/detect_machine.sh | 2 +- ush/forecast_predet.sh | 1 - versions/build.jet.ver | 4 ++-- versions/run.jet.ver | 8 ++++++-- workflow/hosts.py | 2 +- workflow/hosts/jet.yaml | 22 +++++++++++----------- 18 files changed, 34 insertions(+), 31 deletions(-) diff --git a/docs/note_fixfield.txt b/docs/note_fixfield.txt index a7a4001561..32e44b9d98 100644 --- a/docs/note_fixfield.txt +++ b/docs/note_fixfield.txt @@ -4,7 +4,7 @@ They are saved locally on all platforms Hera: /scratch1/NCEPDEV/global/glopara/fix Orion/Hercules: /work/noaa/global/glopara/fix -Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix +Jet: /mnt/lfs5/HFIP/hfv3gfs/glopara/FIX/fix S4: /data/prod/glopara/fix ------------------------------------------------------------------------------ diff --git a/docs/source/components.rst b/docs/source/components.rst index 869ef89bab..f5a60a96af 100644 --- a/docs/source/components.rst +++ b/docs/source/components.rst @@ -59,7 +59,7 @@ Observation data, also known as dump data, is prepared in production and then ar * Hera: ``/scratch1/NCEPDEV/global/glopara/dump`` * Orion/Hercules: ``/work/noaa/rstprod/dump`` -* Jet: ``/mnt/lfs4/HFIP/hfv3gfs/glopara/dump`` +* Jet: ``/mnt/lfs5/HFIP/hfv3gfs/glopara/dump`` * WCOSS2: ``/lfs/h2/emc/global/noscrub/emc.global/dump`` * S4: ``/data/prod/glopara/dump`` diff --git a/docs/source/init.rst b/docs/source/init.rst index e1cabdc8e0..aa71e4e294 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -111,7 +111,7 @@ Warm-start cycled w/ coupled (S2S) model C48 atmosphere C48 enkf (80 members) 5 Hera: /scratch1/NCEPDEV/global/glopara/data/ICSDIR/C48C48mx500 Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C48C48mx500 WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C48C48mx500 - Jet: /lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/C48C48mx500 + Jet: /lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR/C48C48mx500 AWS: https://noaa-nws-global-pds.s3.amazonaws.com/index.html#data/ICSDIR/C48C48mx500 Start date = 2021032312 @@ -227,7 +227,7 @@ Forecast-only P8 prototype initial conditions are made available to users on sup WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED HERA: /scratch1/NCEPDEV/climate/role.ufscpara/IC ORION/Hercules: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs - JET: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs + JET: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs S4: /data/prod/glopara/coupled_ICs These locations are known within the workflow via paths set in ``parm/config/config.coupled_ic``. diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index 2f00c301df..56735a2057 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -52,7 +52,7 @@ setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) prepend_path("MODULEPATH", pathJoin("/lfs5/HFIP/hfv3gfs/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) -prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/lfs5/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run environment") diff --git a/modulefiles/module_gwsetup.jet.lua b/modulefiles/module_gwsetup.jet.lua index bc14b19a79..a722c812a5 100644 --- a/modulefiles/module_gwsetup.jet.lua +++ b/modulefiles/module_gwsetup.jet.lua @@ -4,7 +4,7 @@ Load environment to run GFS workflow setup scripts on Jet load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/gsi-addon-intel/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local python_ver=os.getenv("python_ver") or "3.11.6" diff --git a/parm/config/gfs/config.aero b/parm/config/gfs/config.aero index 2fae019574..f49593a439 100644 --- a/parm/config/gfs/config.aero +++ b/parm/config/gfs/config.aero @@ -24,7 +24,7 @@ case ${machine} in AERO_INPUTS_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions" ;; "JET") - AERO_INPUTS_DIR="/lfs4/HFIP/hfv3gfs/glopara/data/gocart_emissions" + AERO_INPUTS_DIR="/lfs5/HFIP/hfv3gfs/glopara/data/gocart_emissions" ;; *) echo "FATAL ERROR: Machine ${machine} unsupported for aerosols" diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index bd8f13d867..a00cc0949e 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit bd8f13d867721e4ee28de4af437a0de4283609c3 +Subproject commit a00cc0949e2f901e73b58d54834517743916c69a diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 529bb796be..9f44c8798c 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 529bb796bea0e490f186729cd168a91c034bb12d +Subproject commit 9f44c8798c2087aca06df8f629699632e57df431 diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index e1f9f21af1..278ee629e8 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit e1f9f21af16ce912fdc2cd75c5b27094a550a0c5 +Subproject commit 278ee629e87558822e8d13b3fb3b0e16006aa856 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 9426a09ce9..870ddc5eba 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -73,7 +73,7 @@ case "${machine}" in "hera") FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;; "orion") FIX_DIR="/work/noaa/global/glopara/fix" ;; "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;; - "jet") FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" ;; + "jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index fcc9f8461d..6a4e09e947 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit fcc9f8461db5eafbfd1f080da61ea79156ca0145 +Subproject commit 6a4e09e94773ffa39ce7ab6a54a885efada91f21 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 3ef2e6bd72..06eec5b6f6 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 3ef2e6bd725d2662fd6ee95897cb7bac222e5144 +Subproject commit 06eec5b6f636123835e2dfd9fc5229980c006735 diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index b049a6040e..8ad217140a 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -67,7 +67,7 @@ if [[ -d /lfs/h3 ]]; then elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then # We are on NOAA TDS Acorn MACHINE_ID=acorn -elif [[ -d /mnt/lfs1 ]]; then +elif [[ -d /mnt/lfs5 ]]; then # We are on NOAA Jet MACHINE_ID=jet elif [[ -d /scratch1 ]]; then diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index d7c04ea699..5aa9dc9ac7 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -548,7 +548,6 @@ FV3_predet(){ ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt" "${DATA}/postxconfig-NT_FH00.txt" fi fi - } # Disable variable not used warnings diff --git a/versions/build.jet.ver b/versions/build.jet.ver index e103725d41..319a8fa0a6 100644 --- a/versions/build.jet.ver +++ b/versions/build.jet.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev +export spack_env=gsi-addon-intel source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" +export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.jet.ver b/versions/run.jet.ver index 1e41fd0036..90553fb277 100644 --- a/versions/run.jet.ver +++ b/versions/run.jet.ver @@ -1,6 +1,6 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev-rocky8 +export spack_env=gsi-addon-intel export hpss_ver= export ncl_ver=6.6.2 @@ -11,4 +11,8 @@ export gempak_ver=7.4.2 export perl_ver=5.38.0 source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" +export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" + +# Local version of TC_tracker +export ens_tracker_ver=v1.1.15.7 + diff --git a/workflow/hosts.py b/workflow/hosts.py index 34ea067ade..7bde58f95f 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -42,7 +42,7 @@ def detect(cls): machine = 'HERA' elif os.path.exists('/work/noaa'): machine = socket.gethostname().split("-", 1)[0].upper() - elif os.path.exists('/lfs4/HFIP'): + elif os.path.exists('/lfs5/HFIP'): machine = 'JET' elif os.path.exists('/lfs/f1'): machine = 'WCOSS2' diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index 80957083e0..a53224fe52 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -1,12 +1,12 @@ -BASE_GIT: '/lfs4/HFIP/hfv3gfs/glopara/git' -DMPDIR: '/lfs4/HFIP/hfv3gfs/glopara/dump' +BASE_GIT: '/lfs5/HFIP/hfv3gfs/glopara/git' +DMPDIR: '/lfs5/HFIP/hfv3gfs/glopara/dump' BASE_DATA: '/lfs5/HFIP/hfv3gfs/glopara/data' -BASE_IC: '/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR' -PACKAGEROOT: '/lfs4/HFIP/hfv3gfs/glopara/nwpara' -COMINsyn: '/lfs4/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat' -HOMEDIR: '/lfs4/HFIP/hfv3gfs/${USER}' -STMP: '/lfs4/HFIP/hfv3gfs/${USER}/stmp' -PTMP: '/lfs4/HFIP/hfv3gfs/${USER}/ptmp' +BASE_IC: '/mnt/lfs5/HFIP/hfv3gfs/glopara/data/ICSDIR' +PACKAGEROOT: '/lfs5/HFIP/hfv3gfs/glopara/nwpara' +COMINsyn: '/lfs5/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat' +HOMEDIR: '/lfs5/HFIP/hfv3gfs/${USER}' +STMP: '/lfs5/HFIP/hfv3gfs/${USER}/stmp' +PTMP: '/lfs5/HFIP/hfv3gfs/${USER}/ptmp' NOSCRUB: $HOMEDIR ACCOUNT: hfv3gfs SCHEDULER: slurm @@ -25,6 +25,6 @@ ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORTED_RESOLUTIONS: ['C384', 'C192', 'C96', 'C48'] -COMINecmwf: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf -COMINnam: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/nam -COMINukmet: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet +COMINecmwf: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ecmwf +COMINnam: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/nam +COMINukmet: /mnt/lfs5/HFIP/hfv3gfs/glopara/data/external_gempak/ukmet