From a765579358fdfd9247080ed9acdfcdb1404bd950 Mon Sep 17 00:00:00 2001
From: Charles Kawczynski <kawczynski.charles@gmail.com>
Date: Mon, 22 Apr 2024 10:43:28 -0400
Subject: [PATCH] Add nsight report to more jobs

---
 .buildkite/gpu_pipeline/pipeline.yml | 2 +-
 .buildkite/pipeline.yml              | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.buildkite/gpu_pipeline/pipeline.yml b/.buildkite/gpu_pipeline/pipeline.yml
index ed1369bedd..023f2de072 100644
--- a/.buildkite/gpu_pipeline/pipeline.yml
+++ b/.buildkite/gpu_pipeline/pipeline.yml
@@ -76,7 +76,7 @@ steps:
           - mkdir -p gpu_hs_rhoe_equil_55km_nz63_0M_4process
           - >
             srun --cpu-bind=threads --cpus-per-task=4
-            nsys profile --trace=nvtx,mpi,cuda,osrt --output=gpu_hs_rhoe_equil_55km_nz63_0M_4process/output_active/report-%q{PMI_RANK}
+            nsys profile --trace=osrt,nvtx,cuda,mpi,ucx --output=gpu_hs_rhoe_equil_55km_nz63_0M_4process/output_active/report-%q{PMI_RANK}
             julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
             --config_file ${GPU_CONFIG_PATH}gpu_hs_rhoe_equil_55km_nz63_0M_4process.yml
         artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M_4process/output_active/*"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 0a78975de3..94f6bc6e40 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -660,9 +660,10 @@ steps:
       - label: "GPU: baroclinic wave"
         key: "gpu_baroclinic_wave_rhoe"
         command: >
+          nsys profile --trace=nvtx,cuda --output=gpu_baroclinic_wave_rhoe/output_active/report
           julia --color=yes --project=examples examples/hybrid/driver.jl
           --config_file $PERF_CONFIG_PATH/gpu_baroclinic_wave_rhoe.yml
-        artifact_paths: "gpu_implicit_barowave_ref/output_active/*"
+        artifact_paths: "gpu_baroclinic_wave_rhoe/output_active/*"
         agents:
           slurm_mem: 16G
           slurm_gpus: 1
@@ -697,7 +698,7 @@ steps:
       - label: "GPU: GPU dry baroclinic wave - 4 gpus"
         key: "target_gpu_implicit_baroclinic_wave_4process"
         command:
-            # nsys profile --trace=nvtx,cuda,mpi --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK}
+            nsys profile --trace=nvtx,mpi,cuda,osrt --output=target_gpu_implicit_baroclinic_wave_4process/output_active/report-%q{PMI_RANK}
           - mkdir -p target_gpu_implicit_baroclinic_wave_4process
           - >
             srun --cpu-bind=threads --cpus-per-task=4
@@ -756,6 +757,7 @@ steps:
 
       - label: "GPU: Prognostic EDMFX aquaplanet"
         command: >
+          nsys profile --trace=nvtx,cuda --output=gpu_prognostic_edmfx_aquaplanet/output_active/report
           julia --color=yes --project=examples examples/hybrid/driver.jl
           --config_file $GPU_CONFIG_PATH/gpu_prognostic_edmfx_aquaplanet.yml
         artifact_paths: "gpu_prognostic_edmfx_aquaplanet/output_active/*"