From 862f63ef74e37c2ddc66b4fa87354b8f2301e4ec Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 3 Jul 2023 12:07:23 -0600 Subject: [PATCH 1/3] Update submodule pointer for spack for umask check when creating envs (#657) * Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index be4b257ae..c750b0315 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit be4b257aee39e5c4f1f9f2af3512184b37151033 +Subproject commit c750b0315472caab3d6d57c0bd13f60ef6e48926 From 8988176c950c3bf0287093426618950cb53c80a6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 3 Jul 2023 20:08:00 -0600 Subject: [PATCH 2/3] Submodule pointer updates for spack and CMakeModules, remove legacy mirror from aws-pcluster config (#662) * Remove home directory spack-stack source cache * Update submodule pointer for CMakeModules * Update submodule pointer for spack --- configs/sites/aws-pcluster/mirrors.yaml | 17 ----------------- doc/CMakeModules | 2 +- spack | 2 +- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/configs/sites/aws-pcluster/mirrors.yaml b/configs/sites/aws-pcluster/mirrors.yaml index 50cd376e6..52d6f0e4d 100644 --- a/configs/sites/aws-pcluster/mirrors.yaml +++ b/configs/sites/aws-pcluster/mirrors.yaml @@ -1,21 +1,4 @@ mirrors: - local-source-ebs: - fetch: - url: file:///home/ubuntu/jedi/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///home/ubuntu/jedi/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null local-source-efs: fetch: url: file:///mnt/experiments-efs/spack-stack/source-cache diff --git a/doc/CMakeModules b/doc/CMakeModules index cabd7753a..3bd7769ed 160000 --- a/doc/CMakeModules +++ b/doc/CMakeModules @@ -1 +1 @@ -Subproject commit cabd7753ae17f7bfcc6dad56daf10868aa51c3f4 +Subproject commit 3bd7769ed9b81dca951d2943e6fd675812a022bf diff --git a/spack b/spack index c750b0315..5f8f7c59e 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit c750b0315472caab3d6d57c0bd13f60ef6e48926 +Subproject commit 5f8f7c59ec3ac4882756edeb24cc8a41b0c27eb6 From 7a628c7da24664ce8b958c8e03823f4c04ac087e Mon Sep 17 00:00:00 2001 From: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com> Date: Mon, 3 Jul 2023 20:24:59 -0700 Subject: [PATCH 3/3] Add CI checks for util/ utilities (#658) * Add util-tests.yaml * Add util_tests.sh * tweak util-tests.yaml * add checkout version to util-tests.yaml * fix matrix name in util-tests.yaml * fix util CI for MacOS --------- Co-authored-by: Dom Heinzeller --- .github/workflows/util-tests.yaml | 21 ++++++++ util/check_permissions.sh | 10 ++-- util/util_tests.sh | 79 +++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/util-tests.yaml create mode 100755 util/util_tests.sh diff --git a/.github/workflows/util-tests.yaml b/.github/workflows/util-tests.yaml new file mode 100644 index 000000000..d04946faa --- /dev/null +++ b/.github/workflows/util-tests.yaml @@ -0,0 +1,21 @@ +name: util-tests + +on: [pull_request,push,workflow_dispatch] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - name: checkout + uses: actions/checkout@v2 + with: + submodules: false + + - name: run-util-tests + run: | + export SPACK_STACK_DIR=$PWD + cd util/ + ./util_tests.sh diff --git a/util/check_permissions.sh b/util/check_permissions.sh index 7d01121c0..e272cf49c 100755 --- a/util/check_permissions.sh +++ b/util/check_permissions.sh @@ -7,12 +7,12 @@ path=$PWD # Check upstream hierarchy of current directory while [ $path != '/' ]; do - o_perms=$(stat $path --format="%A" | grep -o "...$") - if [ ${o_perms:0:1} != 'r' ]; then + o_perms=$(ls -ld $path | awk '{print $1}' | grep -oE "[r-][w-][x-]" | tail -1) + if [ "${o_perms:0:1}" != 'r' ]; then echo "Path $path is not readable by non-owners; set o+r" 1>&2 iret=1 fi - if [ ${o_perms:2:3} != 'x' ]; then + if [ "${o_perms:2:3}" != 'x' ]; then echo "Path $path is not accessible by non-owners; set o+x" 1&>2 iret=1 fi @@ -20,10 +20,10 @@ while [ $path != '/' ]; do done # Check downstream hierarchy of current directory -n_bad_perms=$(find \( -type d -a -not -perm -005 \) -o \( -type f -a -not -perm -004 \) | wc -l) +n_bad_perms=$(find $PWD \( -type d -and -not -perm -005 \) -or \( -type f -and -not -perm -004 \) | wc -l) if [ $n_bad_perms -gt 0 ]; then echo "There are files under this hierarchy not accessible to non-owning users/groups." - echo "Use 'find \( -type d -a -not -perm -005 \) -o \( -type f -a -not -perm -004 \)' to identify them." + echo 'Use "find $PWD \( -type d -a -not -perm -005 \) -o \( -type f -a -not -perm -004 \)" to identify them.' iret=1 fi diff --git a/util/util_tests.sh b/util/util_tests.sh new file mode 100755 index 000000000..58a105be8 --- /dev/null +++ b/util/util_tests.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# This script is used by the GitHub Actions util-test workflow. + +# This functions runs a command and checks the return code. +function run_and_check(){ + expected=$1 + label=$2 + shift 2 + echo "Running '$*' in $PWD" + eval "$*" &> /tmp/output.$$ + if [ $? -ne $expected ]; then + echo "Test $label failed! Output:" + cat /tmp/output.$$ + rm /tmp/output.$$ + echo + fail=1 + fi +} + +echo "umask:" $(umask) +chmod o+rX $HOME +mkdir -p ${SPACK_STACK_DIR}/util/checks +cd ${SPACK_STACK_DIR}/util/checks + +# Check check_permissions.sh +mkdir -p perm_check1/perm_check2/perm_check3 +cd perm_check1/perm_check2 +chmod 777 ../../perm_check1 +chmod 777 . +chmod 777 ./perm_check3 +run_and_check 0 "check_permissions A" ${SPACK_STACK_DIR}/util/check_permissions.sh +chmod 776 ../../perm_check1 +run_and_check 1 "check_permissions B" ${SPACK_STACK_DIR}/util/check_permissions.sh +chmod 773 ../../perm_check1 +run_and_check 1 "check_permissions C" ${SPACK_STACK_DIR}/util/check_permissions.sh +chmod 770 ../../perm_check1 +run_and_check 1 "check_permissions D" ${SPACK_STACK_DIR}/util/check_permissions.sh +chmod 777 ../../perm_check1 +chmod 776 perm_check3 +run_and_check 1 "check_permissions E" ${SPACK_STACK_DIR}/util/check_permissions.sh +chmod 773 perm_check3 +run_and_check 1 "check_permissions F" ${SPACK_STACK_DIR}/util/check_permissions.sh +chmod 770 perm_check3 +run_and_check 1 "check_permissions G" ${SPACK_STACK_DIR}/util/check_permissions.sh + +# Check show_duplicate_packages.py +cd ${SPACK_STACK_DIR}/util/checks +echo -e " - abcdefg hdf6@1.2.3\n - tuvwxyz hdf6@1.2.3" > fakeconcrete.A +run_and_check 1 "show_duplicate_packages.py A1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.A +run_and_check 1 "show_duplicate_packages.py A2" "cat fakeconcrete.A | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py" +echo -e " - abcdefg hdf6@1.2.3\n - tuvwxyz hdf6@1.2.4" > fakeconcrete.B +run_and_check 1 "show_duplicate_packages.py B1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.B +run_and_check 1 "show_duplicate_packages.py B2" "cat fakeconcrete.B | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py" +echo -e " - abcdefg hdf6@1.2.3\n[+] abcdefg hdf6@1.2.3" > fakeconcrete.C +run_and_check 0 "show_duplicate_packages.py C1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.C +run_and_check 0 "show_duplicate_packages.py C2" "cat fakeconcrete.C | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py" +echo -e " - abcdefg hdf6@1.2.3\n - tuvwxyz hdf6@1.2.3\n - hijklmn mypackage@1.1.1\n[+] opqrstu mypackage@1.1.1" > fakeconcrete.D +run_and_check 0 "show_duplicate_packages.py D1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.D -i hdf6 -i mypackage +run_and_check 0 "show_duplicate_packages.py D2" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.D -i mypackage -i hdf6 +run_and_check 0 "show_duplicate_packages.py D3" "cat fakeconcrete.D | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i hdf6 -i mypackage" +run_and_check 0 "show_duplicate_packages.py D4" "cat fakeconcrete.D | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i mypackage -i hdf6" +run_and_check 1 "show_duplicate_packages.py D5" "cat fakeconcrete.D | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i hdf6" +run_and_check 1 "show_duplicate_packages.py D6" "cat fakeconcrete.D | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i mypackage" + +cmd="${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.A 2>/dev/null | uniq | grep -c hdf6" +echo "Running '$cmd' in $PWD" +if [ $(eval "$cmd") -ne 2 ] ; then + echo "show_duplicate_packages.py E failed!" + fail=1 +fi + +cmd="${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.F -d 2>/dev/null | uniq | grep -c hdf6" +echo -e " - abcdefg hdf6@1.2.3\n - tuvwxyz hdf6@1.2.3\n - a1b2c3d other@1.1.1" > fakeconcrete.F +if [ $(eval "$cmd") -ne 2 ] ; then + echo "show_duplicate_packages.py F failed!" + fail=1 +fi + +exit $fail