Merge branch 'develop' of https://github.com/jcsda/spack-stack into f…

…eature/nautilus_intel_openmpi

.github/workflows/util-tests.yaml

@@ -0,0 +1,21 @@
+name: util-tests
+
+on: [pull_request,push,workflow_dispatch]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: false
+
+      - name: run-util-tests
+        run: |
+          export SPACK_STACK_DIR=$PWD
+          cd util/
+          ./util_tests.sh

configs/sites/aws-pcluster/mirrors.yaml

@@ -1,21 +1,4 @@
 mirrors:
-  local-source-ebs:
-    fetch:
-      url: file:///home/ubuntu/jedi/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///home/ubuntu/jedi/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
   local-source-efs:
     fetch:
       url: file:///mnt/experiments-efs/spack-stack/source-cache

util/check_permissions.sh

@@ -7,23 +7,23 @@ path=$PWD
 
 # Check upstream hierarchy of current directory
 while [ $path != '/' ]; do
-  o_perms=$(stat $path --format="%A" | grep -o "...$")
-  if [ ${o_perms:0:1} != 'r' ]; then
+  o_perms=$(ls -ld $path | awk '{print $1}' | grep -oE "[r-][w-][x-]" | tail -1)
+  if [ "${o_perms:0:1}" != 'r' ]; then
     echo "Path $path is not readable by non-owners; set o+r" 1>&2
     iret=1
   fi
-  if [ ${o_perms:2:3} != 'x' ]; then
+  if [ "${o_perms:2:3}" != 'x' ]; then
     echo "Path $path is not accessible by non-owners; set o+x" 1&>2
     iret=1
   fi
   path=$(dirname $path)
 done
 
 # Check downstream hierarchy of current directory
-n_bad_perms=$(find \( -type d -a -not -perm -005 \) -o \( -type f -a -not -perm -004 \) | wc -l)
+n_bad_perms=$(find $PWD \( -type d -and -not -perm -005 \) -or \( -type f -and -not -perm -004 \) | wc -l)
 if [ $n_bad_perms -gt 0 ]; then
   echo "There are files under this hierarchy not accessible to non-owning users/groups."
-  echo "Use 'find \( -type d -a -not -perm -005 \) -o \( -type f -a -not -perm -004 \)' to identify them."
+  echo 'Use "find $PWD \( -type d -a -not -perm -005 \) -o \( -type f -a -not -perm -004 \)" to identify them.'
   iret=1
 fi
 

util/util_tests.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# This script is used by the GitHub Actions util-test workflow.
+
+# This functions runs a command and checks the return code.
+function run_and_check(){
+  expected=$1
+  label=$2
+  shift 2
+  echo "Running '$*' in $PWD"
+  eval "$*" &> /tmp/output.$$
+  if [ $? -ne $expected ]; then
+    echo "Test $label failed! Output:"
+    cat /tmp/output.$$
+    rm /tmp/output.$$
+    echo
+    fail=1
+  fi
+}
+
+echo "umask:" $(umask)
+chmod o+rX $HOME
+mkdir -p ${SPACK_STACK_DIR}/util/checks
+cd ${SPACK_STACK_DIR}/util/checks
+
+# Check check_permissions.sh
+mkdir -p perm_check1/perm_check2/perm_check3
+cd perm_check1/perm_check2
+chmod 777 ../../perm_check1
+chmod 777 .
+chmod 777 ./perm_check3
+run_and_check 0 "check_permissions A" ${SPACK_STACK_DIR}/util/check_permissions.sh
+chmod 776 ../../perm_check1
+run_and_check 1 "check_permissions B" ${SPACK_STACK_DIR}/util/check_permissions.sh
+chmod 773 ../../perm_check1
+run_and_check 1 "check_permissions C" ${SPACK_STACK_DIR}/util/check_permissions.sh
+chmod 770 ../../perm_check1
+run_and_check 1 "check_permissions D" ${SPACK_STACK_DIR}/util/check_permissions.sh
+chmod 777 ../../perm_check1
+chmod 776 perm_check3
+run_and_check 1 "check_permissions E" ${SPACK_STACK_DIR}/util/check_permissions.sh
+chmod 773 perm_check3
+run_and_check 1 "check_permissions F" ${SPACK_STACK_DIR}/util/check_permissions.sh
+chmod 770 perm_check3
+run_and_check 1 "check_permissions G" ${SPACK_STACK_DIR}/util/check_permissions.sh
+
+# Check show_duplicate_packages.py
+cd ${SPACK_STACK_DIR}/util/checks
+echo -e " -  abcdefg [email protected]\n -  tuvwxyz [email protected]" > fakeconcrete.A
+run_and_check 1 "show_duplicate_packages.py A1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.A
+run_and_check 1 "show_duplicate_packages.py A2" "cat fakeconcrete.A | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py"
+echo -e " -  abcdefg [email protected]\n -  tuvwxyz [email protected]" > fakeconcrete.B
+run_and_check 1 "show_duplicate_packages.py B1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.B
+run_and_check 1 "show_duplicate_packages.py B2" "cat fakeconcrete.B | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py"
+echo -e " -  abcdefg [email protected]\n[+] abcdefg [email protected]" > fakeconcrete.C
+run_and_check 0 "show_duplicate_packages.py C1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.C
+run_and_check 0 "show_duplicate_packages.py C2" "cat fakeconcrete.C | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py"
+echo -e " -  abcdefg [email protected]\n -  tuvwxyz [email protected]\n -  hijklmn [email protected]\n[+] opqrstu [email protected]" > fakeconcrete.D
+run_and_check 0 "show_duplicate_packages.py D1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.D -i hdf6 -i mypackage
+run_and_check 0 "show_duplicate_packages.py D2" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.D -i mypackage -i hdf6
+run_and_check 0 "show_duplicate_packages.py D3" "cat fakeconcrete.D | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i hdf6 -i mypackage"
+run_and_check 0 "show_duplicate_packages.py D4" "cat fakeconcrete.D | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i mypackage -i hdf6"
+run_and_check 1 "show_duplicate_packages.py D5" "cat fakeconcrete.D | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i hdf6"
+run_and_check 1 "show_duplicate_packages.py D6" "cat fakeconcrete.D | ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i mypackage"
+
+cmd="${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.A 2>/dev/null | uniq | grep -c hdf6"
+echo "Running '$cmd' in $PWD"
+if [ $(eval "$cmd") -ne 2 ] ; then
+  echo "show_duplicate_packages.py E failed!"
+  fail=1
+fi
+
+cmd="${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.F -d 2>/dev/null | uniq | grep -c hdf6"
+echo -e " -  abcdefg [email protected]\n -  tuvwxyz [email protected]\n -  a1b2c3d [email protected]" > fakeconcrete.F
+if [ $(eval "$cmd") -ne 2 ] ; then
+  echo "show_duplicate_packages.py F failed!"
+  fail=1
+fi
+
+exit $fail