Merge pull request #1059 from ORNL/1057-trigger-running-foxx-tests-as…

…-part-of-ci

1057 trigger running foxx tests as part of ci

.gitlab/common.yml

@@ -93,7 +93,7 @@
   stage: log
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
-    - docker logs $(docker ps --filter "ancestor=${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --format "{{.Names}}")
+    - docker logs $(docker ps -a --filter "ancestor=${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --format "{{.Names}}")
   rules:
     - when: always
 

.gitlab/end_to_end.yml

@@ -41,10 +41,12 @@ end-to-end-foxx-setup:
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
     - ./scripts/container_stop.sh -n "foxx-" -p
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
+    - CONTAINER_NAME="foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}"
     - echo "#!/bin/bash" > "${RUN_FILE}"
     - echo "docker run -d \\" >> "${RUN_FILE}"
-    - echo "--name \"foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> "${RUN_FILE}"
+    - echo "--name \"${CONTAINER_NAME}\" \\" >> "${RUN_FILE}"
     - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> "${RUN_FILE}"
+    - echo "-e ENABLE_FOXX_TESTS=\"TRUE\" \\" >> "${RUN_FILE}"
     - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> "${RUN_FILE}"
     - echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> "${RUN_FILE}"
     - echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> "${RUN_FILE}"
@@ -60,7 +62,22 @@ end-to-end-foxx-setup:
       # Make sure container is running immediately after because it is meant to
       # be ephermal anyway, this is not the same for the other containers
     - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest"
-    - while [ ! -f "foxx_tmp/.foxx_is_installed" ]; do echo "Waiting for foxx_tmp/.foxx_is_installed"; sleep 10; done
+    - while [ ! -f "foxx_tmp/.foxx_is_installed" ]; do
+      # Container is found to be running
+    -   if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+    -     echo "Waiting for foxx_tmp/.foxx_is_installed"
+    -     docker logs "foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}";
+    -     sleep 10
+    -   else
+      # Container is not running get exit code
+    -     exit_code=$(docker inspect -f '{{.State.ExitCode}}' "$CONTAINER_NAME")
+    -     if [ "$exit_code" != "0" ]; then
+    -       docker logs "foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}";
+    -       exit "$exit_code"
+    -     fi
+    -   fi 
+    - done
+    - docker logs "foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}";
     - ./scripts/ci_database_health_check.sh
   after_script:
     - rm -rf foxx_tmp
@@ -79,6 +96,9 @@ end-to-end-gcs-node-clean:
     - ci-datafed-globus
     - docker
   script:
+    - mkdir -p "${DATAFED_GLOBUS_DIR}"
+    - cp "${CI_DATAFED_GCS_DEPLOYMENT_KEY}" "${DATAFED_GLOBUS_DIR}/deployment-key.json"
+    - cp "${CI_DATAFED_GCS_CLIENT_CRED}" "${DATAFED_GLOBUS_DIR}/client_cred.json"
     - cat "${DATAFED_GLOBUS_DIR}/deployment-key.json" | jq .node_key > "${DATAFED_GLOBUS_DIR}/node_deletion_key.json"
     - SECRET=$(cat "${DATAFED_GLOBUS_DIR}/deployment-key.json" | jq -r .secret)
     - CLIENT=$(cat "${DATAFED_GLOBUS_DIR}/deployment-key.json" | jq -r .client_id)

CHANGELOG.md

@@ -22,6 +22,7 @@
 11. [1012] - Allow customized base build image for Docker dependencies and runtime Dockerfiles
 12. [986] - Design improvement to upload and download transfer box.
 13. [985] - Handles longer than needed timeouts on ui pages.
+14. [1053] - Set CMake to enable foxx tests when built in the CI.
 
 # v2024.6.17.10.40
 

CMakeLists.txt

@@ -35,6 +35,8 @@ setting is turned on DataFed will build it's libraries as shared and try to
 link to shared libraries." OFF)
 OPTION(ENABLE_END_TO_END_API_TESTS "Enable end-to-end API testing" FALSE) 
 OPTION(ENABLE_END_TO_END_WEB_TESTS "Enable end-to-end web testing with Playwright" FALSE)
+OPTION(ENABLE_FOXX_TESTS "Enable Foxx testing, off by default because it
+  will overwrite the test database." FALSE)
 
 set(INSTALL_REPO_SERVER ${BUILD_REPO_SERVER})
 set(INSTALL_AUTHZ ${BUILD_AUTHZ})

docker/entrypoint_foxx.sh

@@ -33,32 +33,58 @@ then
   # Check to see if foxx has previously been installed
   "${PROJECT_ROOT}/scripts/generate_datafed.sh"
 
-  "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" -S. -B build						\
-    -DBUILD_REPO_SERVER=False		\
-    -DBUILD_COMMON=False        \
-    -DBUILD_AUTHZ=False					\
-    -DBUILD_CORE_SERVER=False		\
-    -DBUILD_WEB_SERVER=False		\
-    -DBUILD_DOCS=False					\
-    -DBUILD_PYTHON_CLIENT=False	\
-    -DBUILD_FOXX=True           \
-    -DINSTALL_FOXX=True
-
+  # Should only run this if you are ok with making changes to the database
+  if [ "$ENABLE_FOXX_TESTS" == "TRUE" ]
+  then
+    "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" -S. -B build \
+      -DBUILD_REPO_SERVER=False   \
+      -DBUILD_COMMON=False        \
+      -DBUILD_AUTHZ=False         \
+      -DBUILD_CORE_SERVER=False   \
+      -DBUILD_WEB_SERVER=False    \
+      -DBUILD_DOCS=False          \
+      -DBUILD_PYTHON_CLIENT=False \
+      -DBUILD_FOXX=True           \
+      -DINSTALL_FOXX=True         \
+      -DENABLE_FOXX_TESTS=True
+  else
+    "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" -S. -B build \
+      -DBUILD_REPO_SERVER=False   \
+      -DBUILD_COMMON=False        \
+      -DBUILD_AUTHZ=False         \
+      -DBUILD_CORE_SERVER=False   \
+      -DBUILD_WEB_SERVER=False    \
+      -DBUILD_DOCS=False          \
+      -DBUILD_PYTHON_CLIENT=False \
+      -DBUILD_FOXX=True           \
+      -DINSTALL_FOXX=True
+  fi
 
   "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" --build build
 
   # Give arango container a minute to initialize
   # should be replaced with health check at some point
   sleep 5
   "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" --build build --target install
-
+
+  if [ "$ENABLE_FOXX_TESTS" == "TRUE" ]
+  then
+    "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" \
+      --build build \
+      --target test
+    EXIT_CODE="$?"
+    if [ "$EXIT_CODE" != "0" ]; then exit "$EXIT_CODE"; fi
+  fi
+
+  # Create flag to indicate container has done its job  
   touch "$install_flag"
   chown -R "$UID":"$UID" "/tmp"
-
 else
-  echo "/tmp/.foxx_is_installed has been found skipping reinstall"
+  echo "$install_flag has been found skipping reinstall"
 fi
 
-echo "Sleeping"
-sleep 1000
-
+# Keep container alive for a little bit, the CI pipelines check that the
+# container actually runs. If the container runs to fast the pipeline check
+# might fail because it wasn't able to determine if the container actually
+# ran.
+sleep 60