Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ci] fix shellcheck warnings in CI scripts #6646

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .ci/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ if [[ $OS_NAME == "macos" ]]; then
brew install swig
fi
else # Linux
if type -f apt 2>&1 > /dev/null; then
if { type -f apt > /dev/null; } 2>&1; then
vnherdeiro marked this conversation as resolved.
Show resolved Hide resolved
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
ca-certificates \
Expand All @@ -42,10 +42,10 @@ else # Linux
fi
CMAKE_VERSION="3.30.0"
curl -O -L \
https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh \
"https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" \
|| exit 1
sudo mkdir /opt/cmake || exit 1
sudo sh cmake-${CMAKE_VERSION}-linux-${ARCH}.sh --skip-license --prefix=/opt/cmake || exit 1
sudo sh "cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" --skip-license --prefix=/opt/cmake || exit 1
sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1

if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
Expand Down Expand Up @@ -147,8 +147,8 @@ if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then
curl \
-sL \
-o miniforge.sh \
https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh
sh miniforge.sh -b -p $CONDA
"https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh"
sh miniforge.sh -b -p "${CONDA}"
fi
conda config --set always_yes yes --set changeps1 no
conda update -q -y conda
Expand Down
31 changes: 16 additions & 15 deletions .ci/test-r-package.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ if [[ $R_BUILD_TYPE != "cran" ]]; then
fi

# Get details needed for installing R components
R_MAJOR_VERSION=( ${R_VERSION//./ } )
if [[ "${R_MAJOR_VERSION}" == "3" ]]; then
IFS='.' read -ra R_MAJOR_VERSION <<< "$R_VERSION"
vnherdeiro marked this conversation as resolved.
Show resolved Hide resolved
if [[ "${R_MAJOR_VERSION[0]}" == "3" ]]; then
vnherdeiro marked this conversation as resolved.
Show resolved Hide resolved
export R_MAC_VERSION=3.6.3
export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg
export R_LINUX_VERSION="3.6.3-1bionic"
export R_APT_REPO="bionic-cran35/"
elif [[ "${R_MAJOR_VERSION}" == "4" ]]; then
elif [[ "${R_MAJOR_VERSION[0]}" == "4" ]]; then
vnherdeiro marked this conversation as resolved.
Show resolved Hide resolved
export R_MAC_VERSION=4.3.1
export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/big-sur-${ARCH}/base/R-${R_MAC_VERSION}-${ARCH}.pkg
export R_LINUX_VERSION="4.3.1-1.2204.0"
Expand Down Expand Up @@ -69,7 +69,7 @@ if [[ $OS_NAME == "linux" ]]; then
sudo apt-get install \
--no-install-recommends \
-y \
autoconf=$(cat R-package/AUTOCONF_UBUNTU_VERSION) \
"autoconf=$(cat R-package/AUTOCONF_UBUNTU_VERSION)" \
automake \
|| exit 1
fi
Expand All @@ -90,17 +90,17 @@ if [[ $OS_NAME == "macos" ]]; then
sudo tlmgr --verify-repo=none update --self || exit 1
sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs || exit 1

curl -sL ${R_MAC_PKG_URL} -o R.pkg || exit 1
curl -sL "${R_MAC_PKG_URL}" -o R.pkg || exit 1
sudo installer \
-pkg $(pwd)/R.pkg \
-pkg "$(pwd)/R.pkg" \
-target / || exit 1

# install tidy v5.8.0
# ref: https://groups.google.com/g/r-sig-mac/c/7u_ivEj4zhM
TIDY_URL=https://github.com/htacg/tidy-html5/releases/download/5.8.0/tidy-5.8.0-macos-x86_64+arm64.pkg
curl -sL ${TIDY_URL} -o tidy.pkg
sudo installer \
-pkg $(pwd)/tidy.pkg \
-pkg "$(pwd)/tidy.pkg" \
-target /

# ensure that this newer version of 'tidy' is used by 'R CMD check'
Expand All @@ -110,7 +110,7 @@ fi

# fix for issue where CRAN was not returning {lattice} and {evaluate} when using R 3.6
# "Warning: dependency ‘lattice’ is not available"
if [[ "${R_MAJOR_VERSION}" == "3" ]]; then
if [[ "${R_MAJOR_VERSION[0]}" == "3" ]]; then
vnherdeiro marked this conversation as resolved.
Show resolved Hide resolved
Rscript --vanilla -e "install.packages(c('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', 'https://cran.r-project.org/src/contrib/Archive/evaluate/evaluate_0.23.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}')"
else
# {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}.
Expand Down Expand Up @@ -169,18 +169,19 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then
if [[ "${TASK}" == "r-rchk" ]]; then
echo "Checking R package with rchk"
mkdir -p packages
cp ${PKG_TARBALL} packages
cp "${PKG_TARBALL}" packages
RCHK_LOG_FILE="rchk-logs.txt"
docker run \
-v $(pwd)/packages:/rchk/packages \
{ docker run \
-v "$(pwd)/packages:/rchk/packages" \
kalibera/rchk:latest \
"/rchk/packages/${PKG_TARBALL}" \
2>&1 > ${RCHK_LOG_FILE} \
> ${RCHK_LOG_FILE}; } 2>&1 \
vnherdeiro marked this conversation as resolved.
Show resolved Hide resolved
|| (cat ${RCHK_LOG_FILE} && exit 1)
cat ${RCHK_LOG_FILE}

# the exceptions below are from R itself and not LightGBM:
# https://github.com/kalibera/rchk/issues/22#issuecomment-656036156
# shellcheck disable=SC2046
exit $(
cat ${RCHK_LOG_FILE} \
| grep -v "in function strptime_internal" \
Expand All @@ -194,9 +195,9 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then
# git repo around. This is to protect against the use of relative paths
# like ../../CMakeLists.txt that would only work if you are in the repo
R_CMD_CHECK_DIR="${HOME}/tmp-r-cmd-check/"
mkdir -p ${R_CMD_CHECK_DIR}
mv ${PKG_TARBALL} ${R_CMD_CHECK_DIR}
cd ${R_CMD_CHECK_DIR}
mkdir -p "${R_CMD_CHECK_DIR}"
mv "${PKG_TARBALL}" "${R_CMD_CHECK_DIR}"
cd "${R_CMD_CHECK_DIR}"
fi

declare -i allowed_notes=0
Expand Down
60 changes: 33 additions & 27 deletions .ci/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,17 +56,18 @@ if [[ "$TASK" == "cpp-tests" ]]; then
else
extra_cmake_opts=""
fi
cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON $extra_cmake_opts
cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON "${extra_cmake_opts}"
cmake --build build --target testlightgbm -j4 || exit 1
./testlightgbm || exit 1
exit 0
fi

# including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
CONDA_PYTHON_REQUIREMENT="python=$PYTHON_VERSION[build=*cpython]"
CONDA_PYTHON_REQUIREMENT="python=${PYTHON_VERSION}[build=*cpython]"

if [[ $TASK == "if-else" ]]; then
mamba create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy
mamba create -q -y -n $CONDA_ENV "${CONDA_PYTHON_REQUIREMENT}" numpy
# shellcheck disable=SC1091
source activate $CONDA_ENV
cmake -B build -S . || exit 1
cmake --build build --target lightgbm -j4 || exit 1
Expand All @@ -89,21 +90,22 @@ if [[ $TASK == "swig" ]]; then
python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1
fi
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp ./build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar
cp ./build/lightgbmlib.jar "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbmlib_${OS_NAME}.jar"
fi
exit 0
fi

if [[ $TASK == "lint" ]]; then
mamba create -q -y -n $CONDA_ENV \
${CONDA_PYTHON_REQUIREMENT} \
"${CONDA_PYTHON_REQUIREMENT}" \
'cmakelint>=1.4.3' \
'cpplint>=1.6.0' \
'matplotlib-base>=3.9.1' \
'mypy>=1.11.1' \
'pre-commit>=3.8.0' \
'pyarrow-core>=17.0' \
'r-lintr>=3.1.2'
# shellcheck disable=SC1091
source activate $CONDA_ENV
echo "Linting Python code"
bash ./.ci/lint-python.sh || exit 1
Expand All @@ -125,12 +127,15 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
-n $CONDA_ENV \
'doxygen>=1.10.0' \
'rstcheck>=6.2.4' || exit 1
# shellcheck disable=SC1091
source activate $CONDA_ENV
# check reStructuredText formatting
cd "${BUILD_DIRECTORY}/python-package"
rstcheck --report-level warning $(find . -type f -name "*.rst") || exit 1
find . -type f -name "*.rst" -exec \
rstcheck --report-level warning {} + || exit 1
jameslamb marked this conversation as resolved.
Show resolved Hide resolved
cd "${BUILD_DIRECTORY}/docs"
rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile $(find . -type f -name "*.rst") || exit 1
find . -type f -name "*.rst" -exec \
rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile {} + || exit 1
# build docs
make html || exit 1
if [[ $TASK == "check-links" ]]; then
Expand All @@ -150,30 +155,31 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
fi

if [[ $PYTHON_VERSION == "3.7" ]]; then
CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt"
CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt"
elif [[ $PYTHON_VERSION == "3.8" ]]; then
CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt"
CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt"
else
CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt"
CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt"
jameslamb marked this conversation as resolved.
Show resolved Hide resolved
fi

mamba create \
-y \
-n $CONDA_ENV \
${CONDA_REQUIREMENT_FILES} \
${CONDA_PYTHON_REQUIREMENT} \
--file "${CONDA_REQUIREMENT_FILES}" \
"${CONDA_PYTHON_REQUIREMENT}" \
|| exit 1

# shellcheck disable=SC1091
source activate $CONDA_ENV

cd "${BUILD_DIRECTORY}"

if [[ $TASK == "sdist" ]]; then
sh ./build-python.sh sdist || exit 1
sh .ci/check-python-dists.sh ./dist || exit 1
pip install ./dist/lightgbm-$LGB_VER.tar.gz -v || exit 1
pip install "./dist/lightgbm-${LGB_VER}.tar.gz" -v || exit 1
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp ./dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
cp "./dist/lightgbm-${LGB_VER}.tar.gz" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
fi
pytest ./tests/python_package_test || exit 1
exit 0
Expand All @@ -182,7 +188,7 @@ elif [[ $TASK == "bdist" ]]; then
sh ./build-python.sh bdist_wheel || exit 1
sh .ci/check-python-dists.sh ./dist || exit 1
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
cp dist/lightgbm-"${LGB_VER}"-py3-none-macosx*.whl "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
fi
else
if [[ $ARCH == "x86_64" ]]; then
Expand All @@ -198,10 +204,10 @@ elif [[ $TASK == "bdist" ]]; then
./dist/tmp.whl || exit 1
mv \
./dist/tmp.whl \
./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1
"./dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" || exit 1
sh .ci/check-python-dists.sh ./dist || exit 1
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1
cp "dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
fi
# Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
export LIGHTGBM_TEST_DUAL_CPU_GPU=1
Expand All @@ -220,14 +226,14 @@ if [[ $TASK == "gpu" ]]; then
pip install \
-v \
--config-settings=cmake.define.USE_GPU=ON \
./dist/lightgbm-$LGB_VER.tar.gz \
"./dist/lightgbm-${LGB_VER}.tar.gz" \
|| exit 1
pytest ./tests/python_package_test || exit 1
exit 0
elif [[ $METHOD == "wheel" ]]; then
sh ./build-python.sh bdist_wheel --gpu || exit 1
sh ./.ci/check-python-dists.sh ./dist || exit 1
pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1
pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1
pytest ./tests || exit 1
exit 0
elif [[ $METHOD == "source" ]]; then
Expand All @@ -245,14 +251,14 @@ elif [[ $TASK == "cuda" ]]; then
pip install \
-v \
--config-settings=cmake.define.USE_CUDA=ON \
./dist/lightgbm-$LGB_VER.tar.gz \
"./dist/lightgbm-${LGB_VER}.tar.gz" \
|| exit 1
pytest ./tests/python_package_test || exit 1
exit 0
elif [[ $METHOD == "wheel" ]]; then
sh ./build-python.sh bdist_wheel --cuda || exit 1
sh ./.ci/check-python-dists.sh ./dist || exit 1
pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1
pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1
pytest ./tests || exit 1
exit 0
elif [[ $METHOD == "source" ]]; then
Expand All @@ -265,14 +271,14 @@ elif [[ $TASK == "mpi" ]]; then
pip install \
-v \
--config-settings=cmake.define.USE_MPI=ON \
./dist/lightgbm-$LGB_VER.tar.gz \
"./dist/lightgbm-${LGB_VER}.tar.gz" \
|| exit 1
pytest ./tests/python_package_test || exit 1
exit 0
elif [[ $METHOD == "wheel" ]]; then
sh ./build-python.sh bdist_wheel --mpi || exit 1
sh ./.ci/check-python-dists.sh ./dist || exit 1
pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1
pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1
pytest ./tests || exit 1
exit 0
elif [[ $METHOD == "source" ]]; then
Expand All @@ -290,13 +296,13 @@ pytest ./tests || exit 1
if [[ $TASK == "regular" ]]; then
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
if [[ $OS_NAME == "macos" ]]; then
cp ./lib_lightgbm.dylib $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib
cp ./lib_lightgbm.dylib "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib"
else
if [[ $COMPILER == "gcc" ]]; then
objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1
python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1
fi
cp ./lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so
cp ./lib_lightgbm.so "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.so"
fi
fi
cd "$BUILD_DIRECTORY/examples/python-guide"
Expand All @@ -310,10 +316,10 @@ matplotlib.use\(\"Agg\"\)\
'h5py>=3.10' \
'ipywidgets>=8.1.2' \
'notebook>=7.1.2'
for f in *.py **/*.py; do python $f || exit 1; done # run all examples
for f in *.py **/*.py; do python "${f}" || exit 1; done # run all examples
cd "$BUILD_DIRECTORY/examples/python-guide/notebooks"
sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit 1 # run all notebooks
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace ./*.ipynb || exit 1 # run all notebooks
jameslamb marked this conversation as resolved.
Show resolved Hide resolved

# importing the library should succeed even if all optional dependencies are not present
conda uninstall -n $CONDA_ENV --force --yes \
Expand Down
Loading