Skip to content

Generate conda packages #457

Generate conda packages

Generate conda packages #457

name: Generate conda packages
# This action automatically generate conda packages for the packages in the robotology-superbuild
on:
workflow_dispatch:
inputs:
upload_conda_binaries:
description: 'If true, upload conda binaries, for any other value only generate them without uploading.'
required: true
default: 'true'
test_metapackages_generation:
description: 'If true, also test metapackages generation. Not compatible with upload_conda_binaries set to true.'
required: true
default: 'false'
schedule:
# Run the job once a week
- cron: '0 0 * * 2'
release:
types: [published]
jobs:
# Regardless of the branch on which this action is trigged,
# the CONDA_BUILD_NUMBER CMake option needs to be read from the
# master branch, to avoid that a conda package generation from the
# master branch and one from a releases/YYYY.MM branch use the same
# CONDA_BUILD_NUMBER
get-conda-build-number:
name: "Read Conda Build number from master branch"
runs-on: ubuntu-latest
# Define outputs (see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-defining-outputs-for-a-job)
outputs:
conda_build_number: ${{ steps.step1.outputs.conda_build_number }}
steps:
- uses: actions/checkout@v4
with:
ref: 'master'
- id: step1
name: Get CONDA_BUILD_NUMBER and set it as output
shell: bash
run: |
# Get CONDA_BUILD_NUMBER via grep and set it to an environment variable
export CONDA_BUILD_NUMBER=`grep "CONDA_BUILD_NUMBER" ./conda/cmake/CondaGenerationOptions.cmake | grep -oe '\([0-9.]*\)'`
echo "conda_build_number=${CONDA_BUILD_NUMBER}" >> $GITHUB_OUTPUT
generate-conda-packages:
name: "Generate conda packages @${{ matrix.os }}"
runs-on: ${{ matrix.os }}
needs: get-conda-build-number
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-20.04
conda_platform: linux-64
- os: macos-14
conda_platform: osx-arm64
- os: windows-2019
conda_platform: win-64
steps:
- uses: actions/checkout@v4
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Miniforge3
miniforge-version: latest
- name: Install files to enable compilation of mex files [Conda/Linux]
if: contains(matrix.os, 'ubuntu')
run: |
curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020b_mexa64.zip
unzip msdk_R2020b_mexa64.zip
rm msdk_R2020b_mexa64.zip
echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020b_mexa64" >> $GITHUB_ENV
echo "GHA_Matlab_MEX_EXTENSION=mexa64" >> $GITHUB_ENV
- name: Install files to enable compilation of mex files [Conda/macOS]
if: contains(matrix.os, 'macos')
run: |
curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2023b_mexmaca64.zip
unzip msdk_R2023b_mexmaca64.zip
rm msdk_R2023b_mexmaca64.zip
echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2023b_mexmaca64" >> $GITHUB_ENV
echo "GHA_Matlab_MEX_EXTENSION=mexmaca64" >> $GITHUB_ENV
- name: Install files to enable compilation of mex files [Conda/Windows]
if: contains(matrix.os, 'windows')
shell: bash
run: |
curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020a_mexw64.zip
unzip msdk_R2020a_mexw64.zip
rm msdk_R2020a_mexw64.zip
echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020a_mexw64" >> $GITHUB_ENV
echo "GHA_Matlab_MEX_EXTENSION=mexw64" >> $GITHUB_ENV
- name: Dependencies for conda recipes generation and upload
shell: bash -l {0}
run: |
conda install pyyaml jinja2 conda-build ninja anaconda-client conda-forge-pinning mamba boa multisheller pixi
- name: Print used environment
shell: bash -l {0}
run: |
conda list
env
- name: Generate recipes [Linux&macOS]
if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu')
shell: bash -l {0}
run: |
mkdir build
cd build
cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} ..
- name: Generate recipes [Windows]
if: contains(matrix.os, 'windows')
shell: bash -l {0}
run: |
mkdir build
cd build
cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} ..
- name: Specify additional option if we are in a release and we need to generate robotology-distro metapackages
if: github.event_name == 'release'
shell: bash -l {0}
run: |
cd build
cmake -DROBOTOLOGY_PROJECT_TAGS=Custom -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON .
- name: Specify additional option if we want generate robotology-distro metapackages
if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true'
shell: bash -l {0}
run: |
cd build
cmake -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON .
- name: Fail if both upload_conda_binaries and test_metapackages_generation are passed
if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true'
uses: actions/github-script@v7
with:
script: |
core.setFailed('Build failed. It is not possible to set to true both upload_conda_binaries and test_metapackages_generation.')
- name: If test_metapackages_generation is enabled set an arbitrary version number
if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true'
shell: bash -l {0}
run: |
sed -i -e 's/set(INSTALLER_VERSION "")/set(INSTALLER_VERSION 1234.0)/g' ./packaging/windows/CMakeLists.txt
sed -i -e 's/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "")/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION 1234.0)/g' ./conda/cmake/CondaGenerationOptions.cmake
- name: Set CONDA_BLD_PATH on Windows to avoid long path issues [Windows]
if: contains(matrix.os, 'windows')
shell: bash -l {0}
run: |
echo "CONDA_BLD_PATH=C:\\bld" >> $GITHUB_ENV
- name: Set CONDA_BLD_PATH on non-Windows for consistency [non Windows]
if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu')
shell: bash -l {0}
run: |
echo "CONDA_BLD_PATH=${CONDA_PREFIX}/conda-bld" >> $GITHUB_ENV
- name: Build conda packages
shell: bash -l {0}
run: |
conda config --remove channels defaults
conda config --env --set channel_priority strict
cd build/conda/generated_recipes
# Debug generated recipes
cat */meta.yaml
cat */bld.bat
cat */build.sh
# We use the conda_build_config from conda-forge-pinning, and our local one
# See https://docs.conda.io/projects/conda-build/en/latest/resources/variants.html#creating-conda-build-variant-config-files
# We manually specify the build order as an old workaround for mambabuild, in theory this can be avoided, but it needs to be tested
# see https://github.com/mamba-org/boa/issues/117
# We also manually delete the recipe directory to avoid that it is built again,
# specifically because we can't use --skip-existing as these packages are part of the robotology channel
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml icub-contrib-common
rm -rf icub-contrib-common
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robots-configuration
rm -rf robots-configuration
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robot-log-visualizer
rm -rf robot-log-visualizer
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml idyntree-matlab-bindings
rm -rf idyntree-matlab-bindings
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blockfactory
rm -rf blockfactory
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml yarp-matlab-bindings
rm -rf yarp-matlab-bindings
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml wb-toolbox
rm -rf wb-toolbox
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blocktest
rm -rf blocktest
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml .
- name: Build robotology-distro conda metapackage (if necessary)
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true')
shell: bash -l {0}
run: |
cd build/conda/generated_recipes_metapackages
# Debug generated recipes
cat */meta.yaml
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro
- name: Upload conda packages
shell: bash -l {0}
# Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
env:
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
PREFIX_DEV_TOKEN: ${{ secrets.PREFIX_DEV_TOKEN }}
run: |
cd ${CONDA_BLD_PATH}/${{ matrix.conda_platform}}/
ls *.tar.bz2
anaconda upload --skip-existing *.tar.bz2
pixi auth login https://prefix.dev --token $PREFIX_DEV_TOKEN
for condapackage in *.tar.bz2; do
pixi upload https://prefix.dev/api/v1/upload/robotology "$condapackage"
done
pixi auth logout https://prefix.dev
# Only on linux-64 we upload noarch packages (noarch packages need to be uploaded only once, as they can be reused across platforms)
- name: Upload conda packages (noarch)
shell: bash -l {0}
# Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
if: matrix.conda_platform == 'linux-64' && (github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true'))
env:
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
PREFIX_DEV_TOKEN: ${{ secrets.PREFIX_DEV_TOKEN }}
run: |
cd ${CONDA_BLD_PATH}/noarch/
ls *.tar.bz2
anaconda upload --skip-existing *.tar.bz2
pixi auth login https://prefix.dev --token $PREFIX_DEV_TOKEN
for condapackage in *.tar.bz2; do
pixi upload https://prefix.dev/api/v1/upload/robotology "$condapackage"
done
pixi auth logout https://prefix.dev
- name: Build robotology-distro-all conda metapackage (if necessary)
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true')
shell: bash -l {0}
run: |
cd build/conda/generated_recipes_metapackages
# Debug generated recipes
cat */meta.yaml
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro-all
- name: Upload robotology-distro-all metapackage
shell: bash -l {0}
# Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true')
env:
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
PREFIX_DEV_TOKEN: ${{ secrets.PREFIX_DEV_TOKEN }}
run: |
cd ${CONDA_BLD_PATH}/${{ matrix.conda_platform}}/
ls robotology-distro-all-*.tar.bz2
anaconda upload --skip-existing robotology-distro-all-*.tar.bz2
pixi auth login https://prefix.dev --token $PREFIX_DEV_TOKEN
for condapackage in robotology-distro-all-*.tar.bz2; do
pixi upload https://prefix.dev/api/v1/upload/robotology "$condapackage"
done
pixi auth logout https://prefix.dev
# If the generate-conda-packages completed correctly and binaries are uploaded,
# bump automatically the CONDA_BUILD_NUMBER in conda/cmake/CondaGenerationOptions.cmake
# of master branch for future builds
# the master branch is always used in case the action is triggered by a release on
# a release/vYYYY.MM branch
bump-conda-build-number:
name: "Bump Conda Build number for future builds"
runs-on: ubuntu-latest
needs: generate-conda-packages
if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
steps:
- uses: actions/checkout@v4
with:
ref: 'master'
- name: Bump Conda Build number for future builds
shell: bash
run: |
sh ./scripts/robotologyBumpCondaBuildNumber.sh
- uses: EndBug/[email protected]
with:
default_author: github_actions
message: 'Bump CONDA_BUILD_NUMBER after successful Conda packages build and upload'