This repository has been archived by the owner on Aug 25, 2024. It is now read-only.
Tests #4949
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/arch/**' | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/arch/**' | |
| schedule: | |
| - cron: '0 3 * * *' | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 40 | |
| matrix: | |
| check: [changelog, whitespace, commit, lines] | |
| python-version: [3.7] | |
| node-version: [12.x] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v1 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@v1 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh ${{ matrix.check }} | |
| - name: Run check | |
| run: | | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| SSH_DFFML_GH_PAGES=${{ secrets.SSH_DFFML_GH_PAGES }} ./.ci/run.sh ${{ matrix.check }} | |
| container: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - | |
| sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | |
| sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io | |
| - name: Run check | |
| run: | | |
| ./.ci/run.sh container | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 100 | |
| matrix: | |
| plugin: | |
| - model/daal4py | |
| - model/tensorflow | |
| - model/scratch | |
| - model/scikit | |
| - model/vowpalWabbit | |
| - model/autosklearn | |
| - model/xgboost | |
| - operations/binsec | |
| - operations/data | |
| - operations/deploy | |
| - operations/image | |
| - operations/nlp | |
| - source/mysql | |
| - feature/git | |
| - feature/auth | |
| - service/http | |
| - configloader/yaml | |
| python-version: [3.7] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Remove unused software | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@v1 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: shouldi test binary cache | |
| uses: actions/cache@v1 | |
| with: | |
| path: examples/shouldi/tests/downloads | |
| key: ${{ hashFiles('examples/shouldi/tests/binaries.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-shouldi-test-binaries- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh ${{ matrix.plugin }} | |
| - name: Test | |
| run: | | |
| set -x | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| export PYTHON=python${{ matrix.python-version }} | |
| export PLUGIN=${{ matrix.plugin }} | |
| export LOGGING=debug | |
| export PATH="${HOME}/.local/bin:${PATH}" | |
| export PYPI_TOKENS=$(mktemp) | |
| cat > ${PYPI_TOKENS} <<EOF | |
| examples/shouldi=${{ secrets.PYPI_SHOULDI }} | |
| model/daal4py=${{ secrets.PYPI_MODEL_DAAL4PY }} | |
| model/tensorflow=${{ secrets.PYPI_MODEL_TENSORFLOW }} | |
| model/tensorflow_hub=${{ secrets.PYPI_MODEL_TENSORFLOW_HUB }} | |
| model/scratch=${{ secrets.PYPI_MODEL_SCRATCH }} | |
| model/scikit=${{ secrets.PYPI_MODEL_SCIKIT }} | |
| model/spacy=${{ secrets.PYPI_MODEL_SPACY }} | |
| model/vowpalWabbit=${{ secrets.PYPI_MODEL_VOWPALWABBIT }} | |
| model/autosklearn=${{ secrets.PYPI_MODEL_AUTOSKLEARN }} | |
| model/pytorch=${{ secrets.PYPI_MODEL_PYTORCH }} | |
| model/xgboost=${{ secrets.PYPI_MODEL_XGBOOST }} | |
| source/mysql=${{ secrets.PYPI_SOURCE_MYSQL }} | |
| feature/git=${{ secrets.PYPI_FEATURE_GIT }} | |
| feature/auth=${{ secrets.PYPI_FEATURE_AUTH }} | |
| operations/binsec=${{ secrets.PYPI_OPERATIONS_BINSEC }} | |
| operations/data=${{ secrets.PYPI_OPERATIONS_DATA }} | |
| operations/deploy=${{ secrets.PYPI_OPERATIONS_DEPLOY }} | |
| operations/image=${{ secrets.PYPI_OPERATIONS_IMAGE }} | |
| operations/nlp=${{ secrets.PYPI_OPERATIONS_NLP }} | |
| service/http=${{ secrets.PYPI_SERVICE_HTTP }} | |
| configloader/yaml=${{ secrets.PYPI_CONFIG_YAML }} | |
| configloader/image=${{ secrets.PYPI_CONFIG_IMAGE }} | |
| EOF | |
| export TWINE_USERNAME=__token__ | |
| if [ "x${PLUGIN}" = "x." ]; then | |
| export TWINE_PASSWORD=${{ secrets.PYPI_DFFML }} | |
| else | |
| export TWINE_PASSWORD=$(grep "${PLUGIN}=" "${PYPI_TOKENS}" | sed 's/^[^=]*=//g') | |
| fi | |
| rm ${PYPI_TOKENS} | |
| ./.ci/run.sh "${PLUGIN}" | |
| if [ "x${PLUGIN}" = "x." ]; then | |
| CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} codecov | |
| fi | |
| tutorials: | |
| runs-on: ubuntu-latest | |
| if: false | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 100 | |
| matrix: | |
| python-version: [3.7] | |
| docs: | |
| - docs/cli.rst | |
| - docs/examples/dataflows.rst | |
| - docs/examples/integration.rst | |
| - docs/examples/or_covid_data_by_county.rst | |
| - docs/examples/shouldi.rst | |
| - docs/examples/innersource/swportal.rst | |
| - docs/examples/innersource/microservice.rst | |
| - docs/examples/icecream_sales.rst | |
| - docs/examples/data_cleanup/data_cleanup.rst | |
| - docs/examples/data_cleanup/data_cleanup_classfication.rst | |
| - docs/installation.rst | |
| - docs/troubleshooting.rst | |
| - docs/tutorials/accuracy/mse.rst | |
| - docs/tutorials/dataflows/io.rst | |
| - docs/tutorials/dataflows/nlp.rst | |
| - docs/tutorials/models/iris.rst | |
| - docs/tutorials/models/package.rst | |
| - docs/tutorials/models/docs.rst | |
| - docs/tutorials/models/slr.rst | |
| - docs/tutorials/sources/complex.rst | |
| - docs/tutorials/sources/file.rst | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git fetch --depth=1 origin +refs/tags/*:refs/tags/* | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@v1 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: | | |
| set -x | |
| ./.ci/deps.sh . | |
| - name: Test | |
| run: | | |
| set -x | |
| export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| export PYTHON=python${{ matrix.python-version }} | |
| export PLUGIN=${{ matrix.plugin }} | |
| export LOGGING=debug | |
| export PATH="${HOME}/.local/bin:${PATH}" | |
| ./.ci/run.sh consoletest ${{ matrix.docs }} | |
| macos: | |
| runs-on: macos-latest | |
| if: false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [3.7] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Checkout full upstream repo | |
| run: | | |
| git remote set-url origin https://github.com/intel/dffml | |
| git fetch --prune --unshallow | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "DFFML CI/CD" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get pip cache | |
| id: pip-cache | |
| run: | | |
| python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" | |
| - name: pip cache | |
| uses: actions/cache@v1 | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Setup DFFML | |
| run: | | |
| pip install -U pip setuptools wheel | |
| pip install -e .[dev] | |
| dffml service dev install -skip model/daal4py | |
| # XGBoost requires libomp on OSX | |
| brew install libomp | |
| - name: Test | |
| run: | | |
| python -m unittest discover -v | |
| python -m pip freeze |