upload-data.yml

name: Daily new data upload

on:
  schedule:
    - cron:  '0 4 * * *'

jobs:
  upload:
    name: Upload data
    runs-on: ubuntu-latest
    steps:

      - name: Checkout
        uses: actions/checkout@master

      - name: Checkout data repo
        uses: actions/checkout@v2
        with:
          repository: epidemics/epimodel-covid-data
          path: data-pipeline/data

      - uses: r-lib/actions/setup-r@v1
        with:
          r-version: '4.0.0'

      - name: Install os R dependencies
        run: sudo apt install libcurl4-openssl-dev libxml2-dev libssl-dev

      - name: Cache R packages
        uses: actions/cache@v1
        with:
          path: ${{ env.R_LIBS_USER }}
          key: ${{ runner.os }}-r-${{ hashFiles('**/dependencies.R') }}

      - name: Install R dependencies
        working-directory: data-pipeline
        run: Rscript ./scripts/dependencies.R

      - name: Set up Python 3.7
        uses: actions/setup-python@v1
        with:
          python-version: 3.7

      - name: Get full python version
        id: full-python-version
        run: |
          echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")

      - name: Install and set up Poetry
        working-directory: data-pipeline
        run: |
          python get-poetry.py --preview -y
          source $HOME/.poetry/env
          poetry config virtualenvs.in-project true

      - name: Set up cache
        uses: actions/cache@v1
        with:
          path: data-pipeline/.venv
          key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Install dependencies
        working-directory: data-pipeline
        run: |
          source $HOME/.poetry/env
          poetry install -E 'pymc3 pyro'

      - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
        name: Setup Google Cloud Platform
        with:
          version: '270.0.0'
          service_account_email: ${{ secrets.SA_EMAIL }}
          service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}

      - name: Run upload
        working-directory: data-pipeline
        run: |
          source $HOME/.poetry/env
          poetry run ./run_luigi WebExport --EstimateR-r-executable-path /usr/local/bin/Rscript --export-name main --automatic --UpdateForetold-foretold-channel ${{ secrets.FORETOLD_CHANNEL }}
          poetry run ./run_luigi WebUpload --exported-data data-dir/outputs/web-exports/main/ --channel main --overwrite

      - uses: sonots/slack-notice-action@v3
        with:
          status: ${{ job.status }}
          username: Daily upload status
          icon_emoji: ':octocat:'
          channel: '#covid-prediction'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SLACK_WEBHOOK_URL: ${{ secrets.BLINDSPOT_SLACK_WEBHOOK_URL }}
        if: failure()