ci_build_stubs.sh

#!/bin/bash

GENERATOR_CLI_ARGS=("$@")
REPO_URL="https://github.com/QuantConnect/ironpython-stubs"
REPO_SLUG="${TRAVIS_REPO_SLUG:-$GH_USERNAME/Lean}"
DEPLOYER_EMAIL="stubs-deploy@quantconnect.com"
DEPLOYER_NAME="Python Stubs Deployer"
BRANCH_NAME="master"
LEAN_BIN_PATH="Launcher/bin/Release"
PUSH_BRANCH="feature-python-autogenerated-stubs"

if [[ " ${GENERATOR_CLI_ARGS[@]} " =~ " -h " ]]; then
    echo "STUBS GENERATOR (Debian distros only)"
    echo "  -ipy: Installs IronPython"
    echo "  -g: Generate new stubs"
    echo "  -p: Pushes new stubs to GitHub and opens a PR"
    exit 0
fi

function install_ironpython {
    wget https://github.com/IronLanguages/ironpython2/releases/download/ipy-2.7.10/ironpython_2.7.10.deb
    sudo dpkg -i ironpython_2.7.10.deb
    rm ironpython_2.7.10.deb
}

function generate_python_stubs {
    if [ ! -d ironpython-stubs ]; then
        git clone $REPO_URL
    fi
    
    cd ironpython-stubs
    git fetch origin $BRANCH_NAME
    git checkout $BRANCH_NAME
    cd ../
    
    
    if [ -d "$LEAN_BIN_PATH/ironstubs" ]; then
        rm -rf "./$LEAN_BIN_PATH/ironstubs"
    fi
    if [ -d "$LEAN_BIN_PATH/stubsGenerator" ]; then
        rm -rf "./$LEAN_BIN_PATH/stubsGenerator"
    fi
    
    cp -R ironpython-stubs/ironstubs $LEAN_BIN_PATH
    cp -R ironpython-stubs/stubsGenerator $LEAN_BIN_PATH
    rm -rf ironpython-stubs
    
    cd $LEAN_BIN_PATH
    mkdir -p logs
    
    ipy -m ironstubs make --all --overwrite --keep-partial
    local stubs_generation_exit=$?

    if [ $stubs_generation_exit -ne 0 ]; then
        echo "Generation of stubs failed (ipy) - Exit code $stubs_generation_exit"
        exit 1
    fi
    
    echo "Beginning partial stubs consolidation"
    $HOME/miniconda3/bin/python3.6 -m stubsGenerator --path="./release/stubs/QuantConnect" --partition --size-limit=10000
    if [ $? -ne 0 ]; then
        echo "Parititioning of stubs failed"
        exit 1
    fi
    
    cd ../../../
    if [ -d "Algorithm.Python/stubs/QuantConnect" ]; then
        rm -rf ./Algorithm.Python/stubs/QuantConnect
    fi
                                                                                                     
    mkdir -p ./Algorithm.Python/stubs/
    mv "./$LEAN_BIN_PATH/release/stubs/QuantConnect" ./Algorithm.Python/stubs/QuantConnect
}

function push_to_github {
    # Travis env vars:
    # GH_API_KEY: GitHub API key (personal access key)
    # GH_USERNAME: GitHub username
    # GH_BASE_BRANCH: Branch to merge changes into
    # GH_REPO_ID: Base64 encoded string pointing to repo. Sourced from GH API

    # Deletes any residual files left over by the autogeneration process
    find ./Algorithm.Python/stubs/QuantConnect -type f -name '*.py_[0-9]' -exec rm {} \;
    find ./Algorithm.Python/stubs/QuantConnect -type f -name '*.py_[0-9][0-9]' -exec rm {} \;

    git add ./Algorithm.Python/stubs/QuantConnect
    git status --porcelain | grep -q "Algorithm.Python/stubs/QuantConnect/Algorithm"
    if [ $? -ne 0 ]; then
        echo "New stubs were not generated, skipping creating PR..."

        # Merge commits will not contain any changes. Let's look at the previous commit instead.
        git diff-tree --no-commit-id --name-only -r HEAD~1 | grep -q "Algorithm.Python/stubs/QuantConnect/Algorithm"
        if [ $? -ne 0 ]; then
            echo "HEAD commit contains no new updates to stubs. Exiting..."
            exit 0
        fi
        
        sudo apt-get update
        sudo apt-get install -y awscli

        # Stubs upload for Skylight deployment
        local tag_graphql_query='{ "query": "query { repository(name: \"Lean\", owner: \"QuantConnect\") { refs(refPrefix: \"refs/tags/\", first: 1, orderBy: { field: TAG_COMMIT_DATE, direction: DESC }) { nodes { name } } } }" }'
        local stubs_version_number_response=$(curl -s -H "Authorization: bearer $GH_API_KEY" -X POST --data-raw "$tag_graphql_query" https://api.github.com/graphql 2>/dev/null)
        local stubs_version_number=$(echo "$stubs_version_number_response" | jq .data.repository.refs.nodes[0].name | sed -s 's/"//g')
        local s3_bucket="skylight.quantconnect.com"

        cd ./Algorithm.Python/stubs/
        zip -r stubs.zip QuantConnect/

        aws s3 cp stubs.zip "s3://$s3_bucket/stubs/stubs_v$stubs_version_number.zip" --acl public-read

        if [ $? -ne 0 ]; then
            echo "Pushing stubs to s3 failed"
            exit 1
        fi

        echo "stubs_v$stubs_version_number.zip" > stubs_release.txt
        aws s3 cp stubs_release.txt "s3://$s3_bucket/stubs/stubs_release.txt" --acl public-read

        exit $?;
    fi

    git config --global user.email "$DEPLOYER_EMAIL"
    git config --global user.name "$DEPLOYER_NAME"

    git remote rm origin
    git remote add stubs_deploy https://${GH_USERNAME}:${GH_API_KEY}@github.com/${REPO_SLUG} > /dev/null 2>&1
    git commit -m "Adds autogenerated Python stubs via Travis for QCAlgorithm (Build $TRAVIS_BUILD_NUMBER)"
    # Might give us an error, but it's to be expected
    git branch -D $PUSH_BRANCH
    git checkout -b $PUSH_BRANCH
    git push stubs_deploy $PUSH_BRANCH -f --quiet > /dev/null 2>&1
    
    local graphql_query=$(printf '{"query":"mutation { createPullRequest(input: { title: \\"[Travis #%s] Autogenerated Python Stubs\\", baseRefName: \\"%s\\", headRefName: \\"%s\\", maintainerCanModify: true, draft: false, repositoryId: \\"%s\\"}) { pullRequest { resourcePath }}}","variables":{}}' "$TRAVIS_BUILD_NUMBER" "$GH_BASE_BRANCH" "$PUSH_BRANCH" "$GH_REPO_ID")

    curl -s -H "Authorization: bearer $GH_API_KEY" -X POST --data-raw "$graphql_query" https://api.github.com/graphql > /dev/null 2>&1

    if [ $? -ne 0 ]; then
        echo "GitHub PR creation failed"
        exit 1
    fi
}

CURRENT_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}"

if [[ "$CURRENT_BRANCH" != "master" ]]; then
    exit 0
fi
if [[ " ${GENERATOR_CLI_ARGS[@]} " =~ " -ipy " ]]; then
    install_ironpython
fi
if [[ " ${GENERATOR_CLI_ARGS[@]} " =~ " -g " ]]; then
    generate_python_stubs
fi
if [[ " ${GENERATOR_CLI_ARGS[@]} " =~ " -p " ]]; then
    push_to_github
fi