From b7c9bcd6b7fec4f5535b45834576a4a22d7c708b Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 24 Jun 2023 20:07:00 -0400 Subject: [PATCH] Update runpod --- requirements_runpod.txt | 2 +- setup-runpod.sh | 14 ++ setup.sh | 462 ++++++++++++++++++---------------------- 3 files changed, 225 insertions(+), 253 deletions(-) create mode 100644 setup-runpod.sh diff --git a/requirements_runpod.txt b/requirements_runpod.txt index 71ef35ea4..ded3ea27c 100644 --- a/requirements_runpod.txt +++ b/requirements_runpod.txt @@ -1,4 +1,4 @@ -torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 # no_verify leave this to specify not checking this a verification stage +torch==2.0.1+cu117 torchvision==0.15.2+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 # no_verify leave this to specify not checking this a verification stage xformers==0.0.20 bitsandbytes==0.39.1 accelerate==0.15.0 tensorboard==2.12.1 tensorflow==2.12.0 tensorrt diff --git a/setup-runpod.sh b/setup-runpod.sh new file mode 100644 index 000000000..fa26097ab --- /dev/null +++ b/setup-runpod.sh @@ -0,0 +1,14 @@ +# This gets the directory the script is run from so pathing can work relative to the script where needed. +SCRIPT_DIR="$(cd -- $(dirname -- "$0") && pwd)" + +# Check if the venv folder doesn't exist +if [ ! -d "$SCRIPT_DIR/venv" ]; then + echo "Creating venv..." + python3 -m venv venv +fi + +# Activate the virtual environment +source "$SCRIPT_DIR/venv/bin/activate" + +# Run setup_linux.py script with platform requirements +python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_runpod.txt diff --git a/setup.sh b/setup.sh index cb34d3234..3f521f37c 100755 --- a/setup.sh +++ b/setup.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash -# This file will be the host environment setup file for all operating systems other than base Windows. - -# Set the required package versions here. -TENSORFLOW_MACOS_VERSION="2.12.0" -TENSORFLOW_METAL_VERSION="0.8.0" - +# Function to display help information display_help() { cat <&5 + line=${line##*=} + echo "$line" + return 0 + elif command -v python >/dev/null; then + line="$(python -mplatform)" + echo "$line" + return 0 + elif command -v python3 >/dev/null; then + line="$(python3 -mplatform)" + echo "$line" + return 0 + else + line="None" + echo "$line" + return 1 fi - - case $opt in - b | branch) BRANCH="$OPTARG" ;; - d | dir) DIR="$OPTARG" ;; - g | git-repo) GIT_REPO="$OPTARG" ;; - i | interactive) INTERACTIVE=true ;; - n | no-git-update) SKIP_GIT_UPDATE=true ;; - p | public) PUBLIC=true ;; - r | runpod) RUNPOD=true ;; - s | skip-space-check) SKIP_SPACE_CHECK=true ;; - u | no-gui) SKIP_GUI=true ;; - v) ((VERBOSITY = VERBOSITY + 1)) ;; - h) display_help && exit 0 ;; - *) display_help && exit 0 ;; - esac -done -shift $((OPTIND - 1)) - -# Just in case someone puts in a relative path into $DIR, -# we're going to get the absolute path of that. -if [[ "$DIR" != /* ]] && [[ "$DIR" != ~* ]]; then - DIR="$( - cd "$(dirname "$DIR")" || exit 1 - pwd - )/$(basename "$DIR")" -fi - -for v in $( #Start counting from 3 since 1 and 2 are standards (stdout/stderr). - seq 3 $VERBOSITY -); do - (("$v" <= "$MAXVERBOSITY")) && eval exec "$v>&2" #Don't change anything higher than the maximum verbosity allowed. -done - -for v in $( #From the verbosity level one higher than requested, through the maximum; - seq $((VERBOSITY + 1)) $MAXVERBOSITY -); do - (("$v" > "2")) && eval exec "$v>/dev/null" #Redirect these to bitbucket, provided that they don't match stdout and stderr. -done - -# Example of how to use the verbosity levels. -# printf "%s\n" "This message is seen at verbosity level 1 and above." >&3 -# printf "%s\n" "This message is seen at verbosity level 2 and above." >&4 -# printf "%s\n" "This message is seen at verbosity level 3 and above." >&5 - -# Debug variable dump at max verbosity -echo "BRANCH: $BRANCH -DIR: $DIR -GIT_REPO: $GIT_REPO -INTERACTIVE: $INTERACTIVE -PUBLIC: $PUBLIC -RUNPOD: $RUNPOD -SKIP_SPACE_CHECK: $SKIP_SPACE_CHECK -VERBOSITY: $VERBOSITY -Script directory is ${SCRIPT_DIR}." >&5 - -# This must be set after the getopts loop to account for $DIR changes. -PARENT_DIR="$(dirname "${DIR}")" -VENV_DIR="$DIR/venv" - -if [ -w "$PARENT_DIR" ] && [ ! -d "$DIR" ]; then - echo "Creating install folder ${DIR}." - mkdir "$DIR" -fi - -if [ ! -w "$DIR" ]; then - echo "We cannot write to ${DIR}." - echo "Please ensure the install directory is accurate and you have the correct permissions." - exit 1 -fi +} -# Shared functions -# This checks for free space on the installation drive and returns that in Gb. -size_available() { - local folder - if [ -d "$DIR" ]; then - folder="$DIR" - elif [ -d "$PARENT_DIR" ]; then - folder="$PARENT_DIR" - elif [ -d "$(echo "$DIR" | cut -d "/" -f2)" ]; then - folder="$(echo "$DIR" | cut -d "/" -f2)" +# Function to get the distro family +get_distro_family() { + local line + if [ -f /etc/os-release ]; then + if grep -Eiq '^ID_LIKE=' /etc/os-release >/dev/null; then + line="$(grep -Ei '^ID_LIKE=' /etc/os-release)" + echo "Raw detected os-release distro family line: $line" >&5 + line=${line##*=} + echo "$line" + return 0 + else + line="None" + echo "$line" + return 1 + fi else - echo "We are assuming a root drive install for space-checking purposes." - folder='/' + line="None" + echo "$line" + return 1 fi +} - local FREESPACEINKB - FREESPACEINKB="$(df -Pk "$folder" | sed 1d | grep -v used | awk '{ print $4 "\t" }')" - echo "Detected available space in Kb: $FREESPACEINKB" >&5 - local FREESPACEINGB - FREESPACEINGB=$((FREESPACEINKB / 1024 / 1024)) - echo "$FREESPACEINGB" +# Function to check available storage space +check_storage_space() { + if [ "$SKIP_SPACE_CHECK" = false ]; then + if [ "$(size_available)" -lt 10 ]; then + echo "You have less than 10Gb of free space. This installation may fail." + MSGTIMEOUT=10 # In seconds + MESSAGE="Continuing in..." + echo "Press control-c to cancel the installation." + for ((i = MSGTIMEOUT; i >= 0; i--)); do + printf "\r${MESSAGE} %ss. " "${i}" + sleep 1 + done + fi + fi } -# The expected usage is create_symlinks symlink target_file +# Function to create symlinks create_symlinks() { local symlink="$1" local target_file="$2" @@ -217,17 +178,12 @@ create_symlinks() { fi } - +# Function to install Python dependencies install_python_dependencies() { local TEMP_REQUIREMENTS_FILE # Switch to local virtual env echo "Switching to virtual Python environment." - - if [ "$RUNPOD" = true ]; then - python3 -m venv "$DIR/venv" - fi - if ! inDocker; then if command -v python3.10 >/dev/null; then python3.10 -m venv "$DIR/venv" @@ -243,13 +199,6 @@ install_python_dependencies() { source "$DIR/venv/bin/activate" fi - # Updating pip if there is one - # echo "Checking for pip updates before Python operations." - # pip install --upgrade pip - - # echo "Installing python dependencies. This could take a few minutes as it downloads files." - # echo "If this operation ever runs too long, you can rerun this script in verbose mode to check." - case "$OSTYPE" in "lin"*) if [ "$RUNPOD" = true ]; then @@ -267,29 +216,6 @@ install_python_dependencies() { ;; esac - # DEBUG ONLY (Update this version number to whatever PyCharm recommends) - # pip install pydevd-pycharm~=223.8836.43 - - # Create a temporary requirements file - # TEMP_REQUIREMENTS_FILE=$(mktemp) - - # if [[ "$OSTYPE" == "darwin"* ]]; then - # echo "Copying $DIR/requirements_macos.txt to $TEMP_REQUIREMENTS_FILE" >&3 - # echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 - # awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_macos.txt" >"$TEMP_REQUIREMENTS_FILE" - # else - # echo "Copying $DIR/requirements_linux.txt to $TEMP_REQUIREMENTS_FILE" >&3 - # echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 - # awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_linux.txt" >"$TEMP_REQUIREMENTS_FILE" - # fi - - # # Install the Python dependencies from the temporary requirements file - # if [ $VERBOSITY == 2 ]; then - # python -m pip install --quiet --upgrade -r "$TEMP_REQUIREMENTS_FILE" - # else - # python -m pip install --upgrade -r "$TEMP_REQUIREMENTS_FILE" - # fi - if [ -n "$VIRTUAL_ENV" ] && ! inDocker; then if command -v deactivate >/dev/null; then echo "Exiting Python virtual environment." @@ -300,10 +226,7 @@ install_python_dependencies() { fi } - -# Attempt to non-interactively install a default accelerate config file unless specified otherwise. -# Documentation for order of precedence locations for configuration file for automated installation: -# https://huggingface.co/docs/accelerate/basic_tutorials/launch#custom-configurations +# Function to configure accelerate configure_accelerate() { echo "Source accelerate config location: $DIR/config_files/accelerate/default_config.yaml" >&3 if [ "$INTERACTIVE" = true ]; then @@ -333,50 +256,12 @@ configure_accelerate() { else echo "Could not place the accelerate configuration file. Please configure manually." sleep 2 - bash accelerate config + accelerate config fi fi } -# Offer a warning and opportunity to cancel the installation if < 10Gb of Free Space detected -check_storage_space() { - if [ "$SKIP_SPACE_CHECK" = false ]; then - if [ "$(size_available)" -lt 10 ]; then - echo "You have less than 10Gb of free space. This installation may fail." - MSGTIMEOUT=10 # In seconds - MESSAGE="Continuing in..." - echo "Press control-c to cancel the installation." - for ((i = MSGTIMEOUT; i >= 0; i--)); do - printf "\r${MESSAGE} %ss. " "${i}" - sleep 1 - done - fi - fi -} - -isContainerOrPod() { - local cgroup=/proc/1/cgroup - test -f $cgroup && (grep -qE ':cpuset:/(docker|kubepods)' $cgroup || grep -q ':/docker/' $cgroup) -} - -isDockerBuildkit() { - local cgroup=/proc/1/cgroup - test -f $cgroup && grep -q ':cpuset:/docker/buildkit' $cgroup -} - -isDockerContainer() { - [ -e /.dockerenv ] -} - -inDocker() { - if isContainerOrPod || isDockerBuildkit || isDockerContainer; then - return 0 - else - return 1 - fi -} - -# These are the git operations that will run to update or clone the repo +# Function to update Kohya_SS repo update_kohya_ss() { if [ "$SKIP_GIT_UPDATE" = false ]; then if command -v git >/dev/null; then @@ -412,6 +297,130 @@ update_kohya_ss() { fi } +# Section: Command-line options parsing + +while getopts ":vb:d:g:inprus-:" opt; do + # support long options: https://stackoverflow.com/a/28466267/519360 + if [ "$opt" = "-" ]; then # long option: reformulate OPT and OPTARG + opt="${OPTARG%%=*}" # extract long option name + OPTARG="${OPTARG#$opt}" # extract long option argument (may be empty) + OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=` + fi + + case $opt in + b | branch) BRANCH="$OPTARG" ;; + d | dir) DIR="$OPTARG" ;; + g | git-repo) GIT_REPO="$OPTARG" ;; + i | interactive) INTERACTIVE=true ;; + n | no-git-update) SKIP_GIT_UPDATE=true ;; + p | public) PUBLIC=true ;; + r | runpod) RUNPOD=true ;; + s | skip-space-check) SKIP_SPACE_CHECK=true ;; + u | no-gui) SKIP_GUI=true ;; + v) ((VERBOSITY = VERBOSITY + 1)) ;; + h) display_help && exit 0 ;; + *) display_help && exit 0 ;; + esac +done +shift $((OPTIND - 1)) + +# Just in case someone puts in a relative path into $DIR, +# we're going to get the absolute path of that. +if [[ "$DIR" != /* ]] && [[ "$DIR" != ~* ]]; then + DIR="$( + cd "$(dirname "$DIR")" || exit 1 + pwd + )/$(basename "$DIR")" +fi + +for v in $( #Start counting from 3 since 1 and 2 are standards (stdout/stderr). + seq 3 $VERBOSITY +); do + (("$v" <= "$MAXVERBOSITY")) && eval exec "$v>&2" #Don't change anything higher than the maximum verbosity allowed. +done + +for v in $( #From the verbosity level one higher than requested, through the maximum; + seq $((VERBOSITY + 1)) $MAXVERBOSITY +); do + (("$v" > "2")) && eval exec "$v>/dev/null" #Redirect these to bitbucket, provided that they don't match stdout and stderr. +done + +# Example of how to use the verbosity levels. +# printf "%s\n" "This message is seen at verbosity level 1 and above." >&3 +# printf "%s\n" "This message is seen at verbosity level 2 and above." >&4 +# printf "%s\n" "This message is seen at verbosity level 3 and above." >&5 + +# Debug variable dump at max verbosity +echo "BRANCH: $BRANCH +DIR: $DIR +GIT_REPO: $GIT_REPO +INTERACTIVE: $INTERACTIVE +PUBLIC: $PUBLIC +RUNPOD: $RUNPOD +SKIP_SPACE_CHECK: $SKIP_SPACE_CHECK +VERBOSITY: $VERBOSITY +Script directory is ${SCRIPT_DIR}." >&5 + +# This must be set after the getopts loop to account for $DIR changes. +PARENT_DIR="$(dirname "${DIR}")" +VENV_DIR="$DIR/venv" + +if [ -w "$PARENT_DIR" ] && [ ! -d "$DIR" ]; then + echo "Creating install folder ${DIR}." + mkdir "$DIR" +fi + +if [ ! -w "$DIR" ]; then + echo "We cannot write to ${DIR}." + echo "Please ensure the install directory is accurate and you have the correct permissions." + exit 1 +fi + +# Shared functions +# This checks for free space on the installation drive and returns that in Gb. +size_available() { + local folder + if [ -d "$DIR" ]; then + folder="$DIR" + elif [ -d "$PARENT_DIR" ]; then + folder="$PARENT_DIR" + elif [ -d "$(echo "$DIR" | cut -d "/" -f2)" ]; then + folder="$(echo "$DIR" | cut -d "/" -f2)" + else + echo "We are assuming a root drive install for space-checking purposes." + folder='/' + fi + + local FREESPACEINKB + FREESPACEINKB="$(df -Pk "$folder" | sed 1d | grep -v used | awk '{ print $4 "\t" }')" + echo "Detected available space in Kb: $FREESPACEINKB" >&5 + local FREESPACEINGB + FREESPACEINGB=$((FREESPACEINKB / 1024 / 1024)) + echo "$FREESPACEINGB" +} + +isContainerOrPod() { + local cgroup=/proc/1/cgroup + test -f $cgroup && (grep -qE ':cpuset:/(docker|kubepods)' $cgroup || grep -q ':/docker/' $cgroup) +} + +isDockerBuildkit() { + local cgroup=/proc/1/cgroup + test -f $cgroup && grep -q ':cpuset:/docker/buildkit' $cgroup +} + +isDockerContainer() { + [ -e /.dockerenv ] +} + +inDocker() { + if isContainerOrPod || isDockerBuildkit || isDockerContainer; then + return 0 + else + return 1 + fi +} + # Start OS-specific detection and work if [[ "$OSTYPE" == "lin"* ]]; then # Check if root or sudo @@ -424,55 +433,6 @@ if [[ "$OSTYPE" == "lin"* ]]; then root=true fi - get_distro_name() { - local line - if [ -f /etc/os-release ]; then - # We search for the line starting with ID= - # Then we remove the ID= prefix to get the name itself - line="$(grep -Ei '^ID=' /etc/os-release)" - echo "Raw detected os-release distro line: $line" >&5 - line=${line##*=} - echo "$line" - return 0 - elif command -v python >/dev/null; then - line="$(python -mplatform)" - echo "$line" - return 0 - elif command -v python3 >/dev/null; then - line="$(python3 -mplatform)" - echo "$line" - return 0 - else - line="None" - echo "$line" - return 1 - fi - } - - # We search for the line starting with ID_LIKE= - # Then we remove the ID_LIKE= prefix to get the name itself - # This is the "type" of distro. For example, Ubuntu returns "debian". - get_distro_family() { - local line - if [ -f /etc/os-release ]; then - if grep -Eiq '^ID_LIKE=' /etc/os-release >/dev/null; then - line="$(grep -Ei '^ID_LIKE=' /etc/os-release)" - echo "Raw detected os-release distro family line: $line" >&5 - line=${line##*=} - echo "$line" - return 0 - else - line="None" - echo "$line" - return 1 - fi - else - line="None" - echo "$line" - return 1 - fi - } - check_storage_space update_kohya_ss @@ -489,8 +449,6 @@ if [[ "$OSTYPE" == "lin"* ]]; then echo " " if [ "$RUNPOD" = true ]; then bash apt update -y && apt install -y python3-tk - bash python -m venv venv - source "$DIR/venv/bin/activate" else echo "sudo apt update -y && sudo apt install -y python3-tk" fi @@ -558,21 +516,21 @@ if [[ "$OSTYPE" == "lin"* ]]; then # We need just a little bit more setup for non-interactive environments if [ "$RUNPOD" = true ]; then - # if inDocker; then - # # We get the site-packages from python itself, then cut the string, so no other code changes required. - # VENV_DIR=$(python -c "import site; print(site.getsitepackages()[0])") - # VENV_DIR="${VENV_DIR%/lib/python3.10/site-packages}" - # fi + if inDocker; then + # We get the site-packages from python itself, then cut the string, so no other code changes required. + VENV_DIR=$(python -c "import site; print(site.getsitepackages()[0])") + VENV_DIR="${VENV_DIR%/lib/python3.10/site-packages}" + fi - # # Symlink paths - # libnvinfer_plugin_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.7" - # libnvinfer_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.7" - # libcudart_symlink="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.11.0" + # Symlink paths + libnvinfer_plugin_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.7" + libnvinfer_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.7" + libcudart_symlink="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.11.0" - # #Target file paths - # libnvinfer_plugin_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.8" - # libnvinfer_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.8" - # libcudart_target="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.12" + #Target file paths + libnvinfer_plugin_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.8" + libnvinfer_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.8" + libcudart_target="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.12" # echo "Checking symlinks now." # create_symlinks "$libnvinfer_plugin_symlink" "$libnvinfer_plugin_target"