From 5f6e3aeec64b0772f7a2d8c2b639346a61c1ad26 Mon Sep 17 00:00:00 2001 From: ppppppbz <516988422@qq.com> Date: Sun, 25 Jun 2023 00:10:01 +0800 Subject: [PATCH 1/4] fix group_images bug --- tools/group_images.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/group_images.py b/tools/group_images.py index 8ca1f18af..d94ff1879 100644 --- a/tools/group_images.py +++ b/tools/group_images.py @@ -92,9 +92,9 @@ def resize_and_save_images(self, cropped_images, group_index, source_paths): os.makedirs(self.output_folder, exist_ok=True) original_filename = os.path.basename(source_paths[j]) filename_without_ext = os.path.splitext(original_filename)[0] - output_path = os.path.join(self.output_folder, f"group-{group_index+1}-{filename_without_ext}.jpg") + output_path = os.path.join(self.output_folder, f"group-{group_index+1}-{j+1}-{filename_without_ext}.jpg") log.info(f" Saving processed image to {output_path}") - img.convert('RGB').save(output_path) + img.convert('RGB').save(output_path, quality=100) if self.caption: self.create_caption_file(source_paths[j], group_index, filename_without_ext) @@ -117,7 +117,7 @@ def copy_other_files(self, group, group_index): continue basename, ext = os.path.splitext(filename) if basename == original_basename and ext != original_ext: - shutil.copy2(os.path.join(dirpath, filename), os.path.join(self.output_folder, f"group-{group_index+1}-image-{j+1}{ext}")) + shutil.copy2(os.path.join(dirpath, filename), os.path.join(self.output_folder, f"group-{group_index+1}-{j+1}-{filename}")) def process_images(self): images = self.get_image_paths() From b71818bc67f9fdec669024efa3582c863b152c04 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 24 Jun 2023 13:36:32 -0400 Subject: [PATCH 2/4] Update README --- README.md | 4 +- .../pull kohya_ss sd-scripts updates in.md | 42 +- setup_legacy.sh | 661 ------------------ upgrade.bat | 16 - utilities.cmd | 1 - 5 files changed, 24 insertions(+), 700 deletions(-) delete mode 100755 setup_legacy.sh delete mode 100644 upgrade.bat delete mode 100644 utilities.cmd diff --git a/README.md b/README.md index f17da822a..71985e9ae 100644 --- a/README.md +++ b/README.md @@ -49,9 +49,7 @@ Newer Tutorial: [Generate Studio Quality Realistic Photos By Kohya LoRA Stable D Newer Tutorial: [How To Install And Use Kohya LoRA GUI / Web UI on RunPod IO](https://www.youtube.com/watch?v=3uzCNrQao3o): -[![How To Install And Use Kohya LoRA GUI / Web UI on RunPod IO With Stable Diffusion & Automatic1111](https://github-production-user-asset-6210df.s3.amazonaws.com/19240467/238678226-0c9c3f7d-c308-4793-b790-999fdc271372.png)](https://www - -.youtube.com/watch?v=3uzCNrQao3o) +[![How To Install And Use Kohya LoRA GUI / Web UI on RunPod IO With Stable Diffusion & Automatic1111](https://github-production-user-asset-6210df.s3.amazonaws.com/19240467/238678226-0c9c3f7d-c308-4793-b790-999fdc271372.png)](https://www.youtube.com/watch?v=3uzCNrQao3o) ## Installation diff --git a/examples/pull kohya_ss sd-scripts updates in.md b/examples/pull kohya_ss sd-scripts updates in.md index 8c36243a3..47b1c79ad 100644 --- a/examples/pull kohya_ss sd-scripts updates in.md +++ b/examples/pull kohya_ss sd-scripts updates in.md @@ -1,28 +1,32 @@ -# Pull sd-scripts update in a local branch +## Updating a Local Branch with the Latest sd-scripts Changes -For reference for anyone that would like to pull the latest from kohya/sd-scripts, you can: +To update your local branch with the most recent changes from kohya/sd-scripts, follow these steps: -`git remote add sd-scripts https://github.com/kohya-ss/sd-scripts.git` +1. Add sd-scripts as an alternative remote by executing the following command: -to add it as an alternative remote, then when you want to update: + ``` + git remote add sd-scripts https://github.com/kohya-ss/sd-scripts.git + ``` -``` -git checkout dev -git pull sd-scripts main -``` +2. When you wish to perform an update, execute the following commands: -or, if you want the absolute latest and potentially broken code: + ``` + git checkout dev + git pull sd-scripts main + ``` -``` -git checkout dev -git pull sd-scripts dev -``` + Alternatively, if you want to obtain the latest code, even if it may be unstable: -You'll probably get a conflict for the Readme, but you can get around it with: + ``` + git checkout dev + git pull sd-scripts dev + ``` -``` -git add README.md -git merge --continue -``` +3. If you encounter a conflict with the Readme file, you can resolve it by taking the following steps: -which will probably open a text editor for a commit message, but you can just save and close that and you should be good to go. If there are more merge conflicts than that, you now have a potential learning experience and chance for personal growth. \ No newline at end of file + ``` + git add README.md + git merge --continue + ``` + + This may open a text editor for a commit message, but you can simply save and close it to proceed. Following these steps should resolve the conflict. If you encounter additional merge conflicts, consider them as valuable learning opportunities for personal growth. \ No newline at end of file diff --git a/setup_legacy.sh b/setup_legacy.sh deleted file mode 100755 index a95939959..000000000 --- a/setup_legacy.sh +++ /dev/null @@ -1,661 +0,0 @@ -#!/usr/bin/env bash - -# This file will be the host environment setup file for all operating systems other than base Windows. - -# Set the required package versions here. -TENSORFLOW_MACOS_VERSION="2.12.0" -TENSORFLOW_METAL_VERSION="0.8.0" - -display_help() { - cat <&2" #Don't change anything higher than the maximum verbosity allowed. -done - -for v in $( #From the verbosity level one higher than requested, through the maximum; - seq $((VERBOSITY + 1)) $MAXVERBOSITY -); do - (("$v" > "2")) && eval exec "$v>/dev/null" #Redirect these to bitbucket, provided that they don't match stdout and stderr. -done - -# Example of how to use the verbosity levels. -# printf "%s\n" "This message is seen at verbosity level 1 and above." >&3 -# printf "%s\n" "This message is seen at verbosity level 2 and above." >&4 -# printf "%s\n" "This message is seen at verbosity level 3 and above." >&5 - -# Debug variable dump at max verbosity -echo "BRANCH: $BRANCH -DIR: $DIR -GIT_REPO: $GIT_REPO -INTERACTIVE: $INTERACTIVE -PUBLIC: $PUBLIC -RUNPOD: $RUNPOD -SKIP_SPACE_CHECK: $SKIP_SPACE_CHECK -VERBOSITY: $VERBOSITY -Script directory is ${SCRIPT_DIR}." >&5 - -# This must be set after the getopts loop to account for $DIR changes. -PARENT_DIR="$(dirname "${DIR}")" -VENV_DIR="$DIR/venv" - -if [ -w "$PARENT_DIR" ] && [ ! -d "$DIR" ]; then - echo "Creating install folder ${DIR}." - mkdir "$DIR" -fi - -if [ ! -w "$DIR" ]; then - echo "We cannot write to ${DIR}." - echo "Please ensure the install directory is accurate and you have the correct permissions." - exit 1 -fi - -# Shared functions -# This checks for free space on the installation drive and returns that in Gb. -size_available() { - local folder - if [ -d "$DIR" ]; then - folder="$DIR" - elif [ -d "$PARENT_DIR" ]; then - folder="$PARENT_DIR" - elif [ -d "$(echo "$DIR" | cut -d "/" -f2)" ]; then - folder="$(echo "$DIR" | cut -d "/" -f2)" - else - echo "We are assuming a root drive install for space-checking purposes." - folder='/' - fi - - local FREESPACEINKB - FREESPACEINKB="$(df -Pk "$folder" | sed 1d | grep -v used | awk '{ print $4 "\t" }')" - echo "Detected available space in Kb: $FREESPACEINKB" >&5 - local FREESPACEINGB - FREESPACEINGB=$((FREESPACEINKB / 1024 / 1024)) - echo "$FREESPACEINGB" -} - -# The expected usage is create_symlinks symlink target_file -create_symlinks() { - local symlink="$1" - local target_file="$2" - - echo "Checking symlinks now." - - # Check if the symlink exists - if [ -L "$symlink" ]; then - # Check if the linked file exists and points to the expected file - if [ -e "$symlink" ] && [ "$(readlink "$symlink")" == "$target_file" ]; then - echo "$(basename "$symlink") symlink looks fine. Skipping." - else - if [ -f "$target_file" ]; then - echo "Broken symlink detected. Recreating $(basename "$symlink")." - rm "$symlink" && ln -s "$target_file" "$symlink" - else - echo "$target_file does not exist. Nothing to link." - fi - fi - else - echo "Linking $(basename "$symlink")." - ln -s "$target_file" "$symlink" - fi -} - - -install_python_dependencies() { - local TEMP_REQUIREMENTS_FILE - - # Switch to local virtual env - echo "Switching to virtual Python environment." - if ! inDocker; then - if command -v python3.10 >/dev/null; then - python3.10 -m venv "$DIR/venv" - elif command -v python3 >/dev/null; then - python3 -m venv "$DIR/venv" - else - echo "Valid python3 or python3.10 binary not found." - echo "Cannot proceed with the python steps." - return 1 - fi - - # Activate the virtual environment - source "$DIR/venv/bin/activate" - fi - - # Updating pip if there is one - echo "Checking for pip updates before Python operations." - pip install --upgrade pip - - echo "Installing python dependencies. This could take a few minutes as it downloads files." - echo "If this operation ever runs too long, you can rerun this script in verbose mode to check." - - case "$OSTYPE" in - "linux-gnu"*) - pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 \ - --extra-index-url https://download.pytorch.org/whl/cu118 - pip install --upgrade xformers==0.0.20 - ;; - "darwin"*) - pip install torch==2.0.0 torchvision==0.15.1 \ - -f https://download.pytorch.org/whl/cpu/torch_stable.html - # Check if the processor is Apple Silicon (arm64) - if [[ "$(uname -m)" == "arm64" ]]; then - pip install tensorflow-metal=="$TENSORFLOW_MACOS_VERSION" - else - pip install tensorflow-macos=="$TENSORFLOW_METAL_VERSION" - fi - ;; - esac - - if [ "$RUNPOD" = true ]; then - echo "Installing tenssort." - pip install tensorrt - fi - - # DEBUG ONLY (Update this version number to whatever PyCharm recommends) - # pip install pydevd-pycharm~=223.8836.43 - - # Create a temporary requirements file - TEMP_REQUIREMENTS_FILE=$(mktemp) - - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "Copying $DIR/requirements_macos.txt to $TEMP_REQUIREMENTS_FILE" >&3 - echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 - awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_macos.txt" >"$TEMP_REQUIREMENTS_FILE" - else - echo "Copying $DIR/requirements_linux.txt to $TEMP_REQUIREMENTS_FILE" >&3 - echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 - awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_linux.txt" >"$TEMP_REQUIREMENTS_FILE" - fi - - # Install the Python dependencies from the temporary requirements file - if [ $VERBOSITY == 2 ]; then - python -m pip install --quiet --upgrade -r "$TEMP_REQUIREMENTS_FILE" - else - python -m pip install --upgrade -r "$TEMP_REQUIREMENTS_FILE" - fi - - if [ -n "$VIRTUAL_ENV" ] && ! inDocker; then - if command -v deactivate >/dev/null; then - echo "Exiting Python virtual environment." - deactivate - else - echo "deactivate command not found. Could still be in the Python virtual environment." - fi - fi -} - - -# Attempt to non-interactively install a default accelerate config file unless specified otherwise. -# Documentation for order of precedence locations for configuration file for automated installation: -# https://huggingface.co/docs/accelerate/basic_tutorials/launch#custom-configurations -configure_accelerate() { - echo "Source accelerate config location: $DIR/config_files/accelerate/default_config.yaml" >&3 - if [ "$INTERACTIVE" = true ]; then - accelerate config - else - if env_var_exists HF_HOME; then - if [ ! -f "$HF_HOME/accelerate/default_config.yaml" ]; then - mkdir -p "$HF_HOME/accelerate/" && - echo "Target accelerate config location: $HF_HOME/accelerate/default_config.yaml" >&3 - cp "$DIR/config_files/accelerate/default_config.yaml" "$HF_HOME/accelerate/default_config.yaml" && - echo "Copied accelerate config file to: $HF_HOME/accelerate/default_config.yaml" - fi - elif env_var_exists XDG_CACHE_HOME; then - if [ ! -f "$XDG_CACHE_HOME/huggingface/accelerate" ]; then - mkdir -p "$XDG_CACHE_HOME/huggingface/accelerate" && - echo "Target accelerate config location: $XDG_CACHE_HOME/accelerate/default_config.yaml" >&3 - cp "$DIR/config_files/accelerate/default_config.yaml" "$XDG_CACHE_HOME/huggingface/accelerate/default_config.yaml" && - echo "Copied accelerate config file to: $XDG_CACHE_HOME/huggingface/accelerate/default_config.yaml" - fi - elif env_var_exists HOME; then - if [ ! -f "$HOME/.cache/huggingface/accelerate" ]; then - mkdir -p "$HOME/.cache/huggingface/accelerate" && - echo "Target accelerate config location: $HOME/accelerate/default_config.yaml" >&3 - cp "$DIR/config_files/accelerate/default_config.yaml" "$HOME/.cache/huggingface/accelerate/default_config.yaml" && - echo "Copying accelerate config file to: $HOME/.cache/huggingface/accelerate/default_config.yaml" - fi - else - echo "Could not place the accelerate configuration file. Please configure manually." - sleep 2 - accelerate config - fi - fi -} - -# Offer a warning and opportunity to cancel the installation if < 10Gb of Free Space detected -check_storage_space() { - if [ "$SKIP_SPACE_CHECK" = false ]; then - if [ "$(size_available)" -lt 10 ]; then - echo "You have less than 10Gb of free space. This installation may fail." - MSGTIMEOUT=10 # In seconds - MESSAGE="Continuing in..." - echo "Press control-c to cancel the installation." - for ((i = MSGTIMEOUT; i >= 0; i--)); do - printf "\r${MESSAGE} %ss. " "${i}" - sleep 1 - done - fi - fi -} - -isContainerOrPod() { - local cgroup=/proc/1/cgroup - test -f $cgroup && (grep -qE ':cpuset:/(docker|kubepods)' $cgroup || grep -q ':/docker/' $cgroup) -} - -isDockerBuildkit() { - local cgroup=/proc/1/cgroup - test -f $cgroup && grep -q ':cpuset:/docker/buildkit' $cgroup -} - -isDockerContainer() { - [ -e /.dockerenv ] -} - -inDocker() { - if isContainerOrPod || isDockerBuildkit || isDockerContainer; then - return 0 - else - return 1 - fi -} - -# These are the git operations that will run to update or clone the repo -update_kohya_ss() { - if [ "$SKIP_GIT_UPDATE" = false ]; then - if command -v git >/dev/null; then - # First, we make sure there are no changes that need to be made in git, so no work is lost. - if [ "$(git -C "$DIR" status --porcelain=v1 2>/dev/null | wc -l)" -gt 0 ] && - echo "These files need to be committed or discarded: " >&4 && - git -C "$DIR" status >&4; then - echo "There are changes that need to be committed or discarded in the repo in $DIR." - echo "Commit those changes or run this script with -n to skip git operations entirely." - exit 1 - fi - - echo "Attempting to clone $GIT_REPO." - if [ ! -d "$DIR/.git" ]; then - echo "Cloning and switching to $GIT_REPO:$BRANCH" >&4 - git -C "$PARENT_DIR" clone -b "$BRANCH" "$GIT_REPO" "$(basename "$DIR")" >&3 - git -C "$DIR" switch "$BRANCH" >&4 - else - echo "git repo detected. Attempting to update repository instead." - echo "Updating: $GIT_REPO" - git -C "$DIR" pull "$GIT_REPO" "$BRANCH" >&3 - if ! git -C "$DIR" switch "$BRANCH" >&4; then - echo "Branch $BRANCH did not exist. Creating it." >&4 - git -C "$DIR" switch -c "$BRANCH" >&4 - fi - fi - else - echo "You need to install git." - echo "Rerun this after installing git or run this script with -n to skip the git operations." - fi - else - echo "Skipping git operations." - fi -} - -# Start OS-specific detection and work -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - # Check if root or sudo - root=false - if [ "$EUID" = 0 ]; then - root=true - elif command -v id >/dev/null && [ "$(id -u)" = 0 ]; then - root=true - elif [ "$UID" = 0 ]; then - root=true - fi - - get_distro_name() { - local line - if [ -f /etc/os-release ]; then - # We search for the line starting with ID= - # Then we remove the ID= prefix to get the name itself - line="$(grep -Ei '^ID=' /etc/os-release)" - echo "Raw detected os-release distro line: $line" >&5 - line=${line##*=} - echo "$line" - return 0 - elif command -v python >/dev/null; then - line="$(python -mplatform)" - echo "$line" - return 0 - elif command -v python3 >/dev/null; then - line="$(python3 -mplatform)" - echo "$line" - return 0 - else - line="None" - echo "$line" - return 1 - fi - } - - # We search for the line starting with ID_LIKE= - # Then we remove the ID_LIKE= prefix to get the name itself - # This is the "type" of distro. For example, Ubuntu returns "debian". - get_distro_family() { - local line - if [ -f /etc/os-release ]; then - if grep -Eiq '^ID_LIKE=' /etc/os-release >/dev/null; then - line="$(grep -Ei '^ID_LIKE=' /etc/os-release)" - echo "Raw detected os-release distro family line: $line" >&5 - line=${line##*=} - echo "$line" - return 0 - else - line="None" - echo "$line" - return 1 - fi - else - line="None" - echo "$line" - return 1 - fi - } - - check_storage_space - update_kohya_ss - - distro=get_distro_name - family=get_distro_family - echo "Raw detected distro string: $distro" >&4 - echo "Raw detected distro family string: $family" >&4 - - if "$distro" | grep -qi "Ubuntu" || "$family" | grep -qi "Ubuntu"; then - echo "Ubuntu detected." - if [ $(dpkg-query -W -f='${Status}' python3-tk 2>/dev/null | grep -c "ok installed") = 0 ]; then - # if [ "$root" = true ]; then - echo "This script needs you to install the missing python3-tk packages. Please install with:" - echo " " - echo "sudo apt update -y && sudo apt install -y python3-tk" - exit 1 - # else - # echo "This script needs to be run as root or via sudo to install packages." - # exit 1 - # fi - else - echo "Python TK found..." - fi - elif "$distro" | grep -Eqi "Fedora|CentOS|Redhat"; then - echo "Redhat or Redhat base detected." - if ! rpm -qa | grep -qi python3-tkinter; then - # if [ "$root" = true ]; then - echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n" - echo "sudo dnf install python3-tkinter -y >&3" - exit 1 - # else - # echo "This script needs to be run as root or via sudo to install packages." - # exit 1 - # fi - else - echo "Python TK found..." - fi - elif "$distro" | grep -Eqi "arch" || "$family" | grep -qi "arch"; then - echo "Arch Linux or Arch base detected." - if ! pacman -Qi tk >/dev/null; then - # if [ "$root" = true ]; then - echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n" - echo "pacman --noconfirm -S tk >&3" - exit 1 - # else - # echo "This script needs to be run as root or via sudo to install packages." - # exit 1 - # fi - else - echo "Python TK found..." - fi - elif "$distro" | grep -Eqi "opensuse" || "$family" | grep -qi "opensuse"; then - echo "OpenSUSE detected." - if ! rpm -qa | grep -qi python-tk; then - # if [ "$root" = true ]; then - echo "This script needs you to install the missing python3-tk packages. Please install with:\n\n" - echo "zypper install -y python-tk >&3" - exit 1 - # else - # echo "This script needs to be run as root or via sudo to install packages." - # exit 1 - # fi - else - echo "Python TK found..." - fi - elif [ "$distro" = "None" ] || [ "$family" = "None" ]; then - if [ "$distro" = "None" ]; then - echo "We could not detect your distribution of Linux. Please file a bug report on github with the contents of your /etc/os-release file." - fi - - if [ "$family" = "None" ]; then - echo "We could not detect the family of your Linux distribution. Please file a bug report on github with the contents of your /etc/os-release file." - fi - fi - - install_python_dependencies - - # We need just a little bit more setup for non-interactive environments - if [ "$RUNPOD" = true ]; then - if inDocker; then - # We get the site-packages from python itself, then cut the string, so no other code changes required. - VENV_DIR=$(python -c "import site; print(site.getsitepackages()[0])") - VENV_DIR="${VENV_DIR%/lib/python3.10/site-packages}" - fi - - # Symlink paths - libnvinfer_plugin_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.7" - libnvinfer_symlink="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.7" - libcudart_symlink="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.11.0" - - #Target file paths - libnvinfer_plugin_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer_plugin.so.8" - libnvinfer_target="$VENV_DIR/lib/python3.10/site-packages/tensorrt/libnvinfer.so.8" - libcudart_target="$VENV_DIR/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.12" - - echo "Checking symlinks now." - create_symlinks "$libnvinfer_plugin_symlink" "$libnvinfer_plugin_target" - create_symlinks "$libnvinfer_symlink" "$libnvinfer_target" - create_symlinks "$libcudart_symlink" "$libcudart_target" - - if [ -d "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/" ]; then - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${VENV_DIR}/lib/python3.10/site-packages/tensorrt/" - else - echo "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/ not found; not linking library." - fi - - if [ -d "${VENV_DIR}/lib/python3.10/site-packages/tensorrt/" ]; then - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${VENV_DIR}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/" - else - echo "${VENV_DIR}/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/ not found; not linking library." - fi - - configure_accelerate - - # This is a non-interactive environment, so just directly call gui.sh after all setup steps are complete. - if [ "$SKIP_GUI" = false ]; then - if command -v bash >/dev/null; then - if [ "$PUBLIC" = false ]; then - bash "$DIR"/gui.sh - exit 0 - else - bash "$DIR"/gui.sh --share - exit 0 - fi - else - # This shouldn't happen, but we're going to try to help. - if [ "$PUBLIC" = false ]; then - sh "$DIR"/gui.sh - exit 0 - else - sh "$DIR"/gui.sh --share - exit 0 - fi - fi - fi - fi - - echo -e "Setup finished! Run \e[0;92m./gui.sh\e[0m to start." - echo "Please note if you'd like to expose your public server you need to run ./gui.sh --share" -elif [[ "$OSTYPE" == "darwin"* ]]; then - # The initial setup script to prep the environment on macOS - # xformers has been omitted as that is for Nvidia GPUs only - - if ! command -v brew >/dev/null; then - echo "Please install homebrew first. This is a requirement for the remaining setup." - echo "You can find that here: https://brew.sh" - #shellcheck disable=SC2016 - echo 'The "brew" command should be in $PATH to be detected.' - exit 1 - fi - - check_storage_space - - # Install base python packages - echo "Installing Python 3.10 if not found." - if ! brew ls --versions python@3.10 >/dev/null; then - echo "Installing Python 3.10." - brew install python@3.10 >&3 - else - echo "Python 3.10 found!" - fi - echo "Installing Python-TK 3.10 if not found." - if ! brew ls --versions python-tk@3.10 >/dev/null; then - echo "Installing Python TK 3.10." - brew install python-tk@3.10 >&3 - else - echo "Python Tkinter 3.10 found!" - fi - - update_kohya_ss - - if ! install_python_dependencies; then - echo "You may need to install Python. The command for this is brew install python@3.10." - fi - - configure_accelerate - echo -e "Setup finished! Run ./gui.sh to start." -elif [[ "$OSTYPE" == "cygwin" ]]; then - # Cygwin is a standalone suite of Linux utilities on Windows - echo "This hasn't been validated on cygwin yet." -elif [[ "$OSTYPE" == "msys" ]]; then - # MinGW has the msys environment which is a standalone suite of Linux utilities on Windows - # "git bash" on Windows may also be detected as msys. - echo "This hasn't been validated in msys (mingw) on Windows yet." -fi diff --git a/upgrade.bat b/upgrade.bat deleted file mode 100644 index 3dd329145..000000000 --- a/upgrade.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo off -:: Check if there are any changes that need to be committed -git status --short -if %errorlevel%==1 ( - echo There are changes that need to be committed. Please stash or undo your changes before running this script. - exit -) - -:: Pull the latest changes from the remote repository -git pull - -:: Activate the virtual environment -call .\venv\Scripts\activate.bat - -:: Validate requirements -python.exe .\setup\validate_requirements.py diff --git a/utilities.cmd b/utilities.cmd deleted file mode 100644 index f15f1466b..000000000 --- a/utilities.cmd +++ /dev/null @@ -1 +0,0 @@ -.\venv\Scripts\python.exe library\utilities.py \ No newline at end of file From 570f34a23572cb542da6c266f9f692c668b6e6ab Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 24 Jun 2023 14:07:53 -0400 Subject: [PATCH 3/4] Code refactor --- library/basic_caption_gui.py | 31 ++++++++++++++++++++----------- library/blip_caption_gui.py | 15 ++++++--------- tools/group_images.py | 7 ++++--- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/library/basic_caption_gui.py b/library/basic_caption_gui.py index 1019e35fc..95619ac53 100644 --- a/library/basic_caption_gui.py +++ b/library/basic_caption_gui.py @@ -20,36 +20,43 @@ def caption_images( find_text, replace_text, ): - # Check for images_dir + # Check if images_dir is provided if not images_dir: - msgbox('Image folder is missing...') + msgbox('Image folder is missing. Please provide the directory containing the images to caption.') return + # Check if caption_ext is provided if not caption_ext: msgbox('Please provide an extension for the caption files.') return if caption_text: log.info(f'Captioning files in {images_dir} with {caption_text}...') + + # Build the command to run caption.py run_cmd = f'python "tools/caption.py"' run_cmd += f' --caption_text="{caption_text}"' + + # Add optional flags to the command if overwrite: run_cmd += f' --overwrite' if caption_ext: run_cmd += f' --caption_file_ext="{caption_ext}"' + run_cmd += f' "{images_dir}"' log.info(run_cmd) - # Run the command + # Run the command based on the operating system if os.name == 'posix': os.system(run_cmd) else: subprocess.run(run_cmd) + # Check if overwrite option is enabled if overwrite: if prefix or postfix: - # Add prefix and postfix + # Add prefix and postfix to caption files add_pre_postfix( folder=images_dir, caption_file_ext=caption_ext, @@ -57,6 +64,7 @@ def caption_images( postfix=postfix, ) if find_text: + # Find and replace text in caption files find_replace( folder_path=images_dir, caption_file_ext=caption_ext, @@ -65,18 +73,19 @@ def caption_images( ) else: if prefix or postfix: + # Show a message if modification is not possible without overwrite option enabled msgbox( - 'Could not modify caption files with requested change because the "Overwrite existing captions in folder" option is not selected...' + 'Could not modify caption files with requested change because the "Overwrite existing captions in folder" option is not selected.' ) - log.info('...captioning done') + log.info('Captioning done.') # Gradio UI def gradio_basic_caption_gui_tab(headless=False): with gr.Tab('Basic Captioning'): gr.Markdown( - 'This utility will allow the creation of simple caption files for each image in a folder.' + 'This utility allows you to create simple caption files for each image in a folder.' ) with gr.Row(): images_dir = gr.Textbox( @@ -94,7 +103,7 @@ def gradio_basic_caption_gui_tab(headless=False): ) caption_ext = gr.Textbox( label='Caption file extension', - placeholder='Extension for caption file. eg: .caption, .txt', + placeholder='Extension for caption file (e.g., .caption, .txt)', value='.txt', interactive=True, ) @@ -111,7 +120,7 @@ def gradio_basic_caption_gui_tab(headless=False): ) caption_text = gr.Textbox( label='Caption text', - placeholder='Eg: , by some artist. Leave empty if you just want to add pre or postfix', + placeholder='e.g., "by some artist". Leave empty if you only want to add a prefix or postfix.', interactive=True, ) postfix = gr.Textbox( @@ -122,12 +131,12 @@ def gradio_basic_caption_gui_tab(headless=False): with gr.Row(): find_text = gr.Textbox( label='Find text', - placeholder='Eg: , by some artist. Leave empty if you just want to add pre or postfix', + placeholder='e.g., "by some artist". Leave empty if you only want to add a prefix or postfix.', interactive=True, ) replace_text = gr.Textbox( label='Replacement text', - placeholder='Eg: , by some artist. Leave empty if you just want to replace with nothing', + placeholder='e.g., "by some artist". Leave empty if you want to replace with nothing.', interactive=True, ) caption_button = gr.Button('Caption images') diff --git a/library/blip_caption_gui.py b/library/blip_caption_gui.py index bc2cce34c..3679140c8 100644 --- a/library/blip_caption_gui.py +++ b/library/blip_caption_gui.py @@ -3,7 +3,6 @@ import subprocess import os from .common_gui import get_folder_path, add_pre_postfix - from library.custom_logging import setup_logging # Set up logging @@ -24,21 +23,19 @@ def caption_images( prefix, postfix, ): - # Check for caption_text_input - # if caption_text_input == "": - # msgbox("Caption text is missing...") - # return - - # Check for images_dir_input + # Check if the image folder is provided if train_data_dir == '': msgbox('Image folder is missing...') return + # Check if the caption file extension is provided if caption_file_ext == '': msgbox('Please provide an extension for the caption files.') return log.info(f'Captioning files in {train_data_dir}...') + + # Construct the command to run run_cmd = f'{PYTHON} "finetune/make_captions.py"' run_cmd += f' --batch_size="{int(batch_size)}"' run_cmd += f' --num_beams="{int(num_beams)}"' @@ -79,7 +76,7 @@ def caption_images( def gradio_blip_caption_gui_tab(headless=False): with gr.Tab('BLIP Captioning'): gr.Markdown( - 'This utility will use BLIP to caption files for each images in a folder.' + 'This utility uses BLIP to caption files for each image in a folder.' ) with gr.Row(): train_data_dir = gr.Textbox( @@ -98,7 +95,7 @@ def gradio_blip_caption_gui_tab(headless=False): with gr.Row(): caption_file_ext = gr.Textbox( label='Caption file extension', - placeholder='Extention for caption file. eg: .caption, .txt', + placeholder='Extension for caption file, e.g., .caption, .txt', value='.txt', interactive=True, ) diff --git a/tools/group_images.py b/tools/group_images.py index d94ff1879..4bc420f47 100644 --- a/tools/group_images.py +++ b/tools/group_images.py @@ -92,18 +92,19 @@ def resize_and_save_images(self, cropped_images, group_index, source_paths): os.makedirs(self.output_folder, exist_ok=True) original_filename = os.path.basename(source_paths[j]) filename_without_ext = os.path.splitext(original_filename)[0] - output_path = os.path.join(self.output_folder, f"group-{group_index+1}-{j+1}-{filename_without_ext}.jpg") + final_file_name = f"group-{group_index+1}-{j+1}-{filename_without_ext}" + output_path = os.path.join(self.output_folder, f"{final_file_name}.jpg") log.info(f" Saving processed image to {output_path}") img.convert('RGB').save(output_path, quality=100) if self.caption: - self.create_caption_file(source_paths[j], group_index, filename_without_ext) + self.create_caption_file(source_paths[j], group_index, final_file_name) def create_caption_file(self, source_path, group_index, caption_filename): dirpath = os.path.dirname(source_path) caption = os.path.basename(dirpath).split('_')[-1] caption_filename = caption_filename + self.caption_ext - caption_path = os.path.join(self.output_folder, f"group-{group_index+1}-{caption_filename}") + caption_path = os.path.join(self.output_folder, caption_filename) with open(caption_path, 'w') as f: f.write(caption) From a93c4a353ea2369ed45d879633fbdbd360c2a09b Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 24 Jun 2023 16:15:11 -0400 Subject: [PATCH 4/4] Revert bitsandbytes-windows update --- networks/extract_lora_from_models copy.py | 194 ---------------------- requirements_windows_torch1.txt | 2 +- requirements_windows_torch2.txt | 2 +- setup/setup_windows.py | 4 +- 4 files changed, 4 insertions(+), 198 deletions(-) delete mode 100644 networks/extract_lora_from_models copy.py diff --git a/networks/extract_lora_from_models copy.py b/networks/extract_lora_from_models copy.py deleted file mode 100644 index aacd21b5a..000000000 --- a/networks/extract_lora_from_models copy.py +++ /dev/null @@ -1,194 +0,0 @@ -# extract approximating LoRA by svd from two SD models -# The code is based on https://github.com/cloneofsimo/lora/blob/develop/lora_diffusion/cli_svd.py -# Thanks to cloneofsimo! - -import argparse -import os -import torch -from safetensors.torch import load_file, save_file -from tqdm import tqdm -import library.model_util as model_util -import lora -import numpy as np - - -CLAMP_QUANTILE = 1 # 0.99 -MIN_DIFF = 1e-6 - - -def save_to_file(file_name, model, state_dict, dtype): - if dtype is not None: - for key in list(state_dict.keys()): - if type(state_dict[key]) == torch.Tensor: - state_dict[key] = state_dict[key].to(dtype) - - if os.path.splitext(file_name)[1] == '.safetensors': - save_file(model, file_name) - else: - torch.save(model, file_name) - - -def svd(args): - def str_to_dtype(p): - if p == 'float': - return torch.float - if p == 'fp16': - return torch.float16 - if p == 'bf16': - return torch.bfloat16 - return None - - save_dtype = str_to_dtype(args.save_precision) - - print(f"loading SD model : {args.model_org}") - text_encoder_o, _, unet_o = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_org) - print(f"loading SD model : {args.model_tuned}") - text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_tuned) - - # create LoRA network to extract weights: Use dim (rank) as alpha - lora_network_o = lora.create_network(1.0, args.dim, args.dim * 1.5, None, text_encoder_o, unet_o) - lora_network_t = lora.create_network(1.0, args.dim, args.dim * 1.5, None, text_encoder_t, unet_t) - assert len(lora_network_o.text_encoder_loras) == len( - lora_network_t.text_encoder_loras), f"model version is different (SD1.x vs SD2.x) / それぞれのモデルのバージョンが違います（SD1.xベースとSD2.xベース） " - - # get diffs - diffs = {} - text_encoder_different = False - for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.text_encoder_loras, lora_network_t.text_encoder_loras)): - lora_name = lora_o.lora_name - module_o = lora_o.org_module - module_t = lora_t.org_module - diff = module_t.weight - module_o.weight - - # Text Encoder might be same - if torch.max(torch.abs(diff)) > MIN_DIFF: - text_encoder_different = True - - diff = diff.float() - diffs[lora_name] = diff - - if not text_encoder_different: - print("Text encoder is same. Extract U-Net only.") - lora_network_o.text_encoder_loras = [] - diffs = {} - - for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.unet_loras, lora_network_t.unet_loras)): - lora_name = lora_o.lora_name - module_o = lora_o.org_module - module_t = lora_t.org_module - diff = module_t.weight - module_o.weight - diff = diff.float() - - if args.device: - diff = diff.to(args.device) - - diffs[lora_name] = diff - - # make LoRA with SVD - print("calculating by SVD") - rank = args.dim - lora_weights = {} - with torch.no_grad(): - for lora_name, mat in tqdm(list(diffs.items())): - conv2d = (len(mat.size()) == 4) - if conv2d: - mat = mat.squeeze() - - U, S, Vt = torch.linalg.svd(mat) - - U = U[:, :rank] - S = S[:rank] - U = U @ torch.diag(S) - - Vt = Vt[:rank, :] - - lora_weights[lora_name] = (U, Vt) - - # # make LoRA with svd - # print("calculating by svd") - # rank = args.dim - # lora_weights = {} - # with torch.no_grad(): - # for lora_name, mat in tqdm(list(diffs.items())): - # conv2d = (len(mat.size()) == 4) - # if conv2d: - # mat = mat.squeeze() - - # U, S, Vh = torch.linalg.svd(mat) - - # U = U[:, :rank] - # S = S[:rank] - # U = U @ torch.diag(S) - - # Vh = Vh[:rank, :] - - # # create new tensors directly from the numpy arrays - # U = torch.as_tensor(U) - # Vh = torch.as_tensor(Vh) - - # # dist = torch.cat([U.flatten(), Vh.flatten()]) - # # hi_val = torch.quantile(dist, CLAMP_QUANTILE) - # # low_val = -hi_val - - # # U = U.clamp(low_val, hi_val) - # # Vh = Vh.clamp(low_val, hi_val) - - # # # soft thresholding - # # alpha = S[-1] / 1000.0 # adjust this parameter as needed - # # U = torch.sign(U) * torch.nn.functional.relu(torch.abs(U) - alpha) - # # Vh = torch.sign(Vh) * torch.nn.functional.relu(torch.abs(Vh) - alpha) - - # lora_weights[lora_name] = (U, Vh) - - # make state dict for LoRA - lora_network_o.apply_to(text_encoder_o, unet_o, text_encoder_different, True) # to make state dict - lora_sd = lora_network_o.state_dict() - print(f"LoRA has {len(lora_sd)} weights.") - - for key in list(lora_sd.keys()): - if "alpha" in key: - continue - - lora_name = key.split('.')[0] - i = 0 if "lora_up" in key else 1 - - weights = lora_weights[lora_name][i] - # print(key, i, weights.size(), lora_sd[key].size()) - if len(lora_sd[key].size()) == 4: - weights = weights.unsqueeze(2).unsqueeze(3) - - assert weights.size() == lora_sd[key].size(), f"size unmatch: {key}" - lora_sd[key] = weights - - # load state dict to LoRA and save it - info = lora_network_o.load_state_dict(lora_sd) - print(f"Loading extracted LoRA weights: {info}") - - dir_name = os.path.dirname(args.save_to) - if dir_name and not os.path.exists(dir_name): - os.makedirs(dir_name, exist_ok=True) - - # minimum metadata - metadata = {"ss_network_dim": str(args.dim), "ss_network_alpha": str(args.dim * 1.5)} - - lora_network_o.save_weights(args.save_to, save_dtype, metadata) - print(f"LoRA weights are saved to: {args.save_to}") - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--v2", action='store_true', - help='load Stable Diffusion v2.x model / Stable Diffusion 2.xのモデルを読み込む') - parser.add_argument("--save_precision", type=str, default=None, - choices=[None, "float", "fp16", "bf16"], help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はfloat") - parser.add_argument("--model_org", type=str, default=None, - help="Stable Diffusion original model: ckpt or safetensors file / 元モデル、ckptまたはsafetensors") - parser.add_argument("--model_tuned", type=str, default=None, - help="Stable Diffusion tuned model, LoRA is difference of `original to tuned`: ckpt or safetensors file / 派生モデル（生成されるLoRAは元→派生の差分になります）、ckptまたはsafetensors") - parser.add_argument("--save_to", type=str, default=None, - help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors") - parser.add_argument("--dim", type=int, default=4, help="dimension (rank) of LoRA (default 4) / LoRAの次元数（rank）（デフォルト4）") - parser.add_argument("--device", type=str, default=None, help="device to use, cuda for GPU / 計算を行うデバイス、cuda でGPUを使う") - - args = parser.parse_args() - svd(args) diff --git a/requirements_windows_torch1.txt b/requirements_windows_torch1.txt index 05a5d4bff..5f36315ac 100644 --- a/requirements_windows_torch1.txt +++ b/requirements_windows_torch1.txt @@ -1,5 +1,5 @@ torch==1.12.1+cu116 torchvision==0.13.1+cu116 --index-url https://download.pytorch.org/whl/cu116 # no_verify https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl -U -I --no-deps # no_verify -bitsandbytes-windows +bitsandbytes==0.35.0 accelerate==0.15.0 tensorboard==2.10.1 tensorflow==2.10.1 -r requirements.txt diff --git a/requirements_windows_torch2.txt b/requirements_windows_torch2.txt index fdd4fc7b1..34ae0eabf 100644 --- a/requirements_windows_torch2.txt +++ b/requirements_windows_torch2.txt @@ -1,4 +1,4 @@ torch==2.0.1+cu118 torchvision==0.15.2+cu118 --index-url https://download.pytorch.org/whl/cu118 # no_verify -xformers==0.0.20 bitsandbytes-windows +xformers==0.0.20 bitsandbytes==0.35.0 accelerate==0.19.0 tensorboard==2.12.3 tensorflow==2.12.0 -r requirements.txt diff --git a/setup/setup_windows.py b/setup/setup_windows.py index 39197df0a..179f0f44e 100644 --- a/setup/setup_windows.py +++ b/setup/setup_windows.py @@ -120,7 +120,7 @@ def install_kohya_ss_torch1(): # 'xformers-0.0.14' # ) setup_common.install_requirements('requirements_windows_torch1.txt', check_no_verify_flag=False) - # sync_bits_and_bytes_files() + sync_bits_and_bytes_files() setup_common.configure_accelerate(run_accelerate=True) # run_cmd(f'accelerate config') @@ -144,7 +144,7 @@ def install_kohya_ss_torch2(): # ) setup_common.install_requirements('requirements_windows_torch2.txt', check_no_verify_flag=False) # install('https://huggingface.co/r4ziel/xformers_pre_built/resolve/main/triton-2.0.0-cp310-cp310-win_amd64.whl', 'triton', reinstall=reinstall) - # sync_bits_and_bytes_files() + sync_bits_and_bytes_files() setup_common.configure_accelerate(run_accelerate=True) # run_cmd(f'accelerate config')