Improve setup solution

.gitignore

@@ -20,3 +20,4 @@ test/logs
 test/*.json
 test/ft
 requirements_tmp_for_setup.txt
+0.13.3

pyproject.toml

@@ -1,24 +1,19 @@
 [build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
 
-[project]
-name = "kohya_ss"
+[tool.poetry]
+name = "library"
 version = "1.0.3"
-description = "A GUI wrapper for kohya-ss SD scipts enabling LoRA training with an easy-to-use web application."
-authors = [
-    {name = "bmaltais", email = "[email protected]"},
-]
-readme = "README.md"
-classifiers = [
-    "Development Status :: 5 - Production/Stable",
-    "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
-]
+description = "Libraries required to run kohya_ss GUI"
+authors = ["Bernard Maltais <[email protected]>"]
+license = "Apache-2.0" # Apache Software License
 
-[tool.poetry]
-license = "Apache-2.0"
+[[tool.poetry.source]]
+name = "library"
+path = "library"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.11"
 
-[tool.setuptools.packages.find]
-where = ["library"] # We have to explicitly tell build tools where to look
+[tool.poetry.dev-dependencies]

requirements_macos.txt → requirements.txt

@@ -1,4 +1,3 @@
-accelerate==0.15.0
 albumentations==1.3.0
 altair==4.2.2
 bitsandbytes==0.35.0
@@ -8,20 +7,20 @@ easygui==0.98.3
 einops==0.6.0
 fairscale==0.4.13
 ftfy==6.1.1
-gradio==3.23.0
-huggingface-hub==0.13.0
+gradio==3.33.1
+huggingface-hub>=0.13.3
 lion-pytorch==0.0.6
 lycoris_lora==0.1.6
 opencv-python==4.7.0.68
+prodigyopt==1.0
 pytorch-lightning==1.9.0
 rich==13.4.1
 safetensors==0.2.6
-tensorboard==2.12.1
 timm==0.6.12
 tk==0.1.0
 toml==0.10.2
 transformers==4.26.0
 voluptuous==0.13.1
 wandb==0.15.0
 # for kohya_ss library
-.
+-e . # no_verify leave this to specify not checking this a verification stage

requirements_linux.txt

@@ -1,29 +1,4 @@
-accelerate==0.19.0
-albumentations==1.3.0
-altair==4.2.2
-bitsandbytes==0.35.0
-dadaptation==3.1
-diffusers[torch]==0.10.2
-easygui==0.98.3
-einops==0.6.0
-fairscale==0.4.13
-ftfy==6.1.1
-gradio==3.32.0
-huggingface-hub==0.13.3
-lion-pytorch==0.0.6
-lycoris_lora==0.1.6
-opencv-python==4.7.0.68
-prodigyopt==1.0
-pytorch-lightning==1.9.0
-rich==13.4.1
-safetensors==0.2.6
-tensorboard==2.12.1
-tensorflow==2.12.0
-timm==0.6.12
-tk==0.1.0
-toml==0.10.2
-transformers==4.26.0
-voluptuous==0.13.1
-wandb==0.15.0
-# for kohya_ss library
-.
+torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 # no_verify leave this to specify not checking this a verification stage
+xformers==0.0.20
+accelerate==0.19.0 tensorboard==2.12.1 tensorflow==2.12.0
+-r requirements.txt

requirements_macos_amd64.txt

@@ -0,0 +1,31 @@
+torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+xformers
+accelerate==0.19.0 tensorflow-macos tensorboard==2.12.1
+-r requirements.txt
+
+# accelerate==0.15.0
+# albumentations==1.3.0
+# altair==4.2.2
+# bitsandbytes==0.35.0
+# dadaptation==3.1
+# diffusers[torch]==0.10.2
+# easygui==0.98.3
+# einops==0.6.0
+# fairscale==0.4.13
+# ftfy==6.1.1
+# gradio==3.23.0
+# huggingface-hub==0.13.0
+# lion-pytorch==0.0.6
+# lycoris_lora==0.1.6
+# opencv-python==4.7.0.68
+# pytorch-lightning==1.9.0
+# rich==13.4.1
+# safetensors==0.2.6
+# timm==0.6.12
+# tk==0.1.0
+# toml==0.10.2
+# transformers==4.26.0
+# voluptuous==0.13.1
+# wandb==0.15.0
+# # for kohya_ss library
+# .

requirements_macos_arm64.txt

@@ -0,0 +1,31 @@
+torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+xformers
+accelerate==0.19.0 tensorflow-metal tensorboard==2.12.1
+-r requirements.txt
+
+# accelerate==0.15.0
+# albumentations==1.3.0
+# altair==4.2.2
+# bitsandbytes==0.35.0
+# dadaptation==3.1
+# diffusers[torch]==0.10.2
+# easygui==0.98.3
+# einops==0.6.0
+# fairscale==0.4.13
+# ftfy==6.1.1
+# gradio==3.23.0
+# huggingface-hub==0.13.0
+# lion-pytorch==0.0.6
+# lycoris_lora==0.1.6
+# opencv-python==4.7.0.68
+# pytorch-lightning==1.9.0
+# rich==13.4.1
+# safetensors==0.2.6
+# timm==0.6.12
+# tk==0.1.0
+# toml==0.10.2
+# transformers==4.26.0
+# voluptuous==0.13.1
+# wandb==0.15.0
+# # for kohya_ss library
+# .

requirements_windows_torch1.txt

@@ -1,29 +1,31 @@
-accelerate==0.15.0
-albumentations==1.3.0
-altair==4.2.2
-bitsandbytes==0.35.0
-dadaptation==3.1
-diffusers[torch]==0.10.2
-easygui==0.98.3
-einops==0.6.0
-fairscale==0.4.13
-ftfy==6.1.1
-gradio==3.32.0
-huggingface-hub==0.13.3
-lion-pytorch==0.0.6
-lycoris_lora==0.1.6
-opencv-python==4.7.0.68
-prodigyopt==1.0
-pytorch-lightning==1.9.0
-rich==13.4.1
-safetensors==0.2.6
-tensorboard==2.10.1
-tensorflow==2.10.1
-timm==0.6.12
-tk==0.1.0
-toml==0.10.2
-transformers==4.26.0
-voluptuous==0.13.1
-wandb==0.15.0
-# for kohya_ss library
-.
+torch==1.12.1+cu116 torchvision==0.13.1+cu116 --index-url https://download.pytorch.org/whl/cu116 # no_verify
+https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl -U -I --no-deps # no_verify
+accelerate==0.15.0 tensorboard==2.10.1 tensorflow==2.10.1
+-r requirements.txt
+
+# albumentations==1.3.0
+# altair==4.2.2
+# bitsandbytes==0.35.0
+# dadaptation==3.1
+# diffusers[torch]==0.10.2
+# easygui==0.98.3
+# einops==0.6.0
+# fairscale==0.4.13
+# ftfy==6.1.1
+# gradio==3.32.0
+# huggingface-hub==0.13.3
+# lion-pytorch==0.0.6
+# lycoris_lora==0.1.6
+# opencv-python==4.7.0.68
+# prodigyopt==1.0
+# pytorch-lightning==1.9.0
+# rich==13.4.1
+# safetensors==0.2.6
+# timm==0.6.12
+# tk==0.1.0
+# toml==0.10.2
+# transformers==4.26.0
+# voluptuous==0.13.1
+# wandb==0.15.0
+# # for kohya_ss library
+# .

requirements_windows_torch2.txt

@@ -1,30 +1,32 @@
-accelerate==0.19.0
-albumentations==1.3.0
-altair==4.2.2
-bitsandbytes==0.35.0
-dadaptation==3.1
-diffusers[torch]==0.10.2
-easygui==0.98.3
-einops==0.6.0
-fairscale==0.4.13
-ftfy==6.1.1
-gradio==3.33.1
-huggingface-hub==0.15.1
-lion-pytorch==0.0.6
-lycoris_lora==0.1.6
-opencv-python==4.7.0.68
-prodigyopt==1.0
-pytorch-lightning==1.9.0
-rich==13.4.1
-safetensors==0.2.6
-tensorboard==2.12.3
-tensorflow==2.12.0
-timm==0.6.12
-tk==0.1.0
-toml==0.10.2
-transformers==4.26.0
-voluptuous==0.13.1
-wandb==0.15.0
+torch==2.0.1+cu118 torchvision==0.15.2+cu118 --index-url https://download.pytorch.org/whl/cu118 # no_verify
 xformers==0.0.20
-# for kohya_ss library
-.
+accelerate==0.19.0 tensorboard==2.12.3 tensorflow==2.12.0
+-r requirements.txt
+
+# albumentations==1.3.0
+# altair==4.2.2
+# bitsandbytes==0.35.0
+# dadaptation==3.1
+# diffusers[torch]==0.10.2
+# easygui==0.98.3
+# einops==0.6.0
+# fairscale==0.4.13
+# ftfy==6.1.1
+# gradio==3.33.1
+# huggingface-hub==0.15.1
+# lion-pytorch==0.0.6
+# lycoris_lora==0.1.6
+# opencv-python==4.7.0.68
+# prodigyopt==1.0
+# pytorch-lightning==1.9.0
+# rich==13.4.1
+# safetensors==0.2.6
+# timm==0.6.12
+# tk==0.1.0
+# toml==0.10.2
+# transformers==4.26.0
+# voluptuous==0.13.1
+# wandb==0.15.0
+# xformers==0.0.20
+# # for kohya_ss library
+# .

setup.sh

@@ -239,26 +239,21 @@ install_python_dependencies() {
   fi
 
   # Updating pip if there is one
-  echo "Checking for pip updates before Python operations."
-  pip install --upgrade pip
+  # echo "Checking for pip updates before Python operations."
+  # pip install --upgrade pip
 
-  echo "Installing python dependencies. This could take a few minutes as it downloads files."
-  echo "If this operation ever runs too long, you can rerun this script in verbose mode to check."
+  # echo "Installing python dependencies. This could take a few minutes as it downloads files."
+  # echo "If this operation ever runs too long, you can rerun this script in verbose mode to check."
 
   case "$OSTYPE" in
-    "linux-gnu"*)
-      pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 \
-        --extra-index-url https://download.pytorch.org/whl/cu118
-      pip install --upgrade xformers==0.0.20
+    "lin"*)
+      python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_linux.txt
       ;;
     "darwin"*)
-      pip install torch==2.0.0 torchvision==0.15.1 \
-        -f https://download.pytorch.org/whl/cpu/torch_stable.html
-      # Check if the processor is Apple Silicon (arm64)
       if [[ "$(uname -m)" == "arm64" ]]; then
-        pip install tensorflow-metal=="$TENSORFLOW_MACOS_VERSION"
+        python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_macos_arm64.txt
       else
-        pip install tensorflow-macos=="$TENSORFLOW_METAL_VERSION"
+        python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_macos_amd64.txt
       fi
       ;;
   esac
@@ -272,24 +267,24 @@ install_python_dependencies() {
   # pip install pydevd-pycharm~=223.8836.43
 
   # Create a temporary requirements file
-  TEMP_REQUIREMENTS_FILE=$(mktemp)
-
-  if [[ "$OSTYPE" == "darwin"* ]]; then
-    echo "Copying $DIR/requirements_macos.txt to $TEMP_REQUIREMENTS_FILE" >&3
-    echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3
-    awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_macos.txt" >"$TEMP_REQUIREMENTS_FILE"
-  else
-    echo "Copying $DIR/requirements_linux.txt to $TEMP_REQUIREMENTS_FILE" >&3
-    echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3
-    awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_linux.txt" >"$TEMP_REQUIREMENTS_FILE"
-  fi
-
-  # Install the Python dependencies from the temporary requirements file
-  if [ $VERBOSITY == 2 ]; then
-    python -m pip install --quiet --upgrade -r "$TEMP_REQUIREMENTS_FILE"
-  else
-    python -m pip install --upgrade -r "$TEMP_REQUIREMENTS_FILE"
-  fi
+  # TEMP_REQUIREMENTS_FILE=$(mktemp)
+
+  # if [[ "$OSTYPE" == "darwin"* ]]; then
+  #   echo "Copying $DIR/requirements_macos.txt to $TEMP_REQUIREMENTS_FILE" >&3
+  #   echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3
+  #   awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_macos.txt" >"$TEMP_REQUIREMENTS_FILE"
+  # else
+  #   echo "Copying $DIR/requirements_linux.txt to $TEMP_REQUIREMENTS_FILE" >&3
+  #   echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3
+  #   awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_linux.txt" >"$TEMP_REQUIREMENTS_FILE"
+  # fi
+
+  # # Install the Python dependencies from the temporary requirements file
+  # if [ $VERBOSITY == 2 ]; then
+  #   python -m pip install --quiet --upgrade -r "$TEMP_REQUIREMENTS_FILE"
+  # else
+  #   python -m pip install --upgrade -r "$TEMP_REQUIREMENTS_FILE"
+  # fi
 
   if [ -n "$VIRTUAL_ENV" ] && ! inDocker; then
     if command -v deactivate >/dev/null; then