Merge branch 'dev2' into original-u-net

.gitignore

@@ -18,3 +18,4 @@ uninstall.txt
 test/output
 test/logs
 test/ft
+requirements_tmp_for_setup.txt

Dockerfile

@@ -26,8 +26,8 @@ RUN python3 -m pip install wheel
 ## RUN python3 -m pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers
 
 # Install requirements
-COPY requirements_unix.txt setup.py ./
-RUN python3 -m pip install --use-pep517 -r requirements_unix.txt xformers
+COPY requirements_linux.txt ./setup/setup.py ./
+RUN python3 -m pip install --use-pep517 -r requirements_linux.txt xformers
 
 # Replace pillow with pillow-simd
 RUN python3 -m pip uninstall -y pillow && \

README.md

@@ -102,6 +102,8 @@ If you run on Linux, there is an alternative docker container port with less lim
 
 venv support need to be pre-installed. Can be done on ubuntu 22.04 with `apt install python3.10-venv`
 
+For Linux, make sure to install the cudaNN drivers following the instructions from: `https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64`
+
 Make sure to use a version of python >= 3.10.6 and < 3.11.0
 
 #### Setup
@@ -370,6 +372,10 @@ This will store a backup file with your current locally installed pip packages a
     - If you specify `e` or `E`, all candidates will be selected and the prompt will be repeated multiple times (`--images_per_prompt` is ignored). It may be useful for creating X/Y plots.
     - You can also specify `--am {e$$0.2|0.4|0.6|0.8|1.0},{e$$0.4|0.7|1.0} --d 1234`. In this case, 15 prompts will be generated with 5*3.
     - There is no weighting function.
+* 2023/06/19 (v21.7.10)
+- Quick fix for linux GUI startup where it would try to install darwin requirements on top of linux. Ugly fix but work. Hopefulle some linux user will improve via a PR.
+* 2023/06/18 (v21.7.9)
+- Implement temporary fix for validation of image dataset. Will no longer stop execution but will let training continue... this is changed to avoid stopping training on false positive... yet still raise awaireness that something might be wrong with the image dataset structure.
 * 2023/06/14 (v21.7.8)
 - Add tkinter to dockerised version (thanks to @burdokow)
 - Add option to create caption files from folder names to the `group_images.py` tool.

gen_img_diffusers.py

@@ -3133,6 +3133,9 @@ def process_batch(batch: List[BatchData], highres_fix, highres_1st=False):
                             seed = None
                     elif args.iter_same_seed:
                         seeds = iter_seed
+                    else:
+                        seed = None  # 前のを消す
+
                 if seed is None:
                     seed = random.randint(0, 0x7FFFFFFF)
                 if args.interactive:

gui.bat

@@ -3,14 +3,14 @@
 call .\venv\Scripts\deactivate.bat
 
 :: Calling external python program to check for local modules
-python .\tools\check_local_modules.py --no_question
+python .\setup\check_local_modules.py --no_question
 
 :: Activate the virtual environment
 call .\venv\Scripts\activate.bat
 set PATH=%PATH%;%~dp0venv\Lib\site-packages\torch\lib
 
 :: Validate requirements
-python.exe .\tools\validate_requirements.py
+python.exe .\setup\validate_requirements.py
 
 :: If the exit code is 0, run the kohya_gui.py script with the command-line arguments
 if %errorlevel% equ 0 (

gui.ps1

@@ -29,10 +29,10 @@ if ($pipOutput) {
 $env:PATH += ";$($MyInvocation.MyCommand.Path)\venv\Lib\site-packages\torch\lib"
 
 # Debug info about system
-# python.exe .\tools\debug_info.py
+# python.exe .\setup\debug_info.py
 
 # Validate the requirements and store the exit code
-python.exe .\tools\validate_requirements.py
+python.exe .\setup\validate_requirements.py
 
 # If the exit code is 0, read arguments from gui_parameters.txt (if it exists)
 # and run the kohya_gui.py script with the command-line arguments

gui.sh

@@ -17,6 +17,12 @@ cd "$SCRIPT_DIR"
 source "$SCRIPT_DIR/venv/bin/activate"
 
 # If the requirements are validated, run the kohya_gui.py script with the command-line arguments
-if python  "$SCRIPT_DIR"/tools/validate_requirements.py -r "$SCRIPT_DIR"/requirements_unix.txt; then
-    python "$SCRIPT_DIR/kohya_gui.py" "$@"
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    if python  "$SCRIPT_DIR"/setup/validate_requirements.py -r "$SCRIPT_DIR"/requirements_macos.txt; then
+        python "$SCRIPT_DIR/kohya_gui.py" "$@"
+    fi
+else
+    if python  "$SCRIPT_DIR"/setup/validate_requirements.py -r "$SCRIPT_DIR"/requirements_linux.txt; then
+        python "$SCRIPT_DIR/kohya_gui.py" "$@"
+    fi
 fi

library/common_gui.py

@@ -1287,32 +1287,41 @@ def run_cmd_advanced_training(**kwargs):
     return run_cmd
 
 def verify_image_folder_pattern(folder_path):
+    false_response = True # temporarily set to true to prevent stopping training in case of false positive
+    true_response = True
+
     # Check if the folder exists
     if not os.path.isdir(folder_path):
         log.error(f"The provided path '{folder_path}' is not a valid folder. Please follow the folder structure documentation found at docs\image_folder_structure.md ...")
-        return False
+        return false_response
 
     # Create a regular expression pattern to match the required sub-folder names
+    # The pattern should start with one or more digits (\d+) followed by an underscore (_)
+    # After the underscore, it should match one or more word characters (\w+), which can be letters, numbers, or underscores
+    # Example of a valid pattern matching name: 123_example_folder
     pattern = r'^\d+_\w+'
 
-    # Get the list of sub-folders matching the pattern
-    matching_subfolders = [
-        subfolder
+    # Get the list of sub-folders in the directory
+    subfolders = [
+        os.path.join(folder_path, subfolder)
         for subfolder in os.listdir(folder_path)
-        if os.path.isdir(os.path.join(folder_path, subfolder)) and re.match(pattern, subfolder)
+        if os.path.isdir(os.path.join(folder_path, subfolder))
     ]
 
-    # Check if all sub-folders match the pattern
-    filenames = [filename for filename in os.listdir(folder_path) if not filename.startswith('.')]
-    if len(matching_subfolders) != len(filenames):
-        log.error(f"Not all image folders have proper name patterns <numbre>_<text> in {folder_path}. Please follow the folder structure documentation found at docs/image_folder_structure.md ...")
-        log.error(f"Only folders are allowed in {folder_path}...")
-        return False
+    # Check the pattern of each sub-folder
+    matching_subfolders = [subfolder for subfolder in subfolders if re.match(pattern, os.path.basename(subfolder))]
+
+    # Print non-matching sub-folders
+    non_matching_subfolders = set(subfolders) - set(matching_subfolders)
+    if non_matching_subfolders:
+        log.error(f"The following folders do not match the required pattern <number>_<text>: {', '.join(non_matching_subfolders)}")
+        log.error(f"Please follow the folder structure documentation found at docs\image_folder_structure.md ...")
+        return false_response
 
     # Check if no sub-folders exist
     if not matching_subfolders:
         log.error(f"No image folders found in {folder_path}. Please follow the folder structure documentation found at docs\image_folder_structure.md ...")
-        return False
+        return false_response
 
     log.info(f'Valid image folder names found in: {folder_path}')
-    return True
+    return true_response

library/convert_model_gui.py

@@ -174,6 +174,10 @@ def gradio_convert_model_tab(headless=False):
         gr.Markdown(
             'This utility can be used to convert from one stable diffusion model format to another.'
         )
+
+        model_ext = gr.Textbox(value='*.safetensors *.ckpt', visible=False)
+        model_ext_name = gr.Textbox(value='Model types', visible=False)
+
         with gr.Row():
             source_model_input = gr.Textbox(
                 label='Source model',
@@ -198,7 +202,7 @@ def gradio_convert_model_tab(headless=False):
             )
             button_source_model_file.click(
                 get_file_path,
-                inputs=[source_model_input],
+                inputs=[source_model_input, model_ext, model_ext_name],
                 outputs=source_model_input,
                 show_progress=False,
             )

library/custom_logging.py

@@ -1,6 +1,7 @@
 import os
 import logging
 import time
+import sys
 
 from rich.theme import Theme
 from rich.logging import RichHandler
@@ -23,7 +24,10 @@ def setup_logging(clean=False, debug=False):
     except:
         pass
 
-    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True)
+    if sys.version_info >= (3, 9):
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', encoding='utf-8', force=True)
+    else:
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s | %(levelname)s | %(pathname)s | %(message)s', filename='setup.log', filemode='a', force=True)
 
     console = Console(log_time=True, log_time_format='%H:%M:%S-%f', theme=Theme({
         "traceback.border": "black",

pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "library"
+version = "1.0.3"
+description = "Libraries required to run kohya_ss GUI"
+authors = ["Bernard Maltais <[email protected]>"]
+license = "Apache-2.0" # Apache Software License
+
+[[tool.poetry.source]]
+name = "library"
+path = "library"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.11"
+
+[tool.poetry.dev-dependencies]

requirements_unix.txt → requirements_linux.txt

@@ -8,20 +8,17 @@ easygui==0.98.3
 einops==0.6.0
 fairscale==0.4.13
 ftfy==6.1.1
-gradio==3.23.0; sys_platform == 'darwin'
-gradio==3.32.0; sys_platform != 'darwin'
-huggingface-hub==0.13.3; sys_platform == 'darwin'
-huggingface-hub==0.13.3; sys_platform != 'darwin'
+gradio==3.32.0
+huggingface-hub==0.13.3
 lion-pytorch==0.0.6
 lycoris_lora==0.1.6
 opencv-python==4.7.0.68
 prodigyopt==1.0
 pytorch-lightning==1.9.0
 rich==13.4.1
 safetensors==0.2.6
-tensorboard==2.10.1 ; sys_platform != 'darwin'
-tensorboard==2.12.1 ; sys_platform == 'darwin'
-tensorflow==2.10.1; sys_platform != 'darwin'
+tensorboard==2.12.1
+tensorflow==2.12.0
 timm==0.6.12
 tk==0.1.0
 toml==0.10.2

requirements_macos.txt

@@ -0,0 +1,27 @@
+accelerate==0.15.0
+albumentations==1.3.0
+altair==4.2.2
+bitsandbytes==0.35.0
+dadaptation==3.1
+diffusers[torch]==0.10.2
+easygui==0.98.3
+einops==0.6.0
+fairscale==0.4.13
+ftfy==6.1.1
+gradio==3.23.0
+huggingface-hub==0.13.0
+lion-pytorch==0.0.6
+lycoris_lora==0.1.6
+opencv-python==4.7.0.68
+pytorch-lightning==1.9.0
+rich==13.4.1
+safetensors==0.2.6
+tensorboard==2.12.1
+timm==0.6.12
+tk==0.1.0
+toml==0.10.2
+transformers==4.26.0
+voluptuous==0.13.1
+wandb==0.15.0
+# for kohya_ss library
+.

setup.bat

@@ -20,11 +20,11 @@ mkdir ".\logs\setup" > nul 2>&1
 call .\venv\Scripts\deactivate.bat
 
 :: Calling external python program to check for local modules
-python .\tools\check_local_modules.py
+python .\setup\check_local_modules.py
 
 call .\venv\Scripts\activate.bat
 
-python .\tools\setup_windows.py
+python .\setup\setup_windows.py
 
 :: Deactivate the virtual environment
 call .\venv\Scripts\deactivate.bat

setup.ps1

@@ -17,11 +17,11 @@ $null = New-Item -ItemType Directory -Force -Path ".\logs\setup"
 & .\venv\Scripts\deactivate.bat
 
 # Calling external python program to check for local modules
-& .\venv\Scripts\python.exe .\tools\check_local_modules.py
+& .\venv\Scripts\python.exe .\setup\check_local_modules.py
 
 & .\venv\Scripts\activate.bat
 
-& .\venv\Scripts\python.exe .\tools\setup_windows.py
+& .\venv\Scripts\python.exe .\setup\setup_windows.py
 
 # Deactivate the virtual environment
 & .\venv\Scripts\deactivate.bat