update

TODO.md

@@ -55,6 +55,7 @@ Tech that can be integrated as part of the core workflow...
 - [Null-text inversion](https://github.com/ouhenio/null-text-inversion-colab)
 - [Custom diffusion](https://github.com/guaneec/custom-diffusion-webui)
   - <https://www.cs.cmu.edu/~custom-diffusion/>
+- [Dream artist](https://github.com/7eu7d7/DreamArtist-sd-webui-extension)
 
 ## Video Generation
 
@@ -80,3 +81,22 @@ Cool stuff that is not integrated anywhere...
 - Bunch of stuff:<https://pharmapsychotic.com/tools.html>
 - Prevalent colors to interrogate
 - Auto-Sort inputs by face recognition
+
+
+core library updates:
+- run `./automatic.sh install`
+- note, this is quite a big one so some testing is reccomended after upgrade
+ui updates
+new script:
+- `cli/watermark.py` to a) strip exif from images, b) add invisible watermark to images which persists even if user modifies image so we can always track it
+expose variation seed in main ui
+integrated seed travel functionality into core
+integrated `pix2pix` functionality to standard `img2img` workflow
+- note: requires **pix2pix** model to be loaded
+integrated large `cfg scale` values fix
+tested `aesthetic gradients` training, not worth it
+updated `image browser`
+initial work on **queue management** allowing to submit multiple requests to server
+initial work on `lora` integration (hidden)
+initial work on `custom diffusion` integration (hidden)
+spent quite some time making stable-diffusion compatible with upcomming `pytorch` 2.0 release

automatic.sh

@@ -8,7 +8,7 @@ PYTORCH_CUDA_ALLOC_CONF=garbage_collection_threshold:0.9,max_split_size_mb:512
 CUDA_LAUNCH_BLOCKING=0
 CUDA_CACHE_DISABLE=0
 CUDA_AUTO_BOOST=1
-CUDA_DEVICE_DEFAULT_PERSISTING_L2_CACHE_PERCENTAGE_LIMIT=0
+CUDA_DEVICE_DEFAULT_PERSISTING_L2_CACHE_PERCENTAGE_LIMIT=50
 
 if [ "$PYTHON" == "" ]; then
   PYTHON=`which python`

cli/palette.py

@@ -0,0 +1,148 @@
+# based on <https://towardsdatascience.com/image-color-extraction-with-python-in-4-steps-8d9370d9216e>
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import matplotlib.patches as patches
+import matplotlib.image as mpimg
+
+from PIL import Image
+from matplotlib.offsetbox import OffsetImage, AnnotationBbox
+
+#!pip install easydev                 #version 0.12.0
+#!pip install colormap                #version 1.0.4
+#!pip install opencv-python           #version 4.5.5.64
+#!pip install colorgram.py            #version 1.2.0
+#!pip install extcolors               #version 1.0.0
+
+import cv2
+import extcolors
+
+from colormap import rgb2hex
+
+
+input_name = '<photo location/name>'
+output_width = 900                   #set the output size
+img = Image.open(input_name)
+wpercent = (output_width/float(img.size[0]))
+hsize = int((float(img.size[1])*float(wpercent)))
+img = img.resize((output_width,hsize), Image.ANTIALIAS)
+
+#save
+resize_name = 'resize_' + input_name  #the resized image name
+img.save(resize_name)                 #output location can be specified before resize_name
+
+#read
+plt.figure(figsize=(9, 9))
+img_url = resize_name
+img = plt.imread(img_url)
+plt.imshow(img)
+plt.axis('off')
+plt.show()
+
+colors_x = extcolors.extract_from_path(img_url, tolerance = 12, limit = 12)
+colors_x
+
+
+def color_to_df(input):
+    colors_pre_list = str(input).replace('([(','').split(', (')[0:-1]
+    df_rgb = [i.split('), ')[0] + ')' for i in colors_pre_list]
+    df_percent = [i.split('), ')[1].replace(')','') for i in colors_pre_list]
+    
+    #convert RGB to HEX code
+    df_color_up = [rgb2hex(int(i.split(", ")[0].replace("(","")),
+                          int(i.split(", ")[1]),
+                          int(i.split(", ")[2].replace(")",""))) for i in df_rgb]
+    
+    df = pd.DataFrame(zip(df_color_up, df_percent), columns = ['c_code','occurence'])
+    return df
+
+df_color = color_to_df(colors_x)
+df_color
+
+
+list_color = list(df_color['c_code'])
+list_precent = [int(i) for i in list(df_color['occurence'])]
+text_c = [c + ' ' + str(round(p*100/sum(list_precent),1)) +'%' for c, p in zip(list_color,
+                                                                               list_precent)]
+fig, ax = plt.subplots(figsize=(90,90),dpi=10)
+wedges, text = ax.pie(list_precent,
+                      labels= text_c,
+                      labeldistance= 1.05,
+                      colors = list_color,
+                      textprops={'fontsize': 120, 'color':'black'}
+                     )
+plt.setp(wedges, width=0.3)
+
+#create space in the center
+plt.setp(wedges, width=0.36)
+
+ax.set_aspect("equal")
+fig.set_facecolor('white')
+plt.show()
+
+
+#create background color
+fig, ax = plt.subplots(figsize=(192,108),dpi=10)
+fig.set_facecolor('white')
+plt.savefig('bg.png')
+plt.close(fig)
+
+#create color palette
+bg = plt.imread('bg.png')
+fig = plt.figure(figsize=(90, 90), dpi = 10)
+ax = fig.add_subplot(1,1,1)
+
+x_posi, y_posi, y_posi2 = 320, 25, 25
+for c in list_color:
+    if  list_color.index(c) <= 5:
+        y_posi += 125
+        rect = patches.Rectangle((x_posi, y_posi), 290, 115, facecolor = c)
+        ax.add_patch(rect)
+        ax.text(x = x_posi+360, y = y_posi+80, s = c, fontdict={'fontsize': 150})
+    else:
+        y_posi2 += 125
+        rect = patches.Rectangle((x_posi + 800, y_posi2), 290, 115, facecolor = c)
+        ax.add_artist(rect)
+        ax.text(x = x_posi+1160, y = y_posi2+80, s = c, fontdict={'fontsize': 150})
+        
+ax.axis('off')
+plt.imshow(bg)
+plt.tight_layout()
+
+img = mpimg.imread('<photo location/name>')
+bg = plt.imread('bg.png')
+
+fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(160,120), dpi = 10)
+
+#donut plot
+wedges, text = ax1.pie(list_precent,
+                      labels= text_c,
+                      labeldistance= 1.05,
+                      colors = list_color,
+                      textprops={'fontsize': 160, 'color':'black'})
+plt.setp(wedges, width=0.3)
+
+#add image in the center of donut plot
+imagebox = OffsetImage(img, zoom=2.3)
+ab = AnnotationBbox(imagebox, (0, 0))
+ax1.add_artist(ab)
+
+#color palette
+x_posi, y_posi, y_posi2 = 160, -170, -170
+for c in list_color:
+    if list_color.index(c) <= 5:
+        y_posi += 180
+        rect = patches.Rectangle((x_posi, y_posi), 360, 160, facecolor = c)
+        ax2.add_patch(rect)
+        ax2.text(x = x_posi+400, y = y_posi+100, s = c, fontdict={'fontsize': 190})
+    else:
+        y_posi2 += 180
+        rect = patches.Rectangle((x_posi + 1000, y_posi2), 360, 160, facecolor = c)
+        ax2.add_artist(rect)
+        ax2.text(x = x_posi+1400, y = y_posi2+100, s = c, fontdict={'fontsize': 190})
+
+ax2.axis('off')
+fig.set_facecolor('white')
+plt.imshow(bg)       
+plt.tight_layout()

cli/watermark.py

@@ -0,0 +1,128 @@
+#!/bin/env python
+import os
+import io
+import pathlib
+import argparse
+import filetype
+import numpy as np
+from imwatermark import WatermarkEncoder, WatermarkDecoder
+from PIL import Image
+from PIL.ExifTags import TAGS
+from PIL.TiffImagePlugin import ImageFileDirectory_v2
+from modules.util import log, Map
+import piexif
+import piexif.helper
+
+
+options = Map({ 'method': 'dwtDctSvd', 'type': 'bytes' })
+
+
+def get_exif(image):
+    # using piexif
+    res1 = {}
+    try:
+        exif = piexif.load(image.info["exif"])
+        exif = exif.get("Exif", {})
+        for k, v in exif.items():
+            key = list(vars(piexif.ExifIFD).keys())[list(vars(piexif.ExifIFD).values()).index(k)]
+            res1[key] = piexif.helper.UserComment.load(v)
+    except:
+        pass
+    # using pillow
+    res2 = {}
+    try:
+        res2 = { TAGS[k]: v for k, v in image.getexif().items() if k in TAGS }
+    except:
+        pass
+    return {**res1, **res2}
+
+
+def set_exif(d: dict):
+    ifd = ImageFileDirectory_v2()
+    _TAGS = dict(((v, k) for k, v in TAGS.items())) # enumerate possible exif tags
+    for k, v in d.items():
+        ifd[_TAGS[k]] = v
+    exif_stream = io.BytesIO()
+    ifd.save(exif_stream)
+    bytes = b'Exif\x00\x00' + exif_stream.getvalue()
+    return bytes
+
+
+def get_watermark(image, args):
+    data = np.asarray(image)
+    decoder = WatermarkDecoder(options.type, args.length)
+    bytes = decoder.decode(data, options.method)
+    try:
+        watermark = str(bytes, 'UTF-8').replace('\x00', '')
+    except:
+        watermark = ''
+    return watermark
+
+
+def set_watermark(image, args):
+    data = np.asarray(image)
+    encoder = WatermarkEncoder()
+    encoder.set_watermark(options.type, args.wm.encode('utf-8'))
+    encoded = encoder.encode(data, options.method)
+    image = Image.fromarray(encoded)
+    return image
+
+
+def watermark(args, file):
+    if not os.path.exists(file):
+        log.error({ 'watermark': 'file not found' })
+        return
+    if not filetype.is_image(file):
+        log.error({ 'watermark': 'file is not an image' })
+        return
+    image = Image.open(file)
+    if image.width * image.height < 256 * 256:
+        log.error({ 'watermark': 'image too small' })
+        return
+
+    exif = get_exif(image)
+
+    if args.command == 'read':
+        watermark = get_watermark(image, args)
+        log.info({ 'file': file, 'watermark': watermark, 'exif': exif, 'resolution': f'{image.width}x{image.height}' })
+
+    elif args.command == 'write':
+        metadata = b'' if args.strip else set_exif(exif)
+        if args.output != '':
+            pathlib.Path(args.output).mkdir(parents = True, exist_ok = True)
+        image=set_watermark(image, args)
+        fn = os.path.join(args.output, file)
+        image.save(fn, exif=metadata)
+
+        if args.verify:
+            data = np.asarray(image)
+            decoder = WatermarkDecoder(options.type, args.length)
+            bytes = decoder.decode(data, options.method)
+            if bytes.startswith(b'\xff'):
+                watermark = ''
+            else:
+                watermark = str(bytes, 'UTF-8').replace('\x00', '')
+        else:
+            watermark = args.wm
+
+        log.info({ 'file': fn, 'watermark': watermark, 'exif': None if args.strip else exif, 'resolution': f'{image.width}x{image.height}' })
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description = 'image watermarking')
+    parser.add_argument('command', choices = ['read', 'write'])
+    parser.add_argument('--wm', type=str, required=False, default='mm', help='watermark string')
+    parser.add_argument('--strip', default=False, action='store_true', help = "strip existing exif data")
+    parser.add_argument('--verify', default=False, action='store_true', help = "verify watermark during write")
+    parser.add_argument('--length', type=int, default=16, help="watermark length in bits")
+    parser.add_argument('--output', type=str, required=False, default='', help='folder to store images, default is overwrite in-place')
+    parser.add_argument('input', type=str, nargs='*')
+    args = parser.parse_args()
+    log.info({ 'watermark args': vars(args), 'options': options })
+    for arg in args.input:
+        if os.path.isfile(arg):
+            watermark(args, arg)
+        elif os.path.isdir(arg):
+            for root, _dirs, files in os.walk(arg):
+                for f in files:
+                    watermark(args, os.path.join(root, f))

config.json

@@ -20,9 +20,6 @@
     "directories_max_prompt_words": 8,
     "disable_weights_auto_swap": false,
     "disabled_extensions": [
-        "embedding-inspector",
-        "sd-webui-additional-networks",
-        "stable-diffusion-webui-instruct-pix2pix",
         "ScuNET"
     ],
     "do_not_add_watermark": true,
@@ -134,7 +131,7 @@
     "sd_checkpoint_hash": "cc6cb27103417325ff94f52b7a5d2dde45a7515b25c255d8e396c90014281516",
     "sd_hypernetwork_strength": 1.0,
     "sd_hypernetwork": "None",
-    "sd_lora": "None",
+    "sd_lora": "",
     "sd_model_checkpoint": "sd-v15-runwayml.ckpt [cc6cb27103]",
     "sd_vae_as_default": false,
     "sd_vae_checkpoint_cache": 0,
@@ -185,5 +182,12 @@
         "DPM2 a Karras",
         "LMS Karras"
     ],
-    "images_logger_warning": false
+    "images_logger_warning": false,
+    "images_logger_debug": false,
+    "images_scan_exif": false,
+    "additional_networks_extra_lora_path": "",
+    "additional_networks_sort_models_by": "name",
+    "additional_networks_model_name_filter": "",
+    "additional_networks_xy_grid_model_metadata": "",
+    "additional_networks_hash_thread_count": 1.0
 }

requirements_versions.txt

@@ -23,4 +23,4 @@ scikit-image==0.19.3
 timm==0.6.12
 torchdiffeq==0.2.3
 torchsde==0.2.5
-transformers==4.25.1
+transformers==4.26.0

ui-config.json

@@ -1012,5 +1012,105 @@
     "train/Low-rank approximation sum threshold (lower value means smaller file size, 1 to disable)/value": 0.5,
     "train/Low-rank approximation sum threshold (lower value means smaller file size, 1 to disable)/minimum": 0,
     "train/Low-rank approximation sum threshold (lower value means smaller file size, 1 to disable)/maximum": 1,
-    "train/Low-rank approximation sum threshold (lower value means smaller file size, 1 to disable)/step": 0.01
+    "train/Low-rank approximation sum threshold (lower value means smaller file size, 1 to disable)/step": 0.01,
+    "customscript/prompt_matrix.py/txt2img/Select joining char/visible": true,
+    "customscript/prompt_matrix.py/txt2img/Select joining char/value": "comma",
+    "customscript/prompt_matrix.py/txt2img/Grid margins (px)/visible": true,
+    "customscript/prompt_matrix.py/txt2img/Grid margins (px)/value": 0,
+    "customscript/prompt_matrix.py/txt2img/Grid margins (px)/minimum": 0,
+    "customscript/prompt_matrix.py/txt2img/Grid margins (px)/maximum": 100,
+    "customscript/prompt_matrix.py/txt2img/Grid margins (px)/step": 2,
+    "customscript/xyz_grid.py/txt2img/Grid margins (px)/visible": true,
+    "customscript/xyz_grid.py/txt2img/Grid margins (px)/value": 0,
+    "customscript/xyz_grid.py/txt2img/Grid margins (px)/minimum": 0,
+    "customscript/xyz_grid.py/txt2img/Grid margins (px)/maximum": 100,
+    "customscript/xyz_grid.py/txt2img/Grid margins (px)/step": 2,
+    "customscript/prompt_matrix.py/img2img/Select joining char/visible": true,
+    "customscript/prompt_matrix.py/img2img/Select joining char/value": "comma",
+    "customscript/prompt_matrix.py/img2img/Grid margins (px)/visible": true,
+    "customscript/prompt_matrix.py/img2img/Grid margins (px)/value": 0,
+    "customscript/prompt_matrix.py/img2img/Grid margins (px)/minimum": 0,
+    "customscript/prompt_matrix.py/img2img/Grid margins (px)/maximum": 100,
+    "customscript/prompt_matrix.py/img2img/Grid margins (px)/step": 2,
+    "customscript/xyz_grid.py/img2img/Grid margins (px)/visible": true,
+    "customscript/xyz_grid.py/img2img/Grid margins (px)/value": 0,
+    "customscript/xyz_grid.py/img2img/Grid margins (px)/minimum": 0,
+    "customscript/xyz_grid.py/img2img/Grid margins (px)/maximum": 100,
+    "customscript/xyz_grid.py/img2img/Grid margins (px)/step": 2,
+    "customscript/aesthetic.py/txt2img/Aesthetic weight/visible": true,
+    "customscript/aesthetic.py/txt2img/Aesthetic weight/value": 0.9,
+    "customscript/aesthetic.py/txt2img/Aesthetic weight/minimum": 0,
+    "customscript/aesthetic.py/txt2img/Aesthetic weight/maximum": 1,
+    "customscript/aesthetic.py/txt2img/Aesthetic weight/step": 0.01,
+    "customscript/aesthetic.py/txt2img/Aesthetic steps/visible": true,
+    "customscript/aesthetic.py/txt2img/Aesthetic steps/value": 5,
+    "customscript/aesthetic.py/txt2img/Aesthetic steps/minimum": 0,
+    "customscript/aesthetic.py/txt2img/Aesthetic steps/maximum": 50,
+    "customscript/aesthetic.py/txt2img/Aesthetic steps/step": 1,
+    "customscript/aesthetic.py/txt2img/Aesthetic learning rate/visible": true,
+    "customscript/aesthetic.py/txt2img/Aesthetic learning rate/value": "0.0001",
+    "customscript/aesthetic.py/txt2img/Slerp interpolation/visible": true,
+    "customscript/aesthetic.py/txt2img/Slerp interpolation/value": false,
+    "customscript/aesthetic.py/txt2img/Aesthetic imgs embedding/visible": true,
+    "customscript/aesthetic.py/txt2img/Aesthetic imgs embedding/value": "None",
+    "customscript/aesthetic.py/txt2img/Aesthetic text for imgs/visible": true,
+    "customscript/aesthetic.py/txt2img/Aesthetic text for imgs/value": "",
+    "customscript/aesthetic.py/txt2img/Slerp angle/visible": true,
+    "customscript/aesthetic.py/txt2img/Slerp angle/value": 0.1,
+    "customscript/aesthetic.py/txt2img/Slerp angle/minimum": 0,
+    "customscript/aesthetic.py/txt2img/Slerp angle/maximum": 1,
+    "customscript/aesthetic.py/txt2img/Slerp angle/step": 0.01,
+    "customscript/aesthetic.py/txt2img/Is negative text/visible": true,
+    "customscript/aesthetic.py/txt2img/Is negative text/value": false,
+    "customscript/aesthetic.py/img2img/Aesthetic weight/visible": true,
+    "customscript/aesthetic.py/img2img/Aesthetic weight/value": 0.9,
+    "customscript/aesthetic.py/img2img/Aesthetic weight/minimum": 0,
+    "customscript/aesthetic.py/img2img/Aesthetic weight/maximum": 1,
+    "customscript/aesthetic.py/img2img/Aesthetic weight/step": 0.01,
+    "customscript/aesthetic.py/img2img/Aesthetic steps/visible": true,
+    "customscript/aesthetic.py/img2img/Aesthetic steps/value": 5,
+    "customscript/aesthetic.py/img2img/Aesthetic steps/minimum": 0,
+    "customscript/aesthetic.py/img2img/Aesthetic steps/maximum": 50,
+    "customscript/aesthetic.py/img2img/Aesthetic steps/step": 1,
+    "customscript/aesthetic.py/img2img/Aesthetic learning rate/visible": true,
+    "customscript/aesthetic.py/img2img/Aesthetic learning rate/value": "0.0001",
+    "customscript/aesthetic.py/img2img/Slerp interpolation/visible": true,
+    "customscript/aesthetic.py/img2img/Slerp interpolation/value": false,
+    "customscript/aesthetic.py/img2img/Aesthetic imgs embedding/visible": true,
+    "customscript/aesthetic.py/img2img/Aesthetic imgs embedding/value": "None",
+    "customscript/aesthetic.py/img2img/Aesthetic text for imgs/visible": true,
+    "customscript/aesthetic.py/img2img/Aesthetic text for imgs/value": "",
+    "customscript/aesthetic.py/img2img/Slerp angle/visible": true,
+    "customscript/aesthetic.py/img2img/Slerp angle/value": 0.1,
+    "customscript/aesthetic.py/img2img/Slerp angle/minimum": 0,
+    "customscript/aesthetic.py/img2img/Slerp angle/maximum": 1,
+    "customscript/aesthetic.py/img2img/Slerp angle/step": 0.01,
+    "customscript/aesthetic.py/img2img/Is negative text/visible": true,
+    "customscript/aesthetic.py/img2img/Is negative text/value": false,
+    "train/Regularization dataset directory (optional)/visible": true,
+    "train/Regularization dataset directory (optional)/value": "",
+    "train/Prior-preservation loss weight/visible": true,
+    "train/Prior-preservation loss weight/value": 1.0,
+    "train/Prior-preservation loss weight/minimum": 0.0,
+    "train/Prior-preservation loss weight/maximum": 10.0,
+    "train/Prior-preservation loss weight/step": 0.1,
+    "train/Batch size/minimum": 1,
+    "train/Batch size/maximum": 1024,
+    "train/Batch size/step": 1,
+    "customscript/aesthetic.py/txt2img/Learning rate/visible": true,
+    "customscript/aesthetic.py/txt2img/Learning rate/value": "0.0001",
+    "customscript/aesthetic.py/txt2img/Embedding/visible": true,
+    "customscript/aesthetic.py/txt2img/Embedding/value": "None",
+    "customscript/aesthetic.py/txt2img/Aesthetic text/visible": true,
+    "customscript/aesthetic.py/txt2img/Aesthetic text/value": "",
+    "customscript/aesthetic.py/txt2img/Negative/visible": true,
+    "customscript/aesthetic.py/txt2img/Negative/value": false,
+    "customscript/aesthetic.py/img2img/Learning rate/visible": true,
+    "customscript/aesthetic.py/img2img/Learning rate/value": "0.0001",
+    "customscript/aesthetic.py/img2img/Embedding/visible": true,
+    "customscript/aesthetic.py/img2img/Embedding/value": "None",
+    "customscript/aesthetic.py/img2img/Aesthetic text/visible": true,
+    "customscript/aesthetic.py/img2img/Aesthetic text/value": "",
+    "customscript/aesthetic.py/img2img/Negative/visible": true,
+    "customscript/aesthetic.py/img2img/Negative/value": false
 }