From ade05e521a845954cab585c1686578cfd2fb1bb4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 18 Jan 2023 19:31:40 -0500 Subject: [PATCH] enable pylint --- cli/.pylintrc | 1 + cli/bench.py | 13 +++++++++++-- cli/modules/ffmpeg.py | 1 + cli/modules/grid.py | 1 + cli/modules/ideas.py | 4 ++++ cli/modules/interrogate.py | 2 ++ cli/modules/losschart.py | 3 +++ cli/modules/lossrate.py | 3 +++ cli/modules/process.py | 2 ++ cli/modules/promptist.py | 3 +++ cli/modules/sdapi.py | 7 ++++--- cli/modules/util.py | 3 +++ cli/random/detectmodel.py | 1 + cli/train.py | 8 ++++---- config.json | 3 ++- 15 files changed, 45 insertions(+), 10 deletions(-) diff --git a/cli/.pylintrc b/cli/.pylintrc index cd982dccc..1231dff7a 100644 --- a/cli/.pylintrc +++ b/cli/.pylintrc @@ -1,2 +1,3 @@ # See https://pylint.pycqa.org/en/latest/user_guide/messages/message_control.html [MESSAGES CONTROL] +enable=C,R,W,E,I diff --git a/cli/bench.py b/cli/bench.py index 515f91a60..df0de38eb 100755 --- a/cli/bench.py +++ b/cli/bench.py @@ -2,18 +2,21 @@ """ sd api txt2img benchmark """ -import time -import json import asyncio import base64 import io +import json +import os import sys +import time + from PIL import Image sys.path.append(os.path.join(os.path.dirname(__file__), 'modules')) import modules.sdapi as sdapi from modules.util import Map, log + options = Map({ 'restore_faces': False, 'prompt': 'photo of two dice on a table', @@ -28,10 +31,12 @@ options = Map({ 'height': 512 }) + # batch = [1, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128] batch = [1, 1, 2, 4, 8, 12, 16] oom = 0 + async def txt2img(): t0 = time.perf_counter() data = {} @@ -52,6 +57,7 @@ async def txt2img(): t1 = time.perf_counter() return t1 - t0 + def memstats(): mem = sdapi.getsync('/sdapi/v1/memory') cpu = mem.get('ram', 'unavailable') @@ -68,9 +74,11 @@ def memstats(): gpu.pop('events') return cpu, gpu + def gb(val: float): return round(val / 1024 / 1024 / 1024, 2) + async def main(): log.info({ 'benchmark': { 'batch-sizes': batch } }) sdapi.quiet = True @@ -111,6 +119,7 @@ async def main(): log.info({ 'benchmark': 'ended with oom so you should probably restart your automatic server now' }) await sdapi.close() + if __name__ == '__main__': try: asyncio.run(main()) diff --git a/cli/modules/ffmpeg.py b/cli/modules/ffmpeg.py index 6f4418376..650bfaaeb 100755 --- a/cli/modules/ffmpeg.py +++ b/cli/modules/ffmpeg.py @@ -8,6 +8,7 @@ import subprocess import pathlib import argparse import filetype + from util import log, Map diff --git a/cli/modules/grid.py b/cli/modules/grid.py index b00e5687c..36c245f82 100755 --- a/cli/modules/grid.py +++ b/cli/modules/grid.py @@ -13,6 +13,7 @@ import filetype from PIL import Image, ImageDraw, ImageFont from util import log + params = None diff --git a/cli/modules/ideas.py b/cli/modules/ideas.py index f91ef6f32..9b403f2f9 100755 --- a/cli/modules/ideas.py +++ b/cli/modules/ideas.py @@ -6,12 +6,16 @@ model from: import logging import argparse + from transformers import GPT2Tokenizer, GPT2LMHeadModel + from util import log + tokenizer = None model = None + def prompt(text: str, temp: float = 0.9, top: int = 8, penalty: float = 1.2, alpha: float = 0.6, num: int = 5, length: int = 80): global tokenizer, model # pylint: disable=global-statement if tokenizer is None: diff --git a/cli/modules/interrogate.py b/cli/modules/interrogate.py index 000bea188..7d1bdcf02 100755 --- a/cli/modules/interrogate.py +++ b/cli/modules/interrogate.py @@ -8,8 +8,10 @@ import base64 import sys import os import asyncio + import filetype from PIL import Image + from util import log, Map import sdapi as sdapi diff --git a/cli/modules/losschart.py b/cli/modules/losschart.py index 2ee038c8f..9c5562be2 100755 --- a/cli/modules/losschart.py +++ b/cli/modules/losschart.py @@ -6,12 +6,15 @@ import sys import json import pathlib import logging + import torch import numpy as np from PIL import Image, ImageFont, ImageDraw from matplotlib import pyplot as plt + from util import log, Map + def settings(logdir: str, name: str): filename = os.path.join(logdir, name, 'settings.json') with open(filename, 'r', encoding='utf-8') as f: diff --git a/cli/modules/lossrate.py b/cli/modules/lossrate.py index 2e4d63d3a..f607fe0bd 100755 --- a/cli/modules/lossrate.py +++ b/cli/modules/lossrate.py @@ -5,14 +5,17 @@ auto-generate learn-rate import io import math import logging + import numpy as np from PIL import Image, ImageFont, ImageDraw from matplotlib import pyplot as plt + from util import log, Map loss_types = ['linear', 'log', 'linalg', 'power'] + def gen_steps(steps, step): return [x for x in range(1, steps + step) if x % step == 0] diff --git a/cli/modules/process.py b/cli/modules/process.py index 27e9b452f..924250ae2 100755 --- a/cli/modules/process.py +++ b/cli/modules/process.py @@ -8,9 +8,11 @@ import io import shutil import base64 import pathlib + import numpy as np import mediapipe as mp from PIL import Image, ImageOps + from util import log, Map from sdapi import postsync diff --git a/cli/modules/promptist.py b/cli/modules/promptist.py index f2a2b4cc5..1240cc8c0 100755 --- a/cli/modules/promptist.py +++ b/cli/modules/promptist.py @@ -5,9 +5,12 @@ use microsoft promptist to beautify prompt """ import sys + from transformers import AutoModelForCausalLM, AutoTokenizer + from util import log + def load_prompter(): model = AutoModelForCausalLM.from_pretrained("microsoft/Promptist") # pylint: disable=redefined-outer-name tokenizer = AutoTokenizer.from_pretrained("gpt2") # pylint: disable=redefined-outer-name diff --git a/cli/modules/sdapi.py b/cli/modules/sdapi.py index 101a21b1e..723eef86b 100755 --- a/cli/modules/sdapi.py +++ b/cli/modules/sdapi.py @@ -4,21 +4,22 @@ helper methods that creates HTTP session with managed connection pool provides async HTTP get/post methods and several helper methods """ +import aiohttp import asyncio import logging +import requests import sys -import aiohttp -import requests from util import Map, log + sd_url = "http://127.0.0.1:7860" # automatic1111 api url root use_session = True timeout = aiohttp.ClientTimeout(total = None, sock_connect = 10, sock_read = None) # default value is 5 minutes, we need longer for training - sess = None quiet = False + async def result(req): if req.status != 200: if not quiet: diff --git a/cli/modules/util.py b/cli/modules/util.py index 9402c3d72..6adb00848 100755 --- a/cli/modules/util.py +++ b/cli/modules/util.py @@ -5,9 +5,11 @@ generic helper methods import logging + logging.basicConfig(level = logging.INFO, format = '%(asctime)s %(levelname)s: %(message)s') log = logging.getLogger("sd") + class Map(dict): def __init__(self, *args, **kwargs): super(Map, self).__init__(*args, **kwargs) @@ -45,5 +47,6 @@ class Map(dict): super(Map, self).__delitem__(key) del self.__dict__[key] + if __name__ == "__main__": pass diff --git a/cli/random/detectmodel.py b/cli/random/detectmodel.py index 16a08547f..c104fbea5 100755 --- a/cli/random/detectmodel.py +++ b/cli/random/detectmodel.py @@ -12,6 +12,7 @@ Only difference are some calculations in `ldm/models/diffusion/ddpm.py` and by t import os import sys + import torch def signature(model): diff --git a/cli/train.py b/cli/train.py index 752557781..55c077f7d 100755 --- a/cli/train.py +++ b/cli/train.py @@ -16,10 +16,10 @@ import logging import math import os import sys -import pathlib import time import json -from pathlib import Path +from pathlib import Path, PurePath + import filetype from PIL import Image @@ -383,7 +383,7 @@ async def main(): log.setLevel(logging.DEBUG) log.debug({ 'debug': True }) log.debug({ 'args': params.__dict__ }) - home = pathlib.Path(sys.argv[0]).parent + home = Path(sys.argv[0]).parent global args # pylint: disable=global-statement if os.path.isfile(params.config): try: @@ -453,7 +453,7 @@ async def main(): else: args.train_embedding.template_filename = 'unknown_filewords.txt' if params.name == 'auto': - params.name = pathlib.PurePath(params.src).name + params.name = PurePath(params.src).name log.info({ 'training name': params.name }) if params.dst == "/tmp": params.dst = os.path.join("/tmp/train", params.name) diff --git a/config.json b/config.json index 09be0c939..3e230f202 100644 --- a/config.json +++ b/config.json @@ -161,5 +161,6 @@ "inspiration_rows_num": 4, "inspiration_cols_num": 6, "live_preview_refresh_period": 1000, - "show_progressbar": true + "show_progressbar": true, + "show_warnings": false } \ No newline at end of file