From 4dc5941912cc78bbe986a0c1fcdba6d7b194ab4d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 30 Apr 2023 21:01:49 -0400 Subject: [PATCH] fix embedding logging --- modules/shared.py | 1 - modules/textual_inversion/logging.py | 5 +-- .../textual_inversion/textual_inversion.py | 45 ++----------------- webui.py | 2 +- 4 files changed, 7 insertions(+), 46 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index b2a93542c..611d8af4d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -352,7 +352,6 @@ options_templates.update(options_section(('training', "Training"), { "dataset_filename_word_regex": OptionInfo("", "Filename word regex"), "dataset_filename_join_string": OptionInfo(" ", "Filename join string"), "embeddings_templates_dir": OptionInfo(os.path.join(paths.script_path, 'train', 'templates'), "Embeddings train templates directory"), - "embeddings_train_log": OptionInfo(os.path.join(paths.script_path, 'train', 'log', 'train.csv'), "Embeddings train log file"), "training_image_repeats_per_epoch": OptionInfo(1, "Number of repeats for a single input image per epoch; used only for displaying epoch number", gr.Number, {"precision": 0}), "training_write_csv_every": OptionInfo(0, "Save an csv containing the loss to log directory every N steps, 0 to disable"), "training_enable_tensorboard": OptionInfo(False, "Enable tensorboard logging."), diff --git a/modules/textual_inversion/logging.py b/modules/textual_inversion/logging.py index d0e52beb7..b8440f656 100644 --- a/modules/textual_inversion/logging.py +++ b/modules/textual_inversion/logging.py @@ -16,8 +16,7 @@ def save_settings_to_file(log_directory, all_params): if all_params.get('preview_from_txt2img'): keys = keys | saved_params_previews params.update({k: v for k, v in all_params.items() if k in keys}) - filename = 'settings.json' - fn = os.path.join(log_directory, filename) + filename = f"{params['embedding_name']}-{now.strftime('%Y-%m-%d_%H-%M-%S')}.json" with open(os.path.join(log_directory, filename), "w", encoding='utf-8') as file: - print(f'Training settings file: {fn}') + print(f'Training settings file: {os.path.join(log_directory, filename)}') json.dump(params, file, indent=2) diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index ab412f458..343e55539 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -4,7 +4,7 @@ import csv from collections import namedtuple import torch try: - import intel_extension_for_pytorch as ipex + import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import except: pass import tqdm @@ -283,20 +283,15 @@ def create_embedding(name, num_vectors_per_token, overwrite_old, init_text='*'): def write_loss(log_directory, filename, step, epoch_len, values): if shared.opts.training_write_csv_every == 0: return - - if step % epoch_len != 0: + if step % shared.opts.training_write_csv_every != 0: return write_csv_header = False if os.path.exists(os.path.join(log_directory, filename)) else True - with open(os.path.join(log_directory, filename), "a+", newline='', encoding='utf-8') as fout: csv_writer = csv.DictWriter(fout, fieldnames=["step", "epoch", "epoch_step", *(values.keys())]) - if write_csv_header: csv_writer.writeheader() - epoch = (step - 1) // epoch_len epoch_step = (step - 1) % epoch_len - csv_writer.writerow({ "step": step, "epoch": epoch, @@ -410,16 +405,11 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st tensorboard_writer = tensorboard_setup(log_directory) pin_memory = shared.opts.pin_memory - ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=embedding_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize, use_weight=use_weight) - if shared.opts.save_training_settings_to_txt: save_settings_to_file(log_directory, {**dict(model_name=checkpoint.model_name, model_hash=checkpoint.shorthash, num_of_dataset_images=len(ds), num_vectors_per_token=len(embedding.vec)), **locals()}) - latent_sampling_method = ds.latent_sampling_method - dl = modules.textual_inversion.dataset.PersonalizedDataLoader(ds, latent_sampling_method=latent_sampling_method, batch_size=ds.batch_size, pin_memory=pin_memory) - if unload: shared.parallel_processing_allowed = False shared.sd_model.first_stage_model.to(devices.cpu) @@ -432,7 +422,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st optimizer_saved_dict = torch.load(filename + '.optim', map_location='cpu') if embedding.checksum() == optimizer_saved_dict.get('hash', None): optimizer_state_dict = optimizer_saved_dict.get('optimizer_state_dict', None) - if optimizer_state_dict is not None: optimizer.load_state_dict(optimizer_state_dict) print("Loaded existing optimizer from checkpoint") @@ -451,12 +440,10 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st max_steps_per_epoch = len(ds) // batch_size - (len(ds) // batch_size) % gradient_step loss_step = 0 _loss_step = 0 #internal - last_saved_file = "" last_saved_image = "" forced_filename = "" embedding_yet_to_be_embedded = False - is_training_inpainting_model = shared.sd_model.model.conditioning_key in {'hybrid', 'concat'} img_c = None @@ -478,7 +465,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st break if shared.state.interrupted: break - if clip_grad: clip_grad_sched.step(embedding.step) @@ -487,32 +473,26 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st if use_weight: w = batch.weight.to(devices.device, non_blocking=pin_memory) c = shared.sd_model.cond_stage_model(batch.cond_text) - if is_training_inpainting_model: if img_c is None: img_c = processing.txt2img_image_conditioning(shared.sd_model, c, training_width, training_height) - cond = {"c_concat": [img_c], "c_crossattn": [c]} else: cond = c - if use_weight: loss = shared.sd_model.weighted_forward(x, cond, w)[0] / gradient_step del w else: loss = shared.sd_model.forward(x, cond)[0] / gradient_step del x - _loss_step += loss.item() - scaler.scale(loss).backward() + scaler.scale(loss).backward() # go back until we reach gradient accumulation steps if (j + 1) % gradient_step != 0: continue - if clip_grad: clip_grad(embedding.vec, clip_grad_sched.learn_rate) - scaler.step(optimizer) scaler.update() embedding.step += 1 @@ -520,9 +500,7 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st optimizer.zero_grad(set_to_none=True) loss_step = _loss_step _loss_step = 0 - steps_done = embedding.step + 1 - epoch_num = embedding.step // steps_per_epoch description = f"Training textual inversion step {embedding.step} loss: {loss_step:.5f} lr: {scheduler.learn_rate:.5f}" @@ -534,15 +512,11 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st save_embedding(embedding, optimizer, checkpoint, embedding_name_every, last_saved_file, remove_cached_checksum=True) embedding_yet_to_be_embedded = True - write_loss(log_directory, shared.opts.embeddings_train_log, embedding.step, steps_per_epoch, { - "loss": f"{loss_step:.7f}", - "learn_rate": scheduler.learn_rate - }) + write_loss(log_directory, f"{embedding_name}.csv", embedding.step, steps_per_epoch, { "loss": f"{loss_step:.7f}", "learn_rate": scheduler.learn_rate }) if images_dir is not None and steps_done % create_image_every == 0: forced_filename = f'{embedding_name}-{steps_done}' last_saved_image = os.path.join(images_dir, forced_filename) - shared.sd_model.first_stage_model.to(devices.device) p = processing.StableDiffusionProcessingTxt2Img( @@ -568,7 +542,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st p.height = training_height preview_text = p.prompt - processed = processing.process_images(p) image = processed.images[0] if len(processed.images) > 0 else None @@ -577,35 +550,27 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st if image is not None: shared.state.assign_current_image(image) - last_saved_image, _last_text_info = images.save_image(image, images_dir, "", p.seed, p.prompt, shared.opts.samples_format, processed.infotexts[0], p=p, forced_filename=forced_filename, save_to_dirs=False) last_saved_image += f", prompt: {preview_text}" - if shared.opts.training_enable_tensorboard and shared.opts.training_tensorboard_save_images: tensorboard_add_image(tensorboard_writer, f"Validation at epoch {epoch_num}", image, embedding.step) if save_image_with_stored_embedding and os.path.exists(last_saved_file) and embedding_yet_to_be_embedded: - last_saved_image_chunks = os.path.join(images_embeds_dir, f'{embedding_name}-{steps_done}.png') - info = PngImagePlugin.PngInfo() data = torch.load(last_saved_file) info.add_text("sd-ti-embedding", embedding_to_b64(data)) title = f"<{data.get('name', '???')}>" - try: vectorSize = list(data['string_to_param'].values())[0].shape[0] except Exception: vectorSize = '?' - checkpoint = sd_models.select_checkpoint() footer_left = checkpoint.model_name footer_mid = f'[{checkpoint.shorthash}]' footer_right = f'{vectorSize}v {steps_done}s' - captioned_image = caption_image_overlay(image, title, footer_left, footer_mid, footer_right) captioned_image = insert_image_data_embed(captioned_image, data) - captioned_image.save(last_saved_image_chunks, "PNG", pnginfo=info) embedding_yet_to_be_embedded = False @@ -613,7 +578,6 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st last_saved_image += f", prompt: {preview_text}" shared.state.job_no = embedding.step - shared.state.textinfo = f"""

Loss: {loss_step:.7f}
@@ -633,7 +597,6 @@ Last saved image: {html.escape(last_saved_image)}
shared.sd_model.first_stage_model.to(devices.device) shared.parallel_processing_allowed = old_parallel_processing_allowed sd_hijack_checkpoint.remove() - return embedding, filename diff --git a/webui.py b/webui.py index 8d9e0dc2f..8954cfbfe 100644 --- a/webui.py +++ b/webui.py @@ -212,7 +212,7 @@ def start_ui(): app, _local_url, _share_url = shared.demo.launch( share=cmd_opts.share, server_name=server_name, - server_port=cmd_opts.port, + server_port=cmd_opts.port if cmd_opts.port != 7860 else None, ssl_keyfile=cmd_opts.tls_keyfile, ssl_certfile=cmd_opts.tls_certfile, debug=False,