Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-17 08:19:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into dev

Browse Source
This commit is contained in:
Vladimir Mandic
2024-06-30 14:15:02 -04:00
committed by GitHub
parent 47d865acc1 5af36d2182
commit c2b0124493
7 changed files with 85 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2 -1
View File
@@ -1,12 +1,13 @@
# Change Log for SD.Next
## Update for 2024-06-28
## Update for 2024-06-30
- enable `florence` VLM for all platforms, thanks @lshqqytiger!
- fix executing extensions with zero params
- fix nncf for lora, thanks @Disty0!
- fix diffusers version detection for SD3
- fix current step for higher order samplers
- fix control input type video
- add SD3 with FP16 T5 to list of detected models
- multiple ModernUI fixes
modules/control/run.py
+74 -63
View File
@@ -282,67 +282,72 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
else:
pass
debug(f'Control: run type={unit_type} models={has_models}')
if has_models:
p.ops.append('control')
p.extra_generation_params["Control mode"] = unit_type # overriden later with pretty-print
p.extra_generation_params["Control conditioning"] = control_conditioning if isinstance(control_conditioning, list) else [control_conditioning]
p.extra_generation_params['Control start'] = control_guidance_start if isinstance(control_guidance_start, list) else [control_guidance_start]
p.extra_generation_params['Control end'] = control_guidance_end if isinstance(control_guidance_end, list) else [control_guidance_end]
p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None])
p.extra_generation_params["Control conditioning"] = ';'.join([str(c) for c in p.extra_generation_params["Control conditioning"]])
p.extra_generation_params['Control start'] = ';'.join([str(c) for c in p.extra_generation_params['Control start']])
p.extra_generation_params['Control end'] = ';'.join([str(c) for c in p.extra_generation_params['Control end']])
if unit_type == 't2i adapter' and has_models:
p.extra_generation_params["Control mode"] = 'T2I-Adapter'
p.task_args['adapter_conditioning_scale'] = control_conditioning
instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: T2I-Adapter does not support separate init image')
elif unit_type == 'controlnet' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet'
p.task_args['controlnet_conditioning_scale'] = control_conditioning
p.task_args['control_guidance_start'] = control_guidance_start
p.task_args['control_guidance_end'] = control_guidance_end
p.task_args['guess_mode'] = p.guess_mode
instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
elif unit_type == 'xs' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet-XS'
p.controlnet_conditioning_scale = control_conditioning
p.control_guidance_start = control_guidance_start
p.control_guidance_end = control_guidance_end
instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlNet-XS does not support separate init image')
elif unit_type == 'lite' and has_models:
p.extra_generation_params["Control mode"] = 'ControlLLLite'
p.controlnet_conditioning_scale = control_conditioning
instance = lite.ControlLLitePipeline(shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlLLLite does not support separate init image')
elif unit_type == 'reference' and has_models:
p.extra_generation_params["Control mode"] = 'Reference'
p.extra_generation_params["Control attention"] = p.attention
p.task_args['reference_attn'] = 'Attention' in p.attention
p.task_args['reference_adain'] = 'Adain' in p.attention
p.task_args['attention_auto_machine_weight'] = p.query_weight
p.task_args['gn_auto_machine_weight'] = p.adain_weight
p.task_args['style_fidelity'] = p.fidelity
instance = reference.ReferencePipeline(shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlNet-XS does not support separate init image')
else: # run in txt2img/img2img mode
if len(active_strength) > 0:
p.strength = active_strength[0]
pipe = shared.sd_model
instance = None
def set_pipe():
global pipe, instance # pylint: disable=global-statement
pipe = None
if has_models:
p.ops.append('control')
p.extra_generation_params["Control mode"] = unit_type # overriden later with pretty-print
p.extra_generation_params["Control conditioning"] = control_conditioning if isinstance(control_conditioning, list) else [control_conditioning]
p.extra_generation_params['Control start'] = control_guidance_start if isinstance(control_guidance_start, list) else [control_guidance_start]
p.extra_generation_params['Control end'] = control_guidance_end if isinstance(control_guidance_end, list) else [control_guidance_end]
p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None])
p.extra_generation_params["Control conditioning"] = ';'.join([str(c) for c in p.extra_generation_params["Control conditioning"]])
p.extra_generation_params['Control start'] = ';'.join([str(c) for c in p.extra_generation_params['Control start']])
p.extra_generation_params['Control end'] = ';'.join([str(c) for c in p.extra_generation_params['Control end']])
if unit_type == 't2i adapter' and has_models:
p.extra_generation_params["Control mode"] = 'T2I-Adapter'
p.task_args['adapter_conditioning_scale'] = control_conditioning
instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: T2I-Adapter does not support separate init image')
elif unit_type == 'controlnet' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet'
p.task_args['controlnet_conditioning_scale'] = control_conditioning
p.task_args['control_guidance_start'] = control_guidance_start
p.task_args['control_guidance_end'] = control_guidance_end
p.task_args['guess_mode'] = p.guess_mode
instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
elif unit_type == 'xs' and has_models:
p.extra_generation_params["Control mode"] = 'ControlNet-XS'
p.controlnet_conditioning_scale = control_conditioning
p.control_guidance_start = control_guidance_start
p.control_guidance_end = control_guidance_end
instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlNet-XS does not support separate init image')
elif unit_type == 'lite' and has_models:
p.extra_generation_params["Control mode"] = 'ControlLLLite'
p.controlnet_conditioning_scale = control_conditioning
instance = lite.ControlLLitePipeline(shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlLLLite does not support separate init image')
elif unit_type == 'reference' and has_models:
p.extra_generation_params["Control mode"] = 'Reference'
p.extra_generation_params["Control attention"] = p.attention
p.task_args['reference_attn'] = 'Attention' in p.attention
p.task_args['reference_adain'] = 'Adain' in p.attention
p.task_args['attention_auto_machine_weight'] = p.query_weight
p.task_args['gn_auto_machine_weight'] = p.adain_weight
p.task_args['style_fidelity'] = p.fidelity
instance = reference.ReferencePipeline(shared.sd_model)
pipe = instance.pipeline
if inits is not None:
shared.log.warning('Control: ControlNet-XS does not support separate init image')
else: # run in txt2img/img2img mode
if len(active_strength) > 0:
p.strength = active_strength[0]
pipe = shared.sd_model
instance = None
debug(f'Control: run type={unit_type} models={has_models} pipe={pipe.__class__.__name__ if pipe is not None else None}')
return pipe
pipe = set_pipe()
debug(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}')
t1, t2, t3 = time.time(), 0, 0
status = True
@@ -383,6 +388,7 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
codec = util.decode_fourcc(video.get(cv2.CAP_PROP_FOURCC))
status, frame = video.read()
if status:
shared.state.frame_count = 1 + frames // (video_skip_frames + 1)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
shared.log.debug(f'Control: input video: path={inputs} frames={frames} fps={fps} size={w}x{h} codec={codec}')
except Exception as e:
@@ -390,6 +396,9 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
return [], '', '', 'Error: video open failed'
while status:
if pipe is None: # pipe may have been reset externally
pipe = set_pipe()
debug(f'Control pipeline reinit: class={pipe.__class__.__name__}')
processed_image = None
if frame is not None:
inputs = [Image.fromarray(frame)] # cv2 to pil
@@ -426,9 +435,10 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
else:
debug(f'Control Init image: {i % len(inits) + 1} of {len(inits)}')
init_image = inits[i % len(inits)]
index += 1
if video is not None and index % (video_skip_frames + 1) != 0:
index += 1
continue
index += 1
# resize before
if resize_mode_before != 0 and resize_name_before != 'None':
@@ -593,10 +603,11 @@ def control_run(units: List[unit.Unit] = [], inputs: List[Image.Image] = [], ini
output = None
script_run = False
if pipe is not None: # run new pipeline
pipe.restore_pipeline = restore_pipeline
if not hasattr(pipe, 'restore_pipeline') and video is None:
pipe.restore_pipeline = restore_pipeline
debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}')
debug(f'Control exec pipeline: p={vars(p)}')
debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None) or p.image_mask} ref={p.task_args.get("ref_image", None)}')
# debug(f'Control exec pipeline: p={vars(p)}')
# debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None) or p.image_mask} ref={p.task_args.get("ref_image", None)}')
if sd_models.get_diffusers_task(pipe) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: # force vae back to gpu if not in txt2img mode
sd_models.move_model(pipe.vae, devices.device)
modules/processing.py
+1
View File
@@ -388,6 +388,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed:
devices.torch_gc()
if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None:
print('HERE RESTORE')
shared.sd_model.restore_pipeline()
t1 = time.time()
modules/processing_diffusers.py
+3 -3
View File
@@ -104,7 +104,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
clip_skip=p.clip_skip,
desc='Base',
)
shared.state.sampling_steps = base_args.get('prior_num_inference_steps', None) or base_args.get('num_inference_steps', None) or p.steps
shared.state.sampling_steps = base_args.get('prior_num_inference_steps', None) or p.steps or base_args.get('num_inference_steps', None)
if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1:
p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta
output = None
@@ -215,7 +215,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
desc='Hires',
)
shared.state.job = 'HiRes'
shared.state.sampling_steps = hires_args.get('prior_num_inference_steps', None) or hires_args.get('num_inference_steps', None) or p.steps
shared.state.sampling_steps = hires_args.get('prior_num_inference_steps', None) or p.steps or hires_args.get('num_inference_steps', None)
try:
sd_models_compile.check_deepcache(enable=True)
output = shared.sd_model(**hires_args) # pylint: disable=not-callable
@@ -280,7 +280,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing):
clip_skip=p.clip_skip,
desc='Refiner',
)
shared.state.sampling_steps = refiner_args.get('prior_num_inference_steps', None) or refiner_args.get('num_inference_steps', None) or p.steps
shared.state.sampling_steps = refiner_args.get('prior_num_inference_steps', None) or p.steps or refiner_args.get('num_inference_steps', None)
try:
if 'requires_aesthetics_score' in shared.sd_refiner.config: # sdxl-model needs false and sdxl-refiner needs true
shared.sd_refiner.register_to_config(requires_aesthetics_score = getattr(shared.sd_refiner, 'tokenizer', None) is None)
modules/progress.py
+1 -1
View File
@@ -62,7 +62,7 @@ def progressapi(req: ProgressRequest):
paused = shared.state.paused
if not active:
return InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, id_live_preview=-1, textinfo="Queued..." if queued else "Waiting...")
shared.state.job_count = max(shared.state.job_count, shared.state.job_no)
shared.state.job_count = max(shared.state.frame_count, shared.state.job_count, shared.state.job_no)
batch_x = max(shared.state.job_no, 0)
batch_y = max(shared.state.job_count, 1)
step_x = max(shared.state.sampling_step, 0)
modules/shared_state.py
+3
View File
@@ -12,6 +12,7 @@ class State:
job = ""
job_no = 0
job_count = 0
frame_count = 0
total_jobs = 0
job_timestamp = '0'
sampling_step = 0
@@ -71,6 +72,7 @@ class State:
self.interrupted = False
self.job = title
self.job_count = -1
self.frame_count = -1
self.job_no = 0
self.job_timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
self.paused = False
@@ -93,6 +95,7 @@ class State:
self.job = ""
self.job_count = 0
self.job_no = 0
self.frame_count = 0
self.paused = False
self.interrupted = False
self.skipped = False
wiki
+1 -1
Submodule wiki updated: 8c44b30554...04ecf1fe4b