mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
controlnet fixes and validation
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
+5
-3
@@ -1,8 +1,8 @@
|
||||
# Change Log for SD.Next
|
||||
|
||||
## Update for 2025-10-29
|
||||
## Update for 2025-10-30
|
||||
|
||||
### Highlights for 2025-10-29
|
||||
### Highlights for 2025-10-30
|
||||
|
||||
Less than 2 weeks since last release, here's a service-pack style update with a lot of fixes and improvements:
|
||||
- Reorganization of **Reference Models** into *Base, Quantized, Distilled and Community* sections for easier navigation
|
||||
@@ -20,7 +20,7 @@ Less than 2 weeks since last release, here's a service-pack style update with a
|
||||
|
||||
[ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) | [Sponsor](https://github.com/sponsors/vladmandic)
|
||||
|
||||
### Details for 2025-10-29
|
||||
### Details for 2025-10-30
|
||||
|
||||
- **Reference** networks section is now split into actual *Base* models plus:
|
||||
- **Quantized**: pre-quantized variants of the base models using SDNQ-SVD quantization for optimal quality and smallest possible resource usage
|
||||
@@ -73,6 +73,7 @@ Less than 2 weeks since last release, here's a service-pack style update with a
|
||||
- enhanced LoRA support for **Wan-2.2-14B**
|
||||
- log available attention mechanisms on startup
|
||||
- support for switching back-and-forth **t2i** and **t2v** for *wan-2.x* models
|
||||
- control `api` cache controlnets
|
||||
- **Fixes**
|
||||
- startup error with `--profile` enabled if using `--skip`
|
||||
- restore orig init image for each batch sequence
|
||||
@@ -97,6 +98,7 @@ Less than 2 weeks since last release, here's a service-pack style update with a
|
||||
- avoid unnecessary pipe variant switching
|
||||
- validate pipelines on import
|
||||
- fix `nudenet` process tab operations
|
||||
- `controlnet` input validation
|
||||
|
||||
## Update for 2025-10-18
|
||||
|
||||
|
||||
+18
-4
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env python
|
||||
# example: api-control.py --prompt "anime girl" --control "Canny:Canny:1.0:0.1:0.9:/home/vlado/generative/Samples/anime1.jpg,None:Depth:0.9:0.0:1.0:/home/vlado/generative/Samples/anime1.jpg" --hires --detailer --output /tmp/anime.jpg
|
||||
import os
|
||||
import io
|
||||
import time
|
||||
@@ -70,10 +71,12 @@ def generate(args): # pylint: disable=redefined-outer-name
|
||||
options['negative_prompt'] = args.negative
|
||||
options['steps'] = int(args.steps)
|
||||
options['seed'] = int(args.seed)
|
||||
options['sampler_name'] = args.sampler
|
||||
if args.sampler is not None:
|
||||
options['sampler_name'] = args.sampler
|
||||
|
||||
if args.control is not None:
|
||||
options['unit_type'] = args.type
|
||||
if args.type is not None:
|
||||
options['unit_type'] = args.type
|
||||
options['control'] = []
|
||||
for control in args.control.split(','):
|
||||
u = control.split(':')
|
||||
@@ -86,7 +89,9 @@ def generate(args): # pylint: disable=redefined-outer-name
|
||||
'strength': float(u[2].strip()) if len(u) > 2 else 1.0,
|
||||
'start': float(u[3].strip()) if len(u) > 3 else 0.0,
|
||||
'end': float(u[4].strip()) if len(u) > 4 else 1.0,
|
||||
'override': encode(u[5].strip()) if len(u) > 5 else None,
|
||||
})
|
||||
log.info(f'added control: {options["control"]}')
|
||||
|
||||
if args.ipadapter is not None:
|
||||
options['ip_adapter'] = []
|
||||
@@ -109,6 +114,13 @@ def generate(args): # pylint: disable=redefined-outer-name
|
||||
if args.mask is not None:
|
||||
options['mask'] = encode(args.mask)
|
||||
|
||||
if args.detailer:
|
||||
options['detailer_enabled'] = True
|
||||
|
||||
if args.hires:
|
||||
options['enable_hr'] = True
|
||||
options['hr_force'] = True
|
||||
|
||||
data = post('/sdapi/v1/control', options)
|
||||
t1 = time.time()
|
||||
if 'info' in data:
|
||||
@@ -141,13 +153,15 @@ if __name__ == "__main__":
|
||||
parser.add_argument('--negative', required=False, default='', help='negative prompt text')
|
||||
parser.add_argument('--steps', required=False, default=20, help='number of steps')
|
||||
parser.add_argument('--seed', required=False, default=-1, help='initial seed')
|
||||
parser.add_argument('--sampler', required=False, default='UniPC', help='sampler name')
|
||||
parser.add_argument('--sampler', required=False, default=None, help='sampler name')
|
||||
parser.add_argument('--output', required=False, default=None, help='output image file')
|
||||
parser.add_argument('--processed', required=False, default=None, help='processed output file')
|
||||
parser.add_argument('--model', required=False, help='model name')
|
||||
parser.add_argument('--type', required=False, help='control type')
|
||||
parser.add_argument('--type', required=False, default="controlnet", help='control type')
|
||||
parser.add_argument('--control', required=False, help='control units')
|
||||
parser.add_argument('--ipadapter', required=False, help='ipadapter units')
|
||||
parser.add_argument('--detailer', required=False, default=False, action='store_true', help='force detailer')
|
||||
parser.add_argument('--hires', required=False, default=False, action='store_true', help='force hires')
|
||||
args = parser.parse_args()
|
||||
log.info(f'api-control: {args}')
|
||||
generate(args)
|
||||
|
||||
+26
-38
@@ -37,28 +37,6 @@ ReqControl = models.create_model_from_signature(
|
||||
if not hasattr(ReqControl, "__config__"):
|
||||
ReqControl.__config__ = models.DummyConfig
|
||||
|
||||
"""
|
||||
ReqControl = models.PydanticModelGenerator(
|
||||
"StableDiffusionProcessingControl",
|
||||
StableDiffusionProcessingControl,
|
||||
[
|
||||
{"key": "sampler_index", "type": Union[int, str], "default": 0},
|
||||
{"key": "sampler_name", "type": str, "default": "Default"},
|
||||
{"key": "script_name", "type": Optional[str], "default": ""},
|
||||
{"key": "script_args", "type": list, "default": []},
|
||||
{"key": "send_images", "type": bool, "default": True},
|
||||
{"key": "save_images", "type": bool, "default": False},
|
||||
{"key": "alwayson_scripts", "type": dict, "default": {}},
|
||||
{"key": "ip_adapter", "type": Optional[List[models.ItemIPAdapter]], "default": None, "exclude": True},
|
||||
{"key": "face", "type": Optional[models.ItemFace], "default": None, "exclude": True},
|
||||
{"key": "control", "type": Optional[List[ItemControl]], "default": [], "exclude": True},
|
||||
{"key": "extra", "type": Optional[dict], "default": {}, "exclude": True},
|
||||
]
|
||||
).generate_model()
|
||||
if not hasattr(ReqControl, "__config__"):
|
||||
ReqControl.__config__ = models.DummyConfig
|
||||
"""
|
||||
|
||||
|
||||
class ResControl(BaseModel):
|
||||
images: List[str] = Field(default=None, title="Images", description="")
|
||||
@@ -71,6 +49,7 @@ class APIControl():
|
||||
def __init__(self, queue_lock: Lock):
|
||||
self.queue_lock = queue_lock
|
||||
self.default_script_arg = []
|
||||
self.units = []
|
||||
|
||||
def sanitize_args(self, args: dict):
|
||||
args = vars(args)
|
||||
@@ -142,29 +121,38 @@ class APIControl():
|
||||
from modules.control.unit import Unit, unit_types
|
||||
req.units = []
|
||||
if req.unit_type is None:
|
||||
return req.control
|
||||
req.unit_type = 'controlnet'
|
||||
if req.unit_type not in unit_types:
|
||||
shared.log.error(f'Control uknown unit type: type={req.unit_type} available={unit_types}')
|
||||
return req.control
|
||||
for u in req.control:
|
||||
unit = Unit(
|
||||
enabled = True,
|
||||
unit_type = req.unit_type,
|
||||
model_id = u.model,
|
||||
process_id = u.process,
|
||||
strength = u.strength,
|
||||
start = u.start,
|
||||
end = u.end,
|
||||
)
|
||||
return
|
||||
for i in range(len(req.control)):
|
||||
u = req.control[i]
|
||||
if (len(self.units) > i) and (self.units[i].process_id == u.process) and (self.units[i].model_id == u.model):
|
||||
unit = self.units[i]
|
||||
unit.enabled = True
|
||||
unit.strength = u.strength
|
||||
unit.start = u.start
|
||||
unit.end = u.end
|
||||
else:
|
||||
unit = Unit(
|
||||
enabled = True,
|
||||
unit_type = req.unit_type,
|
||||
model_id = u.model,
|
||||
process_id = u.process,
|
||||
strength = u.strength,
|
||||
start = u.start,
|
||||
end = u.end,
|
||||
)
|
||||
if u.override is not None:
|
||||
unit.override = helpers.decode_base64_to_image(u.override)
|
||||
req.units.append(unit)
|
||||
return req.control
|
||||
self.units = req.units
|
||||
del req.control
|
||||
|
||||
def post_control(self, req: ReqControl):
|
||||
self.prepare_face_module(req)
|
||||
orig_control = self.prepare_control(req)
|
||||
del req.control
|
||||
requested = req.control
|
||||
self.prepare_control(req)
|
||||
|
||||
# prepare args
|
||||
args = req.copy(update={ # Override __init__ params
|
||||
@@ -203,5 +191,5 @@ class APIControl():
|
||||
b64images = list(map(helpers.encode_pil_to_base64, output_images)) if send_images else []
|
||||
b64processed = list(map(helpers.encode_pil_to_base64, output_processed)) if send_images else []
|
||||
self.sanitize_b64(req)
|
||||
req.units = orig_control
|
||||
req.units = requested
|
||||
return ResControl(images=b64images, processed=b64processed, params=vars(req), info=output_info)
|
||||
|
||||
+25
-5
@@ -53,9 +53,22 @@ def is_unified_model():
|
||||
return shared.sd_model.__class__.__name__ in unified_models
|
||||
|
||||
|
||||
def set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits=None):
|
||||
def has_inputs(inputs):
|
||||
current = inputs or []
|
||||
current = current if isinstance(current, list) else [current]
|
||||
current = [input for input in current if input is not None]
|
||||
if current is None or len(current) == 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, active_units, control_conditioning, control_guidance_start, control_guidance_end, inits=None, inputs=None):
|
||||
global pipe, instance # pylint: disable=global-statement
|
||||
pipe = None
|
||||
if has_models and not has_inputs(inits) and not has_inputs(inputs):
|
||||
if not any(has_inputs(u.override) for u in active_units if u.enabled): # check overrides
|
||||
shared.log.error('Control: no input images')
|
||||
return pipe
|
||||
if has_models:
|
||||
p.ops.append('control')
|
||||
p.extra_generation_params["Control type"] = unit_type # overriden later with pretty-print
|
||||
@@ -132,6 +145,7 @@ def check_active(p, unit_type, units):
|
||||
active_strength: List[float] = [] # strength factors for all active models
|
||||
active_start: List[float] = [] # start step for all active models
|
||||
active_end: List[float] = [] # end step for all active models
|
||||
active_units: List[unit.Unit] = [] # all active units
|
||||
num_units = 0
|
||||
for u in units:
|
||||
if u.type != unit_type:
|
||||
@@ -151,6 +165,7 @@ def check_active(p, unit_type, units):
|
||||
active_model.append(u.adapter)
|
||||
active_strength.append(float(u.strength))
|
||||
p.adapter_conditioning_factor = u.factor
|
||||
active_units.append(u)
|
||||
shared.log.debug(f'Control T2I-Adapter unit: i={num_units} process="{u.process.processor_id}" model="{u.adapter.model_id}" strength={u.strength} factor={u.factor}')
|
||||
elif unit_type == 'controlnet' and (u.controlnet.model is not None or is_unified_model()):
|
||||
active_process.append(u.process)
|
||||
@@ -159,6 +174,7 @@ def check_active(p, unit_type, units):
|
||||
active_start.append(float(u.start))
|
||||
active_end.append(float(u.end))
|
||||
p.guess_mode = u.guess
|
||||
active_units.append(u)
|
||||
if isinstance(u.mode, str):
|
||||
if not hasattr(p, 'control_mode'):
|
||||
p.control_mode = []
|
||||
@@ -173,11 +189,13 @@ def check_active(p, unit_type, units):
|
||||
active_strength.append(float(u.strength))
|
||||
active_start.append(float(u.start))
|
||||
active_end.append(float(u.end))
|
||||
active_units.append(u)
|
||||
shared.log.debug(f'Control ControlNet-XS unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}')
|
||||
elif unit_type == 'lite' and u.controlnet.model is not None:
|
||||
active_process.append(u.process)
|
||||
active_model.append(u.controlnet)
|
||||
active_strength.append(float(u.strength))
|
||||
active_units.append(u)
|
||||
shared.log.debug(f'Control ControlLLite unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}')
|
||||
elif unit_type == 'reference':
|
||||
p.override = u.override
|
||||
@@ -185,14 +203,16 @@ def check_active(p, unit_type, units):
|
||||
p.query_weight = float(u.query_weight)
|
||||
p.adain_weight = float(u.adain_weight)
|
||||
p.fidelity = u.fidelity
|
||||
active_units.append(u)
|
||||
shared.log.debug('Control Reference unit')
|
||||
else:
|
||||
if u.process.processor_id is not None:
|
||||
active_process.append(u.process)
|
||||
active_units.append(u)
|
||||
shared.log.debug(f'Control process unit: i={num_units} process={u.process.processor_id}')
|
||||
active_strength.append(float(u.strength))
|
||||
debug_log(f'Control active: process={len(active_process)} model={len(active_model)}')
|
||||
return active_process, active_model, active_strength, active_start, active_end
|
||||
return active_process, active_model, active_strength, active_start, active_end, active_units
|
||||
|
||||
|
||||
def check_enabled(p, unit_type, units, active_model, active_strength, active_start, active_end):
|
||||
@@ -258,7 +278,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
|
||||
guidance_name: str = 'Default', guidance_scale: float = 6.0, guidance_rescale: float = 0.0, guidance_start: float = 0.0, guidance_stop: float = 1.0,
|
||||
cfg_scale: float = 6.0, clip_skip: float = 1.0, image_cfg_scale: float = 6.0, diffusers_guidance_rescale: float = 0.7, pag_scale: float = 0.0, pag_adaptive: float = 0.5, cfg_end: float = 1.0,
|
||||
vae_type: str = 'Full', tiling: bool = False, hidiffusion: bool = False,
|
||||
detailer_enabled: bool = True, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024,
|
||||
detailer_enabled: bool = False, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, detailer_resolution: int = 1024,
|
||||
hdr_mode: int = 0, hdr_brightness: float = 0, hdr_color: float = 0, hdr_sharpen: float = 0, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 0.95,
|
||||
hdr_maximize: bool = False, hdr_max_center: float = 0.6, hdr_max_boundary: float = 1.0, hdr_color_picker: str = None, hdr_tint_ratio: float = 0,
|
||||
resize_mode_before: int = 0, resize_name_before: str = 'None', resize_context_before: str = 'None', width_before: int = 512, height_before: int = 512, scale_by_before: float = 1.0, selected_scale_tab_before: int = 0,
|
||||
@@ -419,7 +439,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
|
||||
return [], '', '', 'Error: model not loaded'
|
||||
|
||||
unit_type = unit_type.strip().lower() if unit_type is not None else ''
|
||||
active_process, active_model, active_strength, active_start, active_end = check_active(p, unit_type, units)
|
||||
active_process, active_model, active_strength, active_start, active_end, active_units = check_active(p, unit_type, units)
|
||||
has_models, selected_models, control_conditioning, control_guidance_start, control_guidance_end = check_enabled(p, unit_type, units, active_model, active_strength, active_start, active_end)
|
||||
|
||||
image_txt = ''
|
||||
@@ -429,7 +449,7 @@ def control_run(state: str = '', # pylint: disable=keyword-arg-before-vararg
|
||||
if is_unified_model():
|
||||
p.init_images = inputs
|
||||
|
||||
pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits)
|
||||
pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, active_units, control_conditioning, control_guidance_start, control_guidance_end, inits, inputs)
|
||||
debug_log(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}')
|
||||
status = True
|
||||
frame = None
|
||||
|
||||
@@ -31,7 +31,7 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation c
|
||||
self.choices = ['default']
|
||||
|
||||
def __str__(self):
|
||||
return f'Unit(index={self.index} enabled={self.enabled} type="{self.type}" strength={self.strength} start={self.start} end={self.end}{self.process}{self.controlnet})'
|
||||
return f'Unit(index={self.index} enabled={self.enabled} type="{self.type}" strength={self.strength} start={self.start} end={self.end}{self.process}{self.controlnet} override={self.override})'
|
||||
|
||||
def __init__(self,
|
||||
# values
|
||||
|
||||
@@ -295,7 +295,7 @@ class ControlNet():
|
||||
self.load_config['original_config_file '] = config_path
|
||||
self.model = cls.from_single_file(model_path, config=config, **self.load_config)
|
||||
|
||||
def load(self, model_id: str = None, force: bool = True) -> str:
|
||||
def load(self, model_id: str = None, force: bool = False) -> str:
|
||||
with load_lock:
|
||||
try:
|
||||
t0 = time.time()
|
||||
@@ -387,9 +387,9 @@ class ControlNet():
|
||||
log.warning(f"Control compile error: {e}")
|
||||
t1 = time.time()
|
||||
self.model_id = model_id
|
||||
log.info(f'Control {what} model loaded: id="{model_id}" path="{model_path}" cls={cls.__name__} time={t1-t0:.2f}')
|
||||
log.info(f'Control {what} model loaded: id="{self.model_id}" path="{model_path}" cls={cls.__name__} time={t1-t0:.2f}')
|
||||
state.end(jobid)
|
||||
return f'{what} loaded model: {model_id}'
|
||||
return f'{what} loaded model: {self.model_id}'
|
||||
except Exception as e:
|
||||
log.error(f'Control {what} model load: id="{model_id}" {e}')
|
||||
errors.display(e, f'Control {what} load')
|
||||
|
||||
@@ -265,7 +265,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output):
|
||||
output.images = [TF.to_pil_image(output.images[i].permute(2,0,1)) for i in range(output.images.shape[0])]
|
||||
|
||||
strength = p.hr_denoising_strength if p.hr_denoising_strength > 0 else p.denoising_strength
|
||||
if (p.hr_upscaler.lower().startswith('latent') or p.hr_force) and strength > 0:
|
||||
if (p.hr_upscaler is not None) and (p.hr_upscaler.lower().startswith('latent') or p.hr_force) and strength > 0:
|
||||
p.ops.append('hires')
|
||||
sd_models_compile.openvino_recompile_model(p, hires=True, refiner=False)
|
||||
if shared.sd_model.__class__.__name__ == "OnnxRawPipeline":
|
||||
|
||||
@@ -66,7 +66,7 @@ def load_hyimage3(checkpoint_info, diffusers_load_config=None): # pylint: disabl
|
||||
)
|
||||
pipe.load_tokenizer(repo_id)
|
||||
|
||||
pipe.pipeline # noqa: B018 # call it to set up pipeline
|
||||
pipe.pipeline # noqa: B018 # call it to set up pipeline # pylint: disable=pointless-statement
|
||||
pipe = HunyuanImage3Wrapper(pipe)
|
||||
|
||||
devices.torch_gc(force=True, reason='load')
|
||||
|
||||
@@ -334,9 +334,9 @@ def apply_control(field):
|
||||
else:
|
||||
run.unit.current = [unit]
|
||||
run.init_units(run.unit.current)
|
||||
active_process, active_model, active_strength, active_start, active_end = run.check_active(p, unit.type, run.unit.current)
|
||||
active_process, active_model, active_strength, active_start, active_end, active_units = run.check_active(p, unit.type, run.unit.current)
|
||||
has_models, selected_models, control_conditioning, control_guidance_start, control_guidance_end = run.check_enabled(p, unit.type, run.unit.current, active_model, active_strength, active_start, active_end)
|
||||
pipe = run.set_pipe(p, has_models, unit.type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end)
|
||||
pipe = run.set_pipe(p, has_models, unit.type, selected_models, active_model, active_strength, active_units, control_conditioning, control_guidance_start, control_guidance_end)
|
||||
_processed_image, _blended_image = processor.preprocess_image(p, pipe, input_image=init_images[0], unit_type=unit.type, active_process=active_process, active_model=active_model, selected_models=selected_models, has_models=has_models)
|
||||
if pipe is not None:
|
||||
shared.sd_model = pipe
|
||||
|
||||
Reference in New Issue
Block a user