mirror of
https://github.com/vladmandic/automatic
synced 2026-09-19 17:24:32 +02:00
@@ -64,9 +64,9 @@ class ControlNetXSOutput(BaseOutput):
|
||||
class ControlNetConditioningEmbedding(nn.Module):
|
||||
"""
|
||||
Quoting from https://arxiv.org/abs/2302.05543: "Stable Diffusion uses a pre-processing method similar to VQ-GAN
|
||||
[11] to convert the entire dataset of 512 x 512 images into smaller 64 x 64 “latent images” for stabilized
|
||||
training. This requires ControlNets to convert image-based conditions to 64 x 64 feature space to match the
|
||||
convolution size. We use a tiny network E(·) of four convolution layers with 4 x 4 kernels and 2 x 2 strides
|
||||
[11] to convert the entire dataset of 512 × 512 images into smaller 64 × 64 “latent images” for stabilized
|
||||
training. This requires ControlNets to convert image-based conditions to 64 × 64 feature space to match the
|
||||
convolution size. We use a tiny network E(·) of four convolution layers with 4 × 4 kernels and 2 × 2 strides
|
||||
(activated by ReLU, channels are 16, 32, 64, 128, initialized with Gaussian weights, trained jointly with the full
|
||||
model) to encode image-space conditions ... into feature maps ..."
|
||||
"""
|
||||
@@ -657,7 +657,7 @@ class ControlNetXSModel(ModelMixin, ConfigMixin):
|
||||
if base_model.config.addition_embed_type == "text":
|
||||
aug_emb = base_model.add_embedding(encoder_hidden_states)
|
||||
elif base_model.config.addition_embed_type == "text_image":
|
||||
raise NotImplementedError
|
||||
raise NotImplementedError()
|
||||
elif base_model.config.addition_embed_type == "text_time":
|
||||
# SDXL - style
|
||||
if "text_embeds" not in added_cond_kwargs:
|
||||
@@ -676,9 +676,9 @@ class ControlNetXSModel(ModelMixin, ConfigMixin):
|
||||
add_embeds = add_embeds.to(temb.dtype)
|
||||
aug_emb = base_model.add_embedding(add_embeds)
|
||||
elif base_model.config.addition_embed_type == "image":
|
||||
raise NotImplementedError
|
||||
raise NotImplementedError()
|
||||
elif base_model.config.addition_embed_type == "image_hint":
|
||||
raise NotImplementedError
|
||||
raise NotImplementedError()
|
||||
|
||||
temb = temb + aug_emb if aug_emb is not None else temb
|
||||
|
||||
|
||||
Reference in New Issue
Block a user