animatediff full latent mode

modules/processing_correction.py

@@ -11,56 +11,74 @@ from modules import shared
 debug = shared.log.info if os.environ.get('SD_HDR_DEBUG', None) is not None else lambda *args, **kwargs: None
 
 
-def soft_clamp_tensor(input_tensor, threshold=0.8, boundary=4):
+def soft_clamp_tensor(tensor, threshold=0.8, boundary=4):
     # shrinking towards the mean; will also remove outliers
-    if max(abs(input_tensor.max()), abs(input_tensor.min())) < boundary or threshold == 0:
-        return input_tensor
+    if max(abs(tensor.max()), abs(tensor.min())) < boundary or threshold == 0:
+        return tensor
     channel_dim = 1
     threshold *= boundary
-    max_vals = input_tensor.max(channel_dim, keepdim=True)[0]
-    max_replace = ((input_tensor - threshold) / (max_vals - threshold)) * (boundary - threshold) + threshold
-    over_mask = input_tensor > threshold
-    min_vals = input_tensor.min(channel_dim, keepdim=True)[0]
-    min_replace = ((input_tensor + threshold) / (min_vals + threshold)) * (-boundary + threshold) - threshold
-    under_mask = input_tensor < -threshold
+    max_vals = tensor.max(channel_dim, keepdim=True)[0]
+    max_replace = ((tensor - threshold) / (max_vals - threshold)) * (boundary - threshold) + threshold
+    over_mask = tensor > threshold
+    min_vals = tensor.min(channel_dim, keepdim=True)[0]
+    min_replace = ((tensor + threshold) / (min_vals + threshold)) * (-boundary + threshold) - threshold
+    under_mask = tensor < -threshold
     debug(f'HDE soft clamp: threshold={threshold} boundary={boundary}')
-    input_tensor = torch.where(over_mask, max_replace, torch.where(under_mask, min_replace, input_tensor))
-    return input_tensor
+    tensor = torch.where(over_mask, max_replace, torch.where(under_mask, min_replace, tensor))
+    return tensor
 
 
-def center_tensor(input_tensor, channel_shift=1.0, full_shift=1.0, channels=[0, 1, 2, 3]): # pylint: disable=dangerous-default-value # noqa: B006
+def center_tensor(tensor, channel_shift=1.0, full_shift=1.0, channels=[0, 1, 2, 3]): # pylint: disable=dangerous-default-value # noqa: B006
     if channel_shift == 0 and full_shift == 0:
-        return input_tensor
+        return tensor
     means = []
     for channel in channels:
-        means.append(input_tensor[0, channel].mean())
-        input_tensor[0, channel] -= means[-1] * channel_shift
+        means.append(tensor[0, channel].mean())
+        tensor[0, channel] -= means[-1] * channel_shift
     debug(f'HDR center: channel-shift{channel_shift} full-shift={full_shift} means={torch.stack(means)}')
-    input_tensor = input_tensor - input_tensor.mean() * full_shift
-    return input_tensor
+    tensor = tensor - tensor.mean() * full_shift
+    return tensor
 
 
-def maximize_tensor(input_tensor, boundary=1.0, channels=[0, 1, 2]): # pylint: disable=dangerous-default-value # noqa: B006
+def maximize_tensor(tensor, boundary=1.0, channels=[0, 1, 2]): # pylint: disable=dangerous-default-value # noqa: B006
     if boundary == 1.0:
-        return input_tensor
+        return tensor
     boundary *= 4
-    min_val = input_tensor.min()
-    max_val = input_tensor.max()
+    min_val = tensor.min()
+    max_val = tensor.max()
     normalization_factor = boundary / max(abs(min_val), abs(max_val))
-    input_tensor[0, channels] *= normalization_factor
+    tensor[0, channels] *= normalization_factor
     debug(f'HDR maximize: boundary={boundary} min={min_val} max={max_val} factor={normalization_factor}')
-    return input_tensor
+    return tensor
 
 
-def correction_callback(p, timestep, kwags):
+def correction(p, timestep, latent):
     if timestep > 950 and p.hdr_clamp:
         p.extra_generation_params["HDR clamp"] = f'{p.hdr_threshold}/{p.hdr_boundary}'
-        kwags["latents"] = soft_clamp_tensor(kwags["latents"], threshold=p.hdr_threshold, boundary=p.hdr_boundary)
+        latent = soft_clamp_tensor(latent, threshold=p.hdr_threshold, boundary=p.hdr_boundary)
     if timestep > 700 and p.hdr_center:
         p.extra_generation_params["HDR center"] = f'{p.hdr_channel_shift}/{p.hdr_full_shift}'
-        kwags["latents"] = center_tensor(kwags["latents"], channel_shift=p.hdr_channel_shift, full_shift=p.hdr_full_shift)
+        latent = center_tensor(latent, channel_shift=p.hdr_channel_shift, full_shift=p.hdr_full_shift)
     if timestep > 1 and timestep < 100 and p.hdr_maximize:
         p.extra_generation_params["HDR max"] = f'{p.hdr_max_center}/p.hdr_max_boundry'
-        kwags["latents"] = center_tensor(kwags["latents"], channel_shift=p.hdr_max_center, full_shift=1.0)
-        kwags["latents"] = maximize_tensor(kwags["latents"], boundary=p.hdr_max_boundry)
-    return kwags
+        latent = center_tensor(latent, channel_shift=p.hdr_max_center, full_shift=1.0)
+        latent = maximize_tensor(latent, boundary=p.hdr_max_boundry)
+    return latent
+
+
+def correction_callback(p, timestep, kwargs):
+    if not p.hdr_clamp and not p.hdr_center and not p.hdr_maximize:
+        return kwargs
+    latents = kwargs["latents"]
+    if len(latents.shape) == 4: # standard batched latent
+        for i in range(latents.shape[0]):
+            latents[i] = correction(p, timestep, latents[i])
+    elif len(latents.shape) == 5 and latents.shape[0] == 1: # probably animatediff
+        latents = latents.squeeze(0).permute(1, 0, 2, 3)
+        for i in range(latents.shape[0]):
+            latents[i] = correction(p, timestep, latents[i])
+        latents = latents.permute(1, 0, 2, 3).unsqueeze(0)
+    else:
+        shared.log.debug(f'HDR correction: unknown latent shape {latents.shape}')
+    kwargs["latents"] = latents
+    return kwargs