mirror of
https://github.com/comfyanonymous/ComfyUI.git
synced 2025-06-06 11:32:09 +08:00
2ab9618732
* Correct calculate_weight and load for OFT * Correct calculate_weight and loading for BOFT
116 lines
3.6 KiB
Python
116 lines
3.6 KiB
Python
import logging
|
|
from typing import Optional
|
|
|
|
import torch
|
|
import comfy.model_management
|
|
from .base import WeightAdapterBase, weight_decompose
|
|
|
|
|
|
class BOFTAdapter(WeightAdapterBase):
|
|
name = "boft"
|
|
|
|
def __init__(self, loaded_keys, weights):
|
|
self.loaded_keys = loaded_keys
|
|
self.weights = weights
|
|
|
|
@classmethod
|
|
def load(
|
|
cls,
|
|
x: str,
|
|
lora: dict[str, torch.Tensor],
|
|
alpha: float,
|
|
dora_scale: torch.Tensor,
|
|
loaded_keys: set[str] = None,
|
|
) -> Optional["BOFTAdapter"]:
|
|
if loaded_keys is None:
|
|
loaded_keys = set()
|
|
blocks_name = "{}.oft_blocks".format(x)
|
|
rescale_name = "{}.rescale".format(x)
|
|
|
|
blocks = None
|
|
if blocks_name in lora.keys():
|
|
blocks = lora[blocks_name]
|
|
if blocks.ndim == 4:
|
|
loaded_keys.add(blocks_name)
|
|
else:
|
|
blocks = None
|
|
if blocks is None:
|
|
return None
|
|
|
|
rescale = None
|
|
if rescale_name in lora.keys():
|
|
rescale = lora[rescale_name]
|
|
loaded_keys.add(rescale_name)
|
|
|
|
weights = (blocks, rescale, alpha, dora_scale)
|
|
return cls(loaded_keys, weights)
|
|
|
|
def calculate_weight(
|
|
self,
|
|
weight,
|
|
key,
|
|
strength,
|
|
strength_model,
|
|
offset,
|
|
function,
|
|
intermediate_dtype=torch.float32,
|
|
original_weight=None,
|
|
):
|
|
v = self.weights
|
|
blocks = v[0]
|
|
rescale = v[1]
|
|
alpha = v[2]
|
|
dora_scale = v[3]
|
|
|
|
blocks = comfy.model_management.cast_to_device(blocks, weight.device, intermediate_dtype)
|
|
if rescale is not None:
|
|
rescale = comfy.model_management.cast_to_device(rescale, weight.device, intermediate_dtype)
|
|
|
|
boft_m, block_num, boft_b, *_ = blocks.shape
|
|
|
|
try:
|
|
# Get r
|
|
I = torch.eye(boft_b, device=blocks.device, dtype=blocks.dtype)
|
|
# for Q = -Q^T
|
|
q = blocks - blocks.transpose(-1, -2)
|
|
normed_q = q
|
|
if alpha > 0: # alpha in boft/bboft is for constraint
|
|
q_norm = torch.norm(q) + 1e-8
|
|
if q_norm > alpha:
|
|
normed_q = q * alpha / q_norm
|
|
# use float() to prevent unsupported type in .inverse()
|
|
r = (I + normed_q) @ (I - normed_q).float().inverse()
|
|
r = r.to(weight)
|
|
inp = org = weight
|
|
|
|
r_b = boft_b//2
|
|
for i in range(boft_m):
|
|
bi = r[i]
|
|
g = 2
|
|
k = 2**i * r_b
|
|
if strength != 1:
|
|
bi = bi * strength + (1-strength) * I
|
|
inp = (
|
|
inp.unflatten(0, (-1, g, k))
|
|
.transpose(1, 2)
|
|
.flatten(0, 2)
|
|
.unflatten(0, (-1, boft_b))
|
|
)
|
|
inp = torch.einsum("b i j, b j ...-> b i ...", bi, inp)
|
|
inp = (
|
|
inp.flatten(0, 1).unflatten(0, (-1, k, g)).transpose(1, 2).flatten(0, 2)
|
|
)
|
|
|
|
if rescale is not None:
|
|
inp = inp * rescale
|
|
|
|
lora_diff = inp - org
|
|
lora_diff = comfy.model_management.cast_to_device(lora_diff, weight.device, intermediate_dtype)
|
|
if dora_scale is not None:
|
|
weight = weight_decompose(dora_scale, weight, lora_diff, alpha, strength, intermediate_dtype, function)
|
|
else:
|
|
weight += function((strength * lora_diff).type(weight.dtype))
|
|
except Exception as e:
|
|
logging.error("ERROR {} {} {}".format(self.name, key, e))
|
|
return weight
|