From fdb44defc19b1e88fa45f1318a2e235eba64e6e6 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Fri, 4 Aug 2023 01:24:09 +0530 Subject: [PATCH 01/17] Added NoisyLatentImage node in nodes.py Diffusers library starts with a random noise latent image, instead of an empty latent image. Therefore, adding this node. --- nodes.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/nodes.py b/nodes.py index 097f9230..fda79df1 100644 --- a/nodes.py +++ b/nodes.py @@ -871,6 +871,27 @@ class EmptyLatentImage: latent = torch.zeros([batch_size, 4, height // 8, width // 8]) return ({"samples":latent}, ) +class NoisyLatentImage: + def __init__(self, device="cpu"): + self.device = device + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), + "width": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 8}), + "height": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 8}), + "batch_size": ("INT", {"default": 1, "min": 1, "max": 64})}} + RETURN_TYPES = ("LATENT",) + FUNCTION = "generate" + + CATEGORY = "latent" + + def generate(self, seed, width, height, batch_size=1): + generator = torch.manual_seed(seed) + latent = torch.randn([batch_size, 4, height // 8, width // 8], generator=generator, device=self.device) + return ({"samples":latent}, ) + class LatentFromBatch: @classmethod From e1349163af1d295a3fe6394aca0a7ad2521169a5 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:08:57 +0530 Subject: [PATCH 02/17] Add ImageAlphaComposite --- nodes.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nodes.py b/nodes.py index 909a3d76..cd5ca322 100644 --- a/nodes.py +++ b/nodes.py @@ -1434,6 +1434,25 @@ class ImageScale: s = s.movedim(1,-1) return (s,) +class ImageAlphaComposite: + @classmethod + def INPUT_TYPES(s): + return {"required": { "image1": ("IMAGE",), + "mask1": ("MASK",), + "image2": ("IMAGE",), + "mask2": ("MASK", {"default": None}), + } + + CATEGORY = "image" + + RETURN_TYPES = ("IMAGE", "MASK") + FUNCTION = "alpha_composite" + def alpha_composite(self, image1, mask1, image2, mask2): + if not mask2: + mask2 = torch.zeros((64,64), dtype=torch.float32, device="cpu") + image = image1 * mask1 + image2 * (1.0 - mask1) + return (image, mask1) + class ImageScaleBy: upscale_methods = ["nearest-exact", "bilinear", "area", "bicubic"] From 0bc25f1e791d7debaa57f8bbc8fd2a1801c3536f Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:13:53 +0530 Subject: [PATCH 03/17] debug --- nodes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes.py b/nodes.py index cd5ca322..b30d5eab 100644 --- a/nodes.py +++ b/nodes.py @@ -1662,6 +1662,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "PreviewImage": "Preview Image", "LoadImage": "Load Image", "LoadImageMask": "Load Image (as Mask)", + "ImageAlphaComposite": "Image overlay using mask", "ImageScale": "Upscale Image", "ImageScaleBy": "Upscale Image By", "ImageUpscaleWithModel": "Upscale Image (using Model)", From 595d9316da7f04f8f68c600887d5fe2ff9c25e5e Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:24:10 +0530 Subject: [PATCH 04/17] debug --- nodes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes.py b/nodes.py index b30d5eab..6148aa3c 100644 --- a/nodes.py +++ b/nodes.py @@ -1570,6 +1570,7 @@ NODE_CLASS_MAPPINGS = { "PreviewImage": PreviewImage, "LoadImage": LoadImage, "LoadImageMask": LoadImageMask, + "ImageAlphaComposite": ImageAlphaComposite, "ImageScale": ImageScale, "ImageScaleBy": ImageScaleBy, "ImageInvert": ImageInvert, From 07d324c8c3a7d4170a09a578a5b1d5cfd0593e7b Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:24:51 +0530 Subject: [PATCH 05/17] debug --- nodes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes.py b/nodes.py index 6148aa3c..655dd278 100644 --- a/nodes.py +++ b/nodes.py @@ -1442,6 +1442,7 @@ class ImageAlphaComposite: "image2": ("IMAGE",), "mask2": ("MASK", {"default": None}), } + } CATEGORY = "image" From bb42e13c526b7d5dacceab80b62375b676e9a1e1 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:30:26 +0530 Subject: [PATCH 06/17] debug --- nodes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nodes.py b/nodes.py index 655dd278..962f41e1 100644 --- a/nodes.py +++ b/nodes.py @@ -1449,8 +1449,6 @@ class ImageAlphaComposite: RETURN_TYPES = ("IMAGE", "MASK") FUNCTION = "alpha_composite" def alpha_composite(self, image1, mask1, image2, mask2): - if not mask2: - mask2 = torch.zeros((64,64), dtype=torch.float32, device="cpu") image = image1 * mask1 + image2 * (1.0 - mask1) return (image, mask1) From 16a862c72229f633d02e6b6ab188a1acb82fb04b Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:33:41 +0530 Subject: [PATCH 07/17] debug --- nodes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 962f41e1..4ac0aa02 100644 --- a/nodes.py +++ b/nodes.py @@ -1449,7 +1449,8 @@ class ImageAlphaComposite: RETURN_TYPES = ("IMAGE", "MASK") FUNCTION = "alpha_composite" def alpha_composite(self, image1, mask1, image2, mask2): - image = image1 * mask1 + image2 * (1.0 - mask1) + print(image.get_shape(), mask1.get_shape()) + image = image1 * mask1 return (image, mask1) class ImageScaleBy: From ce546a369802b7a6183897ee56066eab2f828170 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:35:16 +0530 Subject: [PATCH 08/17] Update nodes.pydebug --- nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 4ac0aa02..0bd41d2f 100644 --- a/nodes.py +++ b/nodes.py @@ -1449,7 +1449,7 @@ class ImageAlphaComposite: RETURN_TYPES = ("IMAGE", "MASK") FUNCTION = "alpha_composite" def alpha_composite(self, image1, mask1, image2, mask2): - print(image.get_shape(), mask1.get_shape()) + print(image1.get_shape(), mask1.get_shape()) image = image1 * mask1 return (image, mask1) From ee6c3b358771b3583ff8df6a43adaf0939914ce6 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:38:53 +0530 Subject: [PATCH 09/17] debug --- nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 0bd41d2f..7da2caf0 100644 --- a/nodes.py +++ b/nodes.py @@ -1449,7 +1449,7 @@ class ImageAlphaComposite: RETURN_TYPES = ("IMAGE", "MASK") FUNCTION = "alpha_composite" def alpha_composite(self, image1, mask1, image2, mask2): - print(image1.get_shape(), mask1.get_shape()) + print(image1.size(), mask1.size()) image = image1 * mask1 return (image, mask1) From 56a5ddd8423796683ea10b64838b24b1b301455c Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:55:17 +0530 Subject: [PATCH 10/17] debug --- nodes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nodes.py b/nodes.py index 7da2caf0..845e6879 100644 --- a/nodes.py +++ b/nodes.py @@ -1439,19 +1439,19 @@ class ImageAlphaComposite: def INPUT_TYPES(s): return {"required": { "image1": ("IMAGE",), "mask1": ("MASK",), - "image2": ("IMAGE",), - "mask2": ("MASK", {"default": None}), + "image2": ("IMAGE",)), } } CATEGORY = "image" - RETURN_TYPES = ("IMAGE", "MASK") + RETURN_TYPES = ("IMAGE",) FUNCTION = "alpha_composite" - def alpha_composite(self, image1, mask1, image2, mask2): + def alpha_composite(self, image1, mask1, image2): + mask1 = mask1.repeat((1, 1, 1, 3)) print(image1.size(), mask1.size()) image = image1 * mask1 - return (image, mask1) + return (image,) class ImageScaleBy: upscale_methods = ["nearest-exact", "bilinear", "area", "bicubic"] From fd61a28a9ce44a825554018f996bde7c495b3871 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 20:55:57 +0530 Subject: [PATCH 11/17] debug --- nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 845e6879..b4fa8e30 100644 --- a/nodes.py +++ b/nodes.py @@ -1439,7 +1439,7 @@ class ImageAlphaComposite: def INPUT_TYPES(s): return {"required": { "image1": ("IMAGE",), "mask1": ("MASK",), - "image2": ("IMAGE",)), + "image2": ("IMAGE",) } } From fcd94aa0434da2c090f60f46eb89f4680f78e2aa Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 21:01:30 +0530 Subject: [PATCH 12/17] debug --- nodes.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nodes.py b/nodes.py index b4fa8e30..789c147e 100644 --- a/nodes.py +++ b/nodes.py @@ -1438,7 +1438,7 @@ class ImageAlphaComposite: @classmethod def INPUT_TYPES(s): return {"required": { "image1": ("IMAGE",), - "mask1": ("MASK",), + "mask": ("MASK",), "image2": ("IMAGE",) } } @@ -1447,8 +1447,12 @@ class ImageAlphaComposite: RETURN_TYPES = ("IMAGE",) FUNCTION = "alpha_composite" - def alpha_composite(self, image1, mask1, image2): - mask1 = mask1.repeat((1, 1, 1, 3)) + def alpha_composite(self, image1, mask, image2): + mask1 = torch.ones_like(image1) + mask1[0, :, :, 0] = mask + mask1[0, :, :, 1] = mask + mask1[0, :, :, 2] = mask + print(image1.size(), mask1.size()) image = image1 * mask1 return (image,) From 2e69927399cb5765ab32afa8aec0858b5646d1ff Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 21:04:47 +0530 Subject: [PATCH 13/17] debug --- nodes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 789c147e..c3ca5845 100644 --- a/nodes.py +++ b/nodes.py @@ -1449,12 +1449,13 @@ class ImageAlphaComposite: FUNCTION = "alpha_composite" def alpha_composite(self, image1, mask, image2): mask1 = torch.ones_like(image1) + mask = torch.ones_like(mask) - mask mask1[0, :, :, 0] = mask mask1[0, :, :, 1] = mask mask1[0, :, :, 2] = mask print(image1.size(), mask1.size()) - image = image1 * mask1 + image = image1 * mask1 + image2 * (torch.ones_like(mask) - mask) return (image,) class ImageScaleBy: From 17b2999195792fbb08ef88bb39b02a41f33a0a07 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 21:07:30 +0530 Subject: [PATCH 14/17] debug --- nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index c3ca5845..b5b79439 100644 --- a/nodes.py +++ b/nodes.py @@ -1455,7 +1455,7 @@ class ImageAlphaComposite: mask1[0, :, :, 2] = mask print(image1.size(), mask1.size()) - image = image1 * mask1 + image2 * (torch.ones_like(mask) - mask) + image = image1 * mask1 + image2 * (torch.ones_like(mask1) - mask1) return (image,) class ImageScaleBy: From 3fe543dedcd832d78eb62d540a6a6de2484abdc7 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 21:12:09 +0530 Subject: [PATCH 15/17] debug --- nodes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nodes.py b/nodes.py index b5b79439..03eddb2c 100644 --- a/nodes.py +++ b/nodes.py @@ -1448,14 +1448,15 @@ class ImageAlphaComposite: RETURN_TYPES = ("IMAGE",) FUNCTION = "alpha_composite" def alpha_composite(self, image1, mask, image2): + # image1 = torch.nn.functional.interpolate(mask[None, None], size=(source.shape[2], source.shape[3]), mode="bilinear") + mask1 = torch.ones_like(image1) - mask = torch.ones_like(mask) - mask mask1[0, :, :, 0] = mask mask1[0, :, :, 1] = mask mask1[0, :, :, 2] = mask print(image1.size(), mask1.size()) - image = image1 * mask1 + image2 * (torch.ones_like(mask1) - mask1) + image = image1 * (torch.ones_like(mask1) - mask1) return (image,) class ImageScaleBy: From 4a25a653dbeb2c58a75e814593f063c4ce981cdb Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 21:17:10 +0530 Subject: [PATCH 16/17] debug --- nodes.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nodes.py b/nodes.py index 03eddb2c..ab9259a7 100644 --- a/nodes.py +++ b/nodes.py @@ -1448,15 +1448,12 @@ class ImageAlphaComposite: RETURN_TYPES = ("IMAGE",) FUNCTION = "alpha_composite" def alpha_composite(self, image1, mask, image2): - # image1 = torch.nn.functional.interpolate(mask[None, None], size=(source.shape[2], source.shape[3]), mode="bilinear") - mask1 = torch.ones_like(image1) mask1[0, :, :, 0] = mask mask1[0, :, :, 1] = mask mask1[0, :, :, 2] = mask - print(image1.size(), mask1.size()) - image = image1 * (torch.ones_like(mask1) - mask1) + image = image1 * (torch.ones_like(mask1) - mask1) + image2 * mask1 return (image,) class ImageScaleBy: From 9fedd24fc49801aa772742c91a8ee2553db20485 Mon Sep 17 00:00:00 2001 From: Saquib Alam Date: Tue, 8 Aug 2023 22:41:52 +0530 Subject: [PATCH 17/17] remove noisy latent code --- nodes.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/nodes.py b/nodes.py index ab9259a7..362eeab7 100644 --- a/nodes.py +++ b/nodes.py @@ -871,28 +871,6 @@ class EmptyLatentImage: latent = torch.zeros([batch_size, 4, height // 8, width // 8]) return ({"samples":latent}, ) -class NoisyLatentImage: - def __init__(self, device="cpu"): - self.device = device - - @classmethod - def INPUT_TYPES(s): - return {"required": { - "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}), - "width": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 8}), - "height": ("INT", {"default": 512, "min": 64, "max": MAX_RESOLUTION, "step": 8}), - "batch_size": ("INT", {"default": 1, "min": 1, "max": 64})}} - RETURN_TYPES = ("LATENT",) - FUNCTION = "generate" - - CATEGORY = "latent" - - def generate(self, seed, width, height, batch_size=1): - generator = torch.manual_seed(seed) - latent = torch.randn([batch_size, 4, height // 8, width // 8], generator=generator, device=self.device) - return ({"samples":latent}, ) - - class LatentFromBatch: @classmethod def INPUT_TYPES(s):