From aa464737be83d7d02716537a1eb41515639335b8 Mon Sep 17 00:00:00 2001 From: Johannes Stelzer Date: Mon, 21 Nov 2022 17:23:16 +0100 Subject: [PATCH] example for inpainting, cleanup --- example1_standard.py | 16 +- example2_inpaint.py | 103 ++++++++ latent_blending.py | 568 +------------------------------------------ 3 files changed, 112 insertions(+), 575 deletions(-) create mode 100644 example2_inpaint.py diff --git a/example1_standard.py b/example1_standard.py index 74f5f4b..99854a3 100644 --- a/example1_standard.py +++ b/example1_standard.py @@ -13,29 +13,21 @@ # limitations under the License. import os, sys -dp_git = "/home/lugo/git/" -sys.path.append(os.path.join(dp_git,'garden4')) -sys.path.append('util') import torch torch.backends.cudnn.benchmark = False import numpy as np import warnings warnings.filterwarnings('ignore') -import time -import subprocess import warnings import torch from tqdm.auto import tqdm -from diffusers import StableDiffusionInpaintPipeline from diffusers import StableDiffusionPipeline from diffusers.schedulers import DDIMScheduler from PIL import Image import matplotlib.pyplot as plt import torch from movie_man import MovieSaver -import datetime from typing import Callable, List, Optional, Union -import inspect from latent_blending import LatentBlending, add_frames_linear_interp torch.set_grad_enabled(False) @@ -60,13 +52,13 @@ pipe = pipe.to(device) #%% Next let's set up all parameters # FIXME below fix numbers -# We want 20 diffusion steps, begin with 2 branches, have 3 branches at step 12 (=0.6*20) +# We want 20 diffusion steps in total, begin with 2 branches, have 3 branches at step 12 (=0.6*20) # 10 branches at step 16 (=0.8*20) and 24 branches at step 18 (=0.9*20) # Furthermore we want seed 993621550 for keyframeA and seed 54878562 for keyframeB () -num_inference_steps = 100 # Number of diffusion interations -list_nmb_branches = [2, 12, 30, 100, 300] # Specify the branching structure -list_injection_strength = [0.0, 0.75, 0.9, 0.93, 0.96] # Specify the branching structure +num_inference_steps = 20 # Number of diffusion interations +list_nmb_branches = [2, 3, 10, 24] # Branching structure: how many branches +list_injection_strength = [0.0, 0.6, 0.8, 0.9] # Branching structure: how deep is the blending width = 512 height = 512 guidance_scale = 5 diff --git a/example2_inpaint.py b/example2_inpaint.py new file mode 100644 index 0000000..ca2552f --- /dev/null +++ b/example2_inpaint.py @@ -0,0 +1,103 @@ +# Copyright 2022 Lunar Ring. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os, sys +import torch +torch.backends.cudnn.benchmark = False +import numpy as np +import warnings +warnings.filterwarnings('ignore') +import time +import subprocess +import warnings +import torch +from tqdm.auto import tqdm +from diffusers import StableDiffusionInpaintPipeline +from PIL import Image +import matplotlib.pyplot as plt +import torch +from movie_man import MovieSaver +from typing import Callable, List, Optional, Union +from latent_blending import LatentBlending, add_frames_linear_interp +torch.set_grad_enabled(False) + +#%% First let us spawn a diffusers pipe using DDIMScheduler +device = "cuda:0" +model_path = "../stable_diffusion_models/stable-diffusion-inpainting" + +pipe = StableDiffusionInpaintPipeline.from_pretrained( + model_path, + revision="fp16", + torch_dtype=torch.float16, + safety_checker=None +) +pipe = pipe.to(device) + + +#%% Let's make a source image and mask. +height = 512 +width = 512 +num_inference_steps = 30 +guidance_scale = 5 +fixed_seeds = [629575320, 670154945] + +lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale) +prompt1 = "photo of a futuristic alien temple in a desert, mystic, glowing, organic, intricate, sci-fi movie, mesmerizing, scary" +lb.set_prompt1(prompt1) +lb.init_inpainting(init_empty=True) +lb.set_seed(fixed_seeds[0]) +image_source = lb.run_diffusion(lb.text_embedding1, return_image=True) +mask_image = 255*np.ones([512,512], dtype=np.uint8) +mask_image[160:250, 200:320] = 0 +mask_image = Image.fromarray(mask_image) + + +#%% Next let's set up all parameters +# FIXME below fix numbers +# We want 20 diffusion steps, begin with 2 branches, have 3 branches at step 12 (=0.6*20) +# 10 branches at step 16 (=0.8*20) and 24 branches at step 18 (=0.9*20) +# Furthermore we want seed 993621550 for keyframeA and seed 54878562 for keyframeB () + +num_inference_steps = 20 # Number of diffusion interations +list_nmb_branches = [2, 3, 10, 24] # Branching structure: how many branches +list_injection_strength = [0.0, 0.6, 0.8, 0.9] # Branching structure: how deep is the blending +width = 512 +height = 512 +guidance_scale = 5 +fixed_seeds = [993621550, 280335986] + +lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale) +prompt1 = "photo of a futuristic alien temple in a desert, mystic, glowing, organic, intricate, sci-fi movie, mesmerizing, scary" +prompt2 = "aerial photo of a futuristic alien temple in a coastal area, waves clashing" +lb.set_prompt1(prompt1) +lb.set_prompt2(prompt2) +lb.init_inpainting(image_source, mask_image) + +imgs_transition = lb.run_transition(list_nmb_branches, list_injection_strength, fixed_seeds=fixed_seeds) + +# let's get more cheap frames via linear interpolation +duration_transition = 12 +fps = 60 +imgs_transition_ext = add_frames_linear_interp(imgs_transition, duration_transition, fps) + +# movie saving +fp_movie = f"/home/lugo/tmp/latentblending/bobo_incoming.mp4" +if os.path.isfile(fp_movie): + os.remove(fp_movie) +ms = MovieSaver(fp_movie, fps=fps, profile='save') +for img in tqdm(imgs_transition_ext): + ms.write_frame(img) +ms.finalize() + + diff --git a/latent_blending.py b/latent_blending.py index 75a6644..adca9b7 100644 --- a/latent_blending.py +++ b/latent_blending.py @@ -928,241 +928,9 @@ def get_time(resolution=None): raise ValueError("bad resolution provided: %s" %resolution) return t -#%% INIT OUTPAINT -# xxxx +#%% le main if __name__ == "__main__": - #%% INIT DEFAULT - - num_inference_steps = 20 - width = 512 - height = 512 - guidance_scale = 5 - seed = 421 - mode = 'standard' - fps_target = 24 - duration_target = 10 - gpu_id = 0 - - device = "cuda:"+str(gpu_id) - model_path = "../stable_diffusion_models/stable-diffusion-v1-5" - - scheduler = DDIMScheduler(beta_start=0.00085, - beta_end=0.012, - beta_schedule="scaled_linear", - clip_sample=False, - set_alpha_to_one=False) - - pipe = StableDiffusionPipeline.from_pretrained( - model_path, - revision="fp16", - # height = height, - # width = width, - torch_dtype=torch.float16, - scheduler=scheduler, - use_auth_token=True - ) - pipe = pipe.to(device) - - #%% standard TRANS RE SANITY - - lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed) - self = lb - prompt1 = "photograph of NYC skyline"#", skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m" - prompt2 = "photograph of NYC skyline at dawn"#", skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m" - self.set_prompt1(prompt1) - self.set_prompt2(prompt2) - - list_nmb_branches = [2, 4] - list_injection_idx = [0, 10] - - fixed_seeds = [421110, 421110] - - ax = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds) - - - #%% EXPERIMENT - prompt1 = "dark painting of a nice house"#", skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m" - prompt2 = "beautiful surreal painting sunset over the ocean"#", skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m" - self.set_prompt1(prompt1) - self.set_prompt2(prompt2) - # we want to run nmb_experiments experiments. all with the same seed - nmb_experiments = 100 - seed1 = 420 - list_seeds2 = [] - list_seeds2.append(seed1) - for j in range(nmb_experiments-1): - list_seeds2.append(np.random.randint(1999912934)) - - # storage - list_latents_exp = [] - list_imgfinal_exp = [] - - - - for j in range(nmb_experiments): - # now run trans alex way - list_nmb_branches = [2, 10] - list_injection_idx = [0, 1] - fixed_seeds = [seed, list_seeds2[j]] - list_imgs_res = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds) - list_latents_exp.append(self.tree_latents[1]) - - # lets run johannes way, just store imgs here - list_nmb_branches = [2, 3, 6, 12] - list_injection_idx = [0, 10, 13, 16] - list_imgs_good = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds) - list_imgfinal_exp.append(list_imgs_good) - - print(f"DONE WITH EXP {j+1}/{nmb_experiments}") - - #%% - for j in range(100): - lx = list_imgfinal_exp[j] - lxx = add_frames_linear_interp(lx, fps_target=24, duration_target=6) - str_idx = f"{j}".zfill(3) - fp_movie = f'/mnt/jamaica/data_lake/bobi_projects/diffusion/exp/trans_{str_idx}.mp4' - ms = MovieSaver(fp_movie, fps=24, profile='save') - for k, img in enumerate(lxx): - ms.write_frame(img) - ms.finalize() - - #%% surgery - t_iter = 15 - rdim = 64*64*4 - res = np.zeros((100, 10, rdim)) - for j in range(100): - for k in range(10): - res[j,k,:] = list_latents_exp[j][k][t_iter].cpu().numpy().ravel() - - - - - - - #%% NEW TRANS - - - prompt1 = "nigth sky reflected on the ocean" - prompt2 = "beautiful forest painting sunset" - - num_inference_steps = 15 - width = 512 - height = 512 - guidance_scale = 5 - seed = 421 - fps_target = 24 - duration_target = 15 - gpu_id = 0 - - # define mask_image - mask_image = 255*np.ones([512,512], dtype=np.uint8) - mask_image[200:300, 200:300] = 0 - mask_image = Image.fromarray(mask_image) - - # load diffusion pipe - device = "cuda:"+str(gpu_id) - model_path = "../stable_diffusion_models/stable-diffusion-inpainting" - - # scheduler = DDIMScheduler(beta_start=0.00085, - # beta_end=0.012, - # beta_schedule="scaled_linear", - # clip_sample=False, - # set_alpha_to_one=False) - - # L - pipe = StableDiffusionInpaintPipeline.from_pretrained( - model_path, - revision="fp16", - torch_dtype=torch.float16, - # scheduler=scheduler, - safety_checker=None - ) - pipe = pipe.to(device) - - lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed) - self = lb - - - - - xxx - # init latentblending & run - self.set_prompt1(prompt1) - self.set_prompt2(prompt2) - - # we first compute img1. we need the full image to do inpainting - self.init_inpainting(init_empty=True) - list_latents = self.run_diffusion_inpaint(self.text_embedding1) - image_source = self.latent2image(list_latents[-1]) - self.init_inpainting(image_source, mask_image) - - img1 = image_source - - #%% INPAINT HALAL - lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed) - self = lb - list_nmb_branches = [2, 4, 6] - list_injection_idx = [0, 4, 12] - list_prompts = [] - list_prompts.append("paiting of a medieval city") - list_prompts.append("paiting of a forest") - list_prompts.append("photo of a desert landscape") - list_prompts.append("photo of a jungle") - - #% INPAINT SANITY 1 - # run empty trans - prompt1 = "nigth sky reflected on the ocean" - prompt2 = "beautiful forest painting sunset" - self.set_prompt1(prompt1) - self.set_prompt2(prompt2) - list_seeds = [420, 420] - self.init_inpainting(init_empty=True) - list_imgs0 = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=list_seeds) - img_source = list_imgs0[-1] - - - - #% INPAINT SANITY 2 - mask_image = 255*np.ones([512,512], dtype=np.uint8) - mask_image[0:222, 0:222] = 0 - - self.swap_forward() - # we provide a new prompt for image2 - prompt2 = list_prompts[0]# "beautiful painting ocean sunset colorful" - # self.swap_forward() - self.randomize_seed() - self.set_prompt2(prompt2) - self.init_inpainting(image_source=img_source, mask_image=mask_image) - list_imgs1 = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, recycle_img1=True, fixed_seeds=list_seeds) - - - plt.imshow(list_imgs0[0]) - plt.show() - plt.imshow(list_imgs0[-1]) - plt.show() - plt.imshow(list_imgs1[0]) - plt.show() - - #%% mini surgery - idx_branch = 5 - img = self.latent2image(self.tree_latents[-1][idx_branch][-1]) - plt.imshow(img) - - #%% INPAINT SANITY 3 - mask_image = 255*np.ones([512,512], dtype=np.uint8) - mask_image[0:222, 0:222] = 0 - - self.swap_forward() - # we provide a new prompt for image2 - prompt2 = list_prompts[1]# "beautiful painting ocean sunset colorful" - # self.swap_forward() - self.randomize_seed() - self.set_prompt2(prompt2) - self.init_inpainting(image_source=img_source, mask_image=mask_image) - list_imgs2 = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=list_seeds) - - #%% LOOP list_prompts = [] list_prompts.append("paiting of a medieval city") @@ -1187,67 +955,8 @@ if __name__ == "__main__": # now we switch them around so image2 becomes image1 img1 = list_imgs[-1] - #%% surg - img = self.latent2image(self.tree_latents[-1][3][-1]) - plt.imshow(img) - - #%% GOOD SINGLE TRANS - height = 512 - width = 512 - lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed) - self = lb - self.randomize_seed() - # init latentblending & run - prompt1 = "photograph of NYC skyline at dawn, skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m" - prompt2 = "photograph of NYC skyline at dusk, skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m" - # prompt1 = "hologram portrait of sumerian god of artificial general intelligence, AI, dystopian, utopia, year 2222, insane detail, incredible machinery, cybernetic power god, sumerian statue, steel clay, silicon masterpiece, futuristic symmetry" - # prompt1 = "photograph of a dark cyberpunk street at night, neon signs, cinematic film still, dark science fiction, highly detailed, bokeh, f5.6, Leica M9, cinematic, iso 100, Kodak Portra" - # prompt2 = "bright photograph of a cyberpunk during daytime, cinematic film still, science fiction, highly detailed, bokeh, f5.6, Leica M9, cinematic, iso 100, Kodak Portra" - # prompt2 = "surreal_painting_of_stylized_sexual_forest" - - - self.set_prompt1(prompt1) - self.set_prompt2(prompt2) - - # list_nmb_branches = [2, 4, 12]#, 15, 200] - # list_injection_idx = [0, 8, 13]#, 24, 28] - list_nmb_branches = [2, 4, 8, 15, 200] - list_injection_idx = [0, 10, 20, 24, 28] - fps_target = 30 - duration_target = 10 - loop_back = True - t0 = time.time() - list_imgs = self.run_transition(list_nmb_branches, list_injection_idx, fixed_seeds='randomize') - - # 1: 2142 - list_imgs_interp = add_frames_linear_interp(list_imgs, fps_target, duration_target) - dt = time.time() - t0 - - - - loop_back = True - # movie saving - str_steps = "" - for s in list_nmb_branches: - str_steps += f"{s}_" - str_steps = str_steps[0:-1] - - str_inject = "" - for k in list_injection_idx: - str_inject += f"{k}_" - str_inject = str_inject[0:-1] - - fp_movie = f"/mnt/jamaica/data_lake/bobi_projects/diffusion/movies/221116_lb/lb_{get_time('second')}_s{str_steps}_k{str_inject}.mp4" - - ms = MovieSaver(fp_movie, fps=fps_target, profile='save') - for img in tqdm(list_imgs_interp): - ms.write_frame(img) - if loop_back: - for img in tqdm(list_imgs_interp[::-1]): - ms.write_frame(img) - ms.finalize() - + #%% GOOD MOVIE ENGINE num_inference_steps = 30 @@ -1269,202 +978,10 @@ if __name__ == "__main__": lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed) dp_movie = "/home/lugo/tmp/movie" - list_parts = [] - for i in range(len(list_prompts)-1): - print(f"Starting movie segment {i+1}/{len(list_prompts)}") - if i==0: - lb.set_prompt1(list_prompts[i]) - lb.set_prompt2(list_prompts[i+1]) - recycle_img1 = False - else: - lb.swap_forward() - lb.set_prompt2(list_prompts[i+1]) - recycle_img1 = True + + - list_imgs = lb.run_transition(list_nmb_branches, list_injection_idx, recycle_img1=recycle_img1) - list_imgs_interp = add_frames_linear_interp(list_imgs, fps_target, duration_target) - - # Save Movie segment - str_idx = f"{i}".zfill(3) - fp_movie = os.path.join(dp_movie, f"{str_idx}.mp4") - ms = MovieSaver(fp_movie, fps=fps_target, profile='save') - for img in tqdm(list_imgs_interp): - ms.write_frame(img) - ms.finalize() - - list_parts.append(fp_movie) - - - list_concat = [] - - for fp_part in list_parts: - list_concat.append(f"""file '{fp_part}'""") - - fp_out = os.path.join(dp_movie, "concat.txt") - - with open(fp_out, "w") as fa: - for item in list_concat: - fa.write("%s\n" % item) - - # str_steps = "" - # for s in list_injection_steps: - # str_steps += f"{s}_" - # str_steps = str_steps[0:-1] - - # str_inject = "" - # for k in list_injection_strength: - # str_inject += f"{k}_" - # str_inject = str_inject[0:-1] - - fp_movie = os.path.join(dp_movie, f'final_movie_{get_time("second")}.mp4') - - cmd = f'ffmpeg -f concat -safe 0 -i {fp_out} -c copy {fp_movie}' - subprocess.call(cmd, shell=True, cwd=dp_movie) - - - - #%% - - list_latents1 = self.run_diffusion(self.text_embedding1) - img1 = self.latent2image(list_latents1[-1]) - - #%% - list_seeds = [] - list_all_latents = [] - list_all_imgs = [] - for i in tqdm(range(100)): - seed = np.random.randint(9999999) - list_seeds.append(seed) - self.seed = seed - list_imgs = self.run_transition(list_nmb_branches, list_injection_idx, True, False) - img2 = list_imgs[-1] - list_all_imgs.append(img2) - list_all_latents.append(self.list_latents_key2) - - - - - #%% - - list_injection_idx = [0, 10, 17, 22, 25] - list_nmb_branches = [3, 6, 10, 30, 60] - - list_imgs_interp = add_frames_linear_interp(list_imgs, fps_target, duration_target) - fp_movie = f"/home/lugo/tmp/lb_new2.mp4" - - ms = MovieSaver(fp_movie, fps=fps_target, profile='save') - for img in tqdm(list_imgs_interp): - ms.write_frame(img) - if True: - for img in tqdm(list_imgs_interp[::-1]): - ms.write_frame(img) - ms.finalize() - - #%% SURGERY - - - - #%% TEST WITH LATENT DIFFS - - # Collect all basic infos - num_inference_steps = 10 - lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed) - self = lb - prompt1 = "magic painting of a oak tree, mystic" - prompt2 = "painting of a trippy lake and symmetry reflections" - self.set_prompt1(prompt1) - self.set_prompt2(prompt2) - - list_latents1 = self.run_diffusion(self.text_embedding1) - img1 = self.latent2image(list_latents1[-1]) - - #%% - list_seeds = [] - list_all_latents = [] - list_all_imgs = [] - for i in tqdm(range(100)): - seed = np.random.randint(9999999) - list_seeds.append(seed) - self.seed = seed - list_latents2 = self.run_diffusion(self.text_embedding2) - img2 = self.latent2image(list_latents2[-1]) - list_all_latents.append(list_latents2) - list_all_imgs.append(img2) - - #%% - # res = np.zeros([100,10]) - # for i in tqdm(range(100)): - # for j in range(num_inference_steps): - - # diff = torch.linalg.norm(list_blocks_prev[bprev][-1]-list_blocks_prev[bprev+1][-1]).item() - - - #%% convert to images - list_imgs = [] - for b in range(len(list_blocks_current)): - img = self.latent2image(list_blocks_current[b][-1]) - list_imgs.append(img) - - - #%% fract - dists_inv = [] - for bprev in range(len(list_blocks_current)-1): - diff = torch.linalg.norm(list_blocks_current[bprev][-1]-list_blocks_current[bprev+1][-1]).item() - dists_inv.append(diff) - plt.plot(dists_inv) - #%% - imgx = self.latent2image(list_blocks_current[1][-1]) - plt.imshow(imgx) - - - #%% SURGERY - dists = [300, 600, 900] - nmb_branches = 20 - list_fract_mixing_prev = [0, 0.4, 0.6, 1] - - - nmb_injection_slots = len(dists) - nmb_injections = nmb_branches-len(list_fract_mixing_prev) - p_samp = get_p(dists) - injection_counter = np.zeros(nmb_injection_slots, dtype=np.int32) - - for j in range(nmb_injections): - idx_injection = np.random.choice(nmb_injection_slots, p=p_samp) - injection_counter[idx_injection] += 1 - - # get linear interpolated injections - list_fract_mixing_current = [] - for j in range(nmb_injection_slots): - fractA = list_fract_mixing_prev[j] - fractB = list_fract_mixing_prev[j+1] - list_fract_mixing_current.extend(np.linspace(fractA, fractB, 2+injection_counter[j])[:-1]) - list_fract_mixing_current.append(1) - - - - - #%% save movie - loop_back = True - # movie saving - str_steps = "" - for s in list_injection_steps: - str_steps += f"{s}_" - str_steps = str_steps[0:-1] - - str_inject = "" - for k in list_injection_strength: - str_inject += f"{k}_" - str_inject = str_inject[0:-1] - - fp_movie = f"/home/lugo/tmp/lb_{get_time('second')}_s{str_steps}_k{str_inject}.mp4" - - ms = MovieSaver(fp_movie, fps=fps_target, profile='save') - for img in tqdm(list_imgs_interp): - ms.write_frame(img) - if loop_back: - for img in tqdm(list_imgs_interp[::-1]): - ms.write_frame(img) - ms.finalize() + #%% EXAMPLE3 MOVIE ENGINE list_injection_steps = [2, 3, 4, 5] @@ -1490,83 +1007,8 @@ if __name__ == "__main__": ) pipe = pipe.to(device) - #%% seed cherrypicking - - prompt1 = "photo of an eerie statue surrounded by ferns and vines" - lb.set_prompt1(prompt1) - - for i in range(4): - seed = np.random.randint(753528763) - lb.set_seed(seed) - txt = f"index {i+1} {seed}: {prompt1}" - img = lb.run_diffusion(lb.text_embedding1, return_image=True) - plt.imshow(img) -# plt.title(txt) - plt.show() - print(txt) - - #%% prompt finetuning - seed = 280335986 - prompt1 = "photo of an eerie statue surrounded by ferns and vines, analog photograph kodak portra, mystical ambience, incredible detail" - lb.set_prompt1(prompt1) - img = lb.run_diffusion(lb.text_embedding1, return_image=True) - plt.imshow(img) - - #%% lets make a nice mask - - - - #%% storage - - #%% make nice images of latents - num_inference_steps = 10 # Number of diffusion interations - list_nmb_branches = [2, 3, 7, 10] # Specify the branching structure - list_injection_idx = [0, 6, 7, 8] # Specify the branching structure - width = 512 - height = 512 - guidance_scale = 5 - fixed_seeds = [993621550, 280335986] - - lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale) - prompt1 = "photo of a beautiful forest covered in white flowers, ambient light, very detailed, magic" - prompt2 = "photo of an eerie statue surrounded by ferns and vines, analog photograph kodak portra, mystical ambience, incredible detail" - lb.set_prompt1(prompt1) - lb.set_prompt2(prompt2) - - imgs_transition = lb.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds) -#%% - dp_tmp= "/home/lugo/tmp/latentblending" - for d in range(len(lb.tree_latents)): - for b in range(list_nmb_branches[d]): - for x in range(len(lb.tree_latents[d][b])): - lati = lb.tree_latents[d][b][x] - img = lb.latent2image(lati) - fn = f"d{d}_b{b}_x{x}.jpg" - ip.save(os.path.join(dp_tmp, fn), img) - - #%% get source img - seed = 280335986 - prompt1 = "photo of a futuristic alien temple resting in the desert, mystic, sunlight" - lb.set_prompt1(prompt1) - lb.init_inpainting(init_empty=True) - - for i in range(5): - seed = np.random.randint(753528763) - lb.set_seed(seed) - txt = f"index {i+1} {seed}: {prompt1}" - img = lb.run_diffusion(lb.text_embedding1, return_image=True) - plt.imshow(img) -# plt.title(txt) - plt.show() - print(txt) - - #%% -""" -index 3 303856737: photo of a futuristic alien temple resting in the desert, mystic, sunlight -""" - #%% """ TODO Coding: