From e4a736a70e21561908d88e8050ecfca2a48d7b61 Mon Sep 17 00:00:00 2001 From: Johannes Stelzer Date: Mon, 21 Nov 2022 16:24:06 +0100 Subject: [PATCH] finding good inpainting --- example1_standard.py | 21 ++++++++--------- latent_blending.py | 54 +++++++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/example1_standard.py b/example1_standard.py index cabd035..74f5f4b 100644 --- a/example1_standard.py +++ b/example1_standard.py @@ -59,32 +59,30 @@ pipe = StableDiffusionPipeline.from_pretrained( pipe = pipe.to(device) #%% Next let's set up all parameters - +# FIXME below fix numbers # We want 20 diffusion steps, begin with 2 branches, have 3 branches at step 12 (=0.6*20) # 10 branches at step 16 (=0.8*20) and 24 branches at step 18 (=0.9*20) # Furthermore we want seed 993621550 for keyframeA and seed 54878562 for keyframeB () -num_inference_steps = 30 # Number of diffusion interations -list_nmb_branches = [2, 6, 30, 100] # Specify the branching structure -list_injection_strength = [0.0, 0.3, 0.73, 0.93] # Specify the branching structure +num_inference_steps = 100 # Number of diffusion interations +list_nmb_branches = [2, 12, 30, 100, 300] # Specify the branching structure +list_injection_strength = [0.0, 0.75, 0.9, 0.93, 0.96] # Specify the branching structure width = 512 height = 512 guidance_scale = 5 -#fixed_seeds = [993621550, 326814432] -#fixed_seeds = [993621550, 888839807] -fixed_seeds = [993621550, 753528763] +fixed_seeds = [993621550, 280335986] lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale) prompt1 = "photo of a beautiful forest covered in white flowers, ambient light, very detailed, magic" -prompt2 = "photo of a mystical sculpture in the middle of the desert, warm sunlight, sand, eery feeling" +prompt2 = "photo of an eerie statue surrounded by ferns and vines, analog photograph kodak portra, mystical ambience, incredible detail" lb.set_prompt1(prompt1) lb.set_prompt2(prompt2) + + imgs_transition = lb.run_transition(list_nmb_branches, list_injection_strength, fixed_seeds=fixed_seeds) -#% - -# let's get more frames +# let's get more cheap frames via linear interpolation duration_transition = 12 fps = 60 imgs_transition_ext = add_frames_linear_interp(imgs_transition, duration_transition, fps) @@ -99,4 +97,3 @@ for img in tqdm(imgs_transition_ext): ms.finalize() -# MOVIE TODO: ueberschreiben! bad prints. \ No newline at end of file diff --git a/latent_blending.py b/latent_blending.py index ba506a9..75a6644 100644 --- a/latent_blending.py +++ b/latent_blending.py @@ -1484,8 +1484,6 @@ if __name__ == "__main__": pipe = StableDiffusionPipeline.from_pretrained( model_path, revision="fp16", - height = height, - width = width, torch_dtype=torch.float16, scheduler=DDIMScheduler(), use_auth_token=True @@ -1494,33 +1492,44 @@ if __name__ == "__main__": #%% seed cherrypicking - prompt1 = "photo of a surreal brutalistic vault that is glowing in the night, futuristic, greek ornaments, spider webs" + prompt1 = "photo of an eerie statue surrounded by ferns and vines" lb.set_prompt1(prompt1) - for i in range(1): - seed = 753528763 #np.random.randint(753528763) + for i in range(4): + seed = np.random.randint(753528763) lb.set_seed(seed) - txt = f"{i} {seed}" + txt = f"index {i+1} {seed}: {prompt1}" img = lb.run_diffusion(lb.text_embedding1, return_image=True) plt.imshow(img) - plt.title(txt) +# plt.title(txt) plt.show() print(txt) + #%% prompt finetuning + seed = 280335986 + prompt1 = "photo of an eerie statue surrounded by ferns and vines, analog photograph kodak portra, mystical ambience, incredible detail" + lb.set_prompt1(prompt1) + img = lb.run_diffusion(lb.text_embedding1, return_image=True) + plt.imshow(img) + #%% lets make a nice mask + + + + #%% storage #%% make nice images of latents num_inference_steps = 10 # Number of diffusion interations - list_nmb_branches = [2, 3, 7, 12] # Specify the branching structure - list_injection_idx = [0, 2, 5, 8] # Specify the branching structure + list_nmb_branches = [2, 3, 7, 10] # Specify the branching structure + list_injection_idx = [0, 6, 7, 8] # Specify the branching structure width = 512 height = 512 guidance_scale = 5 - fixed_seeds = [993621550, 326814432] + fixed_seeds = [993621550, 280335986] lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale) prompt1 = "photo of a beautiful forest covered in white flowers, ambient light, very detailed, magic" - prompt2 = "photo of a mystical sculpture in the middle of the desert, warm sunlight, sand, eery feeling" + prompt2 = "photo of an eerie statue surrounded by ferns and vines, analog photograph kodak portra, mystical ambience, incredible detail" lb.set_prompt1(prompt1) lb.set_prompt2(prompt2) @@ -1535,9 +1544,28 @@ if __name__ == "__main__": fn = f"d{d}_b{b}_x{x}.jpg" ip.save(os.path.join(dp_tmp, fn), img) - - + #%% get source img + seed = 280335986 + prompt1 = "photo of a futuristic alien temple resting in the desert, mystic, sunlight" + lb.set_prompt1(prompt1) + lb.init_inpainting(init_empty=True) + + for i in range(5): + seed = np.random.randint(753528763) + lb.set_seed(seed) + txt = f"index {i+1} {seed}: {prompt1}" + img = lb.run_diffusion(lb.text_embedding1, return_image=True) + plt.imshow(img) +# plt.title(txt) + plt.show() + print(txt) + + #%% +""" +index 3 303856737: photo of a futuristic alien temple resting in the desert, mystic, sunlight + +""" #%% """