diff --git a/example1_standard.py b/example1_standard.py index c13f68f..3de788f 100644 --- a/example1_standard.py +++ b/example1_standard.py @@ -1,4 +1,5 @@ # Copyright 2022 Lunar Ring. All rights reserved. +# Written by Johannes Stelzer @j_stelzer # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/example2_inpaint.py b/example2_inpaint.py index 2fa4c95..e90fdae 100644 --- a/example2_inpaint.py +++ b/example2_inpaint.py @@ -1,4 +1,5 @@ # Copyright 2022 Lunar Ring. All rights reserved. +# Written by Johannes Stelzer @j_stelzer # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/example3_multitrans.py b/example3_multitrans.py index 96e2cc4..73ee65a 100644 --- a/example3_multitrans.py +++ b/example3_multitrans.py @@ -1,4 +1,5 @@ # Copyright 2022 Lunar Ring. All rights reserved. +# Written by Johannes Stelzer @j_stelzer # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/example4_upscaling.py b/example4_upscaling.py index a91caf6..c3c6625 100644 --- a/example4_upscaling.py +++ b/example4_upscaling.py @@ -1,4 +1,5 @@ # Copyright 2022 Lunar Ring. All rights reserved. +# Written by Johannes Stelzer @j_stelzer # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,6 +30,7 @@ from typing import Callable, List, Optional, Union from latent_blending import LatentBlending, add_frames_linear_interp from stable_diffusion_holder import StableDiffusionHolder torch.set_grad_enabled(False) + #%% Define vars for low-resoltion pass dp_img = "upscaling_bleding" # the results will be saved in this folder prompt1 = "photo of mount vesuvius erupting a terrifying pyroclastic ash cloud" diff --git a/gradio_ui.py b/gradio_ui.py index 4dfc558..33f3fd6 100644 --- a/gradio_ui.py +++ b/gradio_ui.py @@ -1,4 +1,5 @@ # Copyright 2022 Lunar Ring. All rights reserved. +# Written by Johannes Stelzer @j_stelzer # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -81,7 +82,7 @@ class BlendingFrontend(): self.nmb_branches_final = 13 self.nmb_imgs_show = 5 self.fps = 30 - self.duration = 5 + self.duration = 10 if not self.use_debug: self.init_diffusion() @@ -302,8 +303,8 @@ with gr.Blocks() as demo: with gr.Row(): num_inference_steps = gr.Slider(5, 100, self.num_inference_steps, step=1, label='num_inference_steps', interactive=True) - guidance_scale = gr.Slider(1, 25, self.guidance_scale, step=0.1, label='guidance_scale', interactive=True) branch1_influence = gr.Slider(0.0, 1.0, self.branch1_influence, step=0.01, label='branch1_influence', interactive=True) + guidance_scale = gr.Slider(1, 25, self.guidance_scale, step=0.1, label='guidance_scale', interactive=True) with gr.Row(): depth_strength = gr.Slider(0.01, 0.99, self.depth_strength, step=0.01, label='depth_strength', interactive=True) @@ -332,7 +333,7 @@ with gr.Blocks() as demo: with gr.Row(): fps = gr.Slider(1, 120, self.fps, step=1, label='fps', interactive=True) - duration = gr.Slider(0.1, 15, self.duration, step=0.1, label='duration', interactive=True) + duration = gr.Slider(0.1, 30, self.duration, step=0.1, label='duration', interactive=True) b_save = gr.Button('save video') with gr.Row(): diff --git a/latent_blending.py b/latent_blending.py index 1519079..09ff518 100644 --- a/latent_blending.py +++ b/latent_blending.py @@ -1,4 +1,5 @@ # Copyright 2022 Lunar Ring. All rights reserved. +# Written by Johannes Stelzer @j_stelzer # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -104,6 +105,8 @@ class LatentBlending(): self.list_injection_idx = None self.list_nmb_branches = None self.branch1_influence = 0.0 + self.branch1_fract_crossfeed = 0.65 + self.branch1_insertion_completed = False self.set_guidance_scale(guidance_scale) self.init_mode() @@ -414,6 +417,18 @@ class LatentBlending(): self.list_nmb_branches_prev = self.list_nmb_branches[:] self.list_injection_idx_prev = self.list_injection_idx[:] + # Split the first block if there is branch1 crossfeeding + if self.branch1_influence > 0.0 and not self.branch1_insertion_completed: + self.list_nmb_branches.insert(1, 2) + idx_crossfeed = int(round(self.list_injection_idx[1]*self.branch1_fract_crossfeed)) + self.list_injection_idx_ext.insert(1, idx_crossfeed) + self.tree_fracts.insert(1, self.tree_fracts[0]) + self.tree_status.insert(1, self.tree_status[0]) + self.tree_latents.insert(1, self.tree_latents[0]) + self.branch1_insertion_completed = True + + + # Pre-define entire branching tree structures self.tree_final_imgs = [None]*self.list_nmb_branches[-1] nmb_blocks_time = len(self.list_injection_idx_ext)-1 @@ -495,7 +510,8 @@ class LatentBlending(): # FIXME: if more than 2 base branches? if idx_branch==1 and self.branch1_influence > 0: fract_base_influence = np.clip(self.branch1_influence, 0, 1) - list_latents[-1] = interpolate_spherical(list_latents[-1], self.tree_latents[0][0][-1], fract_base_influence) + for i in range(len(list_latents)): + list_latents[i] = interpolate_spherical(list_latents[i], self.tree_latents[0][0][i], fract_base_influence) else: # find parents latents b_parent1, b_parent2 = get_closest_idx(fract_mixing, self.tree_fracts[t_block-1]) @@ -786,16 +802,27 @@ class LatentBlending(): img_leaf.save(os.path.join(dp_img, f"lowres_img_{str(i).zfill(4)}.jpg")) # Dump everything relevant into yaml - dict_stuff = {} - dict_stuff['prompt1'] = self.prompt1 - dict_stuff['prompt2'] = self.prompt2 - dict_stuff['seed1'] = int(self.seed1) - dict_stuff['seed2'] = int(self.seed2) - dict_stuff['num_inference_steps'] = self.num_inference_steps - dict_stuff['height'] = self.sdh.height - dict_stuff['width'] = self.sdh.width - dict_stuff['nmb_images'] = len(imgs_transition) - yml_save(os.path.join(dp_img, "lowres.yaml"), dict_stuff) + state_dict = self.get_state_dict() + state_dict['nmb_images'] = len(imgs_transition) + yml_save(os.path.join(dp_img, "lowres.yaml"), state_dict) + + def get_state_dict(self): + state_dict = {} + grab_vars = ['prompt1', 'prompt2', 'seed1', 'seed2', 'height', 'width', + 'num_inference_steps', 'depth_strength', 'guidance_scale', + 'guidance_scale_mid_damper', 'mid_compression_scaler', 'negative_prompt'] + for v in grab_vars: + if hasattr(self, v): + if v == 'seed1' or v == 'seed2': + state_dict[v] = int(getattr(self, v)) + elif v == 'guidance_scale': + state_dict[v] = float(getattr(self, v)) + + else: + state_dict[v] = getattr(self, v) + + + return state_dict def randomize_seed(self): r""" @@ -1110,8 +1137,8 @@ if __name__ == "__main__": #%% First let us spawn a stable diffusion holder device = "cuda" - fp_ckpt = "../stable_diffusion_models/ckpt/v2-1_768-ema-pruned.ckpt" - fp_config = 'configs/v2-inference-v.yaml' + fp_ckpt = "../stable_diffusion_models/ckpt/v2-1_512-ema-pruned.ckpt" + fp_config = 'configs/v2-inference.yaml' sdh = StableDiffusionHolder(fp_ckpt, fp_config, device) @@ -1129,6 +1156,7 @@ if __name__ == "__main__": # Spawn latent blending self = LatentBlending(sdh) + self.branch1_influence = 0.8 self.load_branching_profile(quality=quality, depth_strength=0.3) self.set_prompt1(prompt1) self.set_prompt2(prompt2) diff --git a/movie_util.py b/movie_util.py index e120be9..596b85c 100644 --- a/movie_util.py +++ b/movie_util.py @@ -1,4 +1,5 @@ # Copyright 2022 Lunar Ring. All rights reserved. +# Written by Johannes Stelzer @j_stelzer # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/stable_diffusion_holder.py b/stable_diffusion_holder.py index c2696a9..c96603c 100644 --- a/stable_diffusion_holder.py +++ b/stable_diffusion_holder.py @@ -1,4 +1,5 @@ # Copyright 2022 Lunar Ring. All rights reserved. +# Written by Johannes Stelzer @j_stelzer # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.