technicalkiwi
/
latentblending
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

info

This commit is contained in:
Johannes Stelzer 2023-01-11 11:36:44 +01:00
parent 18d781f8cd
commit 7ebe6aaa66
8 changed files with 52 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
example1_standard.py
View File

@ -1,4 +1,5 @@
# Copyright 2022 Lunar Ring. All rights reserved.
# Written by Johannes Stelzer @j_stelzer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

1
example2_inpaint.py
View File

@ -1,4 +1,5 @@
# Copyright 2022 Lunar Ring. All rights reserved.
# Written by Johannes Stelzer @j_stelzer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

1
example3_multitrans.py
View File

@ -1,4 +1,5 @@
# Copyright 2022 Lunar Ring. All rights reserved.
# Written by Johannes Stelzer @j_stelzer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
example4_upscaling.py
View File

@ -1,4 +1,5 @@
# Copyright 2022 Lunar Ring. All rights reserved.
# Written by Johannes Stelzer @j_stelzer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -29,6 +30,7 @@ from typing import Callable, List, Optional, Union
from latent_blending import LatentBlending, add_frames_linear_interp
from stable_diffusion_holder import StableDiffusionHolder
torch.set_grad_enabled(False)
#%% Define vars for low-resoltion pass
dp_img = "upscaling_bleding" # the results will be saved in this folder
prompt1 = "photo of mount vesuvius erupting a terrifying pyroclastic ash cloud"

7
gradio_ui.py
View File

@ -1,4 +1,5 @@
# Copyright 2022 Lunar Ring. All rights reserved.
# Written by Johannes Stelzer @j_stelzer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -81,7 +82,7 @@ class BlendingFrontend():
self.nmb_branches_final = 13
self.nmb_imgs_show = 5
self.fps = 30
self.duration = 5
self.duration = 10
if not self.use_debug:
self.init_diffusion()
@ -302,8 +303,8 @@ with gr.Blocks() as demo:
with gr.Row():
num_inference_steps = gr.Slider(5, 100, self.num_inference_steps, step=1, label='num_inference_steps', interactive=True)
guidance_scale = gr.Slider(1, 25, self.guidance_scale, step=0.1, label='guidance_scale', interactive=True)
branch1_influence = gr.Slider(0.0, 1.0, self.branch1_influence, step=0.01, label='branch1_influence', interactive=True)
guidance_scale = gr.Slider(1, 25, self.guidance_scale, step=0.1, label='guidance_scale', interactive=True)
with gr.Row():
depth_strength = gr.Slider(0.01, 0.99, self.depth_strength, step=0.01, label='depth_strength', interactive=True)
@ -332,7 +333,7 @@ with gr.Blocks() as demo:
with gr.Row():
fps = gr.Slider(1, 120, self.fps, step=1, label='fps', interactive=True)
duration = gr.Slider(0.1, 15, self.duration, step=0.1, label='duration', interactive=True)
duration = gr.Slider(0.1, 30, self.duration, step=0.1, label='duration', interactive=True)
b_save = gr.Button('save video')
with gr.Row():

54
latent_blending.py
View File

@ -1,4 +1,5 @@
# Copyright 2022 Lunar Ring. All rights reserved.
# Written by Johannes Stelzer @j_stelzer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -104,6 +105,8 @@ class LatentBlending():
self.list_injection_idx = None
self.list_nmb_branches = None
self.branch1_influence = 0.0
self.branch1_fract_crossfeed = 0.65
self.branch1_insertion_completed = False
self.set_guidance_scale(guidance_scale)
self.init_mode()
@ -414,6 +417,18 @@ class LatentBlending():
self.list_nmb_branches_prev = self.list_nmb_branches[:]
self.list_injection_idx_prev = self.list_injection_idx[:]
# Split the first block if there is branch1 crossfeeding
if self.branch1_influence > 0.0 and not self.branch1_insertion_completed:
self.list_nmb_branches.insert(1, 2)
idx_crossfeed = int(round(self.list_injection_idx[1]*self.branch1_fract_crossfeed))
self.list_injection_idx_ext.insert(1, idx_crossfeed)
self.tree_fracts.insert(1, self.tree_fracts[0])
self.tree_status.insert(1, self.tree_status[0])
self.tree_latents.insert(1, self.tree_latents[0])
self.branch1_insertion_completed = True
# Pre-define entire branching tree structures
self.tree_final_imgs = [None]*self.list_nmb_branches[-1]
nmb_blocks_time = len(self.list_injection_idx_ext)-1
@ -495,7 +510,8 @@ class LatentBlending():
# FIXME: if more than 2 base branches?
if idx_branch==1 and self.branch1_influence > 0:
fract_base_influence = np.clip(self.branch1_influence, 0, 1)
list_latents[-1] = interpolate_spherical(list_latents[-1], self.tree_latents[0][0][-1], fract_base_influence)
for i in range(len(list_latents)):
list_latents[i] = interpolate_spherical(list_latents[i], self.tree_latents[0][0][i], fract_base_influence)
else:
# find parents latents
b_parent1, b_parent2 = get_closest_idx(fract_mixing, self.tree_fracts[t_block-1])
@ -786,16 +802,27 @@ class LatentBlending():
img_leaf.save(os.path.join(dp_img, f"lowres_img_{str(i).zfill(4)}.jpg"))
# Dump everything relevant into yaml
dict_stuff = {}
dict_stuff['prompt1'] = self.prompt1
dict_stuff['prompt2'] = self.prompt2
dict_stuff['seed1'] = int(self.seed1)
dict_stuff['seed2'] = int(self.seed2)
dict_stuff['num_inference_steps'] = self.num_inference_steps
dict_stuff['height'] = self.sdh.height
dict_stuff['width'] = self.sdh.width
dict_stuff['nmb_images'] = len(imgs_transition)
yml_save(os.path.join(dp_img, "lowres.yaml"), dict_stuff)
state_dict = self.get_state_dict()
state_dict['nmb_images'] = len(imgs_transition)
yml_save(os.path.join(dp_img, "lowres.yaml"), state_dict)
def get_state_dict(self):
state_dict = {}
grab_vars = ['prompt1', 'prompt2', 'seed1', 'seed2', 'height', 'width',
'num_inference_steps', 'depth_strength', 'guidance_scale',
'guidance_scale_mid_damper', 'mid_compression_scaler', 'negative_prompt']
for v in grab_vars:
if hasattr(self, v):
if v == 'seed1' or v == 'seed2':
state_dict[v] = int(getattr(self, v))
elif v == 'guidance_scale':
state_dict[v] = float(getattr(self, v))
else:
state_dict[v] = getattr(self, v)
return state_dict
def randomize_seed(self):
r"""
@ -1110,8 +1137,8 @@ if __name__ == "__main__":
#%% First let us spawn a stable diffusion holder
device = "cuda"
fp_ckpt = "../stable_diffusion_models/ckpt/v2-1_768-ema-pruned.ckpt"
fp_config = 'configs/v2-inference-v.yaml'
fp_ckpt = "../stable_diffusion_models/ckpt/v2-1_512-ema-pruned.ckpt"
fp_config = 'configs/v2-inference.yaml'
sdh = StableDiffusionHolder(fp_ckpt, fp_config, device)
@ -1129,6 +1156,7 @@ if __name__ == "__main__":
# Spawn latent blending
self = LatentBlending(sdh)
self.branch1_influence = 0.8
self.load_branching_profile(quality=quality, depth_strength=0.3)
self.set_prompt1(prompt1)
self.set_prompt2(prompt2)

1
movie_util.py
View File

@ -1,4 +1,5 @@
# Copyright 2022 Lunar Ring. All rights reserved.
# Written by Johannes Stelzer @j_stelzer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

1
stable_diffusion_holder.py
View File

@ -1,4 +1,5 @@
# Copyright 2022 Lunar Ring. All rights reserved.
# Written by Johannes Stelzer @j_stelzer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.