technicalkiwi
/
latentblending
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

example 1 small upd

This commit is contained in:
Johannes Stelzer 2022-11-21 10:49:33 +01:00
parent f421e4d45a
commit e4f2044973
2 changed files with 157 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
example1_standard.py Normal file
View File

@ -0,0 +1,102 @@
# Copyright 2022 Lunar Ring. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os, sys
dp_git = "/home/lugo/git/"
sys.path.append(os.path.join(dp_git,'garden4'))
sys.path.append('util')
import torch
torch.backends.cudnn.benchmark = False
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import time
import subprocess
import warnings
import torch
from tqdm.auto import tqdm
from diffusers import StableDiffusionInpaintPipeline
from diffusers import StableDiffusionPipeline
from diffusers.schedulers import DDIMScheduler
from PIL import Image
import matplotlib.pyplot as plt
import torch
from movie_man import MovieSaver
import datetime
from typing import Callable, List, Optional, Union
import inspect
from latent_blending import LatentBlending, add_frames_linear_interp
torch.set_grad_enabled(False)
#%% First let us spawn a diffusers pipe using DDIMScheduler
device = "cuda:0"
model_path = "../stable_diffusion_models/stable-diffusion-v1-5"
scheduler = DDIMScheduler(beta_start=0.00085,
beta_end=0.012,
beta_schedule="scaled_linear",
clip_sample=False,
set_alpha_to_one=False)
pipe = StableDiffusionPipeline.from_pretrained(
model_path,
revision="fp16",
torch_dtype=torch.float16,
scheduler=scheduler,
use_auth_token=True
)
pipe = pipe.to(device)
#%% Next let's set up all parameters
# We want 20 diffusion steps, begin with 2 branches, have 3 branches at step 12 (=0.6*20)
# 10 branches at step 16 (=0.8*20) and 24 branches at step 18 (=0.9*20)
# Furthermore we want seed 993621550 for keyframeA and seed 54878562 for keyframeB ()
num_inference_steps = 30 # Number of diffusion interations
list_nmb_branches = [2, 6, 30, 100] # Specify the branching structure
list_injection_strength = [0.0, 0.3, 0.73, 0.93] # Specify the branching structure
width = 512
height = 512
guidance_scale = 5
#fixed_seeds = [993621550, 326814432]
#fixed_seeds = [993621550, 888839807]
fixed_seeds = [993621550, 753528763]
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
prompt1 = "photo of a beautiful forest covered in white flowers, ambient light, very detailed, magic"
prompt2 = "photo of a mystical sculpture in the middle of the desert, warm sunlight, sand, eery feeling"
lb.set_prompt1(prompt1)
lb.set_prompt2(prompt2)
imgs_transition = lb.run_transition(list_nmb_branches, list_injection_strength, fixed_seeds=fixed_seeds)
#%
# let's get more frames
duration_transition = 12
fps = 60
imgs_transition_ext = add_frames_linear_interp(imgs_transition, duration_transition, fps)
# movie saving
fp_movie = f"/home/lugo/tmp/latentblending/bobo_incoming.mp4"
if os.path.isfile(fp_movie):
os.remove(fp_movie)
ms = MovieSaver(fp_movie, fps=fps, profile='save')
for img in tqdm(imgs_transition_ext):
ms.write_frame(img)
ms.finalize()
# MOVIE TODO: ueberschreiben! bad prints.

69
latent_blending.py
View File

@ -115,7 +115,7 @@ class LatentBlending():
self.mask_image = None
self.mode = 'inpaint'
else:
self.mode = 'default'
self.mode = 'standard'
def init_inpainting(
@ -214,14 +214,16 @@ class LatentBlending():
if list_injection_idx is None:
assert list_injection_strength is not None, "Supply either list_injection_idx or list_injection_strength"
# Create the injection indexes
list_injection_idx = [int(round(x*self.num_inference_steps)) for x in list_injection_strength]
assert min(np.diff(list_injection_idx)) > 0, 'Injection idx needs to be increasing'
if min(np.diff(list_injection_idx)) < 2:
print("Warning: your injection spacing is very tight. consider increasing the distances")
assert type(list_injection_strength[0]) is float, "Need to supply floats for list_injection_strength"
assert type(list_injection_strength[1]) is float, "Need to supply floats for list_injection_strength"
# we are checking element 1 in list_injection_strength because "0" is an int... [0, 0.5]
assert max(list_injection_idx) < self.num_inference_steps, "Decrease the injection index or strength"
assert len(list_injection_idx) == len(list_nmb_branches), "Need to have same length"
assert max(list_injection_idx) < self.num_inference_steps,"Injection index cannot happen after last diffusion step! Decrease list_injection_idx or list_injection_strength[-1]"
if fixed_seeds is not None:
@ -364,7 +366,7 @@ class LatentBlending():
return_image: Optional[bool] = False
):
r"""
Wrapper function for run_diffusion_default and run_diffusion_inpaint.
Wrapper function for run_diffusion_standard and run_diffusion_inpaint.
Depending on the mode, the correct one will be executed.
Args:
@ -381,8 +383,8 @@ class LatentBlending():
"""
if self.mode == 'default':
return self.run_diffusion_default(text_embeddings, latents_for_injection=latents_for_injection, idx_start=idx_start, idx_stop=idx_stop, return_image=return_image)
if self.mode == 'standard':
return self.run_diffusion_standard(text_embeddings, latents_for_injection=latents_for_injection, idx_start=idx_start, idx_stop=idx_stop, return_image=return_image)
elif self.mode == 'inpaint':
assert self.image_source is not None, "image_source is None. Please run init_inpainting first."
@ -391,7 +393,7 @@ class LatentBlending():
@torch.no_grad()
def run_diffusion_default(
def run_diffusion_standard(
self,
text_embeddings: torch.FloatTensor,
latents_for_injection: torch.FloatTensor = None,
@ -936,7 +938,7 @@ if __name__ == "__main__":
height = 512
guidance_scale = 5
seed = 421
mode = 'default'
mode = 'standard'
fps_target = 24
duration_target = 10
gpu_id = 0
@ -962,7 +964,7 @@ if __name__ == "__main__":
pipe = pipe.to(device)
#%% DEFAULT TRANS RE SANITY
#%% standard TRANS RE SANITY
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed)
self = lb
@ -1472,7 +1474,7 @@ if __name__ == "__main__":
height = 512
guidance_scale = 5
seed = 421
mode = 'default'
mode = 'standard'
fps_target = 30
duration_target = 15
gpu_id = 0
@ -1490,17 +1492,56 @@ if __name__ == "__main__":
)
pipe = pipe.to(device)
#%% seed cherrypicking
prompt1 = "photo of a surreal brutalistic vault that is glowing in the night, futuristic, greek ornaments, spider webs"
lb.set_prompt1(prompt1)
for i in range(1):
seed = 753528763 #np.random.randint(753528763)
lb.set_seed(seed)
txt = f"{i} {seed}"
img = lb.run_diffusion(lb.text_embedding1, return_image=True)
plt.imshow(img)
plt.title(txt)
plt.show()
print(txt)
#%% make nice images of latents
num_inference_steps = 10 # Number of diffusion interations
list_nmb_branches = [2, 3, 7, 12] # Specify the branching structure
list_injection_idx = [0, 2, 5, 8] # Specify the branching structure
width = 512
height = 512
guidance_scale = 5
fixed_seeds = [993621550, 326814432]
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
prompt1 = "photo of a beautiful forest covered in white flowers, ambient light, very detailed, magic"
prompt2 = "photo of a mystical sculpture in the middle of the desert, warm sunlight, sand, eery feeling"
lb.set_prompt1(prompt1)
lb.set_prompt2(prompt2)
imgs_transition = lb.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds)
#%%
dp_tmp= "/home/lugo/tmp/latentblending"
for d in range(len(lb.tree_latents)):
for b in range(list_nmb_branches[d]):
for x in range(len(lb.tree_latents[d][b])):
lati = lb.tree_latents[d][b][x]
img = lb.latent2image(lati)
fn = f"d{d}_b{b}_x{x}.jpg"
ip.save(os.path.join(dp_tmp, fn), img)
#%%
"""
TODO Coding:
list_nmb_branches > num inference
auto mode (quality settings)
refactor movie man
make movie combiner in movie man