example for inpainting, cleanup
This commit is contained in:
Johannes Stelzer
parent
e4a736a70e
commit
aa464737be
|
|
@ -13,29 +13,21 @@
|
|||
# limitations under the License.
|
||||
|
||||
import os, sys
|
||||
dp_git = "/home/lugo/git/"
|
||||
sys.path.append(os.path.join(dp_git,'garden4'))
|
||||
sys.path.append('util')
|
||||
import torch
|
||||
torch.backends.cudnn.benchmark = False
|
||||
import numpy as np
|
||||
import warnings
|
||||
warnings.filterwarnings('ignore')
|
||||
import time
|
||||
import subprocess
|
||||
import warnings
|
||||
import torch
|
||||
from tqdm.auto import tqdm
|
||||
from diffusers import StableDiffusionInpaintPipeline
|
||||
from diffusers import StableDiffusionPipeline
|
||||
from diffusers.schedulers import DDIMScheduler
|
||||
from PIL import Image
|
||||
import matplotlib.pyplot as plt
|
||||
import torch
|
||||
from movie_man import MovieSaver
|
||||
import datetime
|
||||
from typing import Callable, List, Optional, Union
|
||||
import inspect
|
||||
from latent_blending import LatentBlending, add_frames_linear_interp
|
||||
torch.set_grad_enabled(False)
|
||||
|
||||
|
|
@ -60,13 +52,13 @@ pipe = pipe.to(device)
|
|||
|
||||
#%% Next let's set up all parameters
|
||||
# FIXME below fix numbers
|
||||
# We want 20 diffusion steps, begin with 2 branches, have 3 branches at step 12 (=0.6*20)
|
||||
# We want 20 diffusion steps in total, begin with 2 branches, have 3 branches at step 12 (=0.6*20)
|
||||
# 10 branches at step 16 (=0.8*20) and 24 branches at step 18 (=0.9*20)
|
||||
# Furthermore we want seed 993621550 for keyframeA and seed 54878562 for keyframeB ()
|
||||
|
||||
num_inference_steps = 100 # Number of diffusion interations
|
||||
list_nmb_branches = [2, 12, 30, 100, 300] # Specify the branching structure
|
||||
list_injection_strength = [0.0, 0.75, 0.9, 0.93, 0.96] # Specify the branching structure
|
||||
num_inference_steps = 20 # Number of diffusion interations
|
||||
list_nmb_branches = [2, 3, 10, 24] # Branching structure: how many branches
|
||||
list_injection_strength = [0.0, 0.6, 0.8, 0.9] # Branching structure: how deep is the blending
|
||||
width = 512
|
||||
height = 512
|
||||
guidance_scale = 5
|
||||
|
|
|
|||
|
|
@ -0,0 +1,103 @@
|
|||
# Copyright 2022 Lunar Ring. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os, sys
|
||||
import torch
|
||||
torch.backends.cudnn.benchmark = False
|
||||
import numpy as np
|
||||
import warnings
|
||||
warnings.filterwarnings('ignore')
|
||||
import time
|
||||
import subprocess
|
||||
import warnings
|
||||
import torch
|
||||
from tqdm.auto import tqdm
|
||||
from diffusers import StableDiffusionInpaintPipeline
|
||||
from PIL import Image
|
||||
import matplotlib.pyplot as plt
|
||||
import torch
|
||||
from movie_man import MovieSaver
|
||||
from typing import Callable, List, Optional, Union
|
||||
from latent_blending import LatentBlending, add_frames_linear_interp
|
||||
torch.set_grad_enabled(False)
|
||||
|
||||
#%% First let us spawn a diffusers pipe using DDIMScheduler
|
||||
device = "cuda:0"
|
||||
model_path = "../stable_diffusion_models/stable-diffusion-inpainting"
|
||||
|
||||
pipe = StableDiffusionInpaintPipeline.from_pretrained(
|
||||
model_path,
|
||||
revision="fp16",
|
||||
torch_dtype=torch.float16,
|
||||
safety_checker=None
|
||||
)
|
||||
pipe = pipe.to(device)
|
||||
|
||||
|
||||
#%% Let's make a source image and mask.
|
||||
height = 512
|
||||
width = 512
|
||||
num_inference_steps = 30
|
||||
guidance_scale = 5
|
||||
fixed_seeds = [629575320, 670154945]
|
||||
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
|
||||
prompt1 = "photo of a futuristic alien temple in a desert, mystic, glowing, organic, intricate, sci-fi movie, mesmerizing, scary"
|
||||
lb.set_prompt1(prompt1)
|
||||
lb.init_inpainting(init_empty=True)
|
||||
lb.set_seed(fixed_seeds[0])
|
||||
image_source = lb.run_diffusion(lb.text_embedding1, return_image=True)
|
||||
mask_image = 255*np.ones([512,512], dtype=np.uint8)
|
||||
mask_image[160:250, 200:320] = 0
|
||||
mask_image = Image.fromarray(mask_image)
|
||||
|
||||
|
||||
#%% Next let's set up all parameters
|
||||
# FIXME below fix numbers
|
||||
# We want 20 diffusion steps, begin with 2 branches, have 3 branches at step 12 (=0.6*20)
|
||||
# 10 branches at step 16 (=0.8*20) and 24 branches at step 18 (=0.9*20)
|
||||
# Furthermore we want seed 993621550 for keyframeA and seed 54878562 for keyframeB ()
|
||||
|
||||
num_inference_steps = 20 # Number of diffusion interations
|
||||
list_nmb_branches = [2, 3, 10, 24] # Branching structure: how many branches
|
||||
list_injection_strength = [0.0, 0.6, 0.8, 0.9] # Branching structure: how deep is the blending
|
||||
width = 512
|
||||
height = 512
|
||||
guidance_scale = 5
|
||||
fixed_seeds = [993621550, 280335986]
|
||||
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
|
||||
prompt1 = "photo of a futuristic alien temple in a desert, mystic, glowing, organic, intricate, sci-fi movie, mesmerizing, scary"
|
||||
prompt2 = "aerial photo of a futuristic alien temple in a coastal area, waves clashing"
|
||||
lb.set_prompt1(prompt1)
|
||||
lb.set_prompt2(prompt2)
|
||||
lb.init_inpainting(image_source, mask_image)
|
||||
|
||||
imgs_transition = lb.run_transition(list_nmb_branches, list_injection_strength, fixed_seeds=fixed_seeds)
|
||||
|
||||
# let's get more cheap frames via linear interpolation
|
||||
duration_transition = 12
|
||||
fps = 60
|
||||
imgs_transition_ext = add_frames_linear_interp(imgs_transition, duration_transition, fps)
|
||||
|
||||
# movie saving
|
||||
fp_movie = f"/home/lugo/tmp/latentblending/bobo_incoming.mp4"
|
||||
if os.path.isfile(fp_movie):
|
||||
os.remove(fp_movie)
|
||||
ms = MovieSaver(fp_movie, fps=fps, profile='save')
|
||||
for img in tqdm(imgs_transition_ext):
|
||||
ms.write_frame(img)
|
||||
ms.finalize()
|
||||
|
||||
|
||||
|
|
@ -928,241 +928,9 @@ def get_time(resolution=None):
|
|||
raise ValueError("bad resolution provided: %s" %resolution)
|
||||
return t
|
||||
|
||||
#%% INIT OUTPAINT
|
||||
# xxxx
|
||||
#%% le main
|
||||
if __name__ == "__main__":
|
||||
#%% INIT DEFAULT
|
||||
|
||||
num_inference_steps = 20
|
||||
width = 512
|
||||
height = 512
|
||||
guidance_scale = 5
|
||||
seed = 421
|
||||
mode = 'standard'
|
||||
fps_target = 24
|
||||
duration_target = 10
|
||||
gpu_id = 0
|
||||
|
||||
device = "cuda:"+str(gpu_id)
|
||||
model_path = "../stable_diffusion_models/stable-diffusion-v1-5"
|
||||
|
||||
scheduler = DDIMScheduler(beta_start=0.00085,
|
||||
beta_end=0.012,
|
||||
beta_schedule="scaled_linear",
|
||||
clip_sample=False,
|
||||
set_alpha_to_one=False)
|
||||
|
||||
pipe = StableDiffusionPipeline.from_pretrained(
|
||||
model_path,
|
||||
revision="fp16",
|
||||
# height = height,
|
||||
# width = width,
|
||||
torch_dtype=torch.float16,
|
||||
scheduler=scheduler,
|
||||
use_auth_token=True
|
||||
)
|
||||
pipe = pipe.to(device)
|
||||
|
||||
|
||||
#%% standard TRANS RE SANITY
|
||||
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed)
|
||||
self = lb
|
||||
prompt1 = "photograph of NYC skyline"#", skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m"
|
||||
prompt2 = "photograph of NYC skyline at dawn"#", skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m"
|
||||
self.set_prompt1(prompt1)
|
||||
self.set_prompt2(prompt2)
|
||||
|
||||
list_nmb_branches = [2, 4]
|
||||
list_injection_idx = [0, 10]
|
||||
|
||||
fixed_seeds = [421110, 421110]
|
||||
|
||||
ax = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds)
|
||||
|
||||
|
||||
#%% EXPERIMENT
|
||||
prompt1 = "dark painting of a nice house"#", skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m"
|
||||
prompt2 = "beautiful surreal painting sunset over the ocean"#", skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m"
|
||||
self.set_prompt1(prompt1)
|
||||
self.set_prompt2(prompt2)
|
||||
# we want to run nmb_experiments experiments. all with the same seed
|
||||
nmb_experiments = 100
|
||||
seed1 = 420
|
||||
list_seeds2 = []
|
||||
list_seeds2.append(seed1)
|
||||
for j in range(nmb_experiments-1):
|
||||
list_seeds2.append(np.random.randint(1999912934))
|
||||
|
||||
# storage
|
||||
list_latents_exp = []
|
||||
list_imgfinal_exp = []
|
||||
|
||||
|
||||
|
||||
for j in range(nmb_experiments):
|
||||
# now run trans alex way
|
||||
list_nmb_branches = [2, 10]
|
||||
list_injection_idx = [0, 1]
|
||||
fixed_seeds = [seed, list_seeds2[j]]
|
||||
list_imgs_res = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds)
|
||||
list_latents_exp.append(self.tree_latents[1])
|
||||
|
||||
# lets run johannes way, just store imgs here
|
||||
list_nmb_branches = [2, 3, 6, 12]
|
||||
list_injection_idx = [0, 10, 13, 16]
|
||||
list_imgs_good = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds)
|
||||
list_imgfinal_exp.append(list_imgs_good)
|
||||
|
||||
print(f"DONE WITH EXP {j+1}/{nmb_experiments}")
|
||||
|
||||
#%%
|
||||
for j in range(100):
|
||||
lx = list_imgfinal_exp[j]
|
||||
lxx = add_frames_linear_interp(lx, fps_target=24, duration_target=6)
|
||||
str_idx = f"{j}".zfill(3)
|
||||
fp_movie = f'/mnt/jamaica/data_lake/bobi_projects/diffusion/exp/trans_{str_idx}.mp4'
|
||||
ms = MovieSaver(fp_movie, fps=24, profile='save')
|
||||
for k, img in enumerate(lxx):
|
||||
ms.write_frame(img)
|
||||
ms.finalize()
|
||||
|
||||
#%% surgery
|
||||
t_iter = 15
|
||||
rdim = 64*64*4
|
||||
res = np.zeros((100, 10, rdim))
|
||||
for j in range(100):
|
||||
for k in range(10):
|
||||
res[j,k,:] = list_latents_exp[j][k][t_iter].cpu().numpy().ravel()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#%% NEW TRANS
|
||||
|
||||
|
||||
prompt1 = "nigth sky reflected on the ocean"
|
||||
prompt2 = "beautiful forest painting sunset"
|
||||
|
||||
num_inference_steps = 15
|
||||
width = 512
|
||||
height = 512
|
||||
guidance_scale = 5
|
||||
seed = 421
|
||||
fps_target = 24
|
||||
duration_target = 15
|
||||
gpu_id = 0
|
||||
|
||||
# define mask_image
|
||||
mask_image = 255*np.ones([512,512], dtype=np.uint8)
|
||||
mask_image[200:300, 200:300] = 0
|
||||
mask_image = Image.fromarray(mask_image)
|
||||
|
||||
# load diffusion pipe
|
||||
device = "cuda:"+str(gpu_id)
|
||||
model_path = "../stable_diffusion_models/stable-diffusion-inpainting"
|
||||
|
||||
# scheduler = DDIMScheduler(beta_start=0.00085,
|
||||
# beta_end=0.012,
|
||||
# beta_schedule="scaled_linear",
|
||||
# clip_sample=False,
|
||||
# set_alpha_to_one=False)
|
||||
|
||||
# L
|
||||
pipe = StableDiffusionInpaintPipeline.from_pretrained(
|
||||
model_path,
|
||||
revision="fp16",
|
||||
torch_dtype=torch.float16,
|
||||
# scheduler=scheduler,
|
||||
safety_checker=None
|
||||
)
|
||||
pipe = pipe.to(device)
|
||||
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed)
|
||||
self = lb
|
||||
|
||||
|
||||
|
||||
|
||||
xxx
|
||||
# init latentblending & run
|
||||
self.set_prompt1(prompt1)
|
||||
self.set_prompt2(prompt2)
|
||||
|
||||
# we first compute img1. we need the full image to do inpainting
|
||||
self.init_inpainting(init_empty=True)
|
||||
list_latents = self.run_diffusion_inpaint(self.text_embedding1)
|
||||
image_source = self.latent2image(list_latents[-1])
|
||||
self.init_inpainting(image_source, mask_image)
|
||||
|
||||
img1 = image_source
|
||||
|
||||
#%% INPAINT HALAL
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed)
|
||||
self = lb
|
||||
list_nmb_branches = [2, 4, 6]
|
||||
list_injection_idx = [0, 4, 12]
|
||||
list_prompts = []
|
||||
list_prompts.append("paiting of a medieval city")
|
||||
list_prompts.append("paiting of a forest")
|
||||
list_prompts.append("photo of a desert landscape")
|
||||
list_prompts.append("photo of a jungle")
|
||||
|
||||
#% INPAINT SANITY 1
|
||||
# run empty trans
|
||||
prompt1 = "nigth sky reflected on the ocean"
|
||||
prompt2 = "beautiful forest painting sunset"
|
||||
self.set_prompt1(prompt1)
|
||||
self.set_prompt2(prompt2)
|
||||
list_seeds = [420, 420]
|
||||
self.init_inpainting(init_empty=True)
|
||||
list_imgs0 = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=list_seeds)
|
||||
img_source = list_imgs0[-1]
|
||||
|
||||
|
||||
|
||||
#% INPAINT SANITY 2
|
||||
mask_image = 255*np.ones([512,512], dtype=np.uint8)
|
||||
mask_image[0:222, 0:222] = 0
|
||||
|
||||
self.swap_forward()
|
||||
# we provide a new prompt for image2
|
||||
prompt2 = list_prompts[0]# "beautiful painting ocean sunset colorful"
|
||||
# self.swap_forward()
|
||||
self.randomize_seed()
|
||||
self.set_prompt2(prompt2)
|
||||
self.init_inpainting(image_source=img_source, mask_image=mask_image)
|
||||
list_imgs1 = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, recycle_img1=True, fixed_seeds=list_seeds)
|
||||
|
||||
|
||||
plt.imshow(list_imgs0[0])
|
||||
plt.show()
|
||||
plt.imshow(list_imgs0[-1])
|
||||
plt.show()
|
||||
plt.imshow(list_imgs1[0])
|
||||
plt.show()
|
||||
|
||||
#%% mini surgery
|
||||
idx_branch = 5
|
||||
img = self.latent2image(self.tree_latents[-1][idx_branch][-1])
|
||||
plt.imshow(img)
|
||||
|
||||
#%% INPAINT SANITY 3
|
||||
mask_image = 255*np.ones([512,512], dtype=np.uint8)
|
||||
mask_image[0:222, 0:222] = 0
|
||||
|
||||
self.swap_forward()
|
||||
# we provide a new prompt for image2
|
||||
prompt2 = list_prompts[1]# "beautiful painting ocean sunset colorful"
|
||||
# self.swap_forward()
|
||||
self.randomize_seed()
|
||||
self.set_prompt2(prompt2)
|
||||
self.init_inpainting(image_source=img_source, mask_image=mask_image)
|
||||
list_imgs2 = self.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=list_seeds)
|
||||
|
||||
|
||||
#%% LOOP
|
||||
list_prompts = []
|
||||
list_prompts.append("paiting of a medieval city")
|
||||
|
|
@ -1187,67 +955,8 @@ if __name__ == "__main__":
|
|||
# now we switch them around so image2 becomes image1
|
||||
img1 = list_imgs[-1]
|
||||
|
||||
#%% surg
|
||||
img = self.latent2image(self.tree_latents[-1][3][-1])
|
||||
plt.imshow(img)
|
||||
|
||||
|
||||
#%% GOOD SINGLE TRANS
|
||||
height = 512
|
||||
width = 512
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed)
|
||||
self = lb
|
||||
self.randomize_seed()
|
||||
# init latentblending & run
|
||||
prompt1 = "photograph of NYC skyline at dawn, skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m"
|
||||
prompt2 = "photograph of NYC skyline at dusk, skyscrapers, kodak portra, iso 100, detailed, cinematic, leica m"
|
||||
# prompt1 = "hologram portrait of sumerian god of artificial general intelligence, AI, dystopian, utopia, year 2222, insane detail, incredible machinery, cybernetic power god, sumerian statue, steel clay, silicon masterpiece, futuristic symmetry"
|
||||
# prompt1 = "photograph of a dark cyberpunk street at night, neon signs, cinematic film still, dark science fiction, highly detailed, bokeh, f5.6, Leica M9, cinematic, iso 100, Kodak Portra"
|
||||
# prompt2 = "bright photograph of a cyberpunk during daytime, cinematic film still, science fiction, highly detailed, bokeh, f5.6, Leica M9, cinematic, iso 100, Kodak Portra"
|
||||
# prompt2 = "surreal_painting_of_stylized_sexual_forest"
|
||||
|
||||
|
||||
self.set_prompt1(prompt1)
|
||||
self.set_prompt2(prompt2)
|
||||
|
||||
# list_nmb_branches = [2, 4, 12]#, 15, 200]
|
||||
# list_injection_idx = [0, 8, 13]#, 24, 28]
|
||||
list_nmb_branches = [2, 4, 8, 15, 200]
|
||||
list_injection_idx = [0, 10, 20, 24, 28]
|
||||
fps_target = 30
|
||||
duration_target = 10
|
||||
loop_back = True
|
||||
t0 = time.time()
|
||||
list_imgs = self.run_transition(list_nmb_branches, list_injection_idx, fixed_seeds='randomize')
|
||||
|
||||
# 1: 2142
|
||||
list_imgs_interp = add_frames_linear_interp(list_imgs, fps_target, duration_target)
|
||||
dt = time.time() - t0
|
||||
|
||||
|
||||
|
||||
loop_back = True
|
||||
# movie saving
|
||||
str_steps = ""
|
||||
for s in list_nmb_branches:
|
||||
str_steps += f"{s}_"
|
||||
str_steps = str_steps[0:-1]
|
||||
|
||||
str_inject = ""
|
||||
for k in list_injection_idx:
|
||||
str_inject += f"{k}_"
|
||||
str_inject = str_inject[0:-1]
|
||||
|
||||
fp_movie = f"/mnt/jamaica/data_lake/bobi_projects/diffusion/movies/221116_lb/lb_{get_time('second')}_s{str_steps}_k{str_inject}.mp4"
|
||||
|
||||
ms = MovieSaver(fp_movie, fps=fps_target, profile='save')
|
||||
for img in tqdm(list_imgs_interp):
|
||||
ms.write_frame(img)
|
||||
if loop_back:
|
||||
for img in tqdm(list_imgs_interp[::-1]):
|
||||
ms.write_frame(img)
|
||||
ms.finalize()
|
||||
|
||||
|
||||
|
||||
#%% GOOD MOVIE ENGINE
|
||||
num_inference_steps = 30
|
||||
|
|
@ -1269,202 +978,10 @@ if __name__ == "__main__":
|
|||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed)
|
||||
dp_movie = "/home/lugo/tmp/movie"
|
||||
|
||||
list_parts = []
|
||||
for i in range(len(list_prompts)-1):
|
||||
print(f"Starting movie segment {i+1}/{len(list_prompts)}")
|
||||
if i==0:
|
||||
lb.set_prompt1(list_prompts[i])
|
||||
lb.set_prompt2(list_prompts[i+1])
|
||||
recycle_img1 = False
|
||||
else:
|
||||
lb.swap_forward()
|
||||
lb.set_prompt2(list_prompts[i+1])
|
||||
recycle_img1 = True
|
||||
|
||||
|
||||
|
||||
list_imgs = lb.run_transition(list_nmb_branches, list_injection_idx, recycle_img1=recycle_img1)
|
||||
list_imgs_interp = add_frames_linear_interp(list_imgs, fps_target, duration_target)
|
||||
|
||||
# Save Movie segment
|
||||
str_idx = f"{i}".zfill(3)
|
||||
fp_movie = os.path.join(dp_movie, f"{str_idx}.mp4")
|
||||
ms = MovieSaver(fp_movie, fps=fps_target, profile='save')
|
||||
for img in tqdm(list_imgs_interp):
|
||||
ms.write_frame(img)
|
||||
ms.finalize()
|
||||
|
||||
list_parts.append(fp_movie)
|
||||
|
||||
|
||||
list_concat = []
|
||||
|
||||
for fp_part in list_parts:
|
||||
list_concat.append(f"""file '{fp_part}'""")
|
||||
|
||||
fp_out = os.path.join(dp_movie, "concat.txt")
|
||||
|
||||
with open(fp_out, "w") as fa:
|
||||
for item in list_concat:
|
||||
fa.write("%s\n" % item)
|
||||
|
||||
# str_steps = ""
|
||||
# for s in list_injection_steps:
|
||||
# str_steps += f"{s}_"
|
||||
# str_steps = str_steps[0:-1]
|
||||
|
||||
# str_inject = ""
|
||||
# for k in list_injection_strength:
|
||||
# str_inject += f"{k}_"
|
||||
# str_inject = str_inject[0:-1]
|
||||
|
||||
fp_movie = os.path.join(dp_movie, f'final_movie_{get_time("second")}.mp4')
|
||||
|
||||
cmd = f'ffmpeg -f concat -safe 0 -i {fp_out} -c copy {fp_movie}'
|
||||
subprocess.call(cmd, shell=True, cwd=dp_movie)
|
||||
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
list_latents1 = self.run_diffusion(self.text_embedding1)
|
||||
img1 = self.latent2image(list_latents1[-1])
|
||||
|
||||
#%%
|
||||
list_seeds = []
|
||||
list_all_latents = []
|
||||
list_all_imgs = []
|
||||
for i in tqdm(range(100)):
|
||||
seed = np.random.randint(9999999)
|
||||
list_seeds.append(seed)
|
||||
self.seed = seed
|
||||
list_imgs = self.run_transition(list_nmb_branches, list_injection_idx, True, False)
|
||||
img2 = list_imgs[-1]
|
||||
list_all_imgs.append(img2)
|
||||
list_all_latents.append(self.list_latents_key2)
|
||||
|
||||
|
||||
|
||||
|
||||
#%%
|
||||
|
||||
list_injection_idx = [0, 10, 17, 22, 25]
|
||||
list_nmb_branches = [3, 6, 10, 30, 60]
|
||||
|
||||
list_imgs_interp = add_frames_linear_interp(list_imgs, fps_target, duration_target)
|
||||
fp_movie = f"/home/lugo/tmp/lb_new2.mp4"
|
||||
|
||||
ms = MovieSaver(fp_movie, fps=fps_target, profile='save')
|
||||
for img in tqdm(list_imgs_interp):
|
||||
ms.write_frame(img)
|
||||
if True:
|
||||
for img in tqdm(list_imgs_interp[::-1]):
|
||||
ms.write_frame(img)
|
||||
ms.finalize()
|
||||
|
||||
#%% SURGERY
|
||||
|
||||
|
||||
|
||||
#%% TEST WITH LATENT DIFFS
|
||||
|
||||
# Collect all basic infos
|
||||
num_inference_steps = 10
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed)
|
||||
self = lb
|
||||
prompt1 = "magic painting of a oak tree, mystic"
|
||||
prompt2 = "painting of a trippy lake and symmetry reflections"
|
||||
self.set_prompt1(prompt1)
|
||||
self.set_prompt2(prompt2)
|
||||
|
||||
list_latents1 = self.run_diffusion(self.text_embedding1)
|
||||
img1 = self.latent2image(list_latents1[-1])
|
||||
|
||||
#%%
|
||||
list_seeds = []
|
||||
list_all_latents = []
|
||||
list_all_imgs = []
|
||||
for i in tqdm(range(100)):
|
||||
seed = np.random.randint(9999999)
|
||||
list_seeds.append(seed)
|
||||
self.seed = seed
|
||||
list_latents2 = self.run_diffusion(self.text_embedding2)
|
||||
img2 = self.latent2image(list_latents2[-1])
|
||||
list_all_latents.append(list_latents2)
|
||||
list_all_imgs.append(img2)
|
||||
|
||||
#%%
|
||||
# res = np.zeros([100,10])
|
||||
# for i in tqdm(range(100)):
|
||||
# for j in range(num_inference_steps):
|
||||
|
||||
# diff = torch.linalg.norm(list_blocks_prev[bprev][-1]-list_blocks_prev[bprev+1][-1]).item()
|
||||
|
||||
|
||||
#%% convert to images
|
||||
list_imgs = []
|
||||
for b in range(len(list_blocks_current)):
|
||||
img = self.latent2image(list_blocks_current[b][-1])
|
||||
list_imgs.append(img)
|
||||
|
||||
|
||||
#%% fract
|
||||
dists_inv = []
|
||||
for bprev in range(len(list_blocks_current)-1):
|
||||
diff = torch.linalg.norm(list_blocks_current[bprev][-1]-list_blocks_current[bprev+1][-1]).item()
|
||||
dists_inv.append(diff)
|
||||
plt.plot(dists_inv)
|
||||
#%%
|
||||
imgx = self.latent2image(list_blocks_current[1][-1])
|
||||
plt.imshow(imgx)
|
||||
|
||||
|
||||
#%% SURGERY
|
||||
dists = [300, 600, 900]
|
||||
nmb_branches = 20
|
||||
list_fract_mixing_prev = [0, 0.4, 0.6, 1]
|
||||
|
||||
|
||||
nmb_injection_slots = len(dists)
|
||||
nmb_injections = nmb_branches-len(list_fract_mixing_prev)
|
||||
p_samp = get_p(dists)
|
||||
injection_counter = np.zeros(nmb_injection_slots, dtype=np.int32)
|
||||
|
||||
for j in range(nmb_injections):
|
||||
idx_injection = np.random.choice(nmb_injection_slots, p=p_samp)
|
||||
injection_counter[idx_injection] += 1
|
||||
|
||||
# get linear interpolated injections
|
||||
list_fract_mixing_current = []
|
||||
for j in range(nmb_injection_slots):
|
||||
fractA = list_fract_mixing_prev[j]
|
||||
fractB = list_fract_mixing_prev[j+1]
|
||||
list_fract_mixing_current.extend(np.linspace(fractA, fractB, 2+injection_counter[j])[:-1])
|
||||
list_fract_mixing_current.append(1)
|
||||
|
||||
|
||||
|
||||
|
||||
#%% save movie
|
||||
loop_back = True
|
||||
# movie saving
|
||||
str_steps = ""
|
||||
for s in list_injection_steps:
|
||||
str_steps += f"{s}_"
|
||||
str_steps = str_steps[0:-1]
|
||||
|
||||
str_inject = ""
|
||||
for k in list_injection_strength:
|
||||
str_inject += f"{k}_"
|
||||
str_inject = str_inject[0:-1]
|
||||
|
||||
fp_movie = f"/home/lugo/tmp/lb_{get_time('second')}_s{str_steps}_k{str_inject}.mp4"
|
||||
|
||||
ms = MovieSaver(fp_movie, fps=fps_target, profile='save')
|
||||
for img in tqdm(list_imgs_interp):
|
||||
ms.write_frame(img)
|
||||
if loop_back:
|
||||
for img in tqdm(list_imgs_interp[::-1]):
|
||||
ms.write_frame(img)
|
||||
ms.finalize()
|
||||
|
||||
|
||||
#%% EXAMPLE3 MOVIE ENGINE
|
||||
list_injection_steps = [2, 3, 4, 5]
|
||||
|
|
@ -1490,83 +1007,8 @@ if __name__ == "__main__":
|
|||
)
|
||||
pipe = pipe.to(device)
|
||||
|
||||
#%% seed cherrypicking
|
||||
|
||||
prompt1 = "photo of an eerie statue surrounded by ferns and vines"
|
||||
lb.set_prompt1(prompt1)
|
||||
|
||||
for i in range(4):
|
||||
seed = np.random.randint(753528763)
|
||||
lb.set_seed(seed)
|
||||
txt = f"index {i+1} {seed}: {prompt1}"
|
||||
img = lb.run_diffusion(lb.text_embedding1, return_image=True)
|
||||
plt.imshow(img)
|
||||
# plt.title(txt)
|
||||
plt.show()
|
||||
print(txt)
|
||||
|
||||
#%% prompt finetuning
|
||||
seed = 280335986
|
||||
prompt1 = "photo of an eerie statue surrounded by ferns and vines, analog photograph kodak portra, mystical ambience, incredible detail"
|
||||
lb.set_prompt1(prompt1)
|
||||
img = lb.run_diffusion(lb.text_embedding1, return_image=True)
|
||||
plt.imshow(img)
|
||||
|
||||
#%% lets make a nice mask
|
||||
|
||||
|
||||
|
||||
#%% storage
|
||||
|
||||
#%% make nice images of latents
|
||||
num_inference_steps = 10 # Number of diffusion interations
|
||||
list_nmb_branches = [2, 3, 7, 10] # Specify the branching structure
|
||||
list_injection_idx = [0, 6, 7, 8] # Specify the branching structure
|
||||
width = 512
|
||||
height = 512
|
||||
guidance_scale = 5
|
||||
fixed_seeds = [993621550, 280335986]
|
||||
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
|
||||
prompt1 = "photo of a beautiful forest covered in white flowers, ambient light, very detailed, magic"
|
||||
prompt2 = "photo of an eerie statue surrounded by ferns and vines, analog photograph kodak portra, mystical ambience, incredible detail"
|
||||
lb.set_prompt1(prompt1)
|
||||
lb.set_prompt2(prompt2)
|
||||
|
||||
imgs_transition = lb.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds)
|
||||
#%%
|
||||
dp_tmp= "/home/lugo/tmp/latentblending"
|
||||
for d in range(len(lb.tree_latents)):
|
||||
for b in range(list_nmb_branches[d]):
|
||||
for x in range(len(lb.tree_latents[d][b])):
|
||||
lati = lb.tree_latents[d][b][x]
|
||||
img = lb.latent2image(lati)
|
||||
fn = f"d{d}_b{b}_x{x}.jpg"
|
||||
ip.save(os.path.join(dp_tmp, fn), img)
|
||||
|
||||
#%% get source img
|
||||
seed = 280335986
|
||||
prompt1 = "photo of a futuristic alien temple resting in the desert, mystic, sunlight"
|
||||
lb.set_prompt1(prompt1)
|
||||
lb.init_inpainting(init_empty=True)
|
||||
|
||||
|
||||
for i in range(5):
|
||||
seed = np.random.randint(753528763)
|
||||
lb.set_seed(seed)
|
||||
txt = f"index {i+1} {seed}: {prompt1}"
|
||||
img = lb.run_diffusion(lb.text_embedding1, return_image=True)
|
||||
plt.imshow(img)
|
||||
# plt.title(txt)
|
||||
plt.show()
|
||||
print(txt)
|
||||
|
||||
#%%
|
||||
"""
|
||||
index 3 303856737: photo of a futuristic alien temple resting in the desert, mystic, sunlight
|
||||
|
||||
"""
|
||||
|
||||
#%%
|
||||
"""
|
||||
TODO Coding:
|
||||
|
|
|
|||
Loading…
Reference in New Issue