simpler multitransition functions
This commit is contained in:
parent
a8e2a02f27
commit
2abf976eb2
|
@ -51,22 +51,17 @@ pipe = StableDiffusionPipeline.from_pretrained(
|
|||
pipe = pipe.to(device)
|
||||
|
||||
#%% MULTITRANS
|
||||
# XXX FIXME AssertionError: Need to supply floats for list_injection_strength
|
||||
# GO AS DEEP AS POSSIBLE WITHOUT CAUSING MOTION
|
||||
|
||||
num_inference_steps = 100 # Number of diffusion interations
|
||||
#list_nmb_branches = [2, 12, 24, 55, 77] # Branching structure: how many branches
|
||||
#list_injection_strength = [0.0, 0.35, 0.5, 0.65, 0.95] # Branching structure: how deep is the blending
|
||||
list_nmb_branches = list(np.linspace(2, 600, 15).astype(int)) #
|
||||
list_injection_strength = list(np.linspace(0.45, 0.97, 14).astype(np.float32)) # Branching structure: how deep is the blending
|
||||
list_injection_strength = [float(x) for x in list_injection_strength]
|
||||
num_inference_steps = 30 # Number of diffusion interations
|
||||
list_nmb_branches = [2, 10, 50, 100, 200] #
|
||||
list_injection_strength = list(np.linspace(0.5, 0.95, 4)) # Branching structure: how deep is the blending
|
||||
list_injection_strength.insert(0, 0.0)
|
||||
|
||||
width = 512
|
||||
height = 512
|
||||
guidance_scale = 5
|
||||
fps = 30
|
||||
duration_target = 20
|
||||
duration_single_trans = 20
|
||||
width = 512
|
||||
height = 512
|
||||
|
||||
|
@ -87,31 +82,19 @@ list_prompts.append("statue of an ancient cybernetic messenger annoucing good ne
|
|||
|
||||
list_seeds = [234187386, 422209351, 241845736, 28652396, 783279867, 831049796, 234903931]
|
||||
|
||||
|
||||
fp_movie = "/home/lugo/tmp/latentblending/bubu.mp4"
|
||||
fp_movie = "/home/lugo/tmp/latentblending/bubua.mp4"
|
||||
ms = MovieSaver(fp_movie, fps=fps)
|
||||
|
||||
for i in range(len(list_prompts)-1):
|
||||
print(f"Starting movie segment {i+1}/{len(list_prompts)-1}")
|
||||
lb.run_multi_transition(
|
||||
list_prompts,
|
||||
list_seeds,
|
||||
list_nmb_branches,
|
||||
list_injection_strength=list_injection_strength,
|
||||
ms=ms,
|
||||
fps=fps,
|
||||
duration_single_trans=duration_single_trans
|
||||
)
|
||||
|
||||
if i==0:
|
||||
lb.set_prompt1(list_prompts[i])
|
||||
lb.set_prompt2(list_prompts[i+1])
|
||||
recycle_img1 = False
|
||||
else:
|
||||
lb.swap_forward()
|
||||
lb.set_prompt2(list_prompts[i+1])
|
||||
recycle_img1 = True
|
||||
|
||||
local_seeds = [list_seeds[i], list_seeds[i+1]]
|
||||
list_imgs = lb.run_transition(list_nmb_branches, list_injection_strength, recycle_img1=recycle_img1, fixed_seeds=local_seeds)
|
||||
list_imgs_interp = add_frames_linear_interp(list_imgs, fps, duration_target)
|
||||
|
||||
# Save movie frame
|
||||
for img in list_imgs_interp:
|
||||
ms.write_frame(img)
|
||||
|
||||
ms.finalize()
|
||||
|
||||
#%%
|
||||
#for img in lb.tree_final_imgs:
|
||||
|
|
|
@ -97,6 +97,7 @@ class LatentBlending():
|
|||
self.text_embedding1 = None
|
||||
self.text_embedding2 = None
|
||||
self.stop_diffusion = False
|
||||
self.negative_prompt = None
|
||||
|
||||
|
||||
def check_asserts(self):
|
||||
|
@ -212,7 +213,7 @@ class LatentBlending():
|
|||
|
||||
if list_injection_strength is None:
|
||||
assert list_injection_idx is not None, "Supply either list_injection_idx or list_injection_strength"
|
||||
assert type(list_injection_idx[0]) is int, "Need to supply integers for list_injection_idx"
|
||||
assert isinstance(list_injection_idx[0], int), "Need to supply integers for list_injection_idx"
|
||||
|
||||
if list_injection_idx is None:
|
||||
assert list_injection_strength is not None, "Supply either list_injection_idx or list_injection_strength"
|
||||
|
@ -221,7 +222,7 @@ class LatentBlending():
|
|||
assert min(np.diff(list_injection_idx)) > 0, 'Injection idx needs to be increasing'
|
||||
if min(np.diff(list_injection_idx)) < 2:
|
||||
print("Warning: your injection spacing is very tight. consider increasing the distances")
|
||||
assert type(list_injection_strength[1]) is float, "Need to supply floats for list_injection_strength"
|
||||
assert isinstance(list_injection_strength[1], np.floating), "Need to supply floats for list_injection_strength"
|
||||
# we are checking element 1 in list_injection_strength because "0" is an int... [0, 0.5]
|
||||
|
||||
assert max(list_injection_idx) < self.num_inference_steps, "Decrease the injection index or strength"
|
||||
|
@ -370,6 +371,71 @@ class LatentBlending():
|
|||
return self.tree_final_imgs
|
||||
|
||||
|
||||
def run_multi_transition(
|
||||
self,
|
||||
list_prompts: List[str],
|
||||
list_seeds: List[int] = None,
|
||||
list_nmb_branches: List[int] = None,
|
||||
list_injection_strength: List[float] = None,
|
||||
list_injection_idx: List[int] = None,
|
||||
ms: MovieSaver = None,
|
||||
fps: float = 24,
|
||||
duration_single_trans: float = 15,
|
||||
):
|
||||
r"""
|
||||
Runs multiple transitions and stitches them together. You can supply the seeds for each prompt.
|
||||
Args:
|
||||
list_prompts: List[float]:
|
||||
list of the prompts. There will be a transition starting from the first to the last.
|
||||
list_seeds: List[int] = None:
|
||||
Random Seeds for each prompt.
|
||||
list_nmb_branches: List[int]:
|
||||
list of the number of branches for each injection.
|
||||
list_injection_strength: List[float]:
|
||||
list of injection strengths within interval [0, 1), values need to be increasing.
|
||||
Alternatively you can direclty specify the list_injection_idx.
|
||||
list_injection_idx: List[int]:
|
||||
list of injection strengths within interval [0, 1), values need to be increasing.
|
||||
Alternatively you can specify the list_injection_strength.
|
||||
ms: MovieSaver
|
||||
You need to spawn a moviesaver instance.
|
||||
fps: float:
|
||||
frames per second
|
||||
duration_single_trans: float:
|
||||
The duration of a single transition prompt[i] -> prompt[i+1].
|
||||
The duration of your movie will be duration_single_trans * len(list_prompts)
|
||||
|
||||
"""
|
||||
if list_seeds is None:
|
||||
list_seeds = list(np.random.randint(0, 10e10, len(list_prompts)))
|
||||
|
||||
assert len(list_prompts) == len(list_seeds), "Supply the same number of prompts and seeds"
|
||||
|
||||
for i in range(len(list_prompts)-1):
|
||||
print(f"Starting movie segment {i+1}/{len(list_prompts)-1}")
|
||||
|
||||
if i==0:
|
||||
self.set_prompt1(list_prompts[i])
|
||||
self.set_prompt2(list_prompts[i+1])
|
||||
recycle_img1 = False
|
||||
else:
|
||||
self.swap_forward()
|
||||
self.set_prompt2(list_prompts[i+1])
|
||||
recycle_img1 = True
|
||||
|
||||
local_seeds = [list_seeds[i], list_seeds[i+1]]
|
||||
list_imgs = lb.run_transition(list_nmb_branches, list_injection_strength=list_injection_strength, list_injection_idx=list_injection_idx, recycle_img1=recycle_img1, fixed_seeds=local_seeds)
|
||||
list_imgs_interp = add_frames_linear_interp(list_imgs, fps, duration_single_trans)
|
||||
|
||||
# Save movie frame
|
||||
for img in list_imgs_interp:
|
||||
ms.write_frame(img)
|
||||
|
||||
ms.finalize()
|
||||
|
||||
print("run_multi_transition: All completed.")
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def run_diffusion(
|
||||
self,
|
||||
|
@ -1003,7 +1069,45 @@ if __name__ == "__main__":
|
|||
|
||||
#%%
|
||||
|
||||
num_inference_steps = 30 # Number of diffusion interations
|
||||
list_nmb_branches = [2, 10, 50, 100, 200] #
|
||||
list_injection_strength = list(np.linspace(0.5, 0.95, 4)) # Branching structure: how deep is the blending
|
||||
list_injection_strength.insert(0, 0.0)
|
||||
|
||||
width = 512
|
||||
height = 512
|
||||
guidance_scale = 5
|
||||
fps = 30
|
||||
duration_single_trans = 20
|
||||
width = 512
|
||||
height = 512
|
||||
|
||||
lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
|
||||
|
||||
list_prompts = []
|
||||
list_prompts.append("surrealistic statue made of glitter and dirt, standing in a lake, atmospheric light, strange glow")
|
||||
list_prompts.append("statue of a mix between a tree and human, made of marble, incredibly detailed")
|
||||
list_prompts.append("weird statue of a frog monkey, many colors, standing next to the ruins of an ancient city")
|
||||
list_prompts.append("statue made of hot metal, bizzarre, dark clouds in the sky")
|
||||
list_prompts.append("statue of a spider that looked like a human")
|
||||
list_prompts.append("statue of a bird that looked like a scorpion")
|
||||
list_prompts.append("statue of an ancient cybernetic messenger annoucing good news, golden, futuristic")
|
||||
|
||||
|
||||
list_seeds = [234187386, 422209351, 241845736, 28652396, 783279867, 831049796, 234903931]
|
||||
|
||||
fp_movie = "/home/lugo/tmp/latentblending/bubua.mp4"
|
||||
ms = MovieSaver(fp_movie, fps=fps)
|
||||
|
||||
lb.run_multi_transition(
|
||||
list_prompts,
|
||||
list_seeds,
|
||||
list_nmb_branches,
|
||||
list_injection_strength=list_injection_strength,
|
||||
ms=ms,
|
||||
fps=fps,
|
||||
duration_single_trans=duration_single_trans
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue