latentblending/examples/multi_trans.py

56 lines
1.7 KiB
Python
Raw Normal View History

2023-02-18 07:19:40 +00:00
import torch
import warnings
2024-01-09 14:59:06 +00:00
from diffusers import AutoPipelineForText2Image
2024-01-10 08:47:35 +00:00
from latentblending.movie_util import concatenate_movies
from latentblending.blending_engine import BlendingEngine
2023-11-16 14:37:02 +00:00
torch.set_grad_enabled(False)
torch.backends.cudnn.benchmark = False
warnings.filterwarnings('ignore')
2023-02-18 07:19:40 +00:00
2023-02-22 09:15:03 +00:00
# %% First let us spawn a stable diffusion holder. Uncomment your version of choice.
2024-01-09 14:59:06 +00:00
pipe = AutoPipelineForText2Image.from_pretrained("stabilityai/sdxl-turbo", torch_dtype=torch.float16, variant="fp16")
2023-11-16 12:57:11 +00:00
pipe.to('cuda')
be = BlendingEngine(pipe)
2023-02-18 07:19:40 +00:00
2023-02-22 09:15:03 +00:00
# %% Let's setup the multi transition
2023-02-18 07:19:40 +00:00
fps = 30
2024-01-09 14:59:06 +00:00
duration_single_trans = 10
2023-02-18 07:19:40 +00:00
# Specify a list of prompts below
list_prompts = []
2023-11-16 14:43:58 +00:00
list_prompts.append("Photo of a house, high detail")
list_prompts.append("Photo of an elephant in african savannah")
list_prompts.append("photo of a house, high detail")
2023-10-11 10:17:15 +00:00
2023-02-18 07:19:40 +00:00
# Specify the seeds
list_seeds = np.random.randint(0, 10^9, len(list_prompts))
2023-02-18 07:19:40 +00:00
fp_movie = 'movie_example2.mp4'
2023-02-18 07:19:40 +00:00
2023-02-22 09:15:03 +00:00
list_movie_parts = []
for i in range(len(list_prompts) - 1):
2023-02-18 07:44:28 +00:00
# For a multi transition we can save some computation time and recycle the latents
2023-02-22 09:15:03 +00:00
if i == 0:
2024-01-09 20:07:27 +00:00
be.set_prompt1(list_prompts[i])
be.set_prompt2(list_prompts[i + 1])
2023-02-18 07:44:28 +00:00
recycle_img1 = False
else:
2024-01-09 20:07:27 +00:00
be.swap_forward()
be.set_prompt2(list_prompts[i + 1])
2023-02-22 09:15:03 +00:00
recycle_img1 = True
2023-02-18 07:19:40 +00:00
fp_movie_part = f"tmp_part_{str(i).zfill(3)}.mp4"
2023-02-22 09:15:03 +00:00
fixed_seeds = list_seeds[i:i + 2]
2023-02-18 07:19:40 +00:00
# Run latent blending
2024-01-09 20:07:27 +00:00
be.run_transition(
2023-11-16 14:37:02 +00:00
recycle_img1=recycle_img1,
2023-02-22 09:15:03 +00:00
fixed_seeds=fixed_seeds)
2023-02-18 07:19:40 +00:00
# Save movie
2024-01-09 20:07:27 +00:00
be.write_movie_transition(fp_movie_part, duration_single_trans)
2023-02-18 07:19:40 +00:00
list_movie_parts.append(fp_movie_part)
# Finally, concatente the result
2023-02-22 09:15:03 +00:00
concatenate_movies(fp_movie, list_movie_parts)