new examples

example1_standard.py

@@ -31,11 +31,11 @@ from latent_blending import LatentBlending, add_frames_linear_interp
 from stable_diffusion_holder import StableDiffusionHolder
 torch.set_grad_enabled(False)
 
+
 #%% First let us spawn a stable diffusion holder
 fp_ckpt = "../stable_diffusion_models/ckpt/v2-1_768-ema-pruned.ckpt"
 sdh = StableDiffusionHolder(fp_ckpt)
     
-    
 #%% Next let's set up all parameters
 depth_strength = 0.65 # Specifies how deep (in terms of diffusion iterations the first branching happens)
 t_compute_max_allowed = 15 # Determines the quality of the transition in terms of compute time you grant it
@@ -44,8 +44,8 @@ fixed_seeds = [69731932, 504430820]
 prompt1 = "photo of a beautiful cherry forest covered in white flowers, ambient light, very detailed, magic"
 prompt2 = "photo of an golden statue with a funny hat, surrounded by ferns and vines, grainy analog photograph, mystical ambience, incredible detail"
 
+fp_movie = 'movie_example1.mp4'
 duration_transition = 12 # In seconds
-fps = 30
 
 # Spawn latent blending
 lb = LatentBlending(sdh)
@@ -53,22 +53,11 @@ lb.set_prompt1(prompt1)
 lb.set_prompt2(prompt2)
 
 # Run latent blending
-imgs_transition = lb.run_transition(
+lb.run_transition(
     depth_strength = depth_strength,
     t_compute_max_allowed = t_compute_max_allowed,
     fixed_seeds = fixed_seeds
     )
 
-# Let's get more cheap frames via linear interpolation (duration_transition*fps frames)
+# Save movie
-imgs_transition_ext = add_frames_linear_interp(imgs_transition, duration_transition, fps)
+lb.write_movie_transition(fp_movie, duration_transition)
-
-# Save as MP4
-fp_movie = "movie_example1.mp4"
-if os.path.isfile(fp_movie):
-    os.remove(fp_movie)
-ms = MovieSaver(fp_movie, fps=fps, shape_hw=[sdh.height, sdh.width])
-for img in tqdm(imgs_transition_ext):
-    ms.write_frame(img)
-ms.finalize()
-
-

example2_multitrans.py

@@ -0,0 +1,81 @@
+# Copyright 2022 Lunar Ring. All rights reserved.
+# Written by Johannes Stelzer, email stelzer@lunar-ring.ai twitter @j_stelzer
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, sys
+import torch
+torch.backends.cudnn.benchmark = False
+import numpy as np
+import warnings
+warnings.filterwarnings('ignore')
+import warnings
+import torch
+from tqdm.auto import tqdm
+from PIL import Image
+import torch
+from movie_util import MovieSaver, concatenate_movies
+from typing import Callable, List, Optional, Union
+from latent_blending import LatentBlending, add_frames_linear_interp
+from stable_diffusion_holder import StableDiffusionHolder
+torch.set_grad_enabled(False)
+
+#%% First let us spawn a stable diffusion holder
+fp_ckpt = "../stable_diffusion_models/ckpt/v2-1_512-ema-pruned.ckpt"
+# fp_ckpt = "../stable_diffusion_models/ckpt/v2-1_768-ema-pruned.ckpt"
+sdh = StableDiffusionHolder(fp_ckpt)
+
+    
+#%% Let's setup the multi transition
+fps = 30
+duration_single_trans = 6
+depth_strength = 0.55 #Specifies how deep (in terms of diffusion iterations the first branching happens)
+
+# Specify a list of prompts below
+list_prompts = []
+list_prompts.append("surrealistic statue made of glitter and dirt, standing in a lake, atmospheric light, strange glow")
+list_prompts.append("statue of a mix between a tree and human, made of marble, incredibly detailed")
+list_prompts.append("weird statue of a frog monkey, many colors, standing next to the ruins of an ancient city")
+list_prompts.append("statue of a spider that looked like a human")
+list_prompts.append("statue of a bird that looked like a scorpion")
+list_prompts.append("statue of an ancient cybernetic messenger annoucing good news, golden, futuristic")
+
+# You can optionally specify the seeds
+list_seeds = [954375479, 332539350, 956051013, 408831845, 250009012, 675588737]
+t_compute_max_allowed = 12 # per segment
+fp_movie = 'movie_example2.mp4'
+lb = LatentBlending(sdh)
+
+list_movie_parts = [] #
+for i in range(len(list_prompts)-1):
+    prompt1 = list_prompts[i]
+    prompt2 = list_prompts[i+1]
+    lb.set_prompt1(prompt1)
+    lb.set_prompt2(prompt2)
+    fp_movie_part = f"tmp_part_{str(i).zfill(3)}.mp4"
+    
+    fixed_seeds = list_seeds[i:i+2]
+    
+    # Run latent blending
+    lb.run_transition(
+        depth_strength = depth_strength,
+        t_compute_max_allowed = t_compute_max_allowed,
+        fixed_seeds = fixed_seeds
+        )
+    
+    # Save movie
+    lb.write_movie_transition(fp_movie_part, duration_single_trans)
+    list_movie_parts.append(fp_movie_part)
+
+# Finally, concatente the result
+concatenate_movies(fp_movie, list_movie_parts)

latent_blending.py

@@ -727,64 +727,6 @@ class LatentBlending():
         return torch.randn((1, C, H, W), generator=generator, device=self.sdh.device)
     
 
-    def run_multi_transition(
-            self,
-            fp_movie: str, 
-            list_prompts: List[str],
-            list_seeds: List[int] = None,
-            fps: float = 24,
-            duration_single_trans: float = 15,
-        ):
-        r"""
-        Runs multiple transitions and stitches them together. You can supply the seeds for each prompt.
-        Args:
-            fp_movie: file path for movie saving
-            list_prompts: List[float]:
-                list of the prompts. There will be a transition starting from the first to the last.
-            list_seeds: List[int] = None: 
-                Random Seeds for each prompt.
-            fps: float:
-                frames per second
-            duration_single_trans: float:
-                The duration of a single transition prompt[i] -> prompt[i+1].
-                The duration of your movie will be duration_single_trans * len(list_prompts)
-            
-        """
-        
-        if list_seeds is None:
-            list_seeds = list(np.random.randint(0, 10e10, len(list_prompts)))
-        assert len(list_prompts) == len(list_seeds), "Supply the same number of prompts and seeds"
-        
-        ms = MovieSaver(fp_movie, fps=fps)
-        
-        for i in range(len(list_prompts)-1):
-            print(f"Starting movie segment {i+1}/{len(list_prompts)-1}")
-            
-            if i==0:
-                self.set_prompt1(list_prompts[i])
-                self.set_prompt2(list_prompts[i+1])
-                recycle_img1 = False
-            else:
-                self.swap_forward()
-                self.set_prompt2(list_prompts[i+1])
-                recycle_img1 = True    
-            
-            local_seeds = [list_seeds[i], list_seeds[i+1]]
-            list_imgs = self.run_transition(recycle_img1=recycle_img1, fixed_seeds=local_seeds)
-            list_imgs_interp = add_frames_linear_interp(list_imgs, fps, duration_single_trans)
-            
-            if i==0:
-                self.multi_transition_img_first = list_imgs[0]
-            
-            # Save movie frame
-            for img in list_imgs_interp:
-                ms.write_frame(img)
-                
-        ms.finalize()
-        self.multi_transition_img_last = list_imgs[-1]
-        
-        print("run_multi_transition: All completed.")
-
 
     @torch.no_grad()
     def run_diffusion(
@@ -1018,6 +960,10 @@ class LatentBlending():
     def write_imgs_transition(self, dp_img):
         r"""
         Writes the transition images into the folder dp_img.
+        Requires run_transition to be completed.
+        Args:
+            dp_img: str
+                Directory, into which the transition images, yaml file and latents are written.
         """
         imgs_transition = self.tree_final_imgs
         os.makedirs(dp_img, exist_ok=True)
@@ -1028,6 +974,32 @@ class LatentBlending():
         fp_yml = os.path.join(dp_img, "lowres.yaml") 
         self.save_statedict(fp_yml)
         
+    def write_movie_transition(self, fp_movie, duration_transition, fps=30):
+        r"""
+        Writes the transition movie to fp_movie, using the given duration and fps..
+        The missing frames are linearly interpolated.
+        Args:
+            fp_movie: str
+                file pointer to the final movie.
+            duration_transition: float
+                duration of the movie in seonds
+            fps: int
+                fps of the movie
+                
+        """
+        
+        # Let's get more cheap frames via linear interpolation (duration_transition*fps frames)
+        imgs_transition_ext = add_frames_linear_interp(self.tree_final_imgs, duration_transition, fps)
+
+        # Save as MP4
+        if os.path.isfile(fp_movie):
+            os.remove(fp_movie)
+        ms = MovieSaver(fp_movie, fps=fps, shape_hw=[self.sdh.height, self.sdh.width])
+        for img in tqdm(imgs_transition_ext):
+            ms.write_frame(img)
+        ms.finalize()
+
+        
         
     def save_statedict(self, fp_yml):
         # Dump everything relevant into yaml