simpler multitransition functions

example3_multitrans.py

@@ -51,22 +51,17 @@ pipe = StableDiffusionPipeline.from_pretrained(
 pipe = pipe.to(device)
     
 #%% MULTITRANS
-# XXX FIXME AssertionError: Need to supply floats for list_injection_strength
-# GO AS DEEP AS POSSIBLE WITHOUT CAUSING MOTION
 
-num_inference_steps = 100 # Number of diffusion interations
+num_inference_steps = 30 # Number of diffusion interations
-#list_nmb_branches = [2, 12, 24, 55, 77] # Branching structure: how many branches
+list_nmb_branches = [2, 10, 50, 100, 200] #
-#list_injection_strength = [0.0, 0.35, 0.5, 0.65, 0.95] # Branching structure: how deep is the blending
+list_injection_strength = list(np.linspace(0.5, 0.95, 4)) # Branching structure: how deep is the blending
-list_nmb_branches = list(np.linspace(2, 600, 15).astype(int)) #
+list_injection_strength.insert(0, 0.0)
-list_injection_strength = list(np.linspace(0.45, 0.97, 14).astype(np.float32)) # Branching structure: how deep is the blending
-list_injection_strength = [float(x) for x in list_injection_strength]
-list_injection_strength.insert(0,0.0)
 
 width = 512
 height = 512
 guidance_scale = 5
 fps = 30
-duration_target = 20
+duration_single_trans = 20
 width = 512
 height = 512
 
@@ -87,31 +82,19 @@ list_prompts.append("statue of an ancient cybernetic messenger annoucing good ne
 
 list_seeds = [234187386, 422209351, 241845736, 28652396, 783279867, 831049796, 234903931]
 
-
+fp_movie = "/home/lugo/tmp/latentblending/bubua.mp4"
-fp_movie = "/home/lugo/tmp/latentblending/bubu.mp4"
 ms = MovieSaver(fp_movie, fps=fps)
 
-for i in range(len(list_prompts)-1):
+lb.run_multi_transition(
-    print(f"Starting movie segment {i+1}/{len(list_prompts)-1}")
+        list_prompts, 
+        list_seeds, 
+        list_nmb_branches, 
+        list_injection_strength=list_injection_strength, 
+        ms=ms, 
+        fps=fps, 
+        duration_single_trans=duration_single_trans
+    )
 
-    if i==0:
-        lb.set_prompt1(list_prompts[i])
-        lb.set_prompt2(list_prompts[i+1])
-        recycle_img1 = False    
-    else:
-        lb.swap_forward()
-        lb.set_prompt2(list_prompts[i+1])
-        recycle_img1 = True    
-    
-    local_seeds = [list_seeds[i], list_seeds[i+1]]
-    list_imgs = lb.run_transition(list_nmb_branches, list_injection_strength, recycle_img1=recycle_img1, fixed_seeds=local_seeds)
-    list_imgs_interp = add_frames_linear_interp(list_imgs, fps, duration_target)
-    
-    # Save movie frame
-    for img in list_imgs_interp:
-        ms.write_frame(img)
-        
-ms.finalize()
 
 #%%
 #for img in lb.tree_final_imgs:

latent_blending.py

@@ -97,6 +97,7 @@ class LatentBlending():
         self.text_embedding1 = None
         self.text_embedding2 = None
         self.stop_diffusion = False
+        self.negative_prompt = None
         
     
     def check_asserts(self):
@@ -212,7 +213,7 @@ class LatentBlending():
         
         if list_injection_strength is None:
             assert list_injection_idx is not None, "Supply either list_injection_idx or list_injection_strength"
-            assert type(list_injection_idx[0]) is int, "Need to supply integers for list_injection_idx"
+            assert isinstance(list_injection_idx[0], int), "Need to supply integers for list_injection_idx"
             
         if list_injection_idx is None:
             assert list_injection_strength is not None, "Supply either list_injection_idx or list_injection_strength"
@@ -221,7 +222,7 @@ class LatentBlending():
             assert min(np.diff(list_injection_idx)) > 0, 'Injection idx needs to be increasing'
             if min(np.diff(list_injection_idx)) < 2:
                 print("Warning: your injection spacing is very tight. consider increasing the distances")
-            assert type(list_injection_strength[1]) is float, "Need to supply floats for list_injection_strength"
+            assert isinstance(list_injection_strength[1], np.floating), "Need to supply floats for list_injection_strength"
             # we are checking element 1 in list_injection_strength because "0" is an int... [0, 0.5]
         
         assert max(list_injection_idx) < self.num_inference_steps, "Decrease the injection index or strength"
@@ -370,6 +371,71 @@ class LatentBlending():
         return self.tree_final_imgs
                 
 
+    def run_multi_transition(
+            self,
+            list_prompts: List[str],
+            list_seeds: List[int] = None,
+            list_nmb_branches: List[int] = None, 
+            list_injection_strength: List[float] = None, 
+            list_injection_idx: List[int] = None, 
+            ms: MovieSaver = None,
+            fps: float = 24,
+            duration_single_trans: float = 15,
+        ):
+        r"""
+        Runs multiple transitions and stitches them together. You can supply the seeds for each prompt.
+        Args:
+            list_prompts: List[float]:
+                list of the prompts. There will be a transition starting from the first to the last.
+            list_seeds: List[int] = None: 
+                Random Seeds for each prompt.
+            list_nmb_branches: List[int]:
+                list of the number of branches for each injection.
+            list_injection_strength: List[float]:
+                list of injection strengths within interval [0, 1), values need to be increasing.
+                Alternatively you can direclty specify the list_injection_idx.
+            list_injection_idx: List[int]:
+                list of injection strengths within interval [0, 1), values need to be increasing.
+                Alternatively you can specify the list_injection_strength.
+            ms: MovieSaver
+                You need to spawn a moviesaver instance.
+            fps: float:
+                frames per second
+            duration_single_trans: float:
+                The duration of a single transition prompt[i] -> prompt[i+1].
+                The duration of your movie will be duration_single_trans * len(list_prompts)
+            
+        """
+        if list_seeds is None:
+            list_seeds = list(np.random.randint(0, 10e10, len(list_prompts)))
+            
+        assert len(list_prompts) == len(list_seeds), "Supply the same number of prompts and seeds"
+        
+        for i in range(len(list_prompts)-1):
+            print(f"Starting movie segment {i+1}/{len(list_prompts)-1}")
+            
+            if i==0:
+                self.set_prompt1(list_prompts[i])
+                self.set_prompt2(list_prompts[i+1])
+                recycle_img1 = False    
+            else:
+                self.swap_forward()
+                self.set_prompt2(list_prompts[i+1])
+                recycle_img1 = True    
+            
+            local_seeds = [list_seeds[i], list_seeds[i+1]]
+            list_imgs = lb.run_transition(list_nmb_branches, list_injection_strength=list_injection_strength, list_injection_idx=list_injection_idx, recycle_img1=recycle_img1, fixed_seeds=local_seeds)
+            list_imgs_interp = add_frames_linear_interp(list_imgs, fps, duration_single_trans)
+            
+            # Save movie frame
+            for img in list_imgs_interp:
+                ms.write_frame(img)
+                
+        ms.finalize()
+        
+        print("run_multi_transition: All completed.")
+
+
     @torch.no_grad()
     def run_diffusion(
             self, 
@@ -1003,7 +1069,45 @@ if __name__ == "__main__":
 
 #%%
 
+    num_inference_steps = 30 # Number of diffusion interations
+    list_nmb_branches = [2, 10, 50, 100, 200] #
+    list_injection_strength = list(np.linspace(0.5, 0.95, 4)) # Branching structure: how deep is the blending
+    list_injection_strength.insert(0, 0.0)
     
+    width = 512
+    height = 512
+    guidance_scale = 5
+    fps = 30
+    duration_single_trans = 20
+    width = 512
+    height = 512
+    
+    lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
+    
+    list_prompts = []
+    list_prompts.append("surrealistic statue made of glitter and dirt, standing in a lake, atmospheric light, strange glow")
+    list_prompts.append("statue of a mix between a tree and human, made of marble, incredibly detailed")
+    list_prompts.append("weird statue of a frog monkey, many colors, standing next to the ruins of an ancient city")
+    list_prompts.append("statue made of hot metal, bizzarre, dark clouds in the sky")
+    list_prompts.append("statue of a spider that looked like a human")
+    list_prompts.append("statue of a bird that looked like a scorpion")
+    list_prompts.append("statue of an ancient cybernetic messenger annoucing good news, golden, futuristic")
+    
+    
+    list_seeds = [234187386, 422209351, 241845736, 28652396, 783279867, 831049796, 234903931]
+    
+    fp_movie = "/home/lugo/tmp/latentblending/bubua.mp4"
+    ms = MovieSaver(fp_movie, fps=fps)
+    
+    lb.run_multi_transition(
+            list_prompts, 
+            list_seeds, 
+            list_nmb_branches, 
+            list_injection_strength=list_injection_strength, 
+            ms=ms, 
+            fps=fps, 
+            duration_single_trans=duration_single_trans
+        )