example 1 small upd

example1_standard.py

@@ -0,0 +1,102 @@
+# Copyright 2022 Lunar Ring. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, sys
+dp_git = "/home/lugo/git/"
+sys.path.append(os.path.join(dp_git,'garden4'))
+sys.path.append('util')
+import torch
+torch.backends.cudnn.benchmark = False
+import numpy as np
+import warnings
+warnings.filterwarnings('ignore')
+import time
+import subprocess
+import warnings
+import torch
+from tqdm.auto import tqdm
+from diffusers import StableDiffusionInpaintPipeline
+from diffusers import StableDiffusionPipeline
+from diffusers.schedulers import DDIMScheduler
+from PIL import Image
+import matplotlib.pyplot as plt
+import torch
+from movie_man import MovieSaver
+import datetime
+from typing import Callable, List, Optional, Union
+import inspect
+from latent_blending import LatentBlending, add_frames_linear_interp
+torch.set_grad_enabled(False)
+
+#%% First let us spawn a diffusers pipe using DDIMScheduler
+device = "cuda:0"
+model_path = "../stable_diffusion_models/stable-diffusion-v1-5"
+
+scheduler = DDIMScheduler(beta_start=0.00085,
+            beta_end=0.012,
+            beta_schedule="scaled_linear",
+            clip_sample=False,
+            set_alpha_to_one=False)
+            
+pipe = StableDiffusionPipeline.from_pretrained(
+    model_path,
+    revision="fp16", 
+    torch_dtype=torch.float16,
+    scheduler=scheduler,
+    use_auth_token=True
+)
+pipe = pipe.to(device)
+    
+#%% Next let's set up all parameters
+
+# We want 20 diffusion steps, begin with 2 branches, have 3 branches at step 12 (=0.6*20)
+# 10 branches at step 16 (=0.8*20) and 24 branches at step 18 (=0.9*20)
+# Furthermore we want seed 993621550 for keyframeA and seed 54878562 for keyframeB ()
+
+num_inference_steps = 30 # Number of diffusion interations
+list_nmb_branches = [2, 6, 30, 100] # Specify the branching structure 
+list_injection_strength = [0.0, 0.3, 0.73, 0.93] # Specify the branching structure
+width = 512 
+height = 512
+guidance_scale = 5
+#fixed_seeds = [993621550, 326814432]
+#fixed_seeds = [993621550, 888839807]
+fixed_seeds = [993621550, 753528763]
+    
+lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
+prompt1 = "photo of a beautiful forest covered in white flowers, ambient light, very detailed, magic"
+prompt2 = "photo of a mystical sculpture in the middle of the desert, warm sunlight, sand, eery feeling"
+lb.set_prompt1(prompt1)
+lb.set_prompt2(prompt2)
+
+imgs_transition = lb.run_transition(list_nmb_branches, list_injection_strength, fixed_seeds=fixed_seeds)
+
+#%
+
+# let's get more frames
+duration_transition = 12
+fps = 60
+imgs_transition_ext = add_frames_linear_interp(imgs_transition, duration_transition, fps)
+
+# movie saving
+fp_movie = f"/home/lugo/tmp/latentblending/bobo_incoming.mp4"
+if os.path.isfile(fp_movie):
+    os.remove(fp_movie)
+ms = MovieSaver(fp_movie, fps=fps, profile='save')
+for img in tqdm(imgs_transition_ext):
+    ms.write_frame(img)
+ms.finalize()
+
+
+# MOVIE TODO: ueberschreiben! bad prints.

latent_blending.py

@@ -115,7 +115,7 @@ class LatentBlending():
             self.mask_image = None
             self.mode = 'inpaint'
         else:
-            self.mode = 'default'
+            self.mode = 'standard'
             
         
     def init_inpainting(
@@ -214,14 +214,16 @@ class LatentBlending():
             
         if list_injection_idx is None:
             assert list_injection_strength is not None, "Supply either list_injection_idx or list_injection_strength"
+            # Create the injection indexes
             list_injection_idx = [int(round(x*self.num_inference_steps)) for x in list_injection_strength]
             assert min(np.diff(list_injection_idx)) > 0, 'Injection idx needs to be increasing'
             if min(np.diff(list_injection_idx)) < 2:
                 print("Warning: your injection spacing is very tight. consider increasing the distances")
-            assert type(list_injection_strength[0]) is float, "Need to supply floats for list_injection_strength"
+            assert type(list_injection_strength[1]) is float, "Need to supply floats for list_injection_strength"
+            # we are checking element 1 in list_injection_strength because "0" is an int... [0, 0.5]
         
+        assert max(list_injection_idx) < self.num_inference_steps, "Decrease the injection index or strength"
         assert len(list_injection_idx) == len(list_nmb_branches), "Need to have same length"
-        
         assert max(list_injection_idx) < self.num_inference_steps,"Injection index cannot happen after last diffusion step! Decrease list_injection_idx or list_injection_strength[-1]"
         
         if fixed_seeds is not None:
@@ -364,7 +366,7 @@ class LatentBlending():
             return_image: Optional[bool] = False
         ):
         r"""
-        Wrapper function for run_diffusion_default and run_diffusion_inpaint.
+        Wrapper function for run_diffusion_standard and run_diffusion_inpaint.
         Depending on the mode, the correct one will be executed.
         
         Args:
@@ -381,8 +383,8 @@ class LatentBlending():
         """
         
         
-        if self.mode == 'default':
+        if self.mode == 'standard':
-            return self.run_diffusion_default(text_embeddings, latents_for_injection=latents_for_injection, idx_start=idx_start, idx_stop=idx_stop, return_image=return_image)
+            return self.run_diffusion_standard(text_embeddings, latents_for_injection=latents_for_injection, idx_start=idx_start, idx_stop=idx_stop, return_image=return_image)
         
         elif self.mode == 'inpaint':
             assert self.image_source is not None, "image_source is None. Please run init_inpainting first."
@@ -391,7 +393,7 @@ class LatentBlending():
 
 
     @torch.no_grad()
-    def run_diffusion_default(
+    def run_diffusion_standard(
             self, 
             text_embeddings: torch.FloatTensor, 
             latents_for_injection: torch.FloatTensor = None, 
@@ -936,7 +938,7 @@ if __name__ == "__main__":
     height = 512
     guidance_scale = 5
     seed = 421
-    mode = 'default'
+    mode = 'standard'
     fps_target = 24
     duration_target = 10
     gpu_id = 0
@@ -962,7 +964,7 @@ if __name__ == "__main__":
     pipe = pipe.to(device)
     
 
-    #%% DEFAULT TRANS RE SANITY
+    #%% standard TRANS RE SANITY
     
     lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale, seed)
     self = lb
@@ -1472,7 +1474,7 @@ if __name__ == "__main__":
     height = 512
     guidance_scale = 5
     seed = 421
-    mode = 'default'
+    mode = 'standard'
     fps_target = 30
     duration_target = 15
     gpu_id = 0
@@ -1490,9 +1492,49 @@ if __name__ == "__main__":
     )
     pipe = pipe.to(device)
     
+    #%% seed cherrypicking
+    
+    prompt1 = "photo of a surreal brutalistic vault that is glowing in the night, futuristic, greek ornaments, spider webs"
+    lb.set_prompt1(prompt1)
+    
+    for i in range(1):
+        seed = 753528763 #np.random.randint(753528763)
+        lb.set_seed(seed)
+        txt = f"{i} {seed}"
+        img = lb.run_diffusion(lb.text_embedding1, return_image=True)
+        plt.imshow(img)
+        plt.title(txt)
+        plt.show()
+        print(txt)
         
     
     
+    #%% make nice images of latents
+    num_inference_steps = 10 # Number of diffusion interations
+    list_nmb_branches = [2, 3, 7, 12] # Specify the branching structure 
+    list_injection_idx = [0, 2, 5, 8] # Specify the branching structure
+    width = 512 
+    height = 512
+    guidance_scale = 5
+    fixed_seeds = [993621550, 326814432]
+        
+    lb = LatentBlending(pipe, device, height, width, num_inference_steps, guidance_scale)
+    prompt1 = "photo of a beautiful forest covered in white flowers, ambient light, very detailed, magic"
+    prompt2 = "photo of a mystical sculpture in the middle of the desert, warm sunlight, sand, eery feeling"
+    lb.set_prompt1(prompt1)
+    lb.set_prompt2(prompt2)
+    
+    imgs_transition = lb.run_transition(list_nmb_branches, list_injection_idx=list_injection_idx, fixed_seeds=fixed_seeds)
+#%%
+    dp_tmp= "/home/lugo/tmp/latentblending"
+    for d in range(len(lb.tree_latents)):
+        for b in range(list_nmb_branches[d]):
+            for x in range(len(lb.tree_latents[d][b])):
+                lati = lb.tree_latents[d][b][x]
+                img = lb.latent2image(lati)
+                fn = f"d{d}_b{b}_x{x}.jpg"
+                ip.save(os.path.join(dp_tmp, fn), img)
+                
             
         
 
@@ -1500,7 +1542,6 @@ if __name__ == "__main__":
     #%%
     """
     TODO Coding:
-        list_nmb_branches > num inference
         auto mode (quality settings)
         refactor movie man
         make movie combiner in movie man