From a7270aedfcdf980b37006da5cbe2a285cad6935a Mon Sep 17 00:00:00 2001
From: Johannes Stelzer <stelzer@lunar-ring.ai>
Date: Sat, 18 Feb 2023 08:44:28 +0100
Subject: [PATCH] cleanup
---
example2_inpaint.py | 91 ----------
example2_multitrans.py | 14 +-
example3_multitrans.py | 69 --------
example4_upscaling.py | 67 --------
latent_blending.py | 370 ++++++-----------------------------------
5 files changed, 62 insertions(+), 549 deletions(-)
delete mode 100644 example2_inpaint.py
delete mode 100644 example3_multitrans.py
delete mode 100644 example4_upscaling.py
diff --git a/example2_inpaint.py b/example2_inpaint.py
deleted file mode 100644
index e2de40c..0000000
--- a/example2_inpaint.py
+++ /dev/null
@@ -1,91 +0,0 @@
-# Copyright 2022 Lunar Ring. All rights reserved.
-# Written by Johannes Stelzer, email stelzer@lunar-ring.ai twitter @j_stelzer
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os, sys
-import torch
-torch.backends.cudnn.benchmark = False
-import numpy as np
-import warnings
-warnings.filterwarnings('ignore')
-import time
-import subprocess
-import warnings
-import torch
-from tqdm.auto import tqdm
-from diffusers import StableDiffusionInpaintPipeline
-from PIL import Image
-import torch
-from movie_util import MovieSaver
-from typing import Callable, List, Optional, Union
-from latent_blending import LatentBlending, add_frames_linear_interp
-from stable_diffusion_holder import StableDiffusionHolder
-torch.set_grad_enabled(False)
-
-#%% First let us spawn a stable diffusion holder
-fp_ckpt= "../stable_diffusion_models/ckpt/512-inpainting-ema.ckpt"
-sdh = StableDiffusionHolder(fp_ckpt)
-
-#%% Let's first make a source image and mask.
-quality = 'medium'
-depth_strength = 0.65 #Specifies how deep (in terms of diffusion iterations the first branching happens)
-duration_transition = 7 # In seconds
-fps = 30
-seed0 = 190791709
-
-# Spawn latent blending
-lb = LatentBlending(sdh)
-lb.load_branching_profile(quality=quality, depth_strength=depth_strength)
-prompt1 = "photo of a futuristic alien temple in a desert, mystic, glowing, organic, intricate, sci-fi movie, mesmerizing, scary"
-lb.set_prompt1(prompt1)
-lb.init_inpainting(init_empty=True)
-lb.set_seed(seed0)
-
-# Run diffusion
-list_latents = lb.run_diffusion([lb.text_embedding1])
-image_source = lb.sdh.latent2image(list_latents[-1])
-
-mask_image = 255*np.ones([512,512], dtype=np.uint8)
-mask_image[340:420, 170:280] = 0
-mask_image = Image.fromarray(mask_image)
-
-
-#%% Now let us compute a transition video with inpainting
-# First inject back the latents that we already computed for our source image.
-lb.inject_latents(list_latents, inject_img1=True)
-
-# Then setup the seeds. Keep the one from the first image
-fixed_seeds = [seed0, 6579436]
-
-# Fix the prompts for the target
-prompt2 = "aerial photo of a futuristic alien temple in a blue coastal area, the sun is shining with a bright light"
-lb.set_prompt1(prompt1)
-lb.set_prompt2(prompt2)
-lb.init_inpainting(image_source, mask_image)
-
-# Run latent blending
-imgs_transition = lb.run_transition(recycle_img1=True, fixed_seeds=fixed_seeds)
-
-# Let's get more cheap frames via linear interpolation (duration_transition*fps frames)
-imgs_transition_ext = add_frames_linear_interp(imgs_transition, duration_transition, fps)
-
-# Save as MP4
-fp_movie = "movie_example2.mp4"
-if os.path.isfile(fp_movie):
- os.remove(fp_movie)
-ms = MovieSaver(fp_movie, fps=fps, shape_hw=[lb.height, lb.width])
-for img in tqdm(imgs_transition_ext):
- ms.write_frame(img)
-ms.finalize()
-
diff --git a/example2_multitrans.py b/example2_multitrans.py
index 625d93e..e51a4c8 100644
--- a/example2_multitrans.py
+++ b/example2_multitrans.py
@@ -58,10 +58,16 @@ lb = LatentBlending(sdh)
list_movie_parts = [] #
for i in range(len(list_prompts)-1):
- prompt1 = list_prompts[i]
- prompt2 = list_prompts[i+1]
- lb.set_prompt1(prompt1)
- lb.set_prompt2(prompt2)
+ # For a multi transition we can save some computation time and recycle the latents
+ if i==0:
+ lb.set_prompt1(list_prompts[i])
+ lb.set_prompt2(list_prompts[i+1])
+ recycle_img1 = False
+ else:
+ lb.swap_forward()
+ lb.set_prompt2(list_prompts[i+1])
+ recycle_img1 = True
+
fp_movie_part = f"tmp_part_{str(i).zfill(3)}.mp4"
fixed_seeds = list_seeds[i:i+2]
diff --git a/example3_multitrans.py b/example3_multitrans.py
deleted file mode 100644
index 451bdec..0000000
--- a/example3_multitrans.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Copyright 2022 Lunar Ring. All rights reserved.
-# Written by Johannes Stelzer, email stelzer@lunar-ring.ai twitter @j_stelzer
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os, sys
-import torch
-torch.backends.cudnn.benchmark = False
-import numpy as np
-import warnings
-warnings.filterwarnings('ignore')
-import warnings
-import torch
-from tqdm.auto import tqdm
-from PIL import Image
-import torch
-from movie_util import MovieSaver
-from typing import Callable, List, Optional, Union
-from latent_blending import LatentBlending, add_frames_linear_interp
-from stable_diffusion_holder import StableDiffusionHolder
-torch.set_grad_enabled(False)
-
-#%% First let us spawn a stable diffusion holder
-fp_ckpt = "../stable_diffusion_models/ckpt/v2-1_768-ema-pruned.ckpt"
-sdh = StableDiffusionHolder(fp_ckpt)
-
-
-#%% Let's setup the multi transition
-fps = 30
-duration_single_trans = 15
-quality = 'medium'
-depth_strength = 0.55 #Specifies how deep (in terms of diffusion iterations the first branching happens)
-
-# Specify a list of prompts below
-list_prompts = []
-list_prompts.append("surrealistic statue made of glitter and dirt, standing in a lake, atmospheric light, strange glow")
-list_prompts.append("statue of a mix between a tree and human, made of marble, incredibly detailed")
-list_prompts.append("weird statue of a frog monkey, many colors, standing next to the ruins of an ancient city")
-list_prompts.append("statue of a spider that looked like a human")
-list_prompts.append("statue of a bird that looked like a scorpion")
-list_prompts.append("statue of an ancient cybernetic messenger annoucing good news, golden, futuristic")
-
-# You can optionally specify the seeds
-list_seeds = [954375479, 332539350, 956051013, 408831845, 250009012, 675588737]
-
-lb = LatentBlending(sdh)
-lb.load_branching_profile(quality=quality, depth_strength=depth_strength)
-
-fp_movie = "movie_example3.mp4"
-
-lb.run_multi_transition(
- fp_movie,
- list_prompts,
- list_seeds,
- fps=fps,
- duration_single_trans=duration_single_trans
- )
-
-
diff --git a/example4_upscaling.py b/example4_upscaling.py
deleted file mode 100644
index 5222940..0000000
--- a/example4_upscaling.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# Copyright 2022 Lunar Ring. All rights reserved.
-# Written by Johannes Stelzer, email stelzer@lunar-ring.ai twitter @j_stelzer
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os, sys
-import torch
-torch.backends.cudnn.benchmark = False
-import numpy as np
-import warnings
-warnings.filterwarnings('ignore')
-import warnings
-import torch
-from tqdm.auto import tqdm
-from PIL import Image
-# import matplotlib.pyplot as plt
-import torch
-from movie_util import MovieSaver
-from typing import Callable, List, Optional, Union
-from latent_blending import LatentBlending, add_frames_linear_interp
-from stable_diffusion_holder import StableDiffusionHolder
-torch.set_grad_enabled(False)
-
-#%% Define vars for low-resoltion pass
-dp_img = "upscaling_bleding" # the results will be saved in this folder
-prompt1 = "photo of mount vesuvius erupting a terrifying pyroclastic ash cloud"
-prompt2 = "photo of a inside a building full of ash, fire, death, destruction, explosions"
-fixed_seeds = [5054613, 1168652]
-
-width = 512
-height = 384
-num_inference_steps_lores = 40
-nmb_branches_final_lores = 10
-depth_strength_lores = 0.5
-
-device = "cuda"
-fp_ckpt_lores = "../stable_diffusion_models/ckpt/v2-1_512-ema-pruned.ckpt"
-
-#%% Define vars for high-resoltion pass
-fp_ckpt_hires = "../stable_diffusion_models/ckpt/x4-upscaler-ema.ckpt"
-depth_strength_hires = 0.65
-num_inference_steps_hires = 100
-nmb_branches_final_hires = 6
-
-#%% Run low-res pass
-sdh = StableDiffusionHolder(fp_ckpt_lores)
-lb = LatentBlending(sdh)
-lb.set_prompt1(prompt1)
-lb.set_prompt2(prompt2)
-lb.set_width(width)
-lb.set_height(height)
-lb.run_upscaling_step1(dp_img, depth_strength_lores, num_inference_steps_lores, nmb_branches_final_lores, fixed_seeds)
-
-#%% Run high-res pass
-sdh = StableDiffusionHolder(fp_ckpt_hires)
-lb = LatentBlending(sdh)
-lb.run_upscaling_step2(dp_img, depth_strength_hires, num_inference_steps_hires, nmb_branches_final_hires)
\ No newline at end of file
diff --git a/latent_blending.py b/latent_blending.py
index ec070bb..12ea8ea 100644
--- a/latent_blending.py
+++ b/latent_blending.py
@@ -204,190 +204,6 @@ class LatentBlending():
"""
self.image2_lowres = image
- def load_branching_profile(
- self,
- quality: str = 'medium',
- depth_strength: float = 0.65,
- nmb_frames: int = 100,
- nmb_mindist: int = 3,
- ):
- r"""
- Helper function to set up the branching structure automatically.
-
- Args:
- quality: str
- Determines how many diffusion steps are being made + how many branches in total.
- Tradeoff between quality and speed of computation.
- Choose: lowest, low, medium, high, ultra
- depth_strength: float = 0.65,
- Determines how deep the first injection will happen.
- Deeper injections will cause (unwanted) formation of new structures,
- more shallow values will go into alpha-blendy land.
- nmb_frames: int = 360,
- total number of frames
- nmb_mindist: int = 3
- minimum distance in terms of diffusion iteratinos between subsequent injections
- """
-
- if quality == 'lowest':
- num_inference_steps = 12
- nmb_max_branches = 5
- elif quality == 'low':
- num_inference_steps = 15
- nmb_max_branches = nmb_frames//16
- elif quality == 'medium':
- num_inference_steps = 30
- nmb_max_branches = nmb_frames//8
- elif quality == 'high':
- num_inference_steps = 60
- nmb_max_branches = nmb_frames//4
- elif quality == 'ultra':
- num_inference_steps = 100
- nmb_max_branches = nmb_frames//2
- elif quality == 'upscaling_step1':
- num_inference_steps = 40
- nmb_max_branches = 12
- elif quality == 'upscaling_step2':
- num_inference_steps = 100
- nmb_max_branches = 6
- else:
- raise ValueError(f"quality = '{quality}' not supported")
-
- self.autosetup_branching(depth_strength, num_inference_steps, nmb_max_branches)
-
-
- def autosetup_branching(
- self,
- depth_strength: float = 0.65,
- num_inference_steps: int = 30,
- nmb_max_branches: int = 20,
- nmb_mindist: int = 3,
- ):
- r"""
- Automatically sets up the branching schedule.
-
- Args:
- depth_strength: float = 0.65,
- Determines how deep the first injection will happen.
- Deeper injections will cause (unwanted) formation of new structures,
- more shallow values will go into alpha-blendy land.
- num_inference_steps: int
- Number of diffusion steps. Higher values will take more compute time.
- nmb_max_branches (int): The number of diffusion-generated images
- at the end of the inference.
- nmb_mindist (int): The minimum number of diffusion steps
- between two injections.
- """
-
- idx_injection_first = int(np.round(num_inference_steps*depth_strength))
- idx_injection_last = num_inference_steps - nmb_mindist
- nmb_injections = int(np.floor(num_inference_steps/5)) - 1
-
- list_injection_idx = [0]
- list_injection_idx.extend(np.linspace(idx_injection_first, idx_injection_last, nmb_injections).astype(int))
- list_nmb_branches = np.round(np.logspace(np.log10(2), np.log10(nmb_max_branches), nmb_injections+1)).astype(int)
-
- # Cleanup. There should be at least nmb_mindist diffusion steps between each injection and list_nmb_branches increases
- list_nmb_branches_clean = [list_nmb_branches[0]]
- list_injection_idx_clean = [list_injection_idx[0]]
- for idx_injection, nmb_branches in zip(list_injection_idx[1:], list_nmb_branches[1:]):
- if idx_injection - list_injection_idx_clean[-1] >= nmb_mindist and nmb_branches > list_nmb_branches_clean[-1]:
- list_nmb_branches_clean.append(nmb_branches)
- list_injection_idx_clean.append(idx_injection)
- list_nmb_branches_clean[-1] = nmb_max_branches
-
- list_injection_idx_clean = [int(l) for l in list_injection_idx_clean]
- list_nmb_branches_clean = [int(l) for l in list_nmb_branches_clean]
-
- list_injection_idx = list_injection_idx_clean
- list_nmb_branches = list_nmb_branches_clean
-
- list_nmb_branches = list_nmb_branches
- list_injection_idx = list_injection_idx
- print(f"autosetup_branching: num_inference_steps: {num_inference_steps} list_nmb_branches: {list_nmb_branches} list_injection_idx: {list_injection_idx}")
- self.setup_branching(num_inference_steps, list_nmb_branches=list_nmb_branches, list_injection_idx=list_injection_idx)
-
-
- def setup_branching(self,
- num_inference_steps: int =30,
- list_nmb_branches: List[int] = None,
- list_injection_strength: List[float] = None,
- list_injection_idx: List[int] = None,
- ):
- r"""
- Sets the branching structure for making transitions.
- num_inference_steps: int
- Number of diffusion steps. Larger values will take more compute time.
- list_nmb_branches: List[int]:
- list of the number of branches for each injection.
- list_injection_strength: List[float]:
- list of injection strengths within interval [0, 1), values need to be increasing.
- Alternatively you can direclty specify the list_injection_idx.
- list_injection_idx: List[int]:
- list of injection strengths within interval [0, 1), values need to be increasing.
- Alternatively you can specify the list_injection_strength.
-
- """
- # Assert
- assert not((list_injection_strength is not None) and (list_injection_idx is not None)), "suppyl either list_injection_strength or list_injection_idx"
-
- if list_injection_strength is None:
- assert list_injection_idx is not None, "Supply either list_injection_idx or list_injection_strength"
- assert isinstance(list_injection_idx[0], int) or isinstance(list_injection_idx[0], np.int) , "Need to supply integers for list_injection_idx"
-
- if list_injection_idx is None:
- assert list_injection_strength is not None, "Supply either list_injection_idx or list_injection_strength"
- # Create the injection indexes
- list_injection_idx = [int(round(x*num_inference_steps)) for x in list_injection_strength]
- assert min(np.diff(list_injection_idx)) > 0, 'Injection idx needs to be increasing'
- if min(np.diff(list_injection_idx)) < 2:
- print("Warning: your injection spacing is very tight. consider increasing the distances")
- assert isinstance(list_injection_strength[1], np.floating) or isinstance(list_injection_strength[1], float), "Need to supply floats for list_injection_strength"
- # we are checking element 1 in list_injection_strength because "0" is an int... [0, 0.5]
-
- assert max(list_injection_idx) < num_inference_steps, "Decrease the injection index or strength"
- assert len(list_injection_idx) == len(list_nmb_branches), "Need to have same length"
- assert max(list_injection_idx) < num_inference_steps,"Injection index cannot happen after last diffusion step! Decrease list_injection_idx or list_injection_strength[-1]"
-
-
- # Auto inits
- list_injection_idx_ext = list_injection_idx[:]
- list_injection_idx_ext.append(num_inference_steps)
-
- # If injection at depth 0 not specified, we will start out with 2 branches
- if list_injection_idx_ext[0] != 0:
- list_injection_idx_ext.insert(0,0)
- list_nmb_branches.insert(0,2)
- assert list_nmb_branches[0] == 2, "Need to start with 2 branches. set list_nmb_branches[0]=2"
-
-
- # Set attributes
- self.num_inference_steps = num_inference_steps
- self.sdh.num_inference_steps = num_inference_steps
- self.list_nmb_branches = list_nmb_branches
- self.list_injection_idx = list_injection_idx
- self.list_injection_idx_ext = list_injection_idx_ext
-
- self.init_tree_struct()
-
- def init_tree_struct(self):
- r"""
- Initializes tree variables for holding latents etc.
- """
-
- self.tree_latents = []
- self.tree_fracts = []
- self.tree_status = []
- self.tree_final_imgs_timing = [0]*self.list_nmb_branches[-1]
-
- nmb_blocks_time = len(self.list_injection_idx_ext)-1
- for t_block in range(nmb_blocks_time):
- nmb_branches = self.list_nmb_branches[t_block]
- list_fract_mixing_current = get_spacing(nmb_branches, self.mid_compression_scaler)
- self.tree_fracts.append(list_fract_mixing_current)
- self.tree_latents.append([None]*nmb_branches)
- self.tree_status.append(['untouched']*nmb_branches)
-
def run_transition(
self,
recycle_img1: Optional[bool] = False,
@@ -602,52 +418,6 @@ class LatentBlending():
self.tree_final_imgs.insert(b_parent1+1, self.sdh.latent2image(list_latents[-1]))
self.tree_fracts.insert(b_parent1+1, fract_mixing)
self.tree_idx_injection.insert(b_parent1+1, idx_injection)
-
-
- def compute_latents_mix(self, fract_mixing, b_parent1, b_parent2, idx_injection):
- r"""
- Runs a diffusion trajectory, using the latents from the respective parents
- Args:
- fract_mixing: float
- the fraction along the transition axis [0, 1]
- b_parent1: int
- index of parent1 to be used
- b_parent2: int
- index of parent2 to be used
- idx_injection: int
- the index in terms of diffusion steps, where the next insertion will start.
- """
- list_conditionings = self.get_mixed_conditioning(fract_mixing)
- fract_mixing_parental = (fract_mixing - self.tree_fracts[b_parent1]) / (self.tree_fracts[b_parent2] - self.tree_fracts[b_parent1])
- # idx_reversed = self.num_inference_steps - idx_injection
-
- list_latents_parental_mix = []
- for i in range(self.num_inference_steps):
- latents_p1 = self.tree_latents[b_parent1][i]
- latents_p2 = self.tree_latents[b_parent2][i]
- if latents_p1 is None or latents_p2 is None:
- latents_parental = None
- else:
- latents_parental = interpolate_spherical(latents_p1, latents_p2, fract_mixing_parental)
- list_latents_parental_mix.append(latents_parental)
-
- idx_mixing_stop = int(round(self.num_inference_steps*self.parental_max_depth_influence))
- mixing_coeffs = idx_injection*[self.parental_influence]
- nmb_mixing = idx_mixing_stop - idx_injection
- if nmb_mixing > 0:
- mixing_coeffs.extend(list(np.linspace(self.parental_influence, self.parental_influence*self.parental_influence_decay, nmb_mixing)))
- mixing_coeffs.extend((self.num_inference_steps-len(mixing_coeffs))*[0])
-
- latents_start = list_latents_parental_mix[idx_injection-1]
- list_latents = self.run_diffusion(
- list_conditionings,
- latents_start = latents_start,
- idx_start = idx_injection,
- list_latents_mixing = list_latents_parental_mix,
- mixing_coeffs = mixing_coeffs
- )
-
- return list_latents
def compute_latents1(self, return_image=False):
@@ -706,6 +476,53 @@ class LatentBlending():
return self.sdh.latent2image(list_latents2[-1])
else:
return list_latents2
+
+
+ def compute_latents_mix(self, fract_mixing, b_parent1, b_parent2, idx_injection):
+ r"""
+ Runs a diffusion trajectory, using the latents from the respective parents
+ Args:
+ fract_mixing: float
+ the fraction along the transition axis [0, 1]
+ b_parent1: int
+ index of parent1 to be used
+ b_parent2: int
+ index of parent2 to be used
+ idx_injection: int
+ the index in terms of diffusion steps, where the next insertion will start.
+ """
+ list_conditionings = self.get_mixed_conditioning(fract_mixing)
+ fract_mixing_parental = (fract_mixing - self.tree_fracts[b_parent1]) / (self.tree_fracts[b_parent2] - self.tree_fracts[b_parent1])
+ # idx_reversed = self.num_inference_steps - idx_injection
+
+ list_latents_parental_mix = []
+ for i in range(self.num_inference_steps):
+ latents_p1 = self.tree_latents[b_parent1][i]
+ latents_p2 = self.tree_latents[b_parent2][i]
+ if latents_p1 is None or latents_p2 is None:
+ latents_parental = None
+ else:
+ latents_parental = interpolate_spherical(latents_p1, latents_p2, fract_mixing_parental)
+ list_latents_parental_mix.append(latents_parental)
+
+ idx_mixing_stop = int(round(self.num_inference_steps*self.parental_max_depth_influence))
+ mixing_coeffs = idx_injection*[self.parental_influence]
+ nmb_mixing = idx_mixing_stop - idx_injection
+ if nmb_mixing > 0:
+ mixing_coeffs.extend(list(np.linspace(self.parental_influence, self.parental_influence*self.parental_influence_decay, nmb_mixing)))
+ mixing_coeffs.extend((self.num_inference_steps-len(mixing_coeffs))*[0])
+
+ latents_start = list_latents_parental_mix[idx_injection-1]
+ list_latents = self.run_diffusion(
+ list_conditionings,
+ latents_start = latents_start,
+ idx_start = idx_injection,
+ list_latents_mixing = list_latents_parental_mix,
+ mixing_coeffs = mixing_coeffs
+ )
+
+ return list_latents
+
def get_noise(self, seed):
r"""
@@ -725,7 +542,6 @@ class LatentBlending():
C, H, W = shape_latents
return torch.randn((1, C, H, W), generator=generator, device=self.sdh.device)
-
@torch.no_grad()
@@ -783,55 +599,9 @@ class LatentBlending():
list_latents_mixing = list_latents_mixing,
mixing_coeffs = mixing_coeffs,
return_image=return_image)
+
- # elif self.mode == 'inpaint':
- # text_embeddings = list_conditionings[0]
- # assert self.sdh.image_source is not None, "image_source is None. Please run init_inpainting first."
- # assert self.sdh.mask_image is not None, "image_source is None. Please run init_inpainting first."
- # return self.sdh.run_diffusion_inpaint(text_embeddings, latents_for_injection=latents_for_injection, idx_start=idx_start, idx_stop=idx_stop, return_image=return_image)
-
- # FIXME. new transition engine
- def run_upscaling_step1(
- self,
- dp_img: str,
- depth_strength: float = 0.65,
- num_inference_steps: int = 30,
- nmb_max_branches: int = 10,
- fixed_seeds: Optional[List[int]] = None,
- ):
- r"""
- Initializes inpainting with a source and maks image.
- Args:
- dp_img:
- Path to directory where the low-res images and yaml will be saved to.
- This directory cannot exist and will be created here.
- FIXME
- quality: str
- Determines how many diffusion steps are being made + how many branches in total.
- We suggest to leave it with upscaling_step1 which has 10 final branches.
- depth_strength: float = 0.65,
- Determines how deep the first injection will happen.
- Deeper injections will cause (unwanted) formation of new structures,
- more shallow values will go into alpha-blendy land.
- fixed_seeds: Optional[List[int)]:
- You can supply two seeds that are used for the first and second keyframe (prompt1 and prompt2).
- Otherwise random seeds will be taken.
- """
- assert self.text_embedding1 is not None, 'run set_prompt1(yourprompt1) first'
- assert self.text_embedding2 is not None, 'run set_prompt2(yourprompt2) first'
- assert not os.path.isdir(dp_img), f"directory already exists: {dp_img}"
-
- if fixed_seeds is None:
- fixed_seeds = list(np.random.randint(0, 1000000, 2).astype(np.int32))
-
- # Run latent blending
- imgs_transition = self.run_transition(fixed_seeds=fixed_seeds)
- self.write_imgs_transition(dp_img, imgs_transition)
-
- print(f"run_upscaling_step1: completed! {dp_img}")
-
-
- def run_upscaling_step2(
+ def run_upscaling(
self,
dp_img: str,
depth_strength: float = 0.65,
@@ -839,8 +609,9 @@ class LatentBlending():
nmb_max_branches_highres: int = 5,
nmb_max_branches_lowres: int = 6,
fixed_seeds: Optional[List[int]] = None,
+ duration_single_segment = 3,
):
-
+ #FIXME
fp_yml = os.path.join(dp_img, "lowres.yaml")
fp_movie = os.path.join(dp_img, "movie_highres.mp4")
fps = 24
@@ -864,8 +635,6 @@ class LatentBlending():
text_embeddingA = self.sdh.get_text_embedding(prompt1)
text_embeddingB = self.sdh.get_text_embedding(prompt2)
- #FIXME: have a total length for the whole video section
- duration_single_trans = 3
list_fract_mixing = np.linspace(0, 1, nmb_max_branches_lowres-1)
for i in range(nmb_max_branches_lowres-1):
@@ -891,7 +660,7 @@ class LatentBlending():
nmb_max_branches = nmb_max_branches_highres,
)
- list_imgs_interp = add_frames_linear_interp(list_imgs, fps, duration_single_trans)
+ list_imgs_interp = add_frames_linear_interp(list_imgs, fps, duration_single_segment)
# Save movie frame
for img in list_imgs_interp:
@@ -899,27 +668,6 @@ class LatentBlending():
ms.finalize()
-
-
- def init_inpainting(
- self,
- image_source: Union[Image.Image, np.ndarray] = None,
- mask_image: Union[Image.Image, np.ndarray] = None,
- init_empty: Optional[bool] = False,
- ):
- r"""
- Initializes inpainting with a source and maks image.
- Args:
- image_source: Union[Image.Image, np.ndarray]
- Source image onto which the mask will be applied.
- mask_image: Union[Image.Image, np.ndarray]
- Mask image, value = 0 will stay untouched, value = 255 subjet to diffusion
- init_empty: Optional[bool]:
- Initialize inpainting with an empty image and mask, effectively disabling inpainting,
- useful for generating a first image for transitions using diffusion.
- """
- self.init_mode()
- self.sdh.init_inpainting(image_source, mask_image, init_empty)
@torch.no_grad()
@@ -1060,20 +808,6 @@ class LatentBlending():
self.height = height
self.sdh.height = height
- def inject_latents(self, list_latents, inject_img1=True, inject_img2=False):
- r"""
- Injects list of latents into tree structure.
-
- """
- assert inject_img1 != inject_img2, "Either inject into img1 or img2"
- assert self.tree_latents is not None, "You need to setup the branching beforehand, run autosetup_branching() or setup_branching() before"
-
- for t_block in range(len(self.list_injection_idx)):
- if inject_img1:
- self.tree_latents[t_block][0] = list_latents[self.list_injection_idx_ext[t_block]:self.list_injection_idx_ext[t_block+1]]
- if inject_img2:
- self.tree_latents[t_block][-1] = list_latents[self.list_injection_idx_ext[t_block]:self.list_injection_idx_ext[t_block+1]]
-
def swap_forward(self):
r"""