diff --git a/LICENSE b/LICENSE index 261eeb9..1075f53 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,28 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +BSD 3-Clause License - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Copyright (c) 2023, Lunar Ring - 1. Definitions. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/example1_standard.py b/example1_standard.py deleted file mode 100644 index db82235..0000000 --- a/example1_standard.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2022 Lunar Ring. All rights reserved. -# Written by Johannes Stelzer, email stelzer@lunar-ring.ai twitter @j_stelzer -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import warnings -from latent_blending import LatentBlending -from diffusers_holder import DiffusersHolder -from diffusers import AutoPipelineForText2Image - -warnings.filterwarnings('ignore') -torch.set_grad_enabled(False) -torch.backends.cudnn.benchmark = False - -# %% First let us spawn a stable diffusion holder. Uncomment your version of choice. -pipe = AutoPipelineForText2Image.from_pretrained("stabilityai/sdxl-turbo", torch_dtype=torch.float16, variant="fp16") -pipe.to("cuda") - -dh = DiffusersHolder(pipe) - -lb = LatentBlending(dh) -lb.set_prompt1("photo of underwater landscape, fish, und the sea, incredible detail, high resolution") -lb.set_prompt2("rendering of an alien planet, strange plants, strange creatures, surreal") -lb.set_negative_prompt("blurry, ugly, pale") - -# Run latent blending -lb.run_transition() - -# Save movie -lb.write_movie_transition('movie_example1.mp4', duration_transition=12) diff --git a/latentblending/__init__.py b/latentblending/__init__.py new file mode 100644 index 0000000..d6c8d50 --- /dev/null +++ b/latentblending/__init__.py @@ -0,0 +1,4 @@ +from .blending_engine import BlendingEngine +from .diffusers_holder import DiffusersHolder +from .movie_util import MovieSaver +from .utils import interpolate_spherical, add_frames_linear_interp, interpolate_linear, get_spacing, get_time, yml_load, yml_save diff --git a/latent_blending.py b/latentblending/blending_engine.py similarity index 97% rename from latent_blending.py rename to latentblending/blending_engine.py index 6a58e37..034b173 100644 --- a/latent_blending.py +++ b/latentblending/blending_engine.py @@ -1,18 +1,3 @@ -# Copyright 2022 Lunar Ring. All rights reserved. -# Written by Johannes Stelzer, email stelzer@lunar-ring.ai twitter @j_stelzer -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import os import torch import numpy as np @@ -20,16 +5,16 @@ import warnings import time from tqdm.auto import tqdm from PIL import Image -from movie_util import MovieSaver +from latentblending.movie_util import MovieSaver from typing import List, Optional import lpips -from utils import interpolate_spherical, interpolate_linear, add_frames_linear_interp, yml_load, yml_save +from latentblending.utils import interpolate_spherical, interpolate_linear, add_frames_linear_interp, yml_load, yml_save warnings.filterwarnings('ignore') torch.backends.cudnn.benchmark = False torch.set_grad_enabled(False) -class LatentBlending(): +class BlendingEngine(): def __init__( self, dh: None, diff --git a/diffusers_holder.py b/latentblending/diffusers_holder.py similarity index 96% rename from diffusers_holder.py rename to latentblending/diffusers_holder.py index bb8f615..2f50950 100644 --- a/diffusers_holder.py +++ b/latentblending/diffusers_holder.py @@ -1,24 +1,9 @@ -# Copyright 2022 Lunar Ring. All rights reserved. -# Written by Johannes Stelzer, email stelzer@lunar-ring.ai twitter @j_stelzer -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import torch import numpy as np import warnings from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from utils import interpolate_spherical +from latentblending.utils import interpolate_spherical from diffusers import DiffusionPipeline, StableDiffusionControlNetPipeline, ControlNetModel from diffusers.models.attention_processor import ( AttnProcessor2_0, diff --git a/latentblending/example1_standard.py b/latentblending/example1_standard.py new file mode 100644 index 0000000..aa23d69 --- /dev/null +++ b/latentblending/example1_standard.py @@ -0,0 +1,26 @@ +import torch +import warnings +from blending_engine import BlendingEngine +from diffusers_holder import DiffusersHolder +from diffusers import AutoPipelineForText2Image + +warnings.filterwarnings('ignore') +torch.set_grad_enabled(False) +torch.backends.cudnn.benchmark = False + +# %% First let us spawn a stable diffusion holder. Uncomment your version of choice. +pipe = AutoPipelineForText2Image.from_pretrained("stabilityai/sdxl-turbo", torch_dtype=torch.float16, variant="fp16") +pipe.to("cuda") + +dh = DiffusersHolder(pipe) + +be = BlendingEngine(dh) +be.set_prompt1("photo of underwater landscape, fish, und the sea, incredible detail, high resolution") +be.set_prompt2("rendering of an alien planet, strange plants, strange creatures, surreal") +be.set_negative_prompt("blurry, ugly, pale") + +# Run latent blending +be.run_transition() + +# Save movie +be.write_movie_transition('movie_example1.mp4', duration_transition=12) diff --git a/example2_multitrans.py b/latentblending/example2_multitrans.py similarity index 60% rename from example2_multitrans.py rename to latentblending/example2_multitrans.py index 320923f..f37479b 100644 --- a/example2_multitrans.py +++ b/latentblending/example2_multitrans.py @@ -1,21 +1,6 @@ -# Copyright 2022 Lunar Ring. All rights reserved. -# Written by Johannes Stelzer, email stelzer@lunar-ring.ai twitter @j_stelzer -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import torch import warnings -from latent_blending import LatentBlending +from blending_engine import BlendingEngine from diffusers_holder import DiffusersHolder from diffusers import AutoPipelineForText2Image from movie_util import concatenate_movies @@ -42,29 +27,29 @@ list_prompts.append("photo of a house, high detail") # You can optionally specify the seeds list_seeds = [95437579, 33259350, 956051013] fp_movie = 'movie_example2.mp4' -lb = LatentBlending(dh) +be = BlendingEngine(dh) list_movie_parts = [] for i in range(len(list_prompts) - 1): # For a multi transition we can save some computation time and recycle the latents if i == 0: - lb.set_prompt1(list_prompts[i]) - lb.set_prompt2(list_prompts[i + 1]) + be.set_prompt1(list_prompts[i]) + be.set_prompt2(list_prompts[i + 1]) recycle_img1 = False else: - lb.swap_forward() - lb.set_prompt2(list_prompts[i + 1]) + be.swap_forward() + be.set_prompt2(list_prompts[i + 1]) recycle_img1 = True fp_movie_part = f"tmp_part_{str(i).zfill(3)}.mp4" fixed_seeds = list_seeds[i:i + 2] # Run latent blending - lb.run_transition( + be.run_transition( recycle_img1=recycle_img1, fixed_seeds=fixed_seeds) # Save movie - lb.write_movie_transition(fp_movie_part, duration_single_trans) + be.write_movie_transition(fp_movie_part, duration_single_trans) list_movie_parts.append(fp_movie_part) # Finally, concatente the result diff --git a/gradio_ui.py b/latentblending/gradio_ui.py similarity index 100% rename from gradio_ui.py rename to latentblending/gradio_ui.py diff --git a/movie_util.py b/latentblending/movie_util.py similarity index 100% rename from movie_util.py rename to latentblending/movie_util.py diff --git a/utils.py b/latentblending/utils.py similarity index 100% rename from utils.py rename to latentblending/utils.py diff --git a/requirements.txt b/requirements.txt index db22873..3f4291a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ lpips==0.1.4 -opencv-python==4.7.0.68 +opencv-python ffmpeg-python -diffusers["torch"]==0.23.0 -transformers==4.35.2 +diffusers==0.25.0 +transformers pytest -accelerate diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8268a93 --- /dev/null +++ b/setup.py @@ -0,0 +1,19 @@ +from setuptools import setup, find_packages + +# Read requirements.txt and store its contents in a list +with open('requirements.txt') as f: + required = f.read().splitlines() + +setup( + name='latentblending', + version='0.2', + url='https://github.com/lunarring/latentblending', + description='Butter-smooth video transitions', + long_description=open('README.md').read(), + install_requires=required, + dependency_links=[ + 'git+https://github.com/lunarring/lunar_tools#egg=lunar_tools' + ], + include_package_data=False, +) + diff --git a/test_latentblending.py b/test_latentblending.py deleted file mode 100644 index 78f1688..0000000 --- a/test_latentblending.py +++ /dev/null @@ -1,54 +0,0 @@ -import unittest -from latent_blending import LatentBlending -from diffusers_holder import DiffusersHolder -from diffusers import DiffusionPipeline -import torch - -default_pipe = "stabilityai/stable-diffusion-xl-base-1.0" - - -class TestDiffusersHolder(unittest.TestCase): - - def test_load_diffusers_holder(self): - pipe = DiffusionPipeline.from_pretrained(default_pipe, torch_dtype=torch.float16).to('cuda') - dh = DiffusersHolder(pipe) - self.assertIsNotNone(dh, "Failed to load DiffusersHolder") - - -class TestSingleImageGeneration(unittest.TestCase): - - def test_single_image_generation(self): - pipe = DiffusionPipeline.from_pretrained(default_pipe, torch_dtype=torch.float16).to('cuda') - dh = DiffusersHolder(pipe) - dh.set_dimensions((1024, 704)) - dh.set_num_inference_steps(40) - prompt = "Your prompt here" - text_embeddings = dh.get_text_embedding(prompt) - generator = torch.Generator(device=dh.device).manual_seed(int(420)) - latents_start = dh.get_noise() - list_latents_1 = dh.run_diffusion(text_embeddings, latents_start) - img_orig = dh.latent2image(list_latents_1[-1]) - self.assertIsNotNone(img_orig, "Failed to generate an image") - - -class TestImageTransition(unittest.TestCase): - - def test_image_transition(self): - pipe = DiffusionPipeline.from_pretrained(default_pipe, torch_dtype=torch.float16).to('cuda') - dh = DiffusersHolder(pipe) - lb = LatentBlending(dh) - - lb.set_prompt1('photo of my first prompt1') - lb.set_prompt2('photo of my second prompt') - depth_strength = 0.6 - t_compute_max_allowed = 10 - num_inference_steps = 30 - imgs_transition = lb.run_transition( - depth_strength=depth_strength, - num_inference_steps=num_inference_steps, - t_compute_max_allowed=t_compute_max_allowed) - - self.assertTrue(len(imgs_transition) > 0, "No transition images generated") - -if __name__ == '__main__': - unittest.main()