Migrated from GitHub

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/cut_and_drag_example_1.gif filter=lfs diff=lfs merge=lfs -text
+assets/cut_and_drag_example_2.gif filter=lfs diff=lfs merge=lfs -text
+assets/cut_and_drag_example_3.gif filter=lfs diff=lfs merge=lfs -text
+assets/cut_and_drag_example_4.gif filter=lfs diff=lfs merge=lfs -text
+assets/cut_and_drag_example_5.gif filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

ORIGINAL_README.md

@@ -0,0 +1,127 @@
+<!-- # Go-with-the-Flow: Motion-Controllable Video Diffusion Models Using Real-Time Warped Noise -->
+
+<p align="center">
+  <img src="assets/Logo.png" alt="Go-with-the-Flow: Motion-Controllable Video Diffusion Models Using Real-Time Warped Noise" width="100%">
+</p>
+
+[![Project Page](https://img.shields.io/badge/Project-Page-green?logo=googlechrome&logoColor=green)](https://eyeline-research.github.io/Go-with-the-Flow/)
+[![Paper](https://img.shields.io/badge/Paper-arXiv-b31b1b?logo=arxiv&logoColor=red)](https://arxiv.org/abs/2501.08331)
+[![YouTube Tutorial](https://img.shields.io/badge/YouTube-Tutorial-red?logo=youtube&logoColor=red)](https://www.youtube.com/watch?v=IO3pbQpT5F8)
+[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Go--with--the--Flow-blue)](https://huggingface.co/Eyeline-Research/Go-with-the-Flow/tree/main)
+
+[Ryan Burgert](https://ryanndagreat.github.io)<sup>1,3</sup>, [Yuancheng Xu](https://yuancheng-xu.github.io)<sup>1,4</sup>, [Wenqi Xian](https://www.cs.cornell.edu/~wenqixian/)<sup>1</sup>, [Oliver Pilarski](https://www.linkedin.com/in/oliverpilarski/)<sup>1</sup>, [Pascal Clausen](https://www.linkedin.com/in/pascal-clausen-a179566a/?originalSubdomain=ch)<sup>1</sup>, [Mingming He](https://mingminghe.com/)<sup>1</sup>, [Li Ma](https://limacv.github.io/homepage/)<sup>1</sup>, 
+
+[Yitong Deng](https://yitongdeng.github.io)<sup>2,5</sup>, [Lingxiao Li](https://scholar.google.com/citations?user=rxQDLWcAAAAJ&hl=en)<sup>2</sup>, [Mohsen Mousavi](www.linkedin.com/in/mohsen-mousavi-0516a03)<sup>1</sup>, [Michael Ryoo](http://michaelryoo.com)<sup>3</sup>, [Paul Debevec](https://www.pauldebevec.com)<sup>1</sup>, [Ning Yu](https://ningyu1991.github.io)<sup>1†</sup>
+
+<sup>1</sup>Netflix Eyeline Studios, <sup>2</sup>Netflix, <sup>3</sup>Stony Brook University, <sup>4</sup>University of Maryland, <sup>5</sup>Stanford University<br>
+<sup>†</sup>Project Lead
+
+### Table of Contents
+- [Abstract](#abstract) 
+- [Quick Start: Cut-and-drag Motion Control](#quick-start-cut-and-drag-motion-control)
+  - [Animation Template GUI (Local)](#1-animation-template-gui-local) 
+  - [Running Video Diffusion (GPU)](#2-running-video-diffusion-gpu)
+- [TODO](#todo)  
+- [Citation](#citation)
+
+<a name="abstract"></a>
+## :book: Abstract 
+
+Go-with-the-Flow is an easy and efficient way to control the motion patterns of video diffusion models. It lets a user decide how the camera and objects in a scene will move, and can even let you transfer motion patterns from one video to another.
+
+We simply fine-tune a base model — requiring no changes to the original pipeline or architecture, except: instead of using pure i.i.d. Gaussian noise, we use **warped noise** instead. Inference has exactly the same computational cost as running the base model.
+
+If you create something cool with our model - and want to share it on our website - email [email protected]. We will be creating a user-generated content section, starting with whomever submits the first video! 
+
+If you like this project, please give it a ★!
+
+<a name="quick-start-cut-and-drag-motion-control"></a>
+## :rocket: Quick Start: Cut-and-drag Motion Control
+
+Cut-and-drag motion control lets you take an image, and create a video by cutting out different parts of that image and dragging them around.
+
+For cut-and-drag motion control, there are two parts: an GUI to create a crude animation (no GPU needed), then a diffusion script to turn that crude animation into a pretty one (requires GPU).
+
+**YouTube Tutorial**: [![YouTube Tutorial](https://img.shields.io/badge/YouTube-Tutorial-red?logo=youtube&logoColor=red)](https://www.youtube.com/watch?v=IO3pbQpT5F8)
+
+Examples:
+
+<p align="center">
+  <img src="assets/cut_and_drag_example_1.gif" width="80%">
+  <!-- <img src="assets/cut_and_drag_example_2.gif" width="80%"> -->
+  <img src="assets/cut_and_drag_example_3.gif" width="80%">
+  <img src="assets/cut_and_drag_example_4.gif" width="80%">
+  <img src="assets/cut_and_drag_example_5.gif" width="80%">
+</p>
+
+<a name="1-animation-template-gui-local"></a>
+### 1. Animation Template GUI (Local)
+
+1. Clone this repo, then `cd` into it.  
+2. Install local requirements:
+
+    `pip install -r requirements_local.txt`
+
+3. Run the GUI:
+
+    `python cut_and_drag_gui.py`
+
+4. Follow the instructions shown in the GUI.  
+
+After completion, an MP4 file will be generated. You'll need to move this file to a computer with a decent GPU to continue.
+
+<a name="2-running-video-diffusion-gpu"></a>
+### 2. Running Video Diffusion (GPU)
+
+1. Clone this repo on the machine with the GPU, then `cd` into it.  
+2. Install requirements:
+
+    `pip install -r requirements.txt`
+
+3. Warp the noise (replace `<PATH TO VIDEO OR URL>` accordingly):
+
+    `python make_warped_noise.py <PATH TO VIDEO OR URL> --output_folder noise_warp_output_folder`
+
+4. Run inference:
+
+    ```
+    python cut_and_drag_inference.py noise_warp_output_folder \
+        --prompt "A duck splashing" \
+        --output_mp4_path "output.mp4" \
+        --device "cuda" \
+        --num_inference_steps 5
+    ```
+
+Adjust folder paths, prompts, and other hyperparameters as needed. The output will be saved as `output.mp4`.
+
+<a name="todo"></a>
+## :clipboard: TODO
+
+- [x] Upload All CogVideoX Models
+- [x] Upload Cut-And-Drag Inference Code  
+- [x] Release to Arxiv
+- [ ] Google Colab for people without GPU's
+- [ ] Depth-Warping Inference Code
+- [ ] T2V Motion Transfer Code
+- [ ] ComfyUI Node
+- [ ] Release 3D-to-Video Inference Code + Blender File
+- [ ] Upload AnimateDiff Model
+- [ ] Replicate Instance
+- [ ] Fine-Tuning Code
+
+<a name="citation"></a>
+## :black_nib: Citation
+
+If you use this in your research, please consider citing:
+
+```
+@misc{burgert2025gowiththeflowmotioncontrollablevideodiffusion,
+      title={Go-with-the-Flow: Motion-Controllable Video Diffusion Models Using Real-Time Warped Noise}, 
+      author={Ryan Burgert and Yuancheng Xu and Wenqi Xian and Oliver Pilarski and Pascal Clausen and Mingming He and Li Ma and Yitong Deng and Lingxiao Li and Mohsen Mousavi and Michael Ryoo and Paul Debevec and Ning Yu},
+      year={2025},
+      eprint={2501.08331},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV},
+      url={https://arxiv.org/abs/2501.08331}, 
+}
+```

cut_and_drag_gui.py

@@ -0,0 +1,416 @@
+from rp import *
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.widgets import Slider
+from matplotlib.patches import Polygon as Polygon
+import cv2
+git_import('CommonSource')
+import rp.git.CommonSource.noise_warp as nw
+from easydict import EasyDict
+
+
+def select_polygon(image):
+    fig, ax = plt.subplots()
+    ax.imshow(image)
+    ax.set_title("Left click to add points. Right click to undo. Close the window to finish.")
+
+    path = []
+
+    def onclick(event):
+        if event.button == 1:  # Left click
+            if event.xdata is not None and event.ydata is not None:
+                path.append((event.xdata, event.ydata))
+            ax.clear()
+            ax.imshow(image)
+            ax.set_title("Left click to add points. Right click to undo. Close the window to finish.")
+            for i in range(len(path)):
+                if i > 0:
+                    ax.plot([path[i - 1][0], path[i][0]], [path[i - 1][1], path[i][1]], "r-")
+                ax.plot(path[i][0], path[i][1], "ro")
+            if len(path) > 1:
+                ax.plot([path[-1][0], path[0][0]], [path[-1][1], path[0][1]], "r--")
+            if len(path) > 2:
+                polygon = Polygon(path, closed=True, alpha=0.3, facecolor="r", edgecolor="r")
+                ax.add_patch(polygon)
+            fig.canvas.draw()
+        elif event.button == 3 and path:  # Right click
+            path.pop()
+            ax.clear()
+            ax.imshow(image)
+            ax.set_title("Left click to add points. Right click to undo. Close the window to finish.")
+            for i in range(len(path)):
+                if i > 0:
+                    ax.plot([path[i - 1][0], path[i][0]], [path[i - 1][1], path[i][1]], "r-")
+                ax.plot(path[i][0], path[i][1], "ro")
+            if len(path) > 1:
+                ax.plot([path[-1][0], path[0][0]], [path[-1][1], path[0][1]], "r--")
+            if len(path) > 2:
+                polygon = Polygon(path, closed=True, alpha=0.3, facecolor="r", edgecolor="r")
+                ax.add_patch(polygon)
+            fig.canvas.draw()
+
+    cid = fig.canvas.mpl_connect("button_press_event", onclick)
+    plt.show()
+    fig.canvas.mpl_disconnect(cid)
+
+    return path
+
+
+def select_polygon_and_path(image):
+    fig, ax = plt.subplots()
+    ax.imshow(image)
+    ax.set_title("Left click to add points. Right click to undo. Close the window to finish.")
+
+    polygon_path = []
+    movement_path = []
+
+    cid = fig.canvas.mpl_connect("button_press_event", onclick)
+    plt.show()
+    fig.canvas.mpl_disconnect(cid)
+
+    return polygon_path, movement_path
+
+
+def select_path(image, polygon, num_frames=49):
+    fig, ax = plt.subplots()
+    plt.subplots_adjust(left=0.25, bottom=0.25)
+    ax.imshow(image)
+    ax.set_title("Left click to add points. Right click to undo. Close the window to finish.")
+
+    path = []
+
+    # Add sliders for final scale and rotation
+    ax_scale = plt.axes([0.25, 0.1, 0.65, 0.03])
+    ax_rot = plt.axes([0.25, 0.15, 0.65, 0.03])
+
+    scale_slider = Slider(ax_scale, "Final Scale", 0.1, 5.0, valinit=1)
+    rot_slider = Slider(ax_rot, "Final Rotation", -360, 360, valinit=0)
+
+    scales = []
+    rotations = []
+
+    def interpolate_transformations(n_points):
+        # scales = np.linspace(1, scale_slider.val, n_points)
+        scales = np.exp(np.linspace(0, np.log(scale_slider.val), n_points))
+        rotations = np.linspace(0, rot_slider.val, n_points)
+        return scales, rotations
+
+    def update_display():
+        ax.clear()
+        ax.imshow(image)
+        ax.set_title("Left click to add points. Right click to undo. Close the window to finish.")
+
+        n_points = len(path)
+        if n_points < 1:
+            fig.canvas.draw_idle()
+            return
+
+        # Interpolate scales and rotations over the total number of points
+        scales[:], rotations[:] = interpolate_transformations(n_points)
+
+        origin = np.array(path[0])
+
+        for i in range(n_points):
+            ax.plot(path[i][0], path[i][1], "bo")
+            if i > 0:
+                ax.plot([path[i - 1][0], path[i][0]], [path[i - 1][1], path[i][1]], "b-")
+            # Apply transformation to the polygon
+            transformed_polygon = apply_transformation(np.array(polygon), scales[i], rotations[i], origin)
+            # Offset polygon to the current point relative to the first point
+            position_offset = np.array(path[i]) - origin
+            transformed_polygon += position_offset
+            mpl_poly = Polygon(
+                transformed_polygon,
+                closed=True,
+                alpha=0.3,
+                facecolor="r",
+                edgecolor="r",
+            )
+            ax.add_patch(mpl_poly)
+
+        fig.canvas.draw_idle()
+
+    def onclick(event):
+        if event.inaxes != ax:
+            return
+        if event.button == 1:  # Left click
+            path.append((event.xdata, event.ydata))
+            update_display()
+        elif event.button == 3 and path:  # Right click
+            path.pop()
+            update_display()
+
+    def on_slider_change(val):
+        update_display()
+
+    scale_slider.on_changed(on_slider_change)
+    rot_slider.on_changed(on_slider_change)
+
+    scales, rotations = [], []
+
+    cid_click = fig.canvas.mpl_connect("button_press_event", onclick)
+    plt.show()
+    fig.canvas.mpl_disconnect(cid_click)
+
+    # Final interpolation after the window is closed
+    n_points = num_frames
+    if n_points > 0:
+        scales, rotations = interpolate_transformations(n_points)
+        rotations = [-x for x in rotations]
+        path = as_numpy_array(path)
+        path = as_numpy_array([linterp(path, i) for i in np.linspace(0, len(path) - 1, num=n_points)])
+
+    return path, scales, rotations
+
+
+def animate_polygon(image, polygon, path, scales, rotations,interp=cv2.INTER_LINEAR):
+    frames = []
+    transformed_polygons = []
+    origin = np.array(path[0])
+
+    h, w = image.shape[:2]
+
+    for i in eta(range(len(path)), title="Creating frames for this layer..."):
+        # Compute the affine transformation matrix
+        theta = np.deg2rad(rotations[i])
+        scale = scales[i]
+
+        a11 = scale * np.cos(theta)
+        a12 = -scale * np.sin(theta)
+        a21 = scale * np.sin(theta)
+        a22 = scale * np.cos(theta)
+
+        # Compute translation components
+        tx = path[i][0] - (a11 * origin[0] + a12 * origin[1])
+        ty = path[i][1] - (a21 * origin[0] + a22 * origin[1])
+
+        M = np.array([[a11, a12, tx], [a21, a22, ty]])
+
+        # Apply the affine transformation to the image
+        warped_image = cv2.warpAffine(
+            image,
+            M,
+            (w, h),
+            flags=interp,
+            borderMode=cv2.BORDER_CONSTANT,
+            borderValue=(0, 0, 0),
+        )
+
+        # Transform the polygon points
+        polygon_np = np.array(polygon)
+        ones = np.ones(shape=(len(polygon_np), 1))
+        points_ones = np.hstack([polygon_np, ones])
+        transformed_polygon = M.dot(points_ones.T).T
+        transformed_polygons.append(transformed_polygon)
+
+        # Create a mask for the transformed polygon
+        mask = np.zeros((h, w), dtype=np.uint8)
+        cv2.fillPoly(mask, [np.int32(transformed_polygon)], 255)
+
+        # Extract the polygon area from the warped image
+        rgba_image = cv2.cvtColor(warped_image, cv2.COLOR_BGR2BGRA)
+        alpha_channel = np.zeros((h, w), dtype=np.uint8)
+        alpha_channel[mask == 255] = 255
+        rgba_image[:, :, 3] = alpha_channel
+
+        # Set areas outside the polygon to transparent
+        rgba_image[mask == 0] = (0, 0, 0, 0)
+
+        frames.append(rgba_image)
+
+    # return gather_vars("frames transformed_polygons")
+    return EasyDict(frames=frames,transformed_polygons=transformed_polygons)
+
+
+def apply_transformation(polygon, scale, rotation, origin):
+    # Translate polygon to origin
+    translated_polygon = polygon - origin
+    # Apply scaling
+    scaled_polygon = translated_polygon * scale
+    # Apply rotation
+    theta = np.deg2rad(rotation)
+    rotation_matrix = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
+    rotated_polygon = np.dot(scaled_polygon, rotation_matrix)
+    # Translate back
+    final_polygon = rotated_polygon + origin
+    return final_polygon
+
+
+# def cogvlm_caption_video(video_path, prompt="Please describe this video in detail."):
+#     import rp.web_evaluator as wev
+#
+#     client = wev.Client("100.113.27.133")
+#     result = client.evaluate("run_captioner(x,prompt=prompt)", x=video_path, prompt=prompt)
+#     if result.errored:
+#         raise result.error
+#     return result.value
+
+
+if __name__ == "__main__":
+    fansi_print(big_ascii_text("Go With The Flow!"), "yellow green", "bold")
+
+    image_path = input_conditional(
+        fansi("First Frame: Enter Image Path or URL", "blue cyan", "italic bold underlined"),
+        lambda x: is_a_file(x.strip()) or is_valid_url(x.strip()),
+    ).strip()
+
+    print("Using path: " + fansi_highlight_path(image_path))
+    if is_video_file(image_path):
+        fansi_print('Video path was given. Using first frame as image.')
+        image=load_video(image_path,length=1)[0]
+    else:
+        image = load_image(image_path, use_cache=True)
+        image = resize_image_to_fit(image, height=1440, allow_growth=False)
+
+    rp.fansi_print("PRO TIP: Use this website to help write your captions: https://huggingface.co/spaces/THUDM/CogVideoX-5B-Space", 'blue cyan')
+    prompt=input(fansi('Input the video caption >>> ','blue cyan','bold'))
+
+    SCALE_FACTOR=1
+    #Adjust resolution to 720x480: resize then center-crop
+    HEIGHT=480*SCALE_FACTOR
+    WIDTH=720*SCALE_FACTOR
+    image = resize_image_to_hold(image,height=HEIGHT,width=WIDTH) 
+    image = crop_image(image, height=HEIGHT,width=WIDTH, origin='center')
+    title = input_default(
+        fansi("Enter a title: ", "blue cyan", "italic bold underlined"),
+        get_file_name(
+            image_path,
+            include_file_extension=False,
+        ),
+    )
+    output_folder=make_directory(get_unique_copy_path(title))
+    print("Output folder: " + fansi_highlight_path(output_folder))
+
+    fansi_print("How many layers?", "blue cyan", "italic bold underlined"),
+    num_layers = input_integer(
+        minimum=1,
+    )
+
+    layer_videos = []
+    layer_polygons = []
+    layer_first_frame_masks = []
+    layer_noises = []
+
+    for layer_num in range(num_layers):
+        layer_noise=np.random.randn(HEIGHT,WIDTH,18).astype(np.float32)
+
+        fansi_print(f'You are currently working on layer #{layer_num+1} of {num_layers}','yellow orange','bold')
+        if True or not "polygon" in vars() or input_yes_no("New Polygon?"):
+            polygon = select_polygon(image)
+        if True or not "animation" in vars() or input_yes_no("New Animation?"):
+            animation = select_path(image, polygon)
+
+        
+        animation_output = animate_polygon(image, polygon, *animation)
+
+        noise_output_1 = as_numpy_array(animate_polygon(layer_noise[:,:,3*0:3*1], polygon, *animation, interp=cv2.INTER_NEAREST).frames)
+        noise_output_2 = as_numpy_array(animate_polygon(layer_noise[:,:,3*1:3*2], polygon, *animation, interp=cv2.INTER_NEAREST).frames)
+        noise_output_3 = as_numpy_array(animate_polygon(layer_noise[:,:,3*2:3*3], polygon, *animation, interp=cv2.INTER_NEAREST).frames)
+        noise_output_4 = as_numpy_array(animate_polygon(layer_noise[:,:,3*3:3*4], polygon, *animation, interp=cv2.INTER_NEAREST).frames)
+        noise_output_5 = as_numpy_array(animate_polygon(layer_noise[:,:,3*4:3*5], polygon, *animation, interp=cv2.INTER_NEAREST).frames)
+        noise_output_6 = as_numpy_array(animate_polygon(layer_noise[:,:,3*5:3*6], polygon, *animation, interp=cv2.INTER_NEAREST).frames)
+        noise_warp_output = np.concatenate(
+            [
+                noise_output_1[:,:,:,:3],
+                noise_output_2[:,:,:,:3],
+                noise_output_3[:,:,:,:3],
+                noise_output_4[:,:,:,:3],
+                noise_output_5[:,:,:,:3],
+                noise_output_6[:,:,:,:1],
+            ],
+            axis=3,#THWC
+        )
+
+        frames, transformed_polygons = destructure(animation_output)
+
+        mask = get_image_alpha(frames[0]) > 0
+        
+        layer_polygons.append(transformed_polygons)
+        layer_first_frame_masks.append(mask)
+        layer_videos.append(frames)
+        layer_noises.append(noise_warp_output)
+
+    if True or input_yes_no("Inpaint background?"):
+        total_mask = sum(layer_first_frame_masks).astype(bool)
+        background = cv_inpaint_image(image, mask=total_mask)
+    else:
+        background = "https://t3.ftcdn.net/jpg/02/76/96/64/360_F_276966430_HsEI96qrQyeO4wkcnXtGZOm0Qu4TKCgR.jpg"
+        background = load_image(background, use_cache=True)
+        background = cv_resize_image(background, get_image_dimensions(image))
+        background=as_rgba_image(background)
+
+    ###
+    output_frames = [
+        overlay_images(
+            background,
+            *frame_layers,
+        )
+        for frame_layers in eta(list_transpose(layer_videos),title=fansi("Compositing all frames of the video...",'green','bold'))
+    ]
+    output_frames=as_numpy_array(output_frames)
+
+    
+    output_video_file=save_video_mp4(output_frames, output_folder+'/'+title + ".mp4", video_bitrate="max")
+    output_mask_file = save_video_mp4(
+        [
+            sum([get_image_alpha(x) for x in layers])
+            for layers in list_transpose(layer_videos)
+        ],
+        output_folder + "/" + title + "_mask.mp4",
+        video_bitrate="max",
+    )
+    
+
+    ###
+    fansi_print("Warping noise...",'yellow green','bold italic')
+    output_noises = np.random.randn(1,HEIGHT,WIDTH,16)
+    output_noises=np.repeat(output_noises,49,axis=0)
+    for layer_num in range(num_layers):
+        fansi_print(f'Warping noise for layer #{layer_num+1} of {num_layers}','green','bold')
+        for frame in eta(range(49),title='frame number'):
+            noise_mask = get_image_alpha(layer_videos[layer_num][frame])[:,:,None]>0
+            noise_video_layer = layer_noises[layer_num][frame]
+            output_noises[frame]*=(noise_mask==0)
+            output_noises[frame]+=noise_video_layer*noise_mask
+            #display_image((noise_mask * noise_video_layer)[:,:,:3])
+            display_image(output_noises[frame][:,:,:3]/5+.5)
+    
+    import einops
+    import torch
+    torch_noises=torch.tensor(output_noises)
+    torch_noises=einops.rearrange(torch_noises,'F H W C -> F C H W')        
+    #
+    small_torch_noises=[]
+    for i in eta(range(49),title='Regaussianizing'):
+        torch_noises[i]=nw.regaussianize(torch_noises[i])[0]
+        small_torch_noise=nw.resize_noise(torch_noises[i],(480//8,720//8))
+        small_torch_noises.append(small_torch_noise)
+        #display_image(as_numpy_image(small_torch_noise[:3])/5+.5)
+        display_image(as_numpy_image(torch_noises[i,:3])/5+.5)
+    small_torch_noises=torch.stack(small_torch_noises)#DOWNSAMPLED NOISE FOR CARTRIDGE!
+
+    ###
+    cartridge={}
+    cartridge['instance_noise']=small_torch_noises.bfloat16()
+    cartridge['instance_video']=(as_torch_images(output_frames)*2-1).bfloat16()
+    cartridge['instance_prompt']=prompt
+    output_cartridge_file=object_to_file(cartridge, output_folder + "/" + title + "_cartridge.pkl")
+            
+    ###
+    
+    
+    output_polygons_file=output_folder+'/'+'polygons.npy'
+    polygons=as_numpy_array(layer_polygons)
+    np.save(output_polygons_file,polygons)
+    
+    print()
+    print(fansi('Saved outputs:','green','bold'))
+    print(fansi('    - Saved video: ','green','bold'),fansi_highlight_path(get_relative_path(output_video_file)))
+    print(fansi('    - Saved masks: ','green','bold'),fansi_highlight_path(get_relative_path(output_mask_file)))
+    print(fansi('    - Saved shape: ','green','bold'),fansi_highlight_path(output_polygons_file))
+    print(fansi('    - Saved cartridge: ','green','bold'),fansi_highlight_path(output_cartridge_file))
+
+    print("Press CTRL+C to exit")
+
+
+    display_video(video_with_progress_bar(output_frames), loop=True)

cut_and_drag_inference.py

@@ -0,0 +1,489 @@
+import rp
+# from rp import *
+import torch
+import numpy as np
+import einops
+from diffusers import CogVideoXImageToVideoPipeline
+from diffusers import CogVideoXVideoToVideoPipeline
+from diffusers import CogVideoXPipeline
+from diffusers.utils import export_to_video, load_image
+from icecream import ic
+from diffusers import AutoencoderKLCogVideoX, CogVideoXImageToVideoPipeline, CogVideoXTransformer3DModel 
+from transformers import T5EncoderModel
+
+import rp.git.CommonSource.noise_warp as nw
+
+pipe_ids = dict(
+    T2V5B="THUDM/CogVideoX-5b",
+    T2V2B="THUDM/CogVideoX-2b",
+    I2V5B="THUDM/CogVideoX-5b-I2V",
+)
+
+# From a bird's-eye view, a serene scene unfolds: a herd of deer gracefully navigates shallow, warm-hued waters, their silhouettes stark against the earthy tones. The deer, spread across the frame, cast elongated, well-defined shadows that accentuate their antlers, creating a mesmerizing play of light and dark. This aerial perspective captures the tranquil essence of the setting, emphasizing the harmonious contrast between the deer and their mirror-like reflections on the water's surface. The composition exudes a peaceful stillness, yet the subtle movement suggested by the shadows adds a dynamic layer to the natural beauty and symmetry of the moment.
+base_url = 'https://huggingface.co/Eyeline-Research/Go-with-the-Flow'
+lora_urls = dict(
+    I2V5B_final_i30000_lora_weights                          = base_url+'I2V5B_final_i30000_lora_weights.safetensors',
+    I2V5B_final_i38800_nearest_lora_weights                  = base_url+'I2V5B_final_i38800_nearest_lora_weights.safetensors',
+    I2V5B_resum_blendnorm_0degrad_i13600_DATASET_lora_weights = base_url+'I2V5B_resum_blendnorm_0degrad_i13600_DATASET_lora_weights.safetensors',
+    T2V2B_RDeg_i30000_lora_weights                           = base_url+'T2V2B_RDeg_i30000_lora_weights.safetensors',
+    T2V5B_blendnorm_i18000_DATASET_lora_weights               = base_url+'T2V5B_blendnorm_i18000_DATASET_lora_weights.safetensors',
+    T2V5B_blendnorm_i25000_DATASET_nearest_lora_weights       = base_url+'T2V5B_blendnorm_i25000_DATASET_nearest_lora_weights.safetensors',
+)
+
+dtype=torch.bfloat16
+
+#https://medium.com/@ChatGLM/open-sourcing-cogvideox-a-step-towards-revolutionizing-video-generation-28fa4812699d
+B, F, C, H, W = 1, 13, 16, 60, 90  # The defaults
+num_frames=(F-1)*4+1 #https://miro.medium.com/v2/resize:fit:1400/format:webp/0*zxsAG1xks9pFIsoM
+#Possible num_frames: 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49
+assert num_frames==49
+
+@rp.memoized #Torch never manages to unload it from memory anyway
+def get_pipe(model_name, device=None, low_vram=True):
+    """
+    model_name is like "I2V5B", "T2V2B", or "T2V5B", or a LoRA name like "T2V2B_RDeg_i30000_lora_weights"
+    device is automatically selected if unspecified
+    low_vram, if True, will make the pipeline use CPU offloading
+    """
+
+    if model_name in pipe_ids:
+        lora_name = None
+        pipe_name = model_name
+    else:
+        #By convention, we have lora_paths that start with the pipe names
+        rp.fansi_print(f"Getting pipe name from model_name={model_name}",'cyan','bold')
+        lora_name = model_name
+        pipe_name = lora_name.split('_')[0]
+
+    is_i2v = "I2V" in pipe_name  # This is a convention I'm using right now
+    # is_v2v = "V2V" in pipe_name  # This is a convention I'm using right now
+
+    # if is_v2v:
+    #     old_pipe_name = pipe_name
+    #     old_lora_name = lora_name
+    #     if pipe_name is not None: pipe_name = pipe_name.replace('V2V','T2V')
+    #     if lora_name is not None: lora_name = lora_name.replace('V2V','T2V')
+    #     rp.fansi_print(f"V2V: {old_pipe_name} --> {pipe_name}   &&&   {old_lora_name} --> {lora_name}",'white','bold italic','red')
+
+    pipe_id = pipe_ids[pipe_name]
+    print(f"LOADING PIPE WITH device={device} pipe_name={pipe_name} pipe_id={pipe_id} lora_name={lora_name}" )
+    
+    hub_model_id = pipe_ids[pipe_name]
+
+    transformer = CogVideoXTransformer3DModel.from_pretrained(hub_model_id, subfolder="transformer", torch_dtype=torch.bfloat16)
+    text_encoder = T5EncoderModel.from_pretrained(hub_model_id, subfolder="text_encoder", torch_dtype=torch.bfloat16)
+    vae = AutoencoderKLCogVideoX.from_pretrained(hub_model_id, subfolder="vae", torch_dtype=torch.bfloat16)
+
+    PipeClass = CogVideoXImageToVideoPipeline if is_i2v else CogVideoXPipeline
+    pipe = PipeClass.from_pretrained(hub_model_id, torch_dtype=torch.bfloat16, vae=vae,transformer=transformer,text_encoder=text_encoder)
+
+    if lora_name is not None:
+        lora_folder = rp.make_directory('lora_models')
+        lora_url = lora_urls[lora_name]
+        lora_path = rp.download_url(lora_url, lora_folder, show_progress=True, skip_existing=True)
+        assert rp.file_exists(lora_path), (lora_name, lora_path)
+        print(end="\tLOADING LORA WEIGHTS...",flush=True)
+        pipe.load_lora_weights(lora_path)
+        print("DONE!")
+
+    if device is None:
+        device = rp.select_torch_device()
+
+    if not low_vram:
+        print("\tUSING PIPE DEVICE", device)
+        pipe = pipe.to(device)
+    else:
+        print("\tUSING PIPE DEVICE WITH CPU OFFLOADING",device)
+        pipe=pipe.to('cpu')
+        pipe.enable_sequential_cpu_offload(device=device)
+
+    # pipe.vae.enable_tiling()
+    # pipe.vae.enable_slicing()
+
+    # Metadata
+    pipe.lora_name = lora_name
+    pipe.pipe_name = pipe_name
+    pipe.is_i2v    = is_i2v
+    # pipe.is_v2v    = is_v2v
+    
+    return pipe
+
+def get_downtemp_noise(noise, noise_downtemp_interp):
+    assert noise_downtemp_interp in {'nearest', 'blend', 'blend_norm', 'randn'}, noise_downtemp_interp
+    if   noise_downtemp_interp == 'nearest'    : return                  rp.resize_list(noise, 13)
+    elif noise_downtemp_interp == 'blend'      : return                   downsamp_mean(noise, 13)
+    elif noise_downtemp_interp == 'blend_norm' : return normalized_noises(downsamp_mean(noise, 13))
+    elif noise_downtemp_interp == 'randn'      : return torch.randn_like(rp.resize_list(noise, 13)) #Basically no warped noise, just r
+    else: assert False, 'impossible'
+
+def downsamp_mean(x, l=13):
+    return torch.stack([rp.mean(u) for u in rp.split_into_n_sublists(x, l)])
+
+def normalized_noises(noises):
+    #Noises is in TCHW form
+    return torch.stack([x / x.std(1, keepdim=True) for x in noises])
+
+
+@rp.memoized
+def load_sample_cartridge(
+    sample_path: str,
+    degradation=0,
+    noise_downtemp_interp='nearest',
+    image=None,
+    prompt=None,
+    #SETTINGS:
+    num_inference_steps=30,
+    guidance_scale=6,
+):
+    """
+    COMPLETELY FROM SAMPLE: Generate with /root/micromamba/envs/i2sb/lib/python3.8/site-packages/rp/git/CommonSource/notebooks/CogVidSampleGenerator.ipynb
+    EXAMPLE PATHS:
+        sample_path = '/root/micromamba/envs/i2sb/lib/python3.8/site-packages/rp/git/CommonSource/notebooks/CogVidX_Saved_Train_Samples/plus_pug.pkl'
+        sample_path = '/root/micromamba/envs/i2sb/lib/python3.8/site-packages/rp/git/CommonSource/notebooks/CogVidX_Saved_Train_Samples/amuse_chop.pkl'
+        sample_path = '/root/micromamba/envs/i2sb/lib/python3.8/site-packages/rp/git/CommonSource/notebooks/CogVidX_Saved_Train_Samples/chomp_shop.pkl'
+        sample_path = '/root/micromamba/envs/i2sb/lib/python3.8/site-packages/rp/git/CommonSource/notebooks/CogVidX_Saved_Train_Samples/ahead_job.pkl'
+        sample_path = rp.random_element(glob.glob('/root/micromamba/envs/i2sb/lib/python3.8/site-packages/rp/git/CommonSource/notebooks/CogVidX_Saved_Train_Samples/*.pkl'))
+    """
+
+    #These could be args in the future. I can't think of a use case yet though, so I'll keep the signature clean.
+    noise=None
+    video=None
+
+    if rp.is_a_folder(sample_path):
+        #Was generated using the flow pipeline
+        print(end="LOADING CARTRIDGE FOLDER "+sample_path+"...")
+        
+        noise_file=rp.path_join(sample_path,'noises.npy')
+        instance_noise = np.load(noise_file)
+        instance_noise = torch.tensor(instance_noise)
+        instance_noise = einops.rearrange(instance_noise, 'F H W C -> F C H W')
+
+        video_file=rp.path_join(sample_path,'input.mp4')
+        instance_video = rp.load_video(video_file)
+        instance_video = rp.as_torch_images(instance_video)
+        instance_video = instance_video * 2 - 1
+
+        sample = rp.as_easydict(
+            instance_prompt = '', #Please have some prompt to override this! Ideally the defualt would come from a VLM
+            instance_noise = instance_noise,
+            instance_video = instance_video,
+        )
+
+        print("DONE!")
+        
+    else:
+        #Was generated using the Cut-And-Drag GUI
+        print(end="LOADING CARTRIDGE FILE "+sample_path+"...")
+        sample=rp.file_to_object(sample_path)
+        print("DONE!")
+
+    #SAMPLE EXAMPLE:
+    #    >>> sample=file_to_object('/root/micromamba/envs/i2sb/lib/python3.8/site-packages/rp/git/CommonSource/notebooks/CogVidX_Saved_Train_Samples/ahead_job.pkl')
+    #    >>> list(sample)?s                 -->  ['instance_prompt', 'instance_video', 'instance_noise']
+    #    >>> sample.instance_prompt?s       -->  A group of elk, including a dominant bull, is seen grazing and moving through...
+    #    >>> sample.instance_noise.shape?s  -->  torch.Size([49, 16,  60,  90])
+    #    >>> sample.instance_video.shape?s  -->  torch.Size([49,  3, 480, 720])   # Range: [-1, 1]
+
+    sample_noise  = sample["instance_noise" ].to(dtype)
+    sample_video  = sample["instance_video" ].to(dtype)
+    sample_prompt = sample["instance_prompt"]
+
+    sample_gif_path = sample_path+'.mp4'
+    if not rp.file_exists(sample_gif_path):
+        sample_gif_path = sample_path+'.gif' #The older scripts made this. Backwards compatibility.
+    if not rp.file_exists(sample_gif_path):
+        #Create one!
+        #Clientside warped noise does not come with a nice GIF so we make one here and now!
+        sample_gif_path = sample_path+'.mp4'
+
+        rp.fansi_print("MAKING SAMPLE PREVIEW VIDEO",'light blue green','underlined')
+        preview_sample_video=rp.as_numpy_images(sample_video)/2+.5
+        preview_sample_noise=rp.as_numpy_images(sample_noise)[:,:,:,:3]/5+.5
+        preview_sample_noise = rp.resize_images(preview_sample_noise, size=8, interp="nearest")
+        preview_sample=rp.horizontally_concatenated_videos(preview_sample_video,preview_sample_noise)
+        rp.save_video_mp4(preview_sample,sample_gif_path,video_bitrate='max',framerate=12)
+        rp.fansi_print("DONE MAKING SAMPLE PREVIEW VIDEO!",'light blue green','underlined')
+
+    #prompt=sample.instance_prompt
+    downtemp_noise = get_downtemp_noise(
+        sample_noise,
+        noise_downtemp_interp=noise_downtemp_interp,
+    )
+    downtemp_noise = downtemp_noise[None]
+    downtemp_noise = nw.mix_new_noise(downtemp_noise, degradation)
+
+    assert downtemp_noise.shape == (B, F, C, H, W), (downtemp_noise.shape,(B, F, C, H, W))
+
+    if image is None            : sample_image = rp.as_pil_image(rp.as_numpy_image(sample_video[0].float()/2+.5))
+    elif isinstance(image, str) : sample_image = rp.as_pil_image(rp.as_rgb_image(rp.load_image(image)))
+    else                        : sample_image = rp.as_pil_image(rp.as_rgb_image(image))
+
+    metadata = rp.gather_vars('sample_path degradation downtemp_noise sample_gif_path sample_video sample_noise noise_downtemp_interp')
+    settings = rp.gather_vars('num_inference_steps guidance_scale'+0*'v2v_strength')
+
+    if noise  is None: noise  = downtemp_noise
+    if video  is None: video  = sample_video
+    if image  is None: image  = sample_image
+    if prompt is None: prompt = sample_prompt
+
+    assert noise.shape == (B, F, C, H, W), (noise.shape,(B, F, C, H, W))
+
+    return rp.gather_vars('prompt noise image video metadata settings')
+
+def dict_to_name(d=None, **kwargs):
+    """
+    Used to generate MP4 file names
+    
+    EXAMPLE:
+        >>> dict_to_name(dict(a=5,b='hello',c=None))
+        ans = a=5,b=hello,c=None
+        >>> name_to_dict(ans)
+        ans = {'a': '5', 'b': 'hello', 'c': 'None'}
+    """
+    if d is None:
+        d = {}
+    d.update(kwargs)
+    return ",".join("=".join(map(str, [key, value])) for key, value in d.items())
+
+# def name_to_dict(nam"
+#     Useful for analyzing output MP4 files
+#
+#     EXAMPLE:
+#         >>> dict_to_name(dict(a=5,b='hello',c=None))
+#         ans = a=5,b=hello,c=None
+#         >>> name_to_dict(ans)
+#         ans = {'a': '5', 'b': 'hello', 'c': 'None'}
+#     """
+#     output=rp.as_easydict()
+#     for entry in name.split(','):
+#         key,value=entry.split('=',maxsplit=1)
+#         output[key]=value
+#     return output
+#
+#
+def get_output_path(pipe, cartridge, subfolder:str, output_root:str):
+    """
+    Generates a unique output path for saving a generated video.
+
+    Args:
+        pipe: The video generation pipeline used.
+        cartridge: Data used for generating the video.
+        subfolder (str): Subfolder for saving the video.
+        output_root (str): Root directory for output videos.
+
+    Returns:
+        String representing the unique path to save the video.
+    """
+
+    time = rp.millis()
+
+    output_name = (
+        dict_to_name(
+            t=time,
+            pipe=pipe.pipe_name,
+            lora=pipe.lora_name,
+            steps    =               cartridge.settings.num_inference_steps,
+            # strength =               cartridge.settings.v2v_strength,
+            degrad   =               cartridge.metadata.degradation,
+            downtemp =               cartridge.metadata.noise_downtemp_interp,
+            samp     = rp.get_file_name(rp.get_parent_folder(cartridge.metadata.sample_path), False),
+        )
+        + ".mp4"
+    )
+
+    output_path = rp.get_unique_copy_path(
+        rp.path_join(
+            rp.make_directory(
+                rp.path_join(output_root, subfolder),
+            ),
+            output_name,
+        ),
+    )
+
+    rp.fansi_print(f"OUTPUT PATH: {rp.fansi_highlight_path(output_path)}", "blue", "bold")
+
+    return output_path
+
+def run_pipe(
+    pipe,
+    cartridge,
+    subfolder="first_subfolder",
+    output_root: str = "infer_outputs",
+    output_mp4_path = None, #This overrides subfolder and output_root if specified
+):
+    # output_mp4_path = output_mp4_path or get_output_path(pipe, cartridge, subfolder, output_root)
+
+    if rp.file_exists(output_mp4_path):
+        raise RuntimeError("{output_mp4_path} already exists! Please choose a different output file or delete that one. This script is designed not to clobber previous results.")
+    
+    if pipe.is_i2v:
+        image = cartridge.image
+        if isinstance(image, str):
+            image = rp.load_image(image,use_cache=True)
+        image = rp.as_pil_image(rp.as_rgb_image(image))
+
+    # if pipe.is_v2v:
+    #     print("Making v2v video...")
+    #     v2v_video=cartridge.video
+    #     v2v_video=rp.as_numpy_images(v2v_video) / 2 + .5
+    #     v2v_video=rp.as_pil_images(v2v_video)
+
+    print("NOISE SHAPE",cartridge.noise.shape)
+    print("IMAGE",image)
+
+    video = pipe(
+        prompt=cartridge.prompt,
+        **(dict(image   =image                          ) if pipe.is_i2v else {}),
+        # **(dict(strength=cartridge.settings.v2v_strength) if pipe.is_v2v else {}),
+        # **(dict(video   =v2v_video                      ) if pipe.is_v2v else {}),
+        num_inference_steps=cartridge.settings.num_inference_steps,
+        latents=cartridge.noise,
+
+        guidance_scale=cartridge.settings.guidance_scale,
+        # generator=torch.Generator(device=device).manual_seed(42),
+    ).frames[0]
+
+    export_to_video(video, output_mp4_path, fps=8)
+
+    sample_gif=rp.load_video(cartridge.metadata.sample_gif_path)
+    video=rp.as_numpy_images(video)
+    prevideo = rp.horizontally_concatenated_videos(
+        rp.resize_list(sample_gif, len(video)),
+        video,
+        origin='bottom right',
+    )
+    import textwrap
+    prevideo = rp.labeled_images(
+        prevideo,
+        position="top",
+        labels=cartridge.metadata.sample_path +"\n"+output_mp4_path +"\n\n" + rp.wrap_string_to_width(cartridge.prompt, 250),
+        size_by_lines=True,
+        text_color='light light light blue',
+        # font='G:Lexend'
+    )
+
+    preview_mp4_path = output_mp4_path + "_preview.mp4"
+    preview_gif_path = preview_mp4_path + ".gif"
+    print(end=f"Saving preview MP4 to preview_mp4_path = {preview_mp4_path}...")
+    rp.save_video_mp4(prevideo, preview_mp4_path, framerate=16, video_bitrate="max", show_progress=False)
+    compressed_preview_mp4_path = rp.save_video_mp4(prevideo, output_mp4_path + "_preview_compressed.mp4", framerate=16, show_progress=False)
+    print("done!")
+    print(end=f"Saving preview gif to preview_gif_path = {preview_gif_path}...")
+    rp.convert_to_gif_via_ffmpeg(preview_mp4_path, preview_gif_path, framerate=12,show_progress=False)
+    print("done!")
+
+    return rp.gather_vars('video output_mp4_path preview_mp4_path compressed_preview_mp4_path cartridge subfolder preview_mp4_path preview_gif_path')
+
+
+# #prompt = "A little girl is riding a bicycle at high speed. Focused, detailed, realistic."
+# prompt = "An old house by the lake with wooden plank siding and a thatched roof"
+# prompt = "Soaring through deep space"
+# prompt = "Swimming by the ruins of the titanic"
+# prompt = "A camera flyby of a gigantic ice tower that a princess lives in, zooming in from far away from the castle into her dancing in the window"
+# prompt = "A drone flyby of the grand canyon, aerial view"
+# prompt = "A bunch of puppies running around a front lawn in a giant courtyard "
+# #image = load_image(image=download_url_to_cache("https://media.sciencephoto.com/f0/22/69/89/f0226989-800px-wm.jpg"))
+
+def main(
+    sample_path,
+    output_mp4_path:str,
+    prompt=None,
+    degradation=.5,
+    model_name='I2V5B_final_i38800_nearest_lora_weights',
+
+    low_vram=True,
+    device:str=None,
+    
+    #BROADCASTABLE:
+    noise_downtemp_interp='nearest',
+    image=None,
+    num_inference_steps=30,
+    guidance_scale=6,
+    # v2v_strength=.5,#Timestep for when using Vid2Vid. Only set to not none when using a T2V model!
+):
+    """
+    Main function to run the video generation pipeline with specified parameters.
+
+    Args:
+        model_name (str): Name of the pipeline to use ('T2V5B', 'T2V2B', 'I2V5B', etc).
+        device (str or int, optional): Device to run the model on (e.g., 'cuda:0' or 0). If unspecified, the GPU with the  most free VRAM will be chosen.
+        low_vram (bool): Set to True if you have less than 32GB of VRAM. In enables model cpu offloading, which slows down inference but needs much less vram.
+        sample_path (str or list, optional): Broadcastable. Path(s) to the sample `.pkl` file(s) or folders containing (noise.npy and input.mp4 files)
+        degradation (float or list): Broadcastable. Degradation level(s) for the noise warp (float between 0 and 1).
+        noise_downtemp_interp (str or list): Broadcastable. Interpolation method(s) for down-temporal noise. Options: 'nearest', 'blend', 'blend_norm'.
+        image (str, PIL.Image, or list, optional): Broadcastable. Image(s) to use as the initial frame(s). Can be a URL or a path to an image.
+        prompt (str or list, optional): Broadcastable. Text prompt(s) for video generation.
+        num_inference_steps (int or list): Broadcastable. Number of inference steps for the pipeline.
+    """
+    output_root='infer_outputs', # output_root (str): Root directory where output videos will be saved.
+    subfolder='default_subfolder', # subfolder (str): Subfolder within output_root to save outputs.
+
+    if device is None:
+        device = rp.select_torch_device(reserve=True, prefer_used=True)
+        rp.fansi_print(f"Selected torch device: {device}")
+
+
+    cartridge_kwargs = rp.broadcast_kwargs(
+        rp.gather_vars(
+            "sample_path",
+            "degradation",
+            "noise_downtemp_interp",
+            "image",
+            "prompt",
+            "num_inference_steps",
+            "guidance_scale",
+            # "v2v_strength",
+        )
+    )
+
+    rp.fansi_print("cartridge_kwargs:", "cyan", "bold")
+    print(
+        rp.indentify(
+            rp.with_line_numbers(
+                rp.fansi_pygments(
+                    rp.autoformat_json(cartridge_kwargs),
+                    "json",
+                ),
+                align=True,
+            )
+        ),
+    )
+
+    # cartridges = [load_sample_cartridge(**x) for x in cartridge_kwargs]
+    cartridges = rp.load_files(lambda x:load_sample_cartridge(**x), cartridge_kwargs, show_progress='eta:Loading Cartridges')
+
+    pipe = get_pipe(model_name, device, low_vram=low_vram)
+
+    output=[]
+    for cartridge in cartridges:
+        pipe_out = run_pipe(
+            pipe=pipe,
+            cartridge=cartridge,
+            output_root=output_root,
+            subfolder=subfolder,
+            output_mp4_path=output_mp4_path,
+        )
+
+        output.append(
+            rp.as_easydict(
+                rp.gather(
+                    pipe_out,
+                    [
+                        "output_mp4_path",
+                        "preview_mp4_path",
+                        "compressed_preview_mp4_path",
+                        "preview_mp4_path",
+                        "preview_gif_path",
+                    ],
+                    as_dict=True,
+                )
+            )
+        )
+    return output
+
+if __name__ == '__main__':
+    import fire
+    fire.Fire(main)
+
+
+

make_warped_noise.py

@@ -0,0 +1,93 @@
+#Ryan Burgert 2024
+
+#Setup:
+#    Run this in a Jupyter Notebook on a computer with at least one GPU
+#        `sudo apt install ffmpeg git`
+#        `pip install rp`
+#    The first time you run this it might be a bit slow (it will download necessary models)
+#    The `rp` package will take care of installing the rest of the python packages for you
+
+import rp
+
+rp.r._pip_import_autoyes=True #Automatically install missing packages
+
+rp.pip_import('fire')
+rp.git_import('CommonSource') #If missing, installs code from https://github.com/RyannDaGreat/CommonSource
+import rp.git.CommonSource.noise_warp as nw
+import fire
+
+def main(video:str, output_folder:str):
+    """
+    Takes a video URL or filepath and an output folder path
+    It then resizes that video to height=480, width=720, 49 frames (CogVidX's dimensions)
+    Then it calculates warped noise at latent resolution (i.e. 1/8 of the width and height) with 16 channels
+    It saves that warped noise, optical flows, and related preview videos and images to the output folder
+    The main file you need is <output_folder>/noises.npy which is the gaussian noises in (H,W,C) form
+    """
+
+    if rp.folder_exists(output_folder):
+        raise RuntimeError(f"The given output_folder={repr(output_folder)} already exists! To avoid clobbering what might be in there, please specify a folder that doesn't exist so I can create one for you. Alternatively, you could delete that folder if you don't care whats in it.")
+
+    FRAME = 2**-1 #We immediately resize the input frames by this factor, before calculating optical flow
+                  #The flow is calulated at (input size) × FRAME resolution.
+                  #Higher FLOW values result in slower optical flow calculation and higher intermediate noise resolution
+                  #Larger is not always better - watch the preview in Jupyter to see if it looks good!
+
+    FLOW = 2**3   #Then, we use bilinear interpolation to upscale the flow by this factor
+                  #We warp the noise at (input size) × FRAME × FLOW resolution
+                  #The noise is then downsampled back to (input size)
+                  #Higher FLOW values result in more temporally consistent noise warping at the cost of higher VRAM usage and slower inference time
+    LATENT = 8    #We further downsample the outputs by this amount - because 8 pixels wide corresponds to one latent wide in Stable Diffusion
+                  #The final output size is (input size) ÷ LATENT regardless of FRAME and FLOW
+
+    #LATENT = 1    #Uncomment this line for a prettier visualization! But for latent diffusion models, use LATENT=8
+
+    #You can also use video files or URLs
+    # video = "https://www.shutterstock.com/shutterstock/videos/1100085499/preview/stock-footage-bremen-germany-october-old-style-carousel-moving-on-square-in-city-horses-on-traditional.webm"
+
+    # output_folder = "NoiseWarpOutputFolder"
+
+    if isinstance(video,str):
+        video=rp.load_video(video)
+
+    #Preprocess the video
+    video=rp.resize_list(video,length=49) #Stretch or squash video to 49 frames (CogVideoX's length)
+    video=rp.resize_images_to_hold(video,height=480,width=720)
+    video=rp.crop_images(video,height=480,width=720,origin='center') #Make the resolution 480x720 (CogVideoX's resolution)
+    video=rp.as_numpy_array(video)
+
+
+    #See this function's docstring for more information!
+    output = nw.get_noise_from_video(
+        video,
+        remove_background=False, #Set this to True to matte the foreground - and force the background to have no flow
+        visualize=True,          #Generates nice visualization videos and previews in Jupyter notebook
+        save_files=True,         #Set this to False if you just want the noises without saving to a numpy file
+        
+        noise_channels=16,
+        output_folder=output_folder,
+        resize_frames=FRAME,
+        resize_flow=FLOW,
+        downscale_factor=round(FRAME * FLOW) * LATENT,
+    )
+
+    output.first_frame_path = rp.save_image(video[0],rp.path_join(output_folder,'first_frame.png'))
+
+    rp.save_video_mp4(video, rp.path_join(output_folder, 'input.mp4'), framerate=12, video_bitrate='max')
+
+    #output.numpy_noises_downsampled = as_numpy_images(
+        #nw.resize_noise(
+            #as_torch_images(x),
+            #1 / 8,
+        #)for x 
+    #)
+    #
+    #output.numpy_noises_downsampled_path = path_join(output_folder, 'noises_downsampled.npy')
+    #np.save(numpy_noises_downsampled_path, output.numpy_noises_downsampled)
+
+    print("Noise shape:"  ,output.numpy_noises.shape)
+    print("Flow shape:"   ,output.numpy_flows .shape)
+    print("Output folder:",output.output_folder)
+
+if __name__ == "__main__":
+    fire.Fire(main) 

requirements.txt

@@ -0,0 +1,18 @@
+--index-url https://download.pytorch.org/whl/cu118
+rp
+torch 
+torchvision
+diffusers
+einops
+easydict
+transformers
+accelerate
+oldest-supported-numpy
+sentencepiece
+peft
+opencv-contrib-python
+imageio-ffmpeg
+fire
+moviepy
+icecream
+matplotlib

requirements_local.txt

@@ -0,0 +1,12 @@
+rp
+easydict
+oldest-supported-numpy
+opencv-contrib-python
+imageio-ffmpeg
+fire
+moviepy
+icecream
+matplotlib
+art
+torchvision
+torch