Update README with comprehensive documentation for Real-Time Latent Consistency Model

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+venv/
+public/
+*.pem

Dockerfile

@@ -0,0 +1,48 @@
+FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV PYTHONUNBUFFERED=1
+ENV NODE_MAJOR=20
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    build-essential \
+    python3.9 \
+    python3-pip \
+    python3-dev \
+    git \
+    ffmpeg \
+    google-perftools \
+    ca-certificates curl gnupg \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /code
+
+RUN mkdir -p /etc/apt/keyrings 
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list > /dev/null
+RUN apt-get update && apt-get install nodejs -y
+
+COPY ./requirements.txt /code/requirements.txt
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+    PYTHONUNBUFFERED=1 \
+    SYSTEM=spaces
+
+RUN pip3 install --no-cache-dir --upgrade --pre -r /code/requirements.txt
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
+CMD ["./build-run.sh"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,10 +1,193 @@
 ---
-title: Real Time Model
-emoji: 🐢
-colorFrom: green
-colorTo: yellow
+title: Real-Time Latent Consistency Model Image-to-Image ControlNet
+emoji: 🖼️🖼️
+colorFrom: gray
+colorTo: indigo
 sdk: docker
 pinned: false
+suggested_hardware: a10g-small
+disable_embedding: true
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Real-Time Latent Consistency Model
+
+This demo showcases [Latent Consistency Model (LCM)](https://latent-consistency-models.github.io/) using [Diffusers](https://huggingface.co/docs/diffusers/using-diffusers/lcm) with a MJPEG stream server. You can read more about LCM + LoRAs with diffusers [here](https://huggingface.co/blog/lcm_lora).
+
+You need a webcam to run this demo. 🤗
+
+See a collecting with live demos [here](https://huggingface.co/collections/latent-consistency/latent-consistency-model-demos-654e90c52adb0688a0acbe6f)
+
+## Running Locally
+
+You need CUDA and Python 3.10, Node > 19, Mac with an M1/M2/M3 chip or Intel Arc GPU
+
+
+## Install
+
+```bash
+python -m venv venv
+source venv/bin/activate
+pip3 install -r server/requirements.txt
+cd frontend && npm install && npm run build && cd ..
+python server/main.py --reload --pipeline img2imgSDTurbo 
+ ```
+
+Don't forget to fuild the frontend!!! 
+
+```bash
+cd frontend && npm install && npm run build && cd ..
+```
+
+# Pipelines
+You can build your own pipeline following examples here [here](pipelines),
+
+
+# LCM
+### Image to Image
+
+```bash
+python server/main.py --reload --pipeline img2img 
+```
+
+# LCM
+### Text to Image
+
+```bash
+python server/main.py --reload --pipeline txt2img 
+```
+
+### Image to Image ControlNet Canny
+
+```bash
+python server/main.py --reload --pipeline controlnet 
+```
+
+
+# LCM + LoRa
+
+Using LCM-LoRA, giving it the super power of doing inference in as little as 4 steps. [Learn more here](https://huggingface.co/blog/lcm_lora) or [technical report](https://huggingface.co/papers/2311.05556)
+
+
+### Image to Image ControlNet Canny LoRa
+
+```bash
+python server/main.py --reload --pipeline controlnetLoraSD15
+```
+or SDXL, note that SDXL is slower than SD15 since the inference runs on 1024x1024 images
+
+```bash
+python server/main.py --reload --pipeline controlnetLoraSDXL
+```
+
+### Text to Image
+
+```bash
+python server/main.py --reload --pipeline txt2imgLora
+```
+
+```bash
+python server/main.py --reload --pipeline txt2imgLoraSDXL
+```
+# Available Pipelines
+
+#### [LCM](https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7)
+
+`img2img`  
+`txt2img`   
+`controlnet`   
+`txt2imgLora`   
+`controlnetLoraSD15` 
+
+#### [SD15](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
+`controlnetLoraSDXL`   
+`txt2imgLoraSDXL`   
+
+#### [SDXL Turbo](https://huggingface.co/stabilityai/sd-xl-turbo)
+
+`img2imgSDXLTurbo`    
+`controlnetSDXLTurbo`   
+
+
+#### [SDTurbo](https://huggingface.co/stabilityai/sd-turbo)
+`img2imgSDTurbo`   
+`controlnetSDTurbo`   
+
+#### [Segmind-Vega](https://huggingface.co/segmind/Segmind-Vega)
+`controlnetSegmindVegaRT`   
+`img2imgSegmindVegaRT`   
+
+
+### Setting environment variables
+
+
+* `--host`: Host address (default: 0.0.0.0)  
+* `--port`: Port number (default: 7860)  
+* `--reload`: Reload code on change  
+* `--max-queue-size`: Maximum queue size (optional)
+* `--timeout`: Timeout period (optional)
+* `--safety-checker`: Enable Safety Checker (optional) 
+* `--torch-compile`: Use Torch Compile
+* `--use-taesd` / `--no-taesd`: Use Tiny Autoencoder  
+* `--pipeline`: Pipeline to use (default: "txt2img")  
+* `--ssl-certfile`: SSL Certificate File (optional)
+* `--ssl-keyfile`: SSL Key File (optional)
+* `--debug`: Print Inference time  
+* `--compel`: Compel option  
+* `--sfast`: Enable Stable Fast   
+* `--onediff`: Enable OneDiff
+
+If you run using `bash build-run.sh` you can set `PIPELINE` variables to choose the pipeline you want to run
+
+```bash
+PIPELINE=txt2imgLoraSDXL bash build-run.sh
+```
+
+and setting environment variables
+
+```bash
+TIMEOUT=120 SAFETY_CHECKER=True MAX_QUEUE_SIZE=4 python server/main.py --reload --pipeline txt2imgLoraSDXL
+```
+
+If you're running locally and want to test it on Mobile Safari, the webserver needs to be served over HTTPS, or follow this instruction on my [comment](https://github.com/radames/Real-Time-Latent-Consistency-Model/issues/17#issuecomment-1811957196)
+
+```bash
+openssl req -newkey rsa:4096 -nodes -keyout key.pem -x509 -days 365 -out certificate.pem
+python server/main.py --reload --ssl-certfile=certificate.pem --ssl-keyfile=key.pem
+```
+
+## Docker
+
+You need NVIDIA Container Toolkit for Docker, defaults to `controlnet``
+
+```bash
+docker build -t lcm-live .
+docker run -ti -p 7860:7860 --gpus all lcm-live
+```
+
+reuse models data from host to avoid downloading them again, you can change `~/.cache/huggingface` to any other directory, but if you use hugingface-cli locally, you can share the same cache
+
+```bash
+docker run -ti -p 7860:7860 -e HF_HOME=/data -v ~/.cache/huggingface:/data  --gpus all lcm-live
+```
+ 
+
+or with environment variables
+
+```bash
+docker run -ti -e PIPELINE=txt2imgLoraSDXL -p 7860:7860 --gpus all lcm-live
+```
+
+
+# Demo on Hugging Face
+
+
+* [radames/Real-Time-Latent-Consistency-Model](https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model)  
+* [radames/Real-Time-SD-Turbo](https://huggingface.co/spaces/radames/Real-Time-SD-Turbo)  
+* [latent-consistency/Real-Time-LCM-ControlNet-Lora-SD1.5](https://huggingface.co/spaces/latent-consistency/Real-Time-LCM-ControlNet-Lora-SD1.5)  
+* [latent-consistency/Real-Time-LCM-Text-to-Image-Lora-SD1.5](https://huggingface.co/spaces/latent-consistency/Real-Time-LCM-Text-to-Image-Lora-SD1.5)  
+* [radames/Real-Time-Latent-Consistency-Model-Text-To-Image](https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model-Text-To-Image)  
+
+
+
+
+https://github.com/radames/Real-Time-Latent-Consistency-Model/assets/102277/c4003ac5-e7ff-44c0-97d3-464bb659de70

build-run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+cd frontend
+npm install
+npm run build
+if [ $? -eq 0 ]; then
+    echo -e "\033[1;32m\nfrontend build success \033[0m"
+else
+    echo -e "\033[1;31m\nfrontend build failed\n\033[0m" >&2  exit 1
+fi
+cd ../
+#check if var PIPELINE is set otherwise get default
+if [ -z ${PIPELINE+x} ]; then
+    PIPELINE="controlnet"
+fi
+if [ -z ${COMPILE+x} ]; then
+    COMPILE="--sfast"
+fi
+echo -e "\033[1;32m\npipeline: $PIPELINE \033[0m"
+echo -e "\033[1;32m\ncompile: $COMPILE \033[0m"
+python3 ./server/main.py --port 7860 --host 0.0.0.0 --pipeline $PIPELINE $COMPILE

frontend/.eslintignore

@@ -0,0 +1,13 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+
+# Ignore files for PNPM, NPM and YARN
+pnpm-lock.yaml
+package-lock.json
+yarn.lock

frontend/.eslintrc.cjs

@@ -0,0 +1,30 @@
+module.exports = {
+  root: true,
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:svelte/recommended',
+    'prettier'
+  ],
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint'],
+  parserOptions: {
+    sourceType: 'module',
+    ecmaVersion: 2020,
+    extraFileExtensions: ['.svelte']
+  },
+  env: {
+    browser: true,
+    es2017: true,
+    node: true
+  },
+  overrides: [
+    {
+      files: ['*.svelte'],
+      parser: 'svelte-eslint-parser',
+      parserOptions: {
+        parser: '@typescript-eslint/parser'
+      }
+    }
+  ]
+};

frontend/.gitignore

@@ -0,0 +1,11 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+vite.config.js.timestamp-*
+vite.config.ts.timestamp-*
+public

frontend/.npmrc

@@ -0,0 +1 @@
+engine-strict=true

frontend/.nvmrc

@@ -0,0 +1 @@
+v20.14.0

frontend/.prettierignore

@@ -0,0 +1,13 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+
+# Ignore files for PNPM, NPM and YARN
+pnpm-lock.yaml
+package-lock.json
+yarn.lock

frontend/.prettierrc

@@ -0,0 +1,19 @@
+{
+  "useTabs": false,
+  "singleQuote": true,
+  "trailingComma": "none",
+  "printWidth": 100,
+  "plugins": [
+    "prettier-plugin-svelte",
+    "prettier-plugin-organize-imports",
+    "prettier-plugin-tailwindcss"
+  ],
+  "overrides": [
+    {
+      "files": "*.svelte",
+      "options": {
+        "parser": "svelte"
+      }
+    }
+  ]
+}

frontend/README.md

@@ -0,0 +1,38 @@
+# create-svelte
+
+Everything you need to build a Svelte project, powered by [`create-svelte`](https://github.com/sveltejs/kit/tree/master/packages/create-svelte).
+
+## Creating a project
+
+If you're seeing this, you've probably already done this step. Congrats!
+
+```bash
+# create a new project in the current directory
+npm create svelte@latest
+
+# create a new project in my-app
+npm create svelte@latest my-app
+```
+
+## Developing
+
+Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
+
+```bash
+npm run dev
+
+# or start the server and open the app in a new browser tab
+npm run dev -- --open
+```
+
+## Building
+
+To create a production version of your app:
+
+```bash
+npm run build
+```
+
+You can preview the production build with `npm run preview`.
+
+> To deploy your app, you may need to install an [adapter](https://kit.svelte.dev/docs/adapters) for your target environment.

frontend/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "frontend",
+  "version": "0.0.1",
+  "private": true,
+  "scripts": {
+    "dev": "vite dev",
+    "build": "vite build",
+    "preview": "vite preview",
+    "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
+    "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
+    "lint": "prettier --check . && eslint .",
+    "format": "prettier --write ."
+  },
+  "devDependencies": {
+    "@sveltejs/adapter-auto": "^3.0.0",
+    "@sveltejs/adapter-static": "^3",
+    "@sveltejs/kit": "^2.0.0",
+    "@sveltejs/vite-plugin-svelte": "^3.1.1",
+    "@types/eslint": "^8.56.0",
+    "@typescript-eslint/eslint-plugin": "^7.12.0",
+    "@typescript-eslint/parser": "^7.12.0",
+    "autoprefixer": "^10.4.16",
+    "eslint": "^8.56.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-svelte": "^2.35.1",
+    "postcss": "^8.4.31",
+    "prettier": "^3.3.1",
+    "prettier-plugin-organize-imports": "^3.2.4",
+    "prettier-plugin-svelte": "^3.2.4",
+    "prettier-plugin-tailwindcss": "^0.6.1",
+    "svelte": "^4.2.18",
+    "svelte-check": "^3.8.0",
+    "tailwindcss": "^3.4.4",
+    "tslib": "^2.6.3",
+    "typescript": "^5.0.0",
+    "vite": "^5.0.3"
+  },
+  "type": "module",
+  "dependencies": {
+    "piexifjs": "^1.0.6",
+    "rvfc-polyfill": "^1.0.7"
+  }
+}

frontend/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {}
+  }
+};

frontend/src/app.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

frontend/src/app.d.ts

@@ -0,0 +1,12 @@
+// See https://kit.svelte.dev/docs/types#app
+// for information about these interfaces
+declare global {
+  namespace App {
+    // interface Error {}
+    // interface Locals {}
+    // interface PageData {}
+    // interface Platform {}
+  }
+}
+
+export {};

frontend/src/app.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="%sveltekit.assets%/favicon.png" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    %sveltekit.head%
+  </head>
+  <body data-sveltekit-preload-data="hover">
+    <div style="display: contents">%sveltekit.body%</div>
+  </body>
+</html>

frontend/src/lib/components/AspectRatioSelect.svelte

@@ -0,0 +1,27 @@
+<script lang="ts">
+  import { createEventDispatcher } from 'svelte';
+
+  let options: string[] = ['1:1', '16:9', '4:3', '3:2', '3:4', '9:16'];
+  export let aspectRatio: number = 1;
+  const dispatchEvent = createEventDispatcher();
+
+  function onChange(e: Event) {
+    const target = e.target as HTMLSelectElement;
+    const value = target.value;
+    const [width, height] = value.split(':').map((v) => parseInt(v));
+    aspectRatio = width / height;
+    dispatchEvent('change', aspectRatio);
+  }
+</script>
+
+<div class="relative">
+  <select
+    on:change={onChange}
+    title="Aspect Ratio"
+    class="border-1 block cursor-pointer rounded-md border-gray-800 border-opacity-50 bg-slate-100 bg-opacity-30 p-1 font-medium text-white"
+  >
+    {#each options as option, i}
+      <option value={option}>{option}</option>
+    {/each}
+  </select>
+</div>

frontend/src/lib/components/Button.svelte

@@ -0,0 +1,15 @@
+<script lang="ts">
+  export let classList: string = 'p-2';
+  export let disabled: boolean = false;
+  export let title: string = '';
+</script>
+
+<button class="button {classList}" on:click {disabled} {title}>
+  <slot />
+</button>
+
+<style lang="postcss" scoped>
+  .button {
+    @apply rounded bg-gray-700 font-normal text-white hover:bg-gray-800 disabled:cursor-not-allowed disabled:bg-gray-300 dark:disabled:bg-gray-700 dark:disabled:text-black;
+  }
+</style>

frontend/src/lib/components/Checkbox.svelte

@@ -0,0 +1,14 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = false;
+  export let params: FieldProps;
+  onMount(() => {
+    value = Boolean(params?.default) ?? 8.0;
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center justify-items-start gap-3">
+  <label class="text-sm font-medium" for={params.id}>{params?.title}</label>
+  <input bind:checked={value} type="checkbox" id={params.id} class="cursor-pointer" />
+</div>

frontend/src/lib/components/ImagePlayer.svelte

@@ -0,0 +1,74 @@
+<script lang="ts">
+  import { lcmLiveStatus, LCMLiveStatus, streamId } from '$lib/lcmLive';
+  import { getPipelineValues } from '$lib/store';
+
+  import Button from '$lib/components/Button.svelte';
+  import Floppy from '$lib/icons/floppy.svelte';
+  import Expand from '$lib/icons/expand.svelte';
+  import { snapImage, expandWindow } from '$lib/utils';
+
+  $: isLCMRunning = $lcmLiveStatus !== LCMLiveStatus.DISCONNECTED;
+  $: console.log('isLCMRunning', isLCMRunning);
+  let imageEl: HTMLImageElement;
+  let expandedWindow: Window;
+  let isExpanded = false;
+  async function takeSnapshot() {
+    if (isLCMRunning) {
+      await snapImage(imageEl, {
+        prompt: getPipelineValues()?.prompt,
+        negative_prompt: getPipelineValues()?.negative_prompt,
+        seed: getPipelineValues()?.seed,
+        guidance_scale: getPipelineValues()?.guidance_scale
+      });
+    }
+  }
+  async function toggleFullscreen() {
+    if (isLCMRunning && !isExpanded) {
+      expandedWindow = expandWindow('/api/stream/' + $streamId);
+      expandedWindow.addEventListener('beforeunload', () => {
+        isExpanded = false;
+      });
+      isExpanded = true;
+    } else {
+      expandedWindow?.close();
+      isExpanded = false;
+    }
+  }
+</script>
+
+<div
+  class="relative mx-auto aspect-square max-w-lg self-center overflow-hidden rounded-lg border border-slate-300"
+>
+  <!-- svelte-ignore a11y-missing-attribute -->
+  {#if isLCMRunning}
+    {#if !isExpanded}
+      <img
+        bind:this={imageEl}
+        class="aspect-square w-full rounded-lg"
+        src={'/api/stream/' + $streamId}
+      />
+    {/if}
+    <div class="absolute bottom-1 right-1">
+      <Button
+        on:click={toggleFullscreen}
+        title={'Expand Fullscreen'}
+        classList={'text-sm ml-auto text-white p-1 shadow-lg rounded-lg opacity-50'}
+      >
+        <Expand classList={''} />
+      </Button>
+      <Button
+        on:click={takeSnapshot}
+        disabled={!isLCMRunning}
+        title={'Take Snapshot'}
+        classList={'text-sm ml-auto text-white p-1 shadow-lg rounded-lg opacity-50'}
+      >
+        <Floppy classList={''} />
+      </Button>
+    </div>
+  {:else}
+    <img
+      class="aspect-square w-full rounded-lg"
+      src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
+    />
+  {/if}
+</div>

frontend/src/lib/components/InputRange.svelte

@@ -0,0 +1,50 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = 8.0;
+  export let params: FieldProps;
+  onMount(() => {
+    value = Number(params?.default) ?? 8.0;
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center gap-3">
+  <label class="text-sm font-medium" for={params.id}>{params?.title}</label>
+  <input
+    class="col-span-2 h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-300 dark:bg-gray-500"
+    bind:value
+    type="range"
+    id={params.id}
+    name={params.id}
+    min={params?.min}
+    max={params?.max}
+    step={params?.step ?? 1}
+  />
+  <input
+    type="number"
+    step={params?.step ?? 1}
+    bind:value
+    class="rounded-md border px-1 py-1 text-center text-xs font-bold dark:text-black"
+  />
+</div>
+<!-- 
+<style lang="postcss" scoped>
+  input[type='range']::-webkit-slider-runnable-track {
+    @apply h-2 cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-webkit-slider-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-moz-range-track {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-moz-range-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-ms-track {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+  input[type='range']::-ms-thumb {
+    @apply cursor-pointer rounded-lg dark:bg-gray-50;
+  }
+</style> -->

frontend/src/lib/components/MediaListSwitcher.svelte

@@ -0,0 +1,47 @@
+<script lang="ts">
+  import { mediaDevices, mediaStreamActions } from '$lib/mediaStream';
+  import Screen from '$lib/icons/screen.svelte';
+  import AspectRatioSelect from './AspectRatioSelect.svelte';
+  import { onMount } from 'svelte';
+
+  let deviceId: string = '';
+  let aspectRatio: number = 1;
+
+  onMount(() => {
+    deviceId = $mediaDevices[0].deviceId;
+  });
+  $: {
+    console.log(deviceId);
+  }
+  $: {
+    console.log(aspectRatio);
+  }
+</script>
+
+<div class="flex items-center justify-center text-xs backdrop-blur-sm backdrop-grayscale">
+  <AspectRatioSelect
+    bind:aspectRatio
+    on:change={() => mediaStreamActions.switchCamera(deviceId, aspectRatio)}
+  />
+  <button
+    title="Share your screen"
+    class="border-1 my-1 flex cursor-pointer gap-1 rounded-md border-gray-500 border-opacity-50 bg-slate-100 bg-opacity-30 p-1 font-medium text-white"
+    on:click={() => mediaStreamActions.startScreenCapture()}
+  >
+    <span>Share</span>
+
+    <Screen classList={''} />
+  </button>
+  {#if $mediaDevices}
+    <select
+      bind:value={deviceId}
+      on:change={() => mediaStreamActions.switchCamera(deviceId, aspectRatio)}
+      id="devices-list"
+      class="border-1 block cursor-pointer rounded-md border-gray-800 border-opacity-50 bg-slate-100 bg-opacity-30 p-1 font-medium text-white"
+    >
+      {#each $mediaDevices as device, i}
+        <option value={device.deviceId}>{device.label}</option>
+      {/each}
+    </select>
+  {/if}
+</div>

frontend/src/lib/components/PipelineOptions.svelte

@@ -0,0 +1,60 @@
+<script lang="ts">
+  import type { Fields } from '$lib/types';
+  import { FieldType } from '$lib/types';
+  import InputRange from './InputRange.svelte';
+  import SeedInput from './SeedInput.svelte';
+  import TextArea from './TextArea.svelte';
+  import Checkbox from './Checkbox.svelte';
+  import Selectlist from './Selectlist.svelte';
+  import { pipelineValues } from '$lib/store';
+
+  export let pipelineParams: Fields;
+
+  $: advanceOptions = Object.values(pipelineParams)?.filter((e) => e?.hide == true);
+  $: featuredOptions = Object.values(pipelineParams)?.filter((e) => e?.hide !== true);
+</script>
+
+<div class="flex flex-col gap-3">
+  <div class="grid grid-cols-1 items-center gap-3">
+    {#if featuredOptions}
+      {#each featuredOptions as params}
+        {#if params.field === FieldType.RANGE}
+          <InputRange {params} bind:value={$pipelineValues[params.id]}></InputRange>
+        {:else if params.field === FieldType.SEED}
+          <SeedInput {params} bind:value={$pipelineValues[params.id]}></SeedInput>
+        {:else if params.field === FieldType.TEXTAREA}
+          <TextArea {params} bind:value={$pipelineValues[params.id]}></TextArea>
+        {:else if params.field === FieldType.CHECKBOX}
+          <Checkbox {params} bind:value={$pipelineValues[params.id]}></Checkbox>
+        {:else if params.field === FieldType.SELECT}
+          <Selectlist {params} bind:value={$pipelineValues[params.id]}></Selectlist>
+        {/if}
+      {/each}
+    {/if}
+  </div>
+
+  <details>
+    <summary class="cursor-pointer font-medium">Advanced Options</summary>
+    <div
+      class="grid grid-cols-1 items-center gap-3 {Object.values(pipelineParams).length > 5
+        ? 'sm:grid-cols-2'
+        : ''}"
+    >
+      {#if advanceOptions}
+        {#each advanceOptions as params}
+          {#if params.field === FieldType.RANGE}
+            <InputRange {params} bind:value={$pipelineValues[params.id]}></InputRange>
+          {:else if params.field === FieldType.SEED}
+            <SeedInput {params} bind:value={$pipelineValues[params.id]}></SeedInput>
+          {:else if params.field === FieldType.TEXTAREA}
+            <TextArea {params} bind:value={$pipelineValues[params.id]}></TextArea>
+          {:else if params.field === FieldType.CHECKBOX}
+            <Checkbox {params} bind:value={$pipelineValues[params.id]}></Checkbox>
+          {:else if params.field === FieldType.SELECT}
+            <Selectlist {params} bind:value={$pipelineValues[params.id]}></Selectlist>
+          {/if}
+        {/each}
+      {/if}
+    </div>
+  </details>
+</div>

frontend/src/lib/components/SeedInput.svelte

@@ -0,0 +1,26 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  import Button from './Button.svelte';
+  export let value = 299792458;
+  export let params: FieldProps;
+
+  onMount(() => {
+    value = Number(params?.default ?? '');
+  });
+  function randomize() {
+    value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+  }
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center gap-3">
+  <label class="text-sm font-medium" for="seed">Seed</label>
+  <input
+    bind:value
+    type="number"
+    id="seed"
+    name="seed"
+    class="col-span-2 rounded-md border border-gray-700 p-2 text-right font-light dark:text-black"
+  />
+  <Button on:click={randomize}>Rand</Button>
+</div>

frontend/src/lib/components/Selectlist.svelte

@@ -0,0 +1,24 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value = '';
+  export let params: FieldProps;
+  onMount(() => {
+    value = String(params?.default);
+  });
+</script>
+
+<div class="grid max-w-md grid-cols-4 items-center justify-items-start gap-3">
+  <label for="model-list" class="text-sm font-medium">{params?.title} </label>
+  {#if params?.values}
+    <select
+      bind:value
+      id="model-list"
+      class="cursor-pointer rounded-md border-2 border-gray-500 p-2 font-light dark:text-black"
+    >
+      {#each params.values as model, i}
+        <option value={model} selected={i === 0}>{model}</option>
+      {/each}
+    </select>
+  {/if}
+</div>

frontend/src/lib/components/TextArea.svelte

@@ -0,0 +1,23 @@
+<script lang="ts">
+  import type { FieldProps } from '$lib/types';
+  import { onMount } from 'svelte';
+  export let value: string;
+  export let params: FieldProps;
+  onMount(() => {
+    value = String(params?.default ?? '');
+  });
+</script>
+
+<div class="">
+  <label class="text-sm font-medium" for={params?.title}>
+    {params?.title}
+  </label>
+  <div class="text-normal flex items-center rounded-md border border-gray-700">
+    <textarea
+      class="mx-1 w-full px-3 py-2 font-light outline-none dark:text-black"
+      title={params?.title}
+      placeholder="Add your prompt here..."
+      bind:value
+    ></textarea>
+  </div>
+</div>

frontend/src/lib/components/VideoInput.svelte

@@ -0,0 +1,127 @@
+<script lang="ts">
+  import 'rvfc-polyfill';
+
+  import { onDestroy, onMount } from 'svelte';
+  import {
+    mediaStreamStatus,
+    MediaStreamStatusEnum,
+    onFrameChangeStore,
+    mediaStream,
+    mediaDevices
+  } from '$lib/mediaStream';
+  import MediaListSwitcher from './MediaListSwitcher.svelte';
+  import Button from '$lib/components/Button.svelte';
+  import Expand from '$lib/icons/expand.svelte';
+
+  export let width = 512;
+  export let height = 512;
+  const size = { width, height };
+
+  let videoEl: HTMLVideoElement;
+  let canvasEl: HTMLCanvasElement;
+  let ctx: CanvasRenderingContext2D;
+  let videoFrameCallbackId: number;
+
+  // ajust the throttle time to your needs
+  const THROTTLE = 1000 / 120;
+  let selectedDevice: string = '';
+  let videoIsReady = false;
+
+  onMount(() => {
+    ctx = canvasEl.getContext('2d') as CanvasRenderingContext2D;
+    canvasEl.width = size.width;
+    canvasEl.height = size.height;
+  });
+  $: {
+    console.log(selectedDevice);
+  }
+
+  onDestroy(() => {
+    if (videoFrameCallbackId) videoEl.cancelVideoFrameCallback(videoFrameCallbackId);
+  });
+
+  $: if (videoEl) {
+    videoEl.srcObject = $mediaStream;
+  }
+  let lastMillis = 0;
+  async function onFrameChange(now: DOMHighResTimeStamp, metadata: VideoFrameCallbackMetadata) {
+    if (now - lastMillis < THROTTLE) {
+      videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+      return;
+    }
+    const videoWidth = videoEl.videoWidth;
+    const videoHeight = videoEl.videoHeight;
+    // scale down video to fit canvas,  size.width, size.height
+    const scale = Math.min(size.width / videoWidth, size.height / videoHeight);
+    const width0 = videoWidth * scale;
+    const height0 = videoHeight * scale;
+    const x0 = (size.width - width0) / 2;
+    const y0 = (size.height - height0) / 2;
+    ctx.clearRect(0, 0, size.width, size.height);
+    ctx.drawImage(videoEl, x0, y0, width0, height0);
+
+    const blob = await new Promise<Blob>((resolve) => {
+      canvasEl.toBlob(
+        (blob) => {
+          resolve(blob as Blob);
+        },
+        'image/jpeg',
+        1
+      );
+    });
+    onFrameChangeStore.set({ blob });
+    videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+  }
+
+  $: if ($mediaStreamStatus == MediaStreamStatusEnum.CONNECTED && videoIsReady) {
+    videoFrameCallbackId = videoEl.requestVideoFrameCallback(onFrameChange);
+  }
+
+  function toggleFullscreen() {
+    if (videoIsReady) {
+      if (document.fullscreenElement) {
+        document.exitFullscreen();
+      } else {
+        videoEl.requestFullscreen();
+      }
+    }
+  }
+</script>
+
+<div class="relative mx-auto max-w-lg overflow-hidden rounded-lg border border-slate-300">
+  <div class="relative z-10 flex aspect-square w-full items-center justify-center object-cover">
+    {#if $mediaDevices.length > 0}
+      <div class="absolute bottom-0 right-0 z-10 flex bg-slate-400 bg-opacity-40">
+        <MediaListSwitcher />
+        <Button
+          on:click={toggleFullscreen}
+          title={'Expand Fullscreen'}
+          classList={'text-sm ml-auto text-white p-1 shadow-lg rounded-lg opacity-50'}
+        >
+          <Expand classList={''} />
+        </Button>
+      </div>
+    {/if}
+    <video
+      class="pointer-events-none aspect-square w-full justify-center object-contain"
+      bind:this={videoEl}
+      on:loadeddata={() => {
+        videoIsReady = true;
+      }}
+      playsinline
+      autoplay
+      muted
+      loop
+    ></video>
+    <canvas bind:this={canvasEl} class="absolute left-0 top-0 aspect-square w-full object-cover"
+    ></canvas>
+  </div>
+  <div class="absolute left-0 top-0 flex aspect-square w-full items-center justify-center">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 448" class="w-40 p-5 opacity-20">
+      <path
+        fill="currentColor"
+        d="M224 256a128 128 0 1 0 0-256 128 128 0 1 0 0 256zm-45.7 48A178.3 178.3 0 0 0 0 482.3 29.7 29.7 0 0 0 29.7 512h388.6a29.7 29.7 0 0 0 29.7-29.7c0-98.5-79.8-178.3-178.3-178.3h-91.4z"
+      />
+    </svg>
+  </div>
+</div>

frontend/src/lib/components/Warning.svelte

@@ -0,0 +1,27 @@
+<script lang="ts">
+  export let message: string = '';
+
+  let timeout = 0;
+  $: if (message !== '') {
+    console.log('message', message);
+    clearTimeout(timeout);
+    timeout = setTimeout(() => {
+      message = '';
+    }, 5000);
+  }
+</script>
+
+{#if message}
+  <div class="fixed right-0 top-0 m-4 cursor-pointer" on:click={() => (message = '')}>
+    <div class="rounded bg-red-800 p-4 text-white">
+      {message}
+    </div>
+    <div class="bar transition-all duration-500" style="width: 0;"></div>
+  </div>
+{/if}
+
+<style lang="postcss" scoped>
+  .button {
+    @apply rounded bg-gray-700 font-normal text-white hover:bg-gray-800 disabled:cursor-not-allowed disabled:bg-gray-300 dark:disabled:bg-gray-700 dark:disabled:text-black;
+  }
+</style>

frontend/src/lib/icons/aspect.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string = '';
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" height="16px" class={classList}>
+  <path
+    fill="currentColor"
+    d="M32 32C14.3 32 0 46.3 0 64v96c0 17.7 14.3 32 32 32s32-14.3 32-32V96h64c17.7 0 32-14.3 32-32s-14.3-32-32-32H32zM64 352c0-17.7-14.3-32-32-32s-32 14.3-32 32v96c0 17.7 14.3 32 32 32h96c17.7 0 32-14.3 32-32s-14.3-32-32-32H64V352zM320 32c-17.7 0-32 14.3-32 32s14.3 32 32 32h64v64c0 17.7 14.3 32 32 32s32-14.3 32-32V64c0-17.7-14.3-32-32-32H320zM448 352c0-17.7-14.3-32-32-32s-32 14.3-32 32v64H320c-17.7 0-32 14.3-32 32s14.3 32 32 32h96c17.7 0 32-14.3 32-32V352z"
+  />
+</svg>

frontend/src/lib/icons/expand.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string = '';
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" height="1em" class={classList}>
+  <path
+    fill="currentColor"
+    d="M.3 89.5C.1 91.6 0 93.8 0 96V224 416c0 35.3 28.7 64 64 64l384 0c35.3 0 64-28.7 64-64V224 96c0-35.3-28.7-64-64-64H64c-2.2 0-4.4 .1-6.5 .3c-9.2 .9-17.8 3.8-25.5 8.2C21.8 46.5 13.4 55.1 7.7 65.5c-3.9 7.3-6.5 15.4-7.4 24zM48 224H464l0 192c0 8.8-7.2 16-16 16L64 432c-8.8 0-16-7.2-16-16l0-192z"
+  />
+</svg>

frontend/src/lib/icons/floppy.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string;
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" height="1em" viewBox="0 0 448 512" class={classList}>
+  <path
+    fill="currentColor"
+    d="M48 96v320a16 16 0 0 0 16 16h320a16 16 0 0 0 16-16V170.5a16 16 0 0 0-4.7-11.3l33.9-33.9a64 64 0 0 1 18.7 45.3V416a64 64 0 0 1-64 64H64a64 64 0 0 1-64-64V96a64 64 0 0 1 64-64h245.5a64 64 0 0 1 45.3 18.7l74.5 74.5-33.9 33.9-74.6-74.4-.8-.8V184a24 24 0 0 1-24 24H104a24 24 0 0 1-24-24V80H64a16 16 0 0 0-16 16zm80-16v80h144V80H128zm32 240a64 64 0 1 1 128 0 64 64 0 1 1-128 0z"
+  />
+</svg>

frontend/src/lib/icons/screen.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string = '';
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 -32 576 576" height="16px" class={classList}>
+  <path
+    fill="currentColor"
+    d="M64 0A64 64 0 0 0 0 64v288a64 64 0 0 0 64 64h176l-10.7 32H160a32 32 0 1 0 0 64h256a32 32 0 1 0 0-64h-69.3L336 416h176a64 64 0 0 0 64-64V64a64 64 0 0 0-64-64H64zm448 64v288H64V64h448z"
+  />
+</svg>

frontend/src/lib/icons/spinner.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+  export let classList: string;
+</script>
+
+<svg xmlns="http://www.w3.org/2000/svg" height="1em" viewBox="0 0 512 512" class={classList}>
+  <path
+    fill="currentColor"
+    d="M304 48a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zm0 416a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zM48 304a48 48 0 1 0 0-96 48 48 0 1 0 0 96zm464-48a48 48 0 1 0 -96 0 48 48 0 1 0 96 0zM142.9 437A48 48 0 1 0 75 369.1 48 48 0 1 0 142.9 437zm0-294.2A48 48 0 1 0 75 75a48 48 0 1 0 67.9 67.9zM369.1 437A48 48 0 1 0 437 369.1 48 48 0 1 0 369.1 437z"
+  />
+</svg>

frontend/src/lib/index.ts

@@ -0,0 +1 @@
+// place files you want to import through the `$lib` alias in this folder.

frontend/src/lib/lcmLive.ts

@@ -0,0 +1,97 @@
+import { writable } from 'svelte/store';
+
+export enum LCMLiveStatus {
+  CONNECTED = 'connected',
+  DISCONNECTED = 'disconnected',
+  WAIT = 'wait',
+  SEND_FRAME = 'send_frame',
+  TIMEOUT = 'timeout'
+}
+
+const initStatus: LCMLiveStatus = LCMLiveStatus.DISCONNECTED;
+
+export const lcmLiveStatus = writable<LCMLiveStatus>(initStatus);
+export const streamId = writable<string | null>(null);
+
+let websocket: WebSocket | null = null;
+export const lcmLiveActions = {
+  async start(getSreamdata: () => any[]) {
+    return new Promise((resolve, reject) => {
+      try {
+        const userId = crypto.randomUUID();
+        const websocketURL = `${
+          window.location.protocol === 'https:' ? 'wss' : 'ws'
+        }:${window.location.host}/api/ws/${userId}`;
+
+        websocket = new WebSocket(websocketURL);
+        websocket.onopen = () => {
+          console.log('Connected to websocket');
+        };
+        websocket.onclose = () => {
+          lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+          console.log('Disconnected from websocket');
+        };
+        websocket.onerror = (err) => {
+          console.error(err);
+        };
+        websocket.onmessage = (event) => {
+          const data = JSON.parse(event.data);
+          switch (data.status) {
+            case 'connected':
+              lcmLiveStatus.set(LCMLiveStatus.CONNECTED);
+              streamId.set(userId);
+              resolve({ status: 'connected', userId });
+              break;
+            case 'send_frame':
+              lcmLiveStatus.set(LCMLiveStatus.SEND_FRAME);
+              const streamData = getSreamdata();
+              websocket?.send(JSON.stringify({ status: 'next_frame' }));
+              for (const d of streamData) {
+                this.send(d);
+              }
+              break;
+            case 'wait':
+              lcmLiveStatus.set(LCMLiveStatus.WAIT);
+              break;
+            case 'timeout':
+              console.log('timeout');
+              lcmLiveStatus.set(LCMLiveStatus.TIMEOUT);
+              streamId.set(null);
+              reject(new Error('timeout'));
+              break;
+            case 'error':
+              console.log(data.message);
+              lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+              streamId.set(null);
+              reject(new Error(data.message));
+              break;
+          }
+        };
+      } catch (err) {
+        console.error(err);
+        lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+        streamId.set(null);
+        reject(err);
+      }
+    });
+  },
+  send(data: Blob | { [key: string]: any }) {
+    if (websocket && websocket.readyState === WebSocket.OPEN) {
+      if (data instanceof Blob) {
+        websocket.send(data);
+      } else {
+        websocket.send(JSON.stringify(data));
+      }
+    } else {
+      console.log('WebSocket not connected');
+    }
+  },
+  async stop() {
+    lcmLiveStatus.set(LCMLiveStatus.DISCONNECTED);
+    if (websocket) {
+      websocket.close();
+    }
+    websocket = null;
+    streamId.set(null);
+  }
+};

frontend/src/lib/mediaStream.ts

@@ -0,0 +1,118 @@
+import { get, writable, type Writable } from 'svelte/store';
+
+const BASE_HEIGHT = 720;
+export enum MediaStreamStatusEnum {
+  INIT = 'init',
+  CONNECTED = 'connected',
+  DISCONNECTED = 'disconnected'
+}
+export const onFrameChangeStore: Writable<{ blob: Blob }> = writable({ blob: new Blob() });
+
+export const mediaDevices = writable<MediaDeviceInfo[]>([]);
+export const mediaStreamStatus = writable(MediaStreamStatusEnum.INIT);
+export const mediaStream = writable<MediaStream | null>(null);
+
+export const mediaStreamActions = {
+  async enumerateDevices() {
+    // console.log("Enumerating devices");
+    await navigator.mediaDevices
+      .enumerateDevices()
+      .then((devices) => {
+        const cameras = devices.filter((device) => device.kind === 'videoinput');
+        mediaDevices.set(cameras);
+      })
+      .catch((err) => {
+        console.error(err);
+      });
+  },
+  async start(mediaDevicedID?: string, aspectRatio: number = 1) {
+    const constraints = {
+      audio: false,
+      video: {
+        width: {
+          ideal: BASE_HEIGHT * aspectRatio
+        },
+        height: {
+          ideal: BASE_HEIGHT
+        },
+        deviceId: mediaDevicedID
+      }
+    };
+
+    await navigator.mediaDevices
+      .getUserMedia(constraints)
+      .then((stream) => {
+        mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+        mediaStream.set(stream);
+      })
+      .catch((err) => {
+        console.error(`${err.name}: ${err.message}`);
+        mediaStreamStatus.set(MediaStreamStatusEnum.DISCONNECTED);
+        mediaStream.set(null);
+      });
+  },
+  async startScreenCapture() {
+    const displayMediaOptions = {
+      video: {
+        displaySurface: 'window'
+      },
+      audio: false,
+      surfaceSwitching: 'include'
+    };
+
+    let captureStream = null;
+
+    try {
+      captureStream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
+      const videoTrack = captureStream.getVideoTracks()[0];
+
+      console.log('Track settings:');
+      console.log(JSON.stringify(videoTrack.getSettings(), null, 2));
+      console.log('Track constraints:');
+      console.log(JSON.stringify(videoTrack.getConstraints(), null, 2));
+      mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+      mediaStream.set(captureStream);
+
+      const capabilities = videoTrack.getCapabilities();
+      const aspectRatio = capabilities.aspectRatio;
+      console.log('Aspect Ratio Constraints:', aspectRatio);
+    } catch (err) {
+      console.error(err);
+    }
+  },
+  async switchCamera(mediaDevicedID: string, aspectRatio: number) {
+    console.log('Switching camera');
+    if (get(mediaStreamStatus) !== MediaStreamStatusEnum.CONNECTED) {
+      return;
+    }
+    const constraints = {
+      audio: false,
+      video: {
+        width: {
+          ideal: BASE_HEIGHT * aspectRatio
+        },
+        height: {
+          ideal: BASE_HEIGHT
+        },
+        deviceId: mediaDevicedID
+      }
+    };
+    console.log('Switching camera', constraints);
+    await navigator.mediaDevices
+      .getUserMedia(constraints)
+      .then((stream) => {
+        mediaStreamStatus.set(MediaStreamStatusEnum.CONNECTED);
+        mediaStream.set(stream);
+      })
+      .catch((err) => {
+        console.error(`${err.name}: ${err.message}`);
+      });
+  },
+  async stop() {
+    navigator.mediaDevices.getUserMedia({ video: true }).then((stream) => {
+      stream.getTracks().forEach((track) => track.stop());
+    });
+    mediaStreamStatus.set(MediaStreamStatusEnum.DISCONNECTED);
+    mediaStream.set(null);
+  }
+};

frontend/src/lib/store.ts

@@ -0,0 +1,14 @@
+import { derived, get, writable, type Readable, type Writable } from 'svelte/store';
+
+export const pipelineValues: Writable<Record<string, any>> = writable({});
+export const deboucedPipelineValues: Readable<Record<string, any>> = derived(
+  pipelineValues,
+  ($pipelineValues, set) => {
+    const debounced = setTimeout(() => {
+      set($pipelineValues);
+    }, 100);
+    return () => clearTimeout(debounced);
+  }
+);
+
+export const getPipelineValues = () => get(pipelineValues);

frontend/src/lib/types.ts

@@ -0,0 +1,39 @@
+export const enum FieldType {
+  RANGE = 'range',
+  SEED = 'seed',
+  TEXTAREA = 'textarea',
+  CHECKBOX = 'checkbox',
+  SELECT = 'select'
+}
+export const enum PipelineMode {
+  IMAGE = 'image',
+  VIDEO = 'video',
+  TEXT = 'text'
+}
+
+export interface Fields {
+  [key: string]: FieldProps;
+}
+
+export interface FieldProps {
+  default: number | string;
+  max?: number;
+  min?: number;
+  title: string;
+  field: FieldType;
+  step?: number;
+  disabled?: boolean;
+  hide?: boolean;
+  id: string;
+  values?: string[];
+}
+export interface PipelineInfo {
+  title: {
+    default: string;
+  };
+  name: string;
+  description: string;
+  input_mode: {
+    default: PipelineMode;
+  };
+}

frontend/src/lib/utils.ts

@@ -0,0 +1,98 @@
+import * as piexif from 'piexifjs';
+
+interface IImageInfo {
+  prompt?: string;
+  negative_prompt?: string;
+  seed?: number;
+  guidance_scale?: number;
+}
+export enum windowType {
+  image = 'image',
+  video = 'video'
+}
+
+export function snapImage(imageEl: HTMLImageElement, info: IImageInfo) {
+  try {
+    const zeroth: { [key: string]: any } = {};
+    const exif: { [key: string]: any } = {};
+    const gps: { [key: string]: any } = {};
+    zeroth[piexif.ImageIFD.Make] = 'LCM Image-to-Image ControNet';
+    zeroth[piexif.ImageIFD.ImageDescription] =
+      `prompt: ${info?.prompt} | negative_prompt: ${info?.negative_prompt} | seed: ${info?.seed} | guidance_scale: ${info?.guidance_scale}`;
+    zeroth[piexif.ImageIFD.Software] =
+      'https://github.com/radames/Real-Time-Latent-Consistency-Model';
+    exif[piexif.ExifIFD.DateTimeOriginal] = new Date().toISOString();
+
+    const exifObj = { '0th': zeroth, Exif: exif, GPS: gps };
+    const exifBytes = piexif.dump(exifObj);
+
+    const canvas = document.createElement('canvas');
+    canvas.width = imageEl.naturalWidth;
+    canvas.height = imageEl.naturalHeight;
+    const ctx = canvas.getContext('2d') as CanvasRenderingContext2D;
+    ctx.drawImage(imageEl, 0, 0);
+    const dataURL = canvas.toDataURL('image/jpeg');
+    const withExif = piexif.insert(exifBytes, dataURL);
+
+    const a = document.createElement('a');
+    a.href = withExif;
+    a.download = `lcm_txt_2_img${Date.now()}.png`;
+    a.click();
+  } catch (err) {
+    console.log(err);
+  }
+}
+
+export function expandWindow(streamURL: string, type: windowType = windowType.image) {
+  const newWindow = window.open(
+    '',
+    '_blank',
+    'width=1024,height=1024,scrollbars=0,resizable=1,toolbar=0,menubar=0,location=0,directories=0,status=0'
+  ) as Window;
+
+  const html = `
+      <html>
+          <head>
+              <title>Real-Time Latent Consistency Model</title>
+              <style>
+                  body {
+                      margin: 0;
+                      padding: 0;
+                      background-color: black;
+                  }
+              </style>
+          </head>
+          <body>
+              <script>
+                  let isFullscreen = false;
+                  window.onkeydown = function(event) {
+                      switch (event.code) {
+                          case "Escape":
+                              window.close();
+                              break;
+                          case "Enter":
+                              if (isFullscreen) {
+                                  document.exitFullscreen();
+                                  isFullscreen = false;
+                              } else {
+                                  document.documentElement.requestFullscreen();
+                                  isFullscreen = true;
+                              }
+                              break;
+                      }
+                  }
+              </script>
+          </body>
+      </html>
+      `;
+  newWindow.document.write(html);
+
+  const img = newWindow.document.createElement('img');
+  img.src = streamURL;
+  img.style.width = '100%';
+  img.style.height = '100%';
+  img.style.objectFit = 'contain';
+  newWindow.document.body.appendChild(img);
+
+  return newWindow;
+}

frontend/src/routes/+layout.svelte

@@ -0,0 +1,5 @@
+<script>
+  import '../app.css';
+</script>
+
+<slot />

frontend/src/routes/+page.svelte

@@ -0,0 +1,152 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+  import type { Fields, PipelineInfo } from '$lib/types';
+  import { PipelineMode } from '$lib/types';
+  import ImagePlayer from '$lib/components/ImagePlayer.svelte';
+  import VideoInput from '$lib/components/VideoInput.svelte';
+  import Button from '$lib/components/Button.svelte';
+  import PipelineOptions from '$lib/components/PipelineOptions.svelte';
+  import Spinner from '$lib/icons/spinner.svelte';
+  import Warning from '$lib/components/Warning.svelte';
+  import { lcmLiveStatus, lcmLiveActions, LCMLiveStatus } from '$lib/lcmLive';
+  import { mediaStreamActions, onFrameChangeStore } from '$lib/mediaStream';
+  import { getPipelineValues, deboucedPipelineValues } from '$lib/store';
+
+  let pipelineParams: Fields;
+  let pipelineInfo: PipelineInfo;
+  let pageContent: string;
+  let isImageMode: boolean = false;
+  let maxQueueSize: number = 0;
+  let currentQueueSize: number = 0;
+  let queueCheckerRunning: boolean = false;
+  let warningMessage: string = '';
+  onMount(() => {
+    getSettings();
+  });
+
+  async function getSettings() {
+    const settings = await fetch('/api/settings').then((r) => r.json());
+    pipelineParams = settings.input_params.properties;
+    pipelineInfo = settings.info.properties;
+    isImageMode = pipelineInfo.input_mode.default === PipelineMode.IMAGE;
+    maxQueueSize = settings.max_queue_size;
+    pageContent = settings.page_content;
+    console.log(pipelineParams);
+    toggleQueueChecker(true);
+  }
+  function toggleQueueChecker(start: boolean) {
+    queueCheckerRunning = start && maxQueueSize > 0;
+    if (start) {
+      getQueueSize();
+    }
+  }
+  async function getQueueSize() {
+    if (!queueCheckerRunning) {
+      return;
+    }
+    const data = await fetch('/api/queue').then((r) => r.json());
+    currentQueueSize = data.queue_size;
+    setTimeout(getQueueSize, 10000);
+  }
+
+  function getSreamdata() {
+    if (isImageMode) {
+      return [getPipelineValues(), $onFrameChangeStore?.blob];
+    } else {
+      return [$deboucedPipelineValues];
+    }
+  }
+
+  $: isLCMRunning = $lcmLiveStatus !== LCMLiveStatus.DISCONNECTED;
+  $: if ($lcmLiveStatus === LCMLiveStatus.TIMEOUT) {
+    warningMessage = 'Session timed out. Please try again.';
+  }
+  let disabled = false;
+  async function toggleLcmLive() {
+    try {
+      if (!isLCMRunning) {
+        if (isImageMode) {
+          await mediaStreamActions.enumerateDevices();
+          await mediaStreamActions.start();
+        }
+        disabled = true;
+        await lcmLiveActions.start(getSreamdata);
+        disabled = false;
+        toggleQueueChecker(false);
+      } else {
+        if (isImageMode) {
+          mediaStreamActions.stop();
+        }
+        lcmLiveActions.stop();
+        toggleQueueChecker(true);
+      }
+    } catch (e) {
+      warningMessage = e instanceof Error ? e.message : '';
+      disabled = false;
+      toggleQueueChecker(true);
+    }
+  }
+</script>
+
+<svelte:head>
+  <script
+    src="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.9/iframeResizer.contentWindow.min.js"
+  ></script>
+</svelte:head>
+
+<main class="container mx-auto flex max-w-5xl flex-col gap-3 px-4 py-4">
+  <Warning bind:message={warningMessage}></Warning>
+  <article class="text-center">
+    {#if pageContent}
+      {@html pageContent}
+    {/if}
+    {#if maxQueueSize > 0}
+      <p class="text-sm">
+        There are <span id="queue_size" class="font-bold">{currentQueueSize}</span>
+        user(s) sharing the same GPU, affecting real-time performance. Maximum queue size is {maxQueueSize}.
+        <a
+          href="https://huggingface.co/spaces/radames/Real-Time-Latent-Consistency-Model?duplicate=true"
+          target="_blank"
+          class="text-blue-500 underline hover:no-underline">Duplicate</a
+        > and run it on your own GPU.
+      </p>
+    {/if}
+  </article>
+  {#if pipelineParams}
+    <article class="my-3 grid grid-cols-1 gap-3 sm:grid-cols-4">
+      {#if isImageMode}
+        <div class="col-span-2 sm:col-start-1">
+          <VideoInput
+            width={Number(pipelineParams.width.default)}
+            height={Number(pipelineParams.height.default)}
+          ></VideoInput>
+        </div>
+      {/if}
+      <div class={isImageMode ? 'col-span-2 sm:col-start-3' : 'col-span-4'}>
+        <ImagePlayer />
+      </div>
+      <div class="sm:col-span-4 sm:row-start-2">
+        <Button on:click={toggleLcmLive} {disabled} classList={'text-lg my-1 p-2'}>
+          {#if isLCMRunning}
+            Stop
+          {:else}
+            Start
+          {/if}
+        </Button>
+        <PipelineOptions {pipelineParams}></PipelineOptions>
+      </div>
+    </article>
+  {:else}
+    <!-- loading -->
+    <div class="flex items-center justify-center gap-3 py-48 text-2xl">
+      <Spinner classList={'animate-spin opacity-50'}></Spinner>
+      <p>Loading...</p>
+    </div>
+  {/if}
+</main>
+
+<style lang="postcss">
+  :global(html) {
+    @apply text-black dark:bg-gray-900 dark:text-white;
+  }
+</style>

frontend/src/routes/+page.ts

@@ -0,0 +1 @@
+export const prerender = true;

frontend/svelte.config.js

@@ -0,0 +1,18 @@
+import adapter from '@sveltejs/adapter-static';
+import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
+
+/** @type {import('@sveltejs/kit').Config} */
+const config = {
+  preprocess: vitePreprocess({ postcss: true }),
+  kit: {
+    adapter: adapter({
+      pages: 'public',
+      assets: 'public',
+      fallback: undefined,
+      precompress: false,
+      strict: true
+    })
+  }
+};
+
+export default config;

frontend/tailwind.config.js

@@ -0,0 +1,8 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: ['./src/**/*.{html,js,svelte,ts}', '../pipelines/**/*.py'],
+  theme: {
+    extend: {}
+  },
+  plugins: []
+};

frontend/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "./.svelte-kit/tsconfig.json",
+  "compilerOptions": {
+    "allowJs": true,
+    "checkJs": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "strict": true
+  }
+  // Path aliases are handled by https://kit.svelte.dev/docs/configuration#alias
+  //
+  // If you want to overwrite includes/excludes, make sure to copy over the relevant includes/excludes
+  // from the referenced tsconfig.json - TypeScript does not merge them in
+}

frontend/vite.config.ts

@@ -0,0 +1,15 @@
+import { sveltekit } from '@sveltejs/kit/vite';
+import { defineConfig } from 'vite';
+
+export default defineConfig({
+  plugins: [sveltekit()],
+  server: {
+    proxy: {
+      '/api': 'http://localhost:7860',
+      '/api/ws': {
+        target: 'ws://localhost:7860',
+        ws: true
+      }
+    }
+  }
+});