try to fix the twitter embed

src/app/embed/page.tsx

@@ -0,0 +1,85 @@
+
+import { Metadata, ResolvingMetadata } from "next"
+
+import { AppQueryProps } from "@/types/general"
+
+import { Main } from "../main"
+import { getVideo } from "../server/actions/ai-tube-hf/getVideo"
+
+
+// https://nextjs.org/docs/pages/building-your-application/optimizing/fonts 
+export async function generateMetadata(
+  { params, searchParams: { v: videoId } }: AppQueryProps,
+  parent: ResolvingMetadata
+): Promise<Metadata> {
+  // read route params
+
+  const metadataBase = new URL('https://huggingface.co/spaces/jbilcke-hf/ai-tube')
+
+  try {
+    const video = await getVideo({ videoId, neverThrow: true })
+
+    if (!video) {
+      throw new Error("Video not found")
+    }
+
+    return {
+      title: `${video.label} - AiTube`,
+      metadataBase,
+      openGraph: {
+        // some cool stuff we could use here:
+        // 'video.tv_show' | 'video.other' | 'video.movie' | 'video.episode';
+        type: "video.other",
+        // url: "https://example.com",
+        title: video.label || "", // put the video title here
+        description: video.description || "", // put the video description here
+        siteName: "AiTube",
+        images: [
+          `https://huggingface.co/datasets/jbilcke-hf/ai-tube-index/resolve/main/videos/${video.id}.webp`
+        ],
+        videos: [
+          {
+            "url": video.assetUrl
+          }
+        ],
+        // images: ['/some-specific-page-image.jpg', ...previousImages],
+      },
+      twitter: {
+        card: "player",
+        site: "@flngr",
+        description: video.description || "", 
+        images: `https://huggingface.co/datasets/jbilcke-hf/ai-tube-index/resolve/main/videos/${video.id}.webp`,
+        players: {
+          playerUrl: `https://jbilcke-hf-ai-tube.hf.space/embed?v=${video.id}`,
+          streamUrl: `https://huggingface.co/datasets/jbilcke-hf/ai-tube-index/resolve/main/videos/${video.id}.mp4`,
+          width: 1024,
+          height: 576
+        }
+      }
+    }
+  } catch (err) {
+    return {
+      title: "AiTube",
+      metadataBase,
+      openGraph: {
+        type: "website",
+        // url: "https://example.com",
+        title: "AiTube", // put the video title here
+        description: "", // put the vide description here
+        siteName: "AiTube",
+  
+        videos: [],
+        images: [],
+      },
+    }
+  }
+}
+
+
+export default async function Embed({ searchParams: { v: videoId } }: AppQueryProps) {
+  const publicVideo = await getVideo({ videoId, neverThrow: true })
+  // console.log("WatchPage: --> " + video?.id)
+  return (
+    <Main publicVideo={publicVideo} />
+   )
+}

src/app/interface/top-header/index.tsx

@@ -37,7 +37,9 @@ export function TopHeader() {
 
 
   useEffect(() => {
-    if (view === "public_video" ||  view === "public_channel" || view === "public_music_videos") {
+    if (view === "public_video_embed") {
+      setHeaderMode("hidden")
+    } else if (view === "public_video" || view === "public_channel" || view === "public_music_videos") {
       setHeaderMode("compact")
       setMenuMode("slider_hidden")
     } else {
@@ -53,6 +55,10 @@ export function TopHeader() {
     })
   }, [])
   
+  if (headerMode === "hidden") {
+    return null
+  }
+
   return (
     <div className={cn(
       `flex flex-col`,

src/app/interface/tube-layout/index.tsx

@@ -12,6 +12,15 @@ import { TopHeader } from "../top-header"
 export function TubeLayout({ children }: { children?: ReactNode }) {
   const headerMode = useStore(s => s.headerMode)
   const view = useStore(s => s.view)
+  if (headerMode === "hidden") {
+    return (
+      <div className={cn(
+        `dark flex flex-row h-screen w-screen inset-0 overflow-hidden`
+      )}>
+        {children}
+      </div>
+    ) 
+  }
   return (
     <div className={cn(
       `dark flex flex-row h-screen w-screen inset-0 overflow-hidden`,

src/app/main.tsx

@@ -8,12 +8,13 @@ import { UserChannelView } from "./views/user-channel-view"
 import { PublicVideoView } from "./views/public-video-view"
 import { UserAccountView } from "./views/user-account-view"
 import { NotFoundView } from "./views/not-found-view"
-import { ChannelInfo, InterfaceView, VideoInfo } from "@/types/general"
+import { ChannelInfo, VideoInfo } from "@/types/general"
 import { useEffect } from "react"
 import { usePathname, useRouter } from "next/navigation"
 import { TubeLayout } from "./interface/tube-layout"
 import { PublicMusicVideosView } from "./views/public-music-videos-view"
 import { getCollectionKey } from "@/lib/getCollectionKey"
+import { PublicVideoEmbedView } from "./views/public-video-embed-view"
 
 // this is where we transition from the server-side space
 // and the client-side space
@@ -99,6 +100,7 @@ export function Main({
 
     if (!publicVideo || !publicVideo?.id) { return }
 
+    if (pathname === "/embed") { return }
     // this is a hack for hugging face:
     // we allow the ?v=<id> param on the root of the domain
     if (pathname !== "/watch") {
@@ -138,6 +140,7 @@ export function Main({
   return (
     <TubeLayout>
       {view === "home" && <HomeView />}
+      {view === "public_video_embed" && <PublicVideoEmbedView />}
       {view === "public_video" && <PublicVideoView />}
       {view === "public_music_videos" && <PublicMusicVideosView />}
       {view === "public_channels" && <PublicChannelsView />}

src/app/server/actions/ai-tube-hf/downloadClapProject.ts

@@ -1,24 +1,24 @@
 import { v4 as uuidv4 } from "uuid"
+import { Credentials } from "@/huggingface/hub/src"
 
-import { ClapProject } from "@/types/clap"
+import { ClapProject } from "@/clap/types"
 import { ChannelInfo, VideoInfo, VideoRequest } from "@/types/general"
 import { defaultVideoModel } from "@/app/config"
+import { parseClap } from "@/clap/parseClap"
 
-import { parseClap } from "../utils/parseClap"
 import { parseVideoModelName } from "../utils/parseVideoModelName"
 import { computeOrientationProjectionWidthHeight } from "../utils/computeOrientationProjectionWidthHeight"
 
-import { downloadFileAsText } from "./downloadFileAsText"
 import { downloadFileAsBlob } from "./downloadFileAsBlob"
 
 export async function downloadClapProject({
   path,
-  apiKey,
-  channel
+  channel,
+  credentials,
 }: {
   path: string
-  apiKey?: string
   channel: ChannelInfo
+  credentials: Credentials
 }): Promise<{
   videoRequest: VideoRequest
   videoInfo: VideoInfo
@@ -31,7 +31,7 @@ export async function downloadClapProject({
   const clapString = await downloadFileAsBlob({
     repo,
     path,
-    apiKey,
+    apiKey: credentials.accessToken,
     expectedMimeType: "application/gzip"
   })
 

src/app/server/actions/ai-tube-hf/getVideoRequestsFromChannel.ts

@@ -59,7 +59,7 @@ export async function getVideoRequestsFromChannel({
           const clap = await downloadClapProject({
             path: file.path,
             channel,
-            apiKey
+            credentials,
           })
           console.log("got a clap file:", clap.clapProject.meta)
           

src/app/state/useStore.ts

@@ -91,6 +91,7 @@ export const useStore = create<{
     const routes: Record<string, InterfaceView> = {
       "/": "home",
       "/watch": "public_video",
+      "/embed": "public_video_embed",
       "/music": "public_music_videos",
       "/channels": "public_channels",
       "/channel": "public_channel",

src/app/views/public-video-embed-view/index.tsx

@@ -0,0 +1,71 @@
+"use client"
+
+import { useEffect, useTransition } from "react"
+
+import { useStore } from "@/app/state/useStore"
+import { cn } from "@/lib/utils"
+import { VideoPlayer } from "@/app/interface/video-player"
+
+import { watchVideo } from "@/app/server/actions/stats"
+
+export function PublicVideoEmbedView() {
+  const [_pending, startTransition] = useTransition()
+
+  // current time in the video
+  // note: this is used to *set* the current time, not to read it
+  // EDIT: you know what, let's do this the dirty way for now
+  // const [desiredCurrentTime, setDesiredCurrentTime] = useState()
+
+  const video = useStore(s => s.publicVideo)
+
+  const videoId = `${video?.id || ""}`
+
+  const setPublicVideo = useStore(s => s.setPublicVideo)
+
+  // we inject the current videoId in the URL, if it's not already present
+  // this is a hack for Hugging Face iframes
+  useEffect(() => {
+    const queryString = new URL(location.href).search
+    const searchParams = new URLSearchParams(queryString)
+    if (videoId) {
+      if (searchParams.get("v") !== videoId) {
+        console.log(`current videoId "${videoId}" isn't set in the URL query params.. TODO we should set it`)
+        
+        // searchParams.set("v", videoId)
+        // location.search = searchParams.toString()
+      }
+    } else {
+      // searchParams.delete("v")
+      // location.search = searchParams.toString()
+    }
+  }, [videoId])
+
+  useEffect(() => {
+    startTransition(async () => {
+      if (!video || !video.id) {
+        return
+      }
+      const numberOfViews = await watchVideo(videoId)
+
+      setPublicVideo({
+        ...video,
+        numberOfViews
+      })
+    })
+
+  }, [video?.id])
+
+  if (!video) { return null }
+  
+  return (
+    <div className={cn(
+      `w-full`,
+      `flex flex-col`
+    )}>
+      <VideoPlayer
+        video={video}
+        enableShortcuts={false}
+      />
+    </div>
+  )
+}

src/app/watch/page.tsx

@@ -44,6 +44,18 @@ export async function generateMetadata(
         ],
         // images: ['/some-specific-page-image.jpg', ...previousImages],
       },
+      twitter: {
+        card: "player",
+        site: "@flngr",
+        description: video.description || "", 
+        images: `https://huggingface.co/datasets/jbilcke-hf/ai-tube-index/resolve/main/videos/${video.id}.webp`,
+        players: {
+          playerUrl: `https://jbilcke-hf-ai-tube.hf.space/embed?v=${video.id}`,
+          streamUrl: `https://huggingface.co/datasets/jbilcke-hf/ai-tube-index/resolve/main/videos/${video.id}.mp4`,
+          width: 1024,
+          height: 576
+        }
+      }
     }
   } catch (err) {
     return {

src/{app/server/actions/utils → clap}/parseClap.ts

@@ -1,8 +1,8 @@
 import YAML from "yaml"
 import { v4 as uuidv4 } from "uuid"
 
-import { ClapHeader, ClapMeta, ClapProject, ClapSegment } from "@/types/clap"
-import { getValidNumber } from "@/lib/getValidNumber"
+import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapSegment } from "./types"
+import { getValidNumber } from "@/lib/getValidNumber";
 
 /**
  * import a Clap file (from a plain text string)
@@ -38,6 +38,7 @@ export async function parseClap(inputStringOrBlob: string | Blob): Promise<ClapP
     throw new Error("invalid clap file (sorry, but you can't make up version numbers like that)")
   }
 
+
   const maybeClapMeta = rawData[1] as ClapMeta
 
   const clapMeta: ClapMeta = {
@@ -46,14 +47,71 @@ export async function parseClap(inputStringOrBlob: string | Blob): Promise<ClapP
     description: typeof maybeClapMeta.description === "string" ? maybeClapMeta.description : "",
     licence: typeof maybeClapMeta.licence === "string" ? maybeClapMeta.licence : "",
     orientation: maybeClapMeta.orientation === "portrait" ? "portrait" : maybeClapMeta.orientation === "square" ? "square" : "landscape",
-    width: getValidNumber(maybeClapMeta.width, 256, 4096, 1024),
-    height: getValidNumber(maybeClapMeta.height, 256, 4096, 1024),
+    width: getValidNumber(maybeClapMeta.width, 256, 8192, 1024),
+    height: getValidNumber(maybeClapMeta.height, 256, 8192, 576),
     defaultVideoModel: typeof maybeClapMeta.defaultVideoModel === "string" ? maybeClapMeta.defaultVideoModel : "SVD",
+    extraPositivePrompt: Array.isArray(maybeClapMeta.extraPositivePrompt) ? maybeClapMeta.extraPositivePrompt : [],
   }
 
-  const maybeSegments = rawData.slice(2) as ClapSegment[]
+  /*
+  in case we want to support streaming (mix of models and segments etc), we could do it this way:
+
+  const maybeModelsOrSegments = rawData.slice(2)
+  maybeModelsOrSegments.forEach((unknownElement: any) => {
+    if (isValidNumber(unknownElement?.track)) {
+      maybeSegments.push(unknownElement as ClapSegment)
+    } else {
+      maybeModels.push(unknownElement as ClapModel)
+    }
+  })
+  */
+
+
+  const expectedNumberOfModels = maybeClapHeader.numberOfModels || 0
+  const expectedNumberOfSegments = maybeClapHeader.numberOfSegments || 0
+
+  // note: we assume the order is strictly enforced!
+  // if you implement streaming (mix of models and segments) you will have to rewrite this!
+
+  const afterTheHeaders = 2
+  const afterTheModels = afterTheHeaders + expectedNumberOfModels
+
+  // note: if there are no expected models, maybeModels will be empty
+  const maybeModels = rawData.slice(afterTheHeaders, afterTheModels) as ClapModel[]
+
+  const maybeSegments = rawData.slice(afterTheModels) as ClapSegment[]
+
+  const clapModels: ClapModel[] = maybeModels.map(({
+    id,
+    imageType,
+    audioType,
+    category,
+    triggerName,
+    label,
+    description,
+    author,
+    thumbnailUrl,
+    storageUrl,
+    imagePrompt,
+    audioPrompt,
+  }) => ({
+    // TODO: we should verify each of those, probably
+    id,
+    imageType,
+    audioType,
+    category,
+    triggerName,
+    label,
+    description,
+    author,
+    thumbnailUrl,
+    storageUrl,
+    imagePrompt,
+    audioPrompt,
+  }))
+
 
-  const clapSegments: ClapSegment[] = Array.isArray(maybeSegments) ? maybeSegments.map(({
+  const clapSegments: ClapSegment[] = maybeSegments.map(({
     id,
     track,
     startTimeInMs,
@@ -82,10 +140,11 @@ export async function parseClap(inputStringOrBlob: string | Blob): Promise<ClapP
     assetUrl,
     outputGain,
     seed,
-  })) : []
+  }))
 
   return {
     meta: clapMeta,
+    models: clapModels,
     segments: clapSegments
   }
 }

src/clap/serializeClap.ts

@@ -0,0 +1,114 @@
+import YAML from "yaml"
+import { v4 as uuidv4 } from "uuid"
+
+import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapSegment } from "./types"
+import { getValidNumber } from "@/lib/getValidNumber"
+
+export async function serializeClap({
+  meta, // ClapMeta
+  models, // ClapModel[]
+  segments, // ClapSegment[]
+}: ClapProject): Promise<Blob> {
+  
+  // we play it safe, and we verify the structure of the parameters,
+  // to make sure we generate a valid clap file
+  const clapModels: ClapModel[] = models.map(({
+    id,
+    imageType,
+    audioType,
+    category,
+    triggerName,
+    label,
+    description,
+    author,
+    thumbnailUrl,
+    storageUrl,
+    imagePrompt,
+    audioPrompt,
+  }) => ({
+    id,
+    imageType,
+    audioType,
+    category,
+    triggerName,
+    label,
+    description,
+    author,
+    thumbnailUrl,
+    storageUrl,
+    imagePrompt,
+    audioPrompt,
+  }))
+
+  const clapSegments: ClapSegment[] = segments.map(({
+    id,
+    track,
+    startTimeInMs,
+    endTimeInMs,
+    category,
+    modelId,
+    prompt,
+    outputType,
+    renderId,
+    status,
+    assetUrl,
+    outputGain,
+    seed,
+  }) => ({
+    id,
+    track,
+    startTimeInMs,
+    endTimeInMs,
+    category,
+    modelId,
+    prompt,
+    outputType,
+    renderId,
+    status,
+    assetUrl,
+    outputGain,
+    seed,
+  }))
+
+  const clapHeader: ClapHeader = {
+    format: "clap-0",
+    numberOfModels: clapModels.length,
+    numberOfSegments: clapSegments.length,
+  }
+
+  const clapMeta: ClapMeta = {
+    id: meta.id || uuidv4(),
+    title: typeof meta.title === "string" ? meta.title : "Untitled",
+    description: typeof meta.description === "string" ? meta.description : "",
+    licence: typeof meta.licence === "string" ? meta.licence : "",
+    orientation: meta.orientation === "portrait" ? "portrait" : meta.orientation === "square" ? "square" : "landscape",
+    width: getValidNumber(meta.width, 256, 8192, 1024),
+    height: getValidNumber(meta.height, 256, 8192, 576),
+    defaultVideoModel:  typeof meta.defaultVideoModel === "string" ? meta.defaultVideoModel : "SVD",
+    extraPositivePrompt: Array.isArray(meta.extraPositivePrompt) ? meta.extraPositivePrompt : [],
+  }
+
+  const entries = [
+    clapHeader,
+    clapMeta,
+    ...clapModels,
+    ...clapSegments
+  ]
+
+  const strigifiedResult = YAML.stringify(entries)
+
+  // Convert the string to a Blob
+  const blobResult = new Blob([strigifiedResult], { type: "application/x-yaml" })
+
+   // Create a stream for the blob
+   const readableStream = blobResult.stream();
+
+   // Compress the stream using gzip
+   const compressionStream = new CompressionStream('gzip');
+   const compressedStream = readableStream.pipeThrough(compressionStream);
+
+   // Create a new blob from the compressed stream
+   const compressedBlob = await new Response(compressedStream).blob();
+
+  return compressedBlob
+}

src/{types/clap.ts → clap/types.ts}

@@ -1,5 +1,14 @@
+
+export type ClapSegmentCategory = "render" | "preview" | "characters" | "location" | "time" | "era" | "lighting" | "weather" | "action" | "music" | "sound" | "dialogue" | "style" | "camera" | "generic"
+export type ClapOutputType = "text" | "movement" | "image" | "video" | "audio"
+export type ClapSegmentStatus = "pending" | "completed" | "error"
+export type ClapImageType = "reference_image" | "text_prompt" | "other"
+export type ClapAudioType =  "reference_audio" | "text_prompt" | "other"
+
 export type ClapHeader = {
   format: "clap-0"
+  numberOfModels: number
+  numberOfSegments: number
 }
 
 export type ClapMeta = {
@@ -11,6 +20,7 @@ export type ClapMeta = {
   width: number
   height: number
   defaultVideoModel: string
+  extraPositivePrompt: string[]
 }
 
 export type ClapSegment = {
@@ -18,18 +28,35 @@ export type ClapSegment = {
   track: number
   startTimeInMs: number
   endTimeInMs: number
-  category: "render" | "preview" | "characters" | "location" | "time" | "era" | "lighting" | "weather" | "action" | "music" | "sound" | "dialogue" | "style" | "camera" | "generic"
+  category: ClapSegmentCategory
   modelId: string
   prompt: string
-  outputType: "text" | "movement" | "image" | "video" | "audio"
+  outputType: ClapOutputType
   renderId: string
-  status: "pending" | "completed" | "error"
+  status: ClapSegmentStatus
   assetUrl: string
   outputGain: number
   seed: number
 }
 
+export type ClapModel = {
+  id: string
+  imageType: ClapImageType
+  audioType: ClapAudioType
+  category: ClapSegmentCategory
+  triggerName: string
+  label: string
+  description: string
+  author: string
+  thumbnailUrl: string
+  storageUrl: string
+  imagePrompt: string
+  audioPrompt: string
+}
+
 export type ClapProject = {
   meta: ClapMeta
+  models: ClapModel[]
   segments: ClapSegment[]
+  // let's keep room for other stuff (screenplay etc)
 }

src/lib/extractBase64.ts

@@ -0,0 +1,33 @@
+/**
+ * break a base64 string into sub-components
+ */
+export function extractBase64(base64: string = ""): {
+  mimetype: string;
+  extension: string;
+  data: string;
+  buffer: Buffer;
+  blob: Blob;
+} {
+  // Regular expression to extract the MIME type and the base64 data
+  const matches = base64.match(/^data:([A-Za-z-+/]+);base64,(.+)$/)
+
+  if (!matches || matches.length !== 3) {
+    throw new Error("Invalid base64 string")
+  }
+   
+  const mimetype = matches[1] || ""
+  const data = matches[2] || ""
+  const buffer = Buffer.from(data, "base64")
+  const blob = new Blob([buffer])
+
+  // this should be enough for most media formats (jpeg, png, webp, mp4)
+  const extension = mimetype.split("/").pop() || ""
+
+  return {
+    mimetype,
+    extension,
+    data,
+    buffer,
+    blob,
+  }
+}

src/types/general.ts

@@ -608,6 +608,7 @@ export type InterfaceDisplayMode =
   | "tv"
 
 export type InterfaceHeaderMode =
+  | "hidden"
   | "normal"
   | "compact"
 
@@ -627,6 +628,7 @@ export type InterfaceView =
   | "public_channels"
   | "public_channel" // public view of a channel
   | "public_video" // public view of a video
+  | "public_video_embed" // for integration into twitter etc
   | "public_music_videos" // public music videos - it's a special category, because music is *cool*
   | "not_found"