work on clap file support

src/clap/parseClap.ts

@@ -2,7 +2,7 @@ import YAML from "yaml"
 import { v4 as uuidv4 } from "uuid"
 
 import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapSegment } from "./types"
-import { getValidNumber } from "@/lib/getValidNumber";
+import { getValidNumber } from "@/lib/getValidNumber"
 
 /**
  * import a Clap file (from a plain text string)
@@ -50,7 +50,7 @@ export async function parseClap(inputStringOrBlob: string | Blob): Promise<ClapP
     width: getValidNumber(maybeClapMeta.width, 256, 8192, 1024),
     height: getValidNumber(maybeClapMeta.height, 256, 8192, 576),
     defaultVideoModel: typeof maybeClapMeta.defaultVideoModel === "string" ? maybeClapMeta.defaultVideoModel : "SVD",
-    extraPositivePrompt: Array.isArray(maybeClapMeta.extraPositivePrompt) ? maybeClapMeta.extraPositivePrompt : [],
+    extraPositivePrompt: Array.isArray(maybeClapMeta.extraPositivePrompt) ? maybeClapMeta.extraPositivePrompt : []
   }
 
   /*
@@ -83,31 +83,39 @@ export async function parseClap(inputStringOrBlob: string | Blob): Promise<ClapP
 
   const clapModels: ClapModel[] = maybeModels.map(({
     id,
-    imageType,
-    audioType,
     category,
     triggerName,
     label,
     description,
     author,
     thumbnailUrl,
-    storageUrl,
-    imagePrompt,
-    audioPrompt,
+    seed,
+    assetSourceType,
+    assetUrl,
+    age,
+    gender,
+    region,
+    appearance,
+    voiceVendor,
+    voiceId,
   }) => ({
     // TODO: we should verify each of those, probably
     id,
-    imageType,
-    audioType,
     category,
     triggerName,
     label,
     description,
     author,
     thumbnailUrl,
-    storageUrl,
-    imagePrompt,
-    audioPrompt,
+    seed,
+    assetSourceType,
+    assetUrl,
+    age,
+    gender,
+    region,
+    appearance,
+    voiceVendor,
+    voiceId,
   }))
 
 
@@ -148,3 +156,4 @@ export async function parseClap(inputStringOrBlob: string | Blob): Promise<ClapP
     segments: clapSegments
   }
 }
+

src/clap/serializeClap.ts

@@ -14,30 +14,38 @@ export async function serializeClap({
   // to make sure we generate a valid clap file
   const clapModels: ClapModel[] = models.map(({
     id,
-    imageType,
-    audioType,
     category,
     triggerName,
     label,
     description,
     author,
     thumbnailUrl,
-    storageUrl,
-    imagePrompt,
-    audioPrompt,
+    seed,
+    assetSourceType,
+    assetUrl,
+    age,
+    gender,
+    region,
+    appearance,
+    voiceVendor,
+    voiceId,
   }) => ({
     id,
-    imageType,
-    audioType,
     category,
     triggerName,
     label,
     description,
     author,
     thumbnailUrl,
-    storageUrl,
-    imagePrompt,
-    audioPrompt,
+    seed,
+    assetSourceType,
+    assetUrl,
+    age,
+    gender,
+    region,
+    appearance,
+    voiceVendor,
+    voiceId,
   }))
 
   const clapSegments: ClapSegment[] = segments.map(({

src/clap/types.ts

@@ -2,8 +2,60 @@
 export type ClapSegmentCategory = "render" | "preview" | "characters" | "location" | "time" | "era" | "lighting" | "weather" | "action" | "music" | "sound" | "dialogue" | "style" | "camera" | "generic"
 export type ClapOutputType = "text" | "movement" | "image" | "video" | "audio"
 export type ClapSegmentStatus = "pending" | "completed" | "error"
-export type ClapImageType = "reference_image" | "text_prompt" | "other"
-export type ClapAudioType =  "reference_audio" | "text_prompt" | "other"
+
+export type ClapAssetSource =
+  | "REMOTE" // http:// or https://
+
+    // note that "path" assets are potentially a security risk, they need to be treated with care
+  | "PATH" // a file path eg. /path or ./path/to/ or ../path/to/
+
+  | "DATA" // a data URI, starting with data:
+
+  | "PROMPT" // by default, a plain text prompt
+
+  | "EMPTY"
+
+export type ClapModelGender =
+  | "male"
+  | "female"
+  | "person"
+  | "object"
+
+export type ClapModelAppearance = "serious" | "neutral" | "friendly" | "chill"
+
+// this is used for accent, style..
+export type ClapModelRegion =
+  | "american"
+  | "british"
+  | "australian"
+  | "canadian"
+  | "indian"
+  | "french"
+  | "italian"
+  | "german"
+  | "chinese"
+
+// note: this is all very subjective, so please use good judgment
+//
+// "deep" might indicate a deeper voice tone, thicker, rich in harmonics
+// in this context, it is used to indicate voices that could
+// be associated with African American (AADOS) characters
+//
+// "high" could be used for some other countries, eg. asia
+export type ClapModelTimbre = "high" | "neutral" | "deep"
+
+export type ClapVoiceVendor = "ElevenLabs" | "XTTS"
+
+export type ClapVoice = {
+  name: string
+  gender: ClapModelGender
+  age: number
+  region: ClapModelRegion
+  timbre: ClapModelTimbre
+  appearance: ClapModelAppearance
+  voiceVendor: ClapVoiceVendor
+  voiceId: string
+}
 
 export type ClapHeader = {
   format: "clap-0"
@@ -41,17 +93,24 @@ export type ClapSegment = {
 
 export type ClapModel = {
   id: string
-  imageType: ClapImageType
-  audioType: ClapAudioType
   category: ClapSegmentCategory
   triggerName: string
   label: string
   description: string
   author: string
   thumbnailUrl: string
-  storageUrl: string
-  imagePrompt: string
-  audioPrompt: string
+  seed: number
+
+  assetSourceType: ClapAssetSource
+  assetUrl: string
+  
+  // those are only used by certain types of models
+  age: number
+  gender: ClapModelGender
+  region: ClapModelRegion
+  appearance: ClapModelAppearance
+  voiceVendor: ClapVoiceVendor
+  voiceId: string
 }
 
 export type ClapProject = {

src/lib/getClapAssetSourceType.ts

@@ -0,0 +1,24 @@
+import { ClapAssetSource } from "@/clap/types"
+
+export function getClapAssetSourceSource(input: string): ClapAssetSource {
+  
+  const str = `${input || ""}`
+  if (str.startsWith("https://") || str.startsWith("http://")) {
+    return "REMOTE"
+  }
+
+  // note that "path" assets are potentially a security risk, they need to be treated with care
+  if (str.startsWith("/") || str.startsWith("../") || str.startsWith("./")) {
+    return "PATH"
+  }
+
+  if (str.startsWith("data:")) {
+    return "DATA"
+  }
+
+  if (!str) {
+    return "EMPTY"
+  }
+
+  return "PROMPT"
+}