Create Audio Video combnation

app.py

@@ -1,26 +1,15 @@
 from langchain.llms import HuggingFacePipeline
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, AutoModelForSeq2SeqLM
 
-from components import caption_chain, tag_chain
 from components import pexels, utils
 import os, gc
 import gradio as gr
+from transformers import VitsModel, AutoTokenizer, pipeline
+import torch
 
-model = AutoModelForSeq2SeqLM.from_pretrained("declare-lab/flan-alpaca-gpt4-xl")
-tokenizer = AutoTokenizer.from_pretrained("declare-lab/flan-alpaca-gpt4-xl")
-
-pipe = pipeline(
-    'text2text-generation',
-    model=model,
-    tokenizer= tokenizer,
-    max_length=120
-)
-
-local_llm = HuggingFacePipeline(pipeline=pipe)
+model = VitsModel.from_pretrained("facebook/mms-tts-ind")
+tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-ind")
 
-llm_chain = caption_chain.chain(llm=local_llm)
-sum_llm_chain = tag_chain.chain(llm=local_llm)
 
 pexels_api_key = os.getenv('pexels_api_key')
 
@@ -37,10 +26,15 @@ def pred(product_name, orientation):
         orientation = "square"
         height = 1080
         width = 1080
-    folder_name, sentences = pexels.generate_videos(product_name, pexels_api_key, orientation, height, width, llm_chain, sum_llm_chain)
+    folder_name, sentences = pexels.generate_videos(product_name, pexels_api_key, orientation, height, width, model, tokenizer)
     gc.collect()
     utils.combine_videos(folder_name)
-    return ["\n".join(sentences), os.path.join(folder_name, "Final_Ad_Video.mp4")]
+
+    vid = os.path.join(folder_name,"Final_Ad_Video.mp4")
+    spe = "x.wav"
+
+    utils.combine_audio_video(folder_name, vid, spe)
+    return ["\n".join(sentences), os.path.join(folder_name, "new_filename.mp4")]
         #{'video':os.path.join(folder_name, "Final_Ad_Video.mp4"),
        # 'captions':"\n".join(sentences)}
     
@@ -48,8 +42,8 @@ def pred(product_name, orientation):
 with gr.Blocks() as demo:
     gr.Markdown(
         """
-        # Ads Generator
-        Create video ads based on your product name using AI
+        # Content [Video] Generator
+        Create a short video based on your text input using AI
         ### Note : the video generation takes about 2-4 minutes 
         """
     )
@@ -57,7 +51,7 @@ with gr.Blocks() as demo:
             ["Shorts/Reels/TikTok (1080 x 1920)", "Facebook/Youtube Videos (1920 x 1080)", "Square (1080 x 1080)"], 
             label="Video Dimension", info="Choose dimension"
         )
-    product_name = gr.Textbox(label="product name")
+    product_name = gr.Textbox(label="text story", lines=15, max_lines=100)
     captions = gr.Textbox(label="captions")
     video = gr.Video()
     btn = gr.Button("Submit")

components/caption_chain.py

@@ -1,12 +0,0 @@
-from langchain import PromptTemplate
-from langchain.chains import LLMChain
-
-def chain(llm):
-    template = """Make 5 different advertisement captions about this product. 
-    {product}
-    """
-    
-    prompt = PromptTemplate(template=template, input_variables=["product"])
-    
-    llm_chain = LLMChain(prompt=prompt, llm=llm)
-    return llm_chain

components/pexels.py

@@ -1,5 +1,6 @@
 import requests
 import shutil,os,re
+import datetime
 
 # Searching for the videos
 def search_pexels(keyword, api_key, orientation='potrait', size='medium', endpoint='videos', num_pages=50):
@@ -43,34 +44,42 @@ def download_video(data, parent_path, height, width, links, i):
                     print("Sucessfully saved video in", os.path.join(parent_path,str(i) + '_' + str(v['id'])) + '.mp4')
                     return x['id']
 
+def generate_voice(text, model, tokenizer):
+    speeches = []
+    for x in text:
+    inputs = tokenizer(x, return_tensors="pt")
+
+    with torch.no_grad():
+        output = model(**inputs).waveform
+    speeches.append(output)
+    return speeches
 
 # Utilizing the LLMs to find the relevant videos 
-def generate_videos(product, api_key, orientation, height, width, llm_chain=None, sum_llm_chain=None):
+def generate_videos(text, api_key, orientation, height, width, model, tokenizer):
     prod = product.strip().replace(" ", "_")
     links = []
     try :
         # Split the paragraph by sentences
-            
-        sentences = llm_chain.run(product.strip())
-        print('Sentence :', sentences)
-        
-#         sentences = sentences.split(".")[:-1]
-        sentences = [x.strip() for x in re.split(r'\d+\.', sentences) if len(x) > 6]
-    
+        sentences = list(filter(None,[x.strip() for x in re.split(r'[^A-Za-z0-9 -]', text)]))
         
-        # Create directory with the product's name
-        if os.path.exists(prod):
-            shutil.rmtree(prod)
-        os.mkdir(prod)
+        # Create directory with the name
+        di = str(datetime.datetime.now())
+        if os.path.exists(di):
+            shutil.rmtree(di)
+        os.mkdir(di)
         
         # Generate video for every sentence
         print("Keyword :")
         for i,s in enumerate(sentences):
-            keyword = sum_llm_chain.run(s)
-            print(i+1, ":", keyword)
-            data = search_pexels(keyword, api_key, orientation.lower())
-            link = download_video(data, prod, height, width, links,i)
+            # keyword = sum_llm_chain.run(s)
+            print(i+1, ":", s)
+            data = search_pexels(s, api_key, orientation.lower())
+            link = download_video(data, di, height, width, links,i)
             links.append(link)
+        
+        speeches = generate_voice(sentences, model, tokenizer)
+
+        sf.write("x.wav", torch.cat(speeches, 1)[0], 16500)
             
         print("Success! videos has been generated")
     except Exception as e :

components/tag_chain.py

@@ -1,16 +0,0 @@
-from langchain.prompts import PromptTemplate
-from langchain.chains import LLMChain
-
-def chain(llm):
-  sum_template = """what is the most significant actions or places or things, say it in at most 5 words :
-  
-  {sentence}
-  
-  
-  """
-  
-  sum_prompt = PromptTemplate(template=sum_template, input_variables=["sentence"])
-  
-  sum_llm_chain = LLMChain(prompt=sum_prompt, llm=llm)
-
-  return sum_llm_chain

components/utils.py

@@ -1,4 +1,5 @@
 from moviepy.editor import VideoFileClip, concatenate_videoclips,vfx
+from moviepy.editor import AudioFileClip, CompositeAudioClip
 import os 
 
 def combine_videos(folder_name):
@@ -20,4 +21,12 @@ def combine_videos(folder_name):
 #     else :
 #         combined = concatenate_videoclips([VideoFileClip(os.path.join(folder_name,x)).subclip(0,60//length) for x in os.listdir(folder_name)])
 #         combined.write_videofile(os.path.join(folder_name, "Ad_Video.mp4"))
-    print("Done! Your ads video has been created")
+    print("Done! Your ads video has been created")
+
+def combine_audio_video(folder_name, vid, speech):
+    videoclip = VideoFileClip(vid)
+    audioclip = AudioFileClip(speech)
+
+    new_audioclip = CompositeAudioClip([audioclip])
+    videoclip.audio = new_audioclip
+    videoclip.write_videofile(os.path.join(folder_name,"new_filename.mp4"))

requirements.txt

@@ -2,3 +2,6 @@ langchain
 moviepy
 transformers
 torch
+sentencepiece 
+accelerate 
+sacremoses