#! /usr/bin/env python # coding=utf-8 # Copyright 2023 Bofeng Huang import datetime import logging import os import re import warnings import gradio as gr import librosa # import nltk import pandas as pd import psutil import pytube as pt import torch # import torchaudio from transformers import pipeline, Wav2Vec2ProcessorWithLM, AutoModelForCTC from transformers.utils.logging import disable_progress_bar # nltk.download("punkt") # from nltk.tokenize import sent_tokenize warnings.filterwarnings("ignore") disable_progress_bar() DEFAULT_MODEL_NAME = "bofenghuang/asr-wav2vec2-ctc-french" SAMPLE_RATE = 16_000 GEN_KWARGS = { "chunk_length_s": 30, "stride_length_s": 5, } logging.basicConfig( format="%(asctime)s [%(levelname)s] [%(name)s] %(message)s", datefmt="%Y-%m-%dT%H:%M:%SZ", ) logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) # device = 0 if torch.cuda.is_available() else "cpu" device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") logger.info(f"Model will be loaded on device `{device}`") cached_models = {} def _return_yt_html_embed(yt_url): video_id = yt_url.split("?v=")[-1] HTML_str = ( f'
' "
" ) return HTML_str def download_audio_from_youtube(yt_url, downloaded_filename="audio.wav"): yt = pt.YouTube(yt_url) stream = yt.streams.filter(only_audio=True)[0] # stream.download(filename="audio.mp3") stream.download(filename=downloaded_filename) return downloaded_filename def download_video_from_youtube(yt_url, downloaded_filename="video.mp4"): yt = pt.YouTube(yt_url) stream = yt.streams.filter(progressive=True, file_extension="mp4").order_by("resolution").desc().first() stream.download(filename=downloaded_filename) logger.info(f"Download YouTube video from {yt_url}") return downloaded_filename def _print_memory_info(): memory = psutil.virtual_memory() logger.info( f"Memory info - Free: {memory.available / (1024 ** 3):.2f} Gb, used: {memory.percent}%, total: {memory.total / (1024 ** 3):.2f} Gb" ) def _print_cuda_memory_info(): used_mem, tot_mem = torch.cuda.mem_get_info() logger.info( f"CUDA memory info - Free: {used_mem / 1024 ** 3:.2f} Gb, used: {(tot_mem - used_mem) / 1024 ** 3:.2f} Gb, total: {tot_mem / 1024 ** 3:.2f} Gb" ) def print_memory_info(): _print_memory_info() if torch.cuda.is_available(): _print_cuda_memory_info() def maybe_load_cached_pipeline(model_name): model = cached_models.get(model_name) if model is None: pipe = pipeline(model=model_name, device=device) # model = AutoModelForCTC.from_pretrained(model_name).to(device) # processor = Wav2Vec2ProcessorWithLM.from_pretrained(model_name) # pipe = pipeline( # "automatic-speech-recognition", # model=model, # tokenizer=processor.tokenizer, # feature_extractor=processor.feature_extractor, # decoder=processor.decoder, # ) logger.info(f"`{model_name}` has been loaded on device `{device}`") print_memory_info() cached_models[model_name] = pipe return model def process_audio_file(audio_file): # waveform, sample_rate = torchaudio.load(audio_file) # waveform = waveform.squeeze(axis=0) # mono # # resample # if sample_rate != SAMPLE_RATE: # resampler = torchaudio.transforms.Resample(sample_rate, SAMPLE_RATE) # waveform = resampler(waveform) waveform, sample_rate = librosa.load(audio_file, mono=True) # resample if sample_rate != SAMPLE_RATE: waveform = librosa.resample(waveform, orig_sr=sample_rate, target_sr=SAMPLE_RATE) return waveform def infer(model, filename, return_df=False): audio_data = process_audio_file(filename) text = model(audio_data, **GEN_KWARGS)["text"] if return_df: # return pd.DataFrame({"text": sent_tokenize(text)}) return pd.DataFrame({"text": [text]}) else: return text def transcribe(microphone, file_upload, model_name=DEFAULT_MODEL_NAME): warn_output = "" if (microphone is not None) and (file_upload is not None): warn_output = ( "WARNING: You've uploaded an audio file and used the microphone. " "The recorded file from the microphone will be used and the uploaded audio will be discarded.\n" ) elif (microphone is None) and (file_upload is None): return "ERROR: You have to either use the microphone or upload an audio file" file = microphone if microphone is not None else file_upload model = maybe_load_cached_pipeline(model_name) text = infer(model, file, return_df=True) logger.info(f'Transcription by `{model_name}`:\n{text.to_json(orient="index", force_ascii=False, indent=2)}\n') # return warn_output + text return text def yt_transcribe(yt_url, model_name=DEFAULT_MODEL_NAME): # html_embed_str = _return_yt_html_embed(yt_url) audio_file_path = download_audio_from_youtube(yt_url) model = maybe_load_cached_pipeline(model_name) text = infer(model, audio_file_path, return_df=True) logger.info( f'Transcription by `{model_name}` of "{yt_url}":\n{text.to_json(orient="index", force_ascii=False, indent=2)}\n' ) # return html_embed_str, text return text def video_transcribe(video_file_path, model_name=DEFAULT_MODEL_NAME): if video_file_path is None: raise ValueError("Failed to transcribe video as no video_file_path has been defined") audio_file_path = re.sub(r"\.mp4$", ".wav", video_file_path) os.system( f'ffmpeg -hide_banner -loglevel error -y -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{audio_file_path}"' ) model = maybe_load_cached_pipeline(model_name) text = infer(model, audio_file_path, return_df=True) logger.info(f'Transcription by `{model_name}`:\n{text.to_json(orient="index", force_ascii=False, indent=2)}\n') return text # load default model maybe_load_cached_pipeline(DEFAULT_MODEL_NAME) # default_text_output_df = pd.DataFrame(columns=["start", "end", "text"]) default_text_output_df = pd.DataFrame(columns=["text"]) with gr.Blocks() as demo: with gr.Tab("Transcribe Audio"): gr.Markdown( f"""

Speech-to-Text in French: Transcribe Audio

Transcribe long-form microphone or audio inputs! Demo uses the fine-tuned wav2vec2 model {DEFAULT_MODEL_NAME} and 🤗 Transformers to transcribe audio files of arbitrary length. To achieve improved accuracy and well-punctuated text, please use the [Whisper demo](https://huggingface.co./spaces/bofenghuang/whisper-demo-french). """ ) microphone_input = gr.inputs.Audio(source="microphone", type="filepath", label="Record", optional=True) upload_input = gr.inputs.Audio(source="upload", type="filepath", label="Upload File", optional=True) # with_timestamps_input = gr.Checkbox(label="With timestamps?") microphone_transcribe_btn = gr.Button("Transcribe Audio") # gr.Markdown(''' # Here you will get generated transcrit. # ''') # microphone_text_output = gr.outputs.Textbox(label="Transcription") text_output_df2 = gr.DataFrame( value=default_text_output_df, label="Transcription", row_count=(0, "dynamic"), max_rows=10, wrap=True, overflow_row_behaviour="paginate", ) microphone_transcribe_btn.click(transcribe, inputs=[microphone_input, upload_input], outputs=text_output_df2) # with gr.Tab("Transcribe YouTube"): # gr.Markdown( # f""" #
#

Speech-to-Text in French: Transcribe YouTube

#
# Transcribe long-form YouTube videos! # Demo uses the fine-tuned checkpoint: {DEFAULT_MODEL_NAME} to transcribe video files of arbitrary length. # """ # ) # yt_link_input2 = gr.inputs.Textbox(lines=1, placeholder="Paste the URL to a YouTube video here", label="YouTube URL") # with_timestamps_input2 = gr.Checkbox(label="With timestamps?", value=True) # yt_transcribe_btn = gr.Button("Transcribe YouTube") # # yt_text_output = gr.outputs.Textbox(label="Transcription") # text_output_df3 = gr.DataFrame( # value=default_text_output_df, # label="Transcription", # row_count=(0, "dynamic"), # max_rows=10, # wrap=True, # overflow_row_behaviour="paginate", # ) # # yt_html_output = gr.outputs.HTML(label="YouTube Page") # yt_transcribe_btn.click(yt_transcribe, inputs=[yt_link_input2, with_timestamps_input2], outputs=[text_output_df3]) with gr.Tab("Transcribe Video"): gr.Markdown( f"""

Speech-to-Text in French: Transcribe Video

Transcribe long-form YouTube videos or uploaded video inputs! Demo uses the fine-tuned wav2vec2 model {DEFAULT_MODEL_NAME} and 🤗 Transformers to transcribe audio files of arbitrary length. To achieve improved accuracy and well-punctuated text, please use the [Whisper demo](https://huggingface.co./spaces/bofenghuang/whisper-demo-french). """ ) yt_link_input = gr.inputs.Textbox( lines=1, placeholder="Paste the URL to a YouTube video here", label="YouTube URL" ) download_youtube_btn = gr.Button("Download Youtube video") downloaded_video_output = gr.Video(label="Video file", mirror_webcam=False) download_youtube_btn.click( download_video_from_youtube, inputs=[yt_link_input], outputs=[downloaded_video_output] ) # with_timestamps_input3 = gr.Checkbox(label="With timestamps?", value=True) video_transcribe_btn = gr.Button("Transcribe video") text_output_df = gr.DataFrame( value=default_text_output_df, label="Transcription", row_count=(0, "dynamic"), max_rows=10, wrap=True, overflow_row_behaviour="paginate", ) video_transcribe_btn.click(video_transcribe, inputs=[downloaded_video_output], outputs=[text_output_df]) demo.launch(server_name="0.0.0.0", debug=True) # demo.launch(server_name="0.0.0.0", debug=True, share=True) # demo.launch(enable_queue=True)