import gradio as gr import librosa from PIL import Image, ImageDraw, ImageFont from mutagen.mp3 import MP3 from mutagen.id3 import ID3, APIC, TIT2, TPE1 import io from colorthief import ColorThief import colorsys import math import os from multiprocessing import Pool, cpu_count import tempfile import ffmpeg import subprocess import traceback import shutil import LRC2SRT import sys import re flag = 1 path = "" # Update with your path def safe_read(i: int, a: list): if i >= len(a): return 128 else: return a[i] def getRenderCords(ta: list, idx: int, res: int = 1024, size: tuple = (1280, 720)) -> list: i = idx - res // 2 x, y = size[0] * .9 / -2, (safe_read(i, ta) - 128) * \ (size[1] / 2000) + (size[1] * .7 / -2) c = [] while i < idx + (res // 2): c.append((x, y)) i += 1 y = (safe_read(i, ta) - 128) * (size[1] / 2000) + (size[1] * .7 / -2) x += (size[0] * .9) / res return c def center_to_top_left(coords, width=1280, height=720): new_coords = [] for x, y in coords: new_coords.append(totopleft((x, y), width=width, height=height)) return new_coords def totopleft(coord, width=1280, height=720): return coord[0] + width / 2, height / 2 - coord[1] def getTrigger(ad: int, a: list, max: int = 1024) -> int: i = ad while not (safe_read(i, a) < 126 and safe_read(i+10, a) < 130 or i - ad > max): i += 1 return i def extract_cover_image(mp3_file): audio = MP3(mp3_file, ID3=ID3) if audio.tags == None: return -1 for tag in audio.tags.values(): if isinstance(tag, APIC): image_data = tag.data cover_image = Image.open(io.BytesIO(image_data)) return cover_image print("No cover image found in the MP3 file.") return None def getTitleAndArtist(mp3_file): audio = MP3(mp3_file, ID3=ID3) title = audio.get('TIT2', TIT2(encoding=3, text='')).text[0] artist = audio.get('TPE1', TPE1(encoding=3, text='')).text[0] return title, artist def getColour(img): with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmpfile: img.save(tmpfile.name, format="PNG") color_thief = ColorThief(tmpfile.name) dominant_color = color_thief.get_color(quality=1) os.remove(tmpfile.name) return dominant_color def clamp(number): return max(0, min(number, 1)) def normalizeColour(C): cc = colorsys.rgb_to_hsv(C[0] / 255, C[1] / 255, C[2] / 255) ccc = colorsys.hsv_to_rgb(cc[0], clamp(1.3 * cc[1]), .8) return math.floor(ccc[0] * 255), math.floor(ccc[1] * 255), math.floor(ccc[2] * 255) def normalizeColourBar(C): cc = colorsys.rgb_to_hsv(C[0] / 255, C[1] / 255, C[2] / 255) ccc = colorsys.hsv_to_rgb(cc[0], clamp(1.4 * cc[1]), .6) return math.floor(ccc[0] * 255), math.floor(ccc[1] * 255), math.floor(ccc[2] * 255) def stamp_text(draw, text, font, position, align='left'): text_bbox = draw.textbbox((0, 0), text, font=font) text_width = text_bbox[2] - text_bbox[0] text_height = text_bbox[3] - text_bbox[1] x, y = position y -= text_height // 2 if align == 'center': x -= text_width // 2 elif align == 'right': x -= text_width draw.text((x, y), text, font=font, fill="#fff") def linear_interpolate(start, stop, progress): return start + progress * (stop - start) def filecount(p): files = os.listdir() file_count = len(files) return file_count def render_frame(params): n, samples_array, cover_img, title, artist, dominant_color, width, height, fps, name, oscres, sr = params num_frames = len(samples_array) // (sr // fps) img = Image.new('RGB', (width, height), normalizeColour(dominant_color)) d = ImageDraw.Draw(img) s = math.floor((sr / fps) * n) e = center_to_top_left(getRenderCords(samples_array, getTrigger( s, samples_array, max=oscres), res=oscres, size=(width, height)), width=width, height=height) d.line(e, fill='#fff', width=round(min(2*height/720, 2*width/1280))) cs = math.floor(min(width, height) / 2) cov = cover_img.resize((cs, cs)) img.paste(cov, (((width // 2) - cs // 2), math.floor(height * .1))) fontT = ImageFont.truetype( path+'Lexend-Bold.ttf', 50*(min(width, height)/720)//1) fontA = ImageFont.truetype( path+'Lexend-Bold.ttf', 40*(min(width, height)/720)//1) fontD = ImageFont.truetype( path+'SpaceMono-Bold.ttf', 30*(min(width, height)/720)//1) stamp_text(d, title, fontT, totopleft( (0, min(width, height) * .3 // -2), width=width, height=height), 'center') stamp_text(d, artist, fontA, totopleft( (0, min(width, height) * .44 // -2), width=width, height=height), 'center') d.line(center_to_top_left([(width * .96 // -2, height * .95 // -2), (width * .96 // 2, height * .95 // -2)], width=width, height=height), fill=normalizeColourBar(dominant_color), width=15 * height // 360) d.line(center_to_top_left([(width * .95 // -2, height * .95 // -2), (linear_interpolate(width * .95 // -2, width * .95 // 2, s / len(samples_array)), height * .95 // -2)], width=width, height=height), fill='#fff', width=10 * height // 360) img.save(path+f'out/{name}/{str(n)}.png', 'PNG') return 1 # Indicate one frame processed def RenderVid(af, n, fps=30): (ffmpeg .input(path+f'out/{n}/%d.png', framerate=fps) .input(af) .output(n + '.mp4', vcodec='libx264', r=fps, pix_fmt='yuv420p', acodec='aac', shortest=None) .run() ) gr.Interface.download(f"{n}.mp4") invisible_chars = ["\u200B", "\uFEFF"] def remove_bom(data: str) -> str: BOM = '\ufeff' return data.lstrip(BOM) def stripinvisibles(s): e = remove_bom(s) for i in invisible_chars: e.replace(i, "") return e def start_progress(title): global progress_x sys.stdout.write(f"{title}:\n") sys.stdout.write(f"[{'-'*60}] 0%\r") sys.stdout.flush() progress_x = 0 def progress(x): global progress_x sys.stdout.write(f"[{'#'*round(60*x)}{'-'*(60-round(60*x))}] {x:.2%}\r") sys.stdout.flush() progress_x = x def end_progress(): sys.stdout.write(f"[{'#'*60}] 100.00%\r\n") sys.stdout.flush() haslyrics = False def main(file, name, fps=30, res: tuple = (1280, 720), oscres=512, sr=11025, lyrics=None, img=None, tit=None, ast=None): global flag p = gr.Progress() LRC2SRT.clear() if os.path.exists("out.srt"): os.remove("out.srt") global haslyrics haslyrics = False if lyrics: p(0.5, "parsing lyrics") try: outf = open("out.srt", mode="x", encoding="UTF8") sf = stripinvisibles(open(lyrics, encoding="UTF8").read()) if sf[0] == '[': gr.Info("Lyrics of LRC type was detected, converting to SRT") LRC2SRT.convert_to_srt(sf) outf.write('\n'.join(LRC2SRT.SRT)) haslyrics = True elif sf[0].isdigit(): outf.write(sf) gr.Info("Lyrics of SRT type was detected") haslyrics = True else: gr.Warning("Lyrics file is invalid, skipping") except Exception as e: print(traceback.format_exc()) gr.Warning( "Failed to parse lyrics, ensure there are no blank lines in between, you may use Lyrics Editor to ensure compatability") os.makedirs(path + f'out/{name}/', exist_ok=True) global iii iii = 0 # Load the audio file if flag: gr.Info("This is the first file since startup, this may take some time") flag = 0 p(0.25, "loading file") audio_path = file y, sr = librosa.load(audio_path, sr=sr) # Resample to 11025 Hz y_u8 = (y * 128 + 128).astype('uint8') samples_array = y_u8.tolist() p(0.5, "extracting metadata") # Extract cover image, title, and artist cover_file = None if img: cover_file = Image.open(img) cover_img = extract_cover_image(audio_path) if img: cover_img = cover_file if cover_img is None: raise gr.Error( "Mp3 must have a cover image, upload the image under the 'Metadata' section", duration=None) elif cover_img == -1 and not (tit or ast or img): raise gr.Error( "Mp3 is missing tags, add the info under the 'Metadata' section", duration=None) title, artist = getTitleAndArtist(audio_path) if tit and ast: title, artist = tit, ast if title == '' or artist == '': gr.Warning('Missing Title or Artist') if img: color_thief = ColorThief(img) dominant_color = color_thief.get_color(quality=1) cover_img = cover_file else: dominant_color = getColour(cover_img) # Frame rendering parameters width, height, fps = res[0], res[1], fps num_frames = len(samples_array) // (sr // fps) # Prepare parameters for each frame params = [(n, samples_array, cover_img, title, artist, dominant_color, width, height, fps, name, oscres, sr) for n in range(num_frames)] print('-'*50) print('Info:' + "External" if img else "ID3") print("Title: " + title) print("Artist: " + artist) print(f'Resolution: {str(width)}x{str(height)}') print("Background Colour: " + str(dominant_color)) print('Framerate: ' + str(fps)) print('Frame Count: ' + str(num_frames)) print('Segments per frame: ' + str(oscres)) print('-'*50) #start_progress("Rendering:") try: with Pool(cpu_count()-1) as pool: num_frames = len(samples_array) // (sr // fps) # Use imap to get progress updates for _ in pool.imap_unordered(render_frame, params): iii += 1 # Increment frame count for progress p((iii, num_frames), desc="Rendering Frames") #progress(iii/num_frames) except Exception as e: raise gr.Error("Something went wrong whilst rendering") #finally: #end_progress() p = gr.Progress() p(0, desc="Compiling video") print('-'*50) print('FFMPEG') if haslyrics: ffmpeg_cmd = [ "ffmpeg", '-framerate', str(fps), '-i', path + f'out/{name}/%d.png', # Input PNG images '-i', file, # Input MP3 audio '-i', path + 'out.srt', # Input SRT subtitles '-c:v', 'libx264', '-r', str(fps), '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-c:s', 'mov_text', # Use mov_text codec for subtitles '-y', path + f'{name}.mp4' # Output MP4 filename ] else: ffmpeg_cmd = [ "ffmpeg", '-framerate', str(fps), '-i', path + f'out/{name}/%d.png', # Input PNG images '-i', f'{file}', # Input MP3 audio '-c:v', 'libx264', '-r', str(fps), '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-y', path + f'{name}.mp4' # Output MP4 filename ] process = subprocess.Popen(ffmpeg_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) # Regular expression to match frame information frame_re = re.compile(r"frame=\s*(\d+)") p = gr.Progress() #start_progress('Compliling video') while True: output = process.stderr.readline() if output == '' and process.poll() is not None: break if output: # Check if the output line contains frame information match = frame_re.search(output) if match: frame = match.group(1) p((int(frame), num_frames), desc="Compiling Video") #progress(int(frame)/num_frames) # Wait for the process to complete process.wait() #end_progress() print('-'*50) return f"{name}.mp4", haslyrics def gradio_interface(audio_file, lyrics, output_name, fps=30, vidwidth=1280, vidheight=720, oscres=512, img=None, tit=None, ast=None): if audio_file is None: raise gr.Error("Please Provide an Audio File") if not str(audio_file).endswith(".mp3"): raise gr.Error("Only MP3 Files are supported at this time") resolution = f"{vidwidth}x{vidheight}" res = tuple(map(int, resolution.split('x'))) video_file, haslyrics = main(audio_file, output_name, fps=fps, res=res, oscres=oscres, lyrics=lyrics, img=img, tit=tit, ast=ast) # Clean up the temporary file shutil.rmtree("out") srt_output = "out.srt" if haslyrics else None return video_file, srt_output, haslyrics def update_srt_output_visibility(haslyrics): return gr.update(visible=haslyrics) with gr.Blocks() as demo: gr.Markdown( 'Upload an MP3 file and configure parameters to create a visualization video.') gr.Markdown( 'Optionally upload a word or line synced lyric file in the advanced section.') with gr.Row(): # Inputs on the left with gr.Column(): with gr.Accordion(label="Audio Settings", open=True): gr.Markdown('## Load your mp3 file here') audio_file = gr.File( label="Upload your MP3 file", file_count='single', file_types=['mp3']) with gr.Accordion(label="Mp3 Metadata", open=False): gr.Markdown( '## Add Metadata here if your mp3 does not have one') cover_img = gr.Image(label='Cover Art', type="filepath") title_input = gr.Textbox(label='Title') artist_input = gr.Textbox(label='Artists') with gr.Accordion(label="Video Output Settings", open=False): gr.Markdown('## Configure Video Output Here') output_name = gr.Textbox( label="Output Video Name", value='Output') fps_slider = gr.Slider( label="Frames per Second", minimum=20, maximum=60, step=1, value=30) vidwidth_slider = gr.Slider( label="Output Video Width", minimum=100, maximum=2000, value=1280, step=2) vidheight_slider = gr.Slider( label="Output Video Height", minimum=100, maximum=2000, value=720, step=2) with gr.Accordion(label="Advanced Options", open=False): oscres_slider = gr.Slider( label="Number of Visualization Segments", minimum=256, maximum=2048, step=2, value=1024) gr.Markdown( 'If uploading LRC, ensure a blank timed line at the end to avoid conversion errors') lyrics_file = gr.File(label="(Optional) Upload Lyrics as LRC or SRT", file_count='single', file_types=['lrc', 'srt']) # Add a submit button submit_btn = gr.Button("Generate Video") # Outputs on the right with gr.Column(): output_video = gr.Video(label="Output") with gr.Accordion(label="SRT File (Only used if lyrics is provided)", open=False): srt_output = gr.File(label="SRT Output") # Bind the button to the function submit_btn.click( fn=gradio_interface, inputs=[audio_file, lyrics_file, output_name, fps_slider, vidwidth_slider, vidheight_slider, oscres_slider, cover_img, title_input, artist_input], outputs=[output_video, srt_output] ) # Launch Gradio interface if __name__ == '__main__': demo.launch()