Spaces:
Paused
Paused
import time | |
import base64 | |
import gradio as gr | |
from sentence_transformers import SentenceTransformer | |
import httpx | |
import json | |
import os | |
import requests | |
import urllib | |
from os import path | |
from pydub import AudioSegment | |
MUBERT_LICENSE = os.environ.get('MUBERT_LICENSE') | |
MUBERT_TOKEN = os.environ.get('MUBERT_TOKEN') | |
#img_to_text = gr.Blocks.load(name="spaces/pharma/CLIP-Interrogator") | |
img_to_text = gr.Blocks.load(name="spaces/fffiloni/CLIP-Interrogator-2") | |
from share_btn import community_icon_html, loading_icon_html, share_js | |
from utils import get_tags_for_prompts, get_mubert_tags_embeddings | |
minilm = SentenceTransformer('all-MiniLM-L6-v2') | |
mubert_tags_embeddings = get_mubert_tags_embeddings(minilm) | |
def get_prompts(uploaded_image, track_duration, gen_intensity, gen_mode): | |
print("calling clip interrogator") | |
#prompt = img_to_text(uploaded_image, "ViT-L (best for Stable Diffusion 1.*)", "fast", fn_index=1)[0] | |
prompt = img_to_text(uploaded_image, 'fast', 4, fn_index=1)[0] | |
print(prompt) | |
pat = get_pat_token() | |
#music_result = get_music(pat, prompt, track_duration, gen_intensity, gen_mode) | |
music_result = generate_track_by_prompt(pat, prompt, track_duration, gen_intensity, gen_mode) | |
#print(pat) | |
return music_result, gr.update(visible=True), gr.update(visible=True), gr.update(visible=True) | |
def get_pat_token(): | |
r = httpx.post('https://api-b2b.mubert.com/v2/GetServiceAccess', | |
json={ | |
"method": "GetServiceAccess", | |
"params": { | |
"email":"[email protected]", | |
"phone":"+11234567890", | |
"license": MUBERT_LICENSE, | |
"token": MUBERT_TOKEN, | |
} | |
}) | |
rdata = json.loads(r.text) | |
#print(rdata) | |
#assert rdata['status'] == 1, "probably incorrect e-mail" | |
#pat = rdata['data']['pat'] | |
print(rdata['data']['pat']) | |
return rdata['data']['pat'] | |
def get_music(pat, prompt, track_duration, gen_intensity, gen_mode): | |
r = httpx.post('https://api-b2b.mubert.com/v2/TTMRecordTrack', | |
json={ | |
"method": "TTMRecordTrack", | |
"params": | |
{ | |
"text":"jazz music", | |
"pat": pat, | |
"mode":"track", | |
"duration":track_duration, | |
} | |
}) | |
rdata = json.loads(r.text) | |
print(rdata['data']['tasks'][0]['download_link']) | |
#assert rdata['status'] == 1, "probably incorrect e-mail" | |
#track = rdata['data']['tasks']['download_link'] | |
#print(track) | |
return "done" | |
def get_track_by_tags(tags, pat, duration, gen_intensity, gen_mode, maxit=20): | |
r = httpx.post('https://api-b2b.mubert.com/v2/RecordTrackTTM', | |
json={ | |
"method": "RecordTrackTTM", | |
"params": { | |
"pat": pat, | |
"duration": duration, | |
"format": "wav", | |
"intensity":gen_intensity, | |
"tags": tags, | |
"mode": gen_mode | |
} | |
}) | |
rdata = json.loads(r.text) | |
print(rdata) | |
#assert rdata['status'] == 1, rdata['error']['text'] | |
trackurl = rdata['data']['tasks'][0] | |
print('Generating track ', end='') | |
for i in range(maxit): | |
r = httpx.get(trackurl) | |
if r.status_code == 200: | |
return trackurl | |
time.sleep(1) | |
def generate_track_by_prompt(pat, prompt, duration, gen_intensity, gen_mode): | |
try: | |
_, tags = get_tags_for_prompts(minilm, mubert_tags_embeddings, prompt)[0] | |
result = get_track_by_tags(tags, pat, int(duration), gen_intensity, gen_mode) | |
print(result) | |
return result, ",".join(tags), "Success" | |
except Exception as e: | |
return None, "", str(e) | |
def convert_mp3_to_wav(mp3_filepath): | |
url = mp3_filepath | |
save_as = "file.mp3" | |
data = urllib.request.urlopen(url) | |
f = open(save_as,'wb') | |
f.write(data.read()) | |
f.close() | |
wave_file="file.wav" | |
sound = AudioSegment.from_mp3(save_as) | |
sound.export(wave_file, format="wav") | |
return wave_file | |
article = """ | |
<div class="footer"> | |
<p> | |
Follow <a href="https://twitter.com/fffiloni" target="_blank">Sylvain Filoni</a> for future updates π€ | |
</p> | |
</div> | |
<div id="may-like-container" style="display: flex;justify-content: center;flex-direction: column;align-items: center;margin-bottom: 30px;"> | |
<p style="font-size: 0.8em;margin-bottom: 4px;">You may also like: </p> | |
<div id="may-like" style="display: flex;flex-wrap: wrap;align-items: center;height: 20px;"> | |
<svg height="20" width="122" style="margin-left:4px;margin-bottom: 6px;"> | |
<a href="https://huggingface.co./spaces/fffiloni/spectrogram-to-music" target="_blank"> | |
<image href="https://img.shields.io/badge/π€ Spaces-Riffusion-blue" src="https://img.shields.io/badge/π€ Spaces-Riffusion-blue.png" height="20"/> | |
</a> | |
</svg> | |
</div> | |
</div> | |
""" | |
with gr.Blocks(css="style.css") as demo: | |
with gr.Column(elem_id="col-container"): | |
gr.HTML("""<div style="text-align: center; max-width: 700px; margin: 0 auto;"> | |
<div | |
style=" | |
display: inline-flex; | |
align-items: center; | |
gap: 0.8rem; | |
font-size: 1.75rem; | |
" | |
> | |
<h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;"> | |
Image to Music | |
</h1> | |
</div> | |
<p style="margin-bottom: 10px; font-size: 94%"> | |
Sends an image in to <a href="https://huggingface.co./spaces/pharma/CLIP-Interrogator" target="_blank">CLIP Interrogator</a> | |
to generate a text prompt which is then run through | |
<a href="https://huggingface.co./Mubert" target="_blank">Mubert</a> text-to-music to generate music from the input image! | |
</p> | |
</div>""") | |
input_img = gr.Image(type="filepath", elem_id="input-img") | |
music_output = gr.Audio(label="Result", type="filepath", elem_id="music-output").style(height="5rem") | |
text_status = gr.Textbox(label="status") | |
with gr.Group(elem_id="share-btn-container"): | |
community_icon = gr.HTML(community_icon_html, visible=False) | |
loading_icon = gr.HTML(loading_icon_html, visible=False) | |
share_button = gr.Button("Share to community", elem_id="share-btn", visible=False) | |
with gr.Accordion(label="Music Generation Options", open=False): | |
track_duration = gr.Slider(minimum=20, maximum=120, value=30, step=5, label="Track duration", elem_id="duration-inp") | |
with gr.Row(): | |
gen_intensity = gr.Dropdown(choices=["low", "medium", "high"], value="medium", label="Intensity") | |
gen_mode = gr.Radio(label="mode", choices=["track", "loop"], value="track") | |
generate = gr.Button("Generate Music from Image") | |
gr.HTML(article) | |
generate.click(get_prompts, inputs=[input_img,track_duration,gen_intensity,gen_mode], outputs=[text_status, share_button, community_icon, loading_icon], api_name="i2m") | |
share_button.click(None, [], [], _js=share_js) | |
demo.queue(max_size=32, concurrency_count=20).launch() |