init

app.py

@@ -0,0 +1,117 @@
+from transformers import ClapModel, ClapProcessor
+import gradio as gr
+import torch
+import torchaudio
+import os
+import numpy as np
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import Distance, VectorParams
+from qdrant_client.http import models
+
+
+
+
+
+class ClapSSGradio():
+
+    def __init__(
+        self,
+        name,
+        k=10,
+    ):
+
+        self.name = name
+        self.k = k
+
+        print("Env?!")
+        print(os.getenv('HUGGINGFACE_API_TOKEN')[:2])
+
+        self.model = ClapModel.from_pretrained(
+            f"Audiogen/{name}", use_auth_token=os.getenv('HUGGINGFACE_API_TOKEN'))
+        self.tokenizer = ClapProcessor.from_pretrained(
+            f"Audiogen/{name}", use_auth_token=os.getenv('HUGGINGFACE_API_TOKEN'))
+
+        self.sas_token = os.environ['AZURE_SAS_TOKEN']
+        self.account_name = 'Audiogen'
+        self.storage_name = 'audiogentrainingdataeun'
+
+        self._start_qdrant()
+
+    def _start_qdrant(self):
+        self.client = QdrantClient(url=os.getenv(
+            "QDRANT_URL"), api_key=os.getenv('QDRANT_API_KEY'))
+        # print(self.client.get_collection(collection_name=self.name))
+
+    @torch.no_grad()
+    def _embed_query(self, query):
+        inputs = self.tokenizer(
+            query, return_tensors="pt", padding='max_length', max_length=77, truncation=True)
+        return self.model.get_text_features(**inputs).cpu().numpy().tolist()[0]
+
+    def _similarity_search(self, query):
+        results = self.client.search(
+            collection_name=self.name,
+            query_vector=self._embed_query(query),
+            limit=self.k,
+            score_threshold=0.5,
+        )
+
+        containers = [result.payload['container'] for result in results]
+        filenames = [result.id for result in results]
+        captions = [result.payload['caption'] for result in results]
+        scores = [result.score for result in results]
+
+        # print to stdout
+        print(f"\nQuery: {query}\n")
+        for i, (container, filename, caption, score) in enumerate(zip(containers, filenames, captions, scores)):
+            print(f"{i}: {container} - {caption}. Score: {score}")
+
+        waveforms = self._download_results(containers, filenames)
+
+        if len(waveforms) == 0:
+            print("\nNo results found")
+
+        if len(waveforms) < self.k:
+            waveforms.extend([(int(48000), np.zeros((480000, 2)))
+                             for _ in range(self.k - len(waveforms))])
+
+        return waveforms
+
+    def _download_results(self, containers: list, filenames: list):
+
+        # construct url
+        urls = [f"https://{self.storage_name}.blob.core.windows.net/{container}/{file_name}.flac?{self.sas_token}" for container,
+                file_name in zip(containers, filenames)]
+
+        # make requests
+        waveforms = []
+        for url in urls:
+            waveform, sample_rate = torchaudio.load(url)
+            waveforms.append(tuple([sample_rate, waveform.numpy().T]))
+
+        return waveforms
+
+    def launch(self, share=False):
+        # gradio app structure
+        with gr.Blocks(title='Clap Semantic Search') as ui:
+
+            with gr.Row():
+                with gr.Column(variant='panel'):
+                    search = gr.Textbox(placeholder='Search Samples')
+
+                with gr.Column():
+                    audioboxes = []
+                    gr.Markdown("Output")
+                    for i in range(self.k):
+                        t = gr.components.Audio(label=f"{i}", visible=True)
+                        audioboxes.append(t)
+
+            search.submit(fn=self._similarity_search, inputs=[
+                          search], outputs=audioboxes)
+
+            ui.launch(share=share)
+
+
+if __name__ == "__main__":
+    app = ClapSSGradio("clap-2")
+    app.launch(share=False)

requirements.txt

@@ -0,0 +1,89 @@
+aiofiles==23.2.1
+altair==5.1.2
+annotated-types==0.5.0
+anyio==3.7.1
+attrs==23.1.0
+certifi==2023.7.22
+charset-normalizer==3.3.0
+click==8.1.7
+contourpy==1.1.1
+cycler==0.12.0
+exceptiongroup==1.1.3
+fastapi==0.103.2
+ffmpy==0.3.1
+filelock==3.12.4
+fonttools==4.43.0
+fsspec==2023.9.2
+gradio==3.46.1
+gradio_client==0.5.3
+grpcio==1.59.0
+grpcio-tools==1.59.0
+h11==0.14.0
+h2==4.1.0
+hpack==4.0.0
+httpcore==0.18.0
+httpx==0.25.0
+huggingface-hub==0.16.4
+hyperframe==6.0.1
+idna==3.4
+importlib-resources==6.1.0
+Jinja2==3.1.2
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+MarkupSafe==2.1.3
+matplotlib==3.8.0
+mpmath==1.3.0
+networkx==3.1
+numpy==1.26.0
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.2.140
+nvidia-nvtx-cu12==12.1.105
+orjson==3.9.7
+packaging==23.2
+pandas==2.1.1
+Pillow==10.0.1
+portalocker==2.8.2
+protobuf==4.24.4
+pydantic==2.4.2
+pydantic_core==2.10.1
+pydub==0.25.1
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+qdrant-client==1.5.4
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+rpds-py==0.10.3
+safetensors==0.3.3
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.27.0
+sympy==1.12
+tokenizers==0.14.0
+toolz==0.12.0
+torch==2.1.0
+torchaudio==2.1.0
+tqdm==4.66.1
+transformers==4.34.0
+triton==2.1.0
+typing_extensions==4.8.0
+tzdata==2023.3
+urllib3==1.26.17
+uvicorn==0.23.2
+websockets==11.0.3
+zipp==3.17.0