Update: CSV, HDF5 and MIDI outputs

app.py

@@ -1,9 +1,9 @@
 import gradio as gr
-import cq2m_utils
-cq2m_interface = gr.Interface(fn=cq2m_utils.cq2m,
-                              inputs=gr.Audio(type='filepath', format='wav'),
-                              outputs=gr.File(type='file'),
-                              title="Choral Quartets to Midi",
-                              description="An application that uses Multi-Pitch Estimation and Voice Assignment to transform .WAV files with Choral Quartets recordings into MIDI files, with a separate track for each voice.")
+from cqfe_utils import cqfe
+cqfe_interface = gr.Interface(fn=cqfe,
+                              inputs=gr.Audio(type='filepath', format='wav', label='Audio Input File'),
+                              outputs=gr.File(type='file', label='F0 Output Files'),
+                              title="Choral Quartets F0 Extractor",
+                              description="An application that uses Multi-Pitch Estimation and Voice Assignment to transform audio files with Choral Quartets recordings into files (CSV, HDF5 and MIDI) containing F0 estimations for each voice (Soprano, Alto, Tenor and Bass). The processing may take a few minutes.")
 
-cq2m_interface.launch()
+cqfe_interface.launch()

app_test.ipynb

@@ -7,52 +7,22 @@
    "outputs": [],
    "source": [
     "import gradio as gr\n",
-    "import cq2m_utils"
+    "from cqfe_utils import cqfe\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Running on local URL:  http://127.0.0.1:7860\n",
-      "\n",
-      "To create a public link, set `share=True` in `launch()`.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div><iframe src=\"http://127.0.0.1:7860/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": []
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "cq2m_interface = gr.Interface(fn=cq2m_utils.cq2m,\n",
-    "                              inputs=gr.Audio(type='filepath', format='wav'),\n",
-    "                              outputs=gr.File(type='file'),\n",
-    "                              title=\"Choral Quartets to Midi\",\n",
-    "                              description=\"An application that uses Multi-Pitch Estimation and Voice Assignment to transform .WAV files with Choral Quartets recordings into MIDI files, with a separate track for each voice.\")\n",
+    "cqfe_interface = gr.Interface(fn=cqfe,\n",
+    "                              inputs=gr.Audio(type='filepath', format='wav', label='Audio Input File'),\n",
+    "                              outputs=gr.File(type='file', label='F0 Output Files'),\n",
+    "                              title=\"Choral Quartets F0 Extractor\",\n",
+    "                              description=\"An application that uses Multi-Pitch Estimation and Voice Assignment to transform audio files with Choral Quartets recordings into files (CSV, HDF5 and MIDI) containing F0 estimations for each voice (Soprano, Alto, Tenor and Bass). The processing may take a few minutes.\")\n",
     "\n",
-    "cq2m_interface.launch()"
+    "cqfe_interface.launch()"
    ]
   }
  ],

cq2m_utils.py → cqfe_utils.py

@@ -2,9 +2,20 @@ import os
 import math
 import mido
 import pumpp
+import librosa
 import numpy as np
+import pandas as pd
 from scipy.ndimage import gaussian_filter1d
-from cq2m_models import mask_voas_cnn_model, late_deep_cnn_model
+from cqfe_models import mask_voas_cnn_v2_model, late_deep_cnn_model
+
+############################################################
+
+freqscale = librosa.cqt_frequencies(n_bins=360, fmin=32.7, bins_per_octave=60)
+
+def bin_to_freq(bin):
+    return freqscale[bin]
+
+vec_bin_to_freq = np.vectorize(bin_to_freq)
 
 ############################################################
 
@@ -134,6 +145,28 @@ def song_to_midi(sop, alto, ten, bass):
 
 ############################################################
 
+def song_to_tables(sop, alto, ten, bass):
+
+    savepath_csv = './output.csv'
+    savepath_hdf5 = './output.hdf5'
+    timescale = np.arange(0, 0.011609977 * (sop.shape[1]), 0.011609977)[:sop.shape[1]]
+
+    s_argmax = vec_bin_to_freq(np.argmax(sop, axis=0))
+    a_argmax = vec_bin_to_freq(np.argmax(alto, axis=0))
+    t_argmax = vec_bin_to_freq(np.argmax(ten, axis=0))
+    b_argmax = vec_bin_to_freq(np.argmax(bass, axis=0))
+
+    data = np.array([timescale, s_argmax, a_argmax, t_argmax, b_argmax], dtype=np.float32).T
+    columns = ['Timestep', 'Soprano', 'Alto', 'Tenor', 'Bass']
+
+    df = pd.DataFrame(data, columns=columns)
+    df.to_csv(savepath_csv, mode='w', header=True)
+    df.to_hdf(savepath_hdf5, key='F0', mode='w', complevel=9, complib='blosc', append=False, format='table')
+
+    return savepath_csv, savepath_hdf5
+
+############################################################
+
 def prediction_postproc(input_array, argmax_and_threshold=True, gaussian_blur=True):
     prediction = np.moveaxis(input_array, 0, 1).reshape(360, -1)
     if(argmax_and_threshold):
@@ -190,7 +223,7 @@ def get_mpe_prediction(model, audio_file=None):
        Part of this function is part of deepsalience
     """
 
-    split_value = 2500
+    split_value = 4000
 
     if audio_file is not None:
 
@@ -250,10 +283,11 @@ def get_va_prediction(model, f0_matrix):
 
 ############################################################
 
-def cq2m(audiofile, mpe=late_deep_cnn_model(), va=mask_voas_cnn_model()):
+def cqfe(audiofile, mpe=late_deep_cnn_model(), va=mask_voas_cnn_v2_model()):
     mpe_pred = get_mpe_prediction(mpe, audiofile)
     s_pred, a_pred, t_pred, b_pred = get_va_prediction(va, mpe_pred)
-    midi = song_to_midi(s_pred, a_pred, t_pred, b_pred)
-    return midi
+    output_midi = song_to_midi(s_pred, a_pred, t_pred, b_pred)
+    output_csv, output_hdf5 = song_to_tables(s_pred, a_pred, t_pred, b_pred)
+    return [output_midi, output_csv, output_hdf5]
 
 ############################################################

pyproject.toml

@@ -1,11 +1,11 @@
 [tool.poetry]
-name = "choral-quartets-to-midi"
-version = "0.1.0"
-description = "An application that uses Multi-Pitch Estimation and Voice Assignment to transform .WAV files with Choral Quartets recordings into MIDI files, with a separate track for each voice. Based on Late/DeepCNN by Helena Cuesta and MaskVoasCNN by André Paiva."
+name = "Choral-Quartets-F0-Extractor"
+version = "0.1.1"
+description = "An application that uses Multi-Pitch Estimation and Voice Assignment to transform audio files with Choral Quartets recordings into files (CSV, HDF5 and MIDI) containing F0 estimations for each voice (Soprano, Alto, Tenor and Bass)."
 authors = ["André Paiva (Xornotor) <[email protected]>"]
 license = "cc"
 readme = "README.md"
-packages = [{include = "choral_quartets_to_midi"}]
+packages = [{include = "Choral_Quartets_F0_Extractor"}]
 
 [tool.poetry.dependencies]
 python = "^3.11"
@@ -16,6 +16,8 @@ mido
 pumpp
 numpy
 scipy
+pandas
+librosa
 
 
 [build-system]

requirements.txt

@@ -4,4 +4,6 @@ typing-extensions
 mido
 pumpp
 numpy
-scipy
+scipy
+pandas
+librosa