Update README.md

README.md

@@ -23,7 +23,7 @@ model-index:
     metrics:
        - name: Test WER
          type: wer
-         value: ??
+         value: 47.99
 ---
 
 # Wav2Vec2-Large-XLSR-53-rw
@@ -41,6 +41,7 @@ import torchaudio
 from datasets import load_dataset
 from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
 
+# WARNING! This will download and extract to use about 80GB on disk.
 test_dataset = load_dataset("common_voice", "rw", split="test[:2%]") 
 
 processor = Wav2Vec2Processor.from_pretrained("lucio/wav2vec2-large-xlsr-kinyarwanda") 
@@ -70,10 +71,11 @@ print("Reference:", test_dataset["sentence"][:2])
 
 ## Evaluation
 
-The model can be evaluated as follows on the Kinyarwanda test data of Common Voice.
+The model can be evaluated as follows on the Kinyarwanda test data of Common Voice. Note that to even load the test data, the whole 40GB Kinyarwanda dataset will be downloaded and extracted into another 40GB directory, so you will need that space available on disk (e.g. not possible in the free tier of Google Colab). This script uses the `chunked_wer` function from [pcuenq](https://huggingface.co/pcuenq/wav2vec2-large-xlsr-53-es).
 
 
 ```python
+import jiwer
 import torch
 import torchaudio
 from datasets import load_dataset, load_metric
@@ -87,7 +89,7 @@ processor = Wav2Vec2Processor.from_pretrained("lucio/wav2vec2-large-xlsr-kinyarw
 model = Wav2Vec2ForCTC.from_pretrained("lucio/wav2vec2-large-xlsr-kinyarwanda")
 model.to("cuda")
 
-chars_to_ignore_regex = '[\[\],?.!;:%\'"‘’“”(){}‟ˮ´ʺ″«»/…‽�–-]'
+chars_to_ignore_regex = '[\\[\\],?.!;:%\\'"‘’“”(){}‟ˮ´ʺ″«»/…‽�–-]'
 resampler = torchaudio.transforms.Resample(48_000, 16_000)
 
 # Preprocessing the datasets.
@@ -98,7 +100,7 @@ def speech_file_to_array_fn(batch):
     batch["speech"] = resampler(speech_array).squeeze().numpy()
     return batch
 
-test_dataset = test_dataset.map(speech_file_to_array_fn)
+test_dataset = test_dataset.map(speech_file_to_array_fn, remove_columns=['path'])
 
 # Preprocessing the datasets.
 # We need to read the audio files as arrays
@@ -114,13 +116,28 @@ def evaluate(batch):
 
 result = test_dataset.map(evaluate, batched=True, batch_size=8)
 
-print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
+def chunked_wer(targets, predictions, chunk_size=None):                                          
+    if chunk_size is None: return jiwer.wer(targets, predictions)                                
+    start = 0                                                                                    
+    end = chunk_size                                                                             
+    H, S, D, I = 0, 0, 0, 0                                                                      
+    while start < len(targets):                                                                  
+        chunk_metrics = jiwer.compute_measures(targets[start:end], predictions[start:end])       
+        H = H + chunk_metrics["hits"]                                                            
+        S = S + chunk_metrics["substitutions"]                                                   
+        D = D + chunk_metrics["deletions"]                                                       
+        I = I + chunk_metrics["insertions"]                                                      
+        start += chunk_size                                                                      
+        end += chunk_size                                                                        
+    return float(S + D + I) / float(H + S + D)
+
+print("WER: {:2f}".format(100 * chunked_wer(result["sentence"], result["pred_strings"], chunk_size=4000)))
 ```
 
-**Test Result**: ?? % 
+**Test Result**: 47.99 % 
 
 ## Training
 
 The Common Voice `validation` dataset was used for training, with 12% of the test dataset used for validation, trained on 1 V100 GPU for 48 hours (20 epochs).
 
-The script used for training was just the `run_finetuning.py` script provided in OVHcloud's databuzzword/hf-wav2vec image.
+The script used for training was just the `run_finetuning.py` script provided in OVHcloud's `databuzzword/hf-wav2vec` image.