Saving train state of step 50000

checkpoint-50000-epoch-3/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09c35b07606e97495a37616ed994df43889eb4a598bc9c960d376b454e7d394f
+size 3652769047

checkpoint-50000-epoch-3/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc60973ca81c4e6978d54a7a38afe8b605630b7e92e8b6ba71ce2ccd7f647ca3
+size 2605239710

checkpoint-50000-epoch-3/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3932b085d086fdd7cda745ea5df5da9de938dd02afb55c6d8c4ea852d0e007
+size 16100

checkpoint-50000-epoch-3/random_states_1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d069517a6ab2bc4e3db00708c5709cf8f82627bfb66e373187266da80434e03d
+size 16100

checkpoint-50000-epoch-3/random_states_2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cc8acd2f855e9e41b27844d52e03888930c309dd4a91cec294b211d96da5d9f
+size 16100

checkpoint-50000-epoch-3/random_states_3.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f69188709da17af83e3184e136e61fb2ebe97237ba7fab6e70c2fa41f6d4a223
+size 16100

checkpoint-50000-epoch-3/random_states_4.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f746cbf158872f2299407683e74fb90936fa3a4f250b8a34aa1aeaf6029b7ac
+size 16100

checkpoint-50000-epoch-3/random_states_5.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f9914ec22970a5b5c485e5d27c7e6f9f666fd0e6380fbaf59419527e4259f81
+size 16100

checkpoint-50000-epoch-3/random_states_6.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87e40ce9aab0771cb31fe95c7514c08ef76b2943053aba2836aea1fbbed10e28
+size 16100

checkpoint-50000-epoch-3/random_states_7.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d4562fde2759a668f94d65ab8fd4671e5e6c655f2461ad561a6bff168593b32
+size 16100

checkpoint-50000-epoch-3/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:585a68aa005b84644c654a9d79b3a7ee935259f1376345794bbe5821b4f7ac85
+size 1000

starting_point_0.01.json

@@ -10,7 +10,7 @@
     "prompt_tokenizer_name":"google/flan-t5-base",
 
     "report_to": ["wandb"],
-    "overwrite_output_dir": true,
+    "overwrite_output_dir": false,
     "output_dir": "./",
 
     "train_dataset_name": "blabble-io/libritts_r+blabble-io/libritts_r+blabble-io/libritts_r+parler-tts/mls_eng_10k",

training/eval.py

@@ -1,6 +1,6 @@
 import torch
 import evaluate
-from transformers import AutoModel, AutoProcessor, pipeline
+from transformers import AutoModel, AutoProcessor, pipeline, WhisperForConditionalGeneration, WhisperTokenizer, WhisperTokenizerFast
 
 
 def clap_similarity(clap_model_name_or_path, texts, audios, device):
@@ -24,13 +24,36 @@ def clap_similarity(clap_model_name_or_path, texts, audios, device):
 def wer(asr_model_name_or_path, prompts, audios, device, per_device_eval_batch_size, sampling_rate):
     metric = evaluate.load("wer")
     asr_pipeline = pipeline(model=asr_model_name_or_path, device=device)
+
+    return_language = None
+    if isinstance(asr_pipeline.model, WhisperForConditionalGeneration):
+        return_language = True
+
     transcriptions = asr_pipeline(
         [{"raw": audio, "sampling_rate": sampling_rate} for audio in audios],
         batch_size=int(per_device_eval_batch_size),
+        return_language=return_language,
     )
 
-    word_error = 100 * metric.compute(
-        predictions=[t["text"].lower() for t in transcriptions], references=[t.lower() for t in prompts]
-    )
+    if isinstance(asr_pipeline.tokenizer, (WhisperTokenizer, WhisperTokenizerFast)):
+        tokenizer = asr_pipeline.tokenizer
+    else:
+        tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-large-v3")
+
+    english_normalizer = tokenizer.normalize
+    basic_normalizer = tokenizer.basic_normalize
+
+    normalized_predictions = []
+    normalized_references = []
+
+    for pred, ref in zip(transcriptions, prompts):
+        normalizer = english_normalizer if hasattr(pred, "language") and pred["language"] == "english" else basic_normalizer
+        norm_ref = normalizer(ref)
+        if len(norm_ref) > 0:
+            norm_pred = normalizer(pred["text"])
+            normalized_predictions.append(norm_pred)
+            normalized_references.append(norm_pred)
+
+    word_error = 100 * metric.compute(predictions=normalized_predictions, references=normalized_references)
 
     return word_error, [t["text"] for t in transcriptions]