shtpgshus commited on
Commit
507cf49
·
verified ·
1 Parent(s): 607bf62

Safe suffix removal

Browse files

Replace `rstrip()` to `removesuffix()` to safely remove suffix without unintended character stripping.

Files changed (1) hide show
  1. sample_finetune_speech.py +1 -1
sample_finetune_speech.py CHANGED
@@ -284,7 +284,7 @@ def evaluate(
284
  for _pred_ids, _stop_tokens_idx in zip(generated_ids, stop_tokens_idx)
285
  ]
286
  all_generated_texts.extend(generated_text)
287
- labels = [processor.decode(_label_ids[_label_ids != 0]).rstrip(ANSWER_SUFFIX) for _label_ids in inputs["labels"]]
288
  all_labels.extend(labels)
289
 
290
  all_generated_texts = gather_object(all_generated_texts)
 
284
  for _pred_ids, _stop_tokens_idx in zip(generated_ids, stop_tokens_idx)
285
  ]
286
  all_generated_texts.extend(generated_text)
287
+ labels = [processor.decode(_label_ids[_label_ids != 0]).removesuffix(ANSWER_SUFFIX) for _label_ids in inputs["labels"]]
288
  all_labels.extend(labels)
289
 
290
  all_generated_texts = gather_object(all_generated_texts)